From 4ff47f5b2e8b1ee6dff35016d4c91464c8d32f75 Mon Sep 17 00:00:00 2001 From: Tao He Date: Mon, 24 Apr 2023 18:00:39 +0800 Subject: [PATCH] Prefix etcd pod and services with the vineyardd name to avoid conflicts (#1339) See `k8s/pkg/templates/etcd/service.yaml` and `k8s/pkg/templates/etcd/etcd.yaml`. Refactor some unnecessary global functions and changed the default etcd replicas to 1 as well. --------- Signed-off-by: Tao He --- .../templates/deployment.yaml | 2 +- .../templates/serving-cert.yaml | 2 +- .../templates/sidecar-crd.yaml | 2 +- .../templates/vineyardd-crd.yaml | 2 +- docs/notes/cloud-native/deploy-kubernetes.rst | 11 ++-- docs/notes/cloud-native/vineyard-operator.rst | 2 +- k8s/Makefile | 42 ++++++------- k8s/apis/k8s/v1alpha1/sidecar_types.go | 2 +- k8s/apis/k8s/v1alpha1/vineyardd_types.go | 2 +- k8s/cmd/README.md | 12 ++-- k8s/cmd/commands/flags/vineyardd_flags.go | 2 +- k8s/cmd/commands/sidecar/inject.go | 16 ++--- k8s/cmd/commands/util/template.go | 2 +- k8s/config/certmanager/certificate.yaml | 2 +- k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml | 2 +- .../crd/bases/k8s.v6d.io_vineyardds.yaml | 2 +- k8s/config/default/manager_webhook_patch.yaml | 2 +- k8s/controllers/k8s/backup_controller.go | 53 +++++++--------- k8s/controllers/k8s/operation_controller.go | 3 +- k8s/controllers/k8s/recover_controller.go | 38 +++++------ k8s/controllers/k8s/sidecar_controller.go | 28 +++------ k8s/controllers/k8s/utils.go | 32 ++++++++++ k8s/controllers/k8s/vineyardd_controller.go | 63 ++++--------------- k8s/pkg/templates/backup/job.yaml | 2 +- k8s/pkg/templates/etcd/etcd.yaml | 19 +++--- k8s/pkg/templates/etcd/service.yaml | 7 ++- k8s/pkg/templates/recover/job.yaml | 4 +- k8s/pkg/templates/vineyardd/deployment.yaml | 1 + k8s/pkg/templates/vineyardd/etcd-service.yaml | 5 +- k8s/pkg/templates/vineyardd/service.yaml | 1 + src/server/memory/malloc.cc | 5 ++ src/server/services/meta_service.h | 16 +++-- 32 files changed, 192 insertions(+), 192 deletions(-) diff --git a/charts/vineyard-operator/templates/deployment.yaml b/charts/vineyard-operator/templates/deployment.yaml index bb3a5c600..3fe861cfd 100644 --- a/charts/vineyard-operator/templates/deployment.yaml +++ b/charts/vineyard-operator/templates/deployment.yaml @@ -86,4 +86,4 @@ spec: - name: cert secret: defaultMode: 420 - secretName: webhook-server-cert \ No newline at end of file + secretName: vineyard-operator-webhook-server-cert \ No newline at end of file diff --git a/charts/vineyard-operator/templates/serving-cert.yaml b/charts/vineyard-operator/templates/serving-cert.yaml index 2c1209625..d90dcfa8c 100644 --- a/charts/vineyard-operator/templates/serving-cert.yaml +++ b/charts/vineyard-operator/templates/serving-cert.yaml @@ -16,4 +16,4 @@ spec: issuerRef: kind: Issuer name: '{{ include "vineyard-operator.fullname" . }}-selfsigned-issuer' - secretName: webhook-server-cert \ No newline at end of file + secretName: vineyard-operator-webhook-server-cert \ No newline at end of file diff --git a/charts/vineyard-operator/templates/sidecar-crd.yaml b/charts/vineyard-operator/templates/sidecar-crd.yaml index 8b03a9570..82f942522 100644 --- a/charts/vineyard-operator/templates/sidecar-crd.yaml +++ b/charts/vineyard-operator/templates/sidecar-crd.yaml @@ -47,7 +47,7 @@ spec: spec: properties: etcdReplicas: - default: 3 + default: 1 type: integer metric: default: diff --git a/charts/vineyard-operator/templates/vineyardd-crd.yaml b/charts/vineyard-operator/templates/vineyardd-crd.yaml index 25f0303ea..5f2611d38 100644 --- a/charts/vineyard-operator/templates/vineyardd-crd.yaml +++ b/charts/vineyard-operator/templates/vineyardd-crd.yaml @@ -47,7 +47,7 @@ spec: spec: properties: etcdReplicas: - default: 3 + default: 1 type: integer metric: default: diff --git a/docs/notes/cloud-native/deploy-kubernetes.rst b/docs/notes/cloud-native/deploy-kubernetes.rst index b1a4ae98e..4dc624bcd 100644 --- a/docs/notes/cloud-native/deploy-kubernetes.rst +++ b/docs/notes/cloud-native/deploy-kubernetes.rst @@ -56,16 +56,17 @@ Option #2: Install form source code 3. Install the cert-manager -.. code:: bash + .. code:: bash - $ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml + $ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml -.. note:: + .. note:: - Please wait the cert-manager for a while until it is ready before installing the - vineyard operator. + Please wait the cert-manager for a while until it is ready before installing the + vineyard operator. 4. Next, deploy the vineyard operator: + .. code:: bash $ make -C k8s deploy diff --git a/docs/notes/cloud-native/vineyard-operator.rst b/docs/notes/cloud-native/vineyard-operator.rst index c8e3777de..c8abf4bd0 100644 --- a/docs/notes/cloud-native/vineyard-operator.rst +++ b/docs/notes/cloud-native/vineyard-operator.rst @@ -106,7 +106,7 @@ The detailed configuration entries for creating a vineyard cluster are listed as * - | etcdReplicas - int - The etcd replicas of vineyard - - 3 + - 1 * - | vineyard. | image diff --git a/k8s/Makefile b/k8s/Makefile index fa6ffc82f..7273c38e7 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -38,11 +38,11 @@ else GOBIN=$(shell go env GOBIN) endif -GOLINT = $(shell command -v ${GOBIN}/golangci-lint > /dev/null) -HELMIFY = $(shell command -v ${GOBIN}/helmify > /dev/null) -E2E = $(shell command -v ${GOBIN}/e2e > /dev/null) -KUSTOMIZE = $(shell command -v ${GOBIN}/kustomize > /dev/null) -CONTROLLER_GEN = $(shell command -v ${GOBIN}/controller-gen > /dev/null) +GOLINT = $(shell command -v $(GOBIN)/golangci-lint 2> /dev/null) +HELMIFY = $(shell command -v $(GOBIN)/helmify 2> /dev/null) +E2E = $(shell command -v $(GOBIN)/e2e 2> /dev/null) +KUSTOMIZE = $(shell command -v $(GOBIN)/kustomize 2> /dev/null) +CONTROLLER_GEN = $(shell command -v $(GOBIN)/controller-gen 2> /dev/null) ## Tool Versions KUSTOMIZE_VERSION ?= v4@v4.5.5 @@ -91,7 +91,7 @@ uninstall: manifests kustomize # Deploy controller in the configured Kubernetes cluster in ~/.kube/config .PHONY: deploy deploy: manifests kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) $(KUSTOMIZE) build config/default | kubectl apply -f - # Undeploy controller in the configured Kubernetes cluster in ~/.kube/config @@ -102,32 +102,32 @@ undeploy: kustomize # Undeploy controller and all created resources in the configured Kubernetes cluster in ~/.kube/config .PHONY: undeploy-all undeploy-all: kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) $(KUSTOMIZE) build config/default | kubectl delete -f - # Deploy namespace, CRDs, rbac, etc. except the controller in the configured Kubernetes cluster in ~/.kube/config .PHONY: predeploy predeploy: manifests kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) .PHONY: unpredeploy unpredeploy: manifests kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) # update the operator image's registry to localhost .PHONY: update-registry update-registry: kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) # recover the operator image's registry to vineyardcloudnative .PHONY: recover-registry recover-registry: kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) # dump the deployment configuration .PHONY: dry-run dry-run: manifests kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) $(KUSTOMIZE) build config/default > controller.yaml # Generate manifests e.g. CRD, RBAC etc. @@ -152,7 +152,7 @@ modules = ./apis/... \ # Run golangci-lint .PHONY: golint golint: - ifeq (${GOLINT},) + ifeq ($(GOLINT),) @echo "golangci-lint not exist, installing it..." go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) endif @@ -162,7 +162,7 @@ golint: # Install helmify if not exist .PHONY: helmify helmify: - ifeq (${HELMIFY},) + ifeq ($(HELMIFY),) @echo "helmify not exist, installing it..." go install github.com/arttor/helmify/cmd/helmify@$(HELMIFY_VERSION) endif @@ -172,7 +172,7 @@ helmify: # Install e2e if not exist .PHONY: e2e e2e: - ifeq (${E2E},) + ifeq ($(E2E),) @echo "e2e not exist, installing it..." go install github.com/apache/skywalking-infra-e2e/cmd/e2e@$(E2E_VERSION) endif @@ -210,24 +210,24 @@ vendor: .PHONY: docker-build docker-build: if docker build --help | grep -q load; then \ - docker build --load . -t ${IMG}; \ + docker build --load . -t $(IMG); \ else \ - docker build . -t ${IMG}; \ + docker build . -t $(IMG); \ fi docker-build-push-multi-arch: - docker buildx build . -t ${IMG} --platform linux/amd64,linux/arm64 --push + docker buildx build . -t $(IMG) --platform linux/amd64,linux/arm64 --push # Push the docker image .PHONY: docker-push docker-push: - docker push ${IMG} + docker push $(IMG) # find or download controller-gen # download controller-gen if necessary .PHONY: controller-gen controller-gen: - ifeq (${CONTROLLER_GEN},) + ifeq ($(CONTROLLER_GEN),) @{ \ set -e ;\ CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ @@ -241,7 +241,7 @@ controller-gen: .PHONY: kustomize kustomize: - ifeq (${KUSTOMIZE},) + ifeq ($(KUSTOMIZE),) @{ \ set -e ;\ KUSTOMIZE_GEN_TMP_DIR=$$(mktemp -d) ;\ diff --git a/k8s/apis/k8s/v1alpha1/sidecar_types.go b/k8s/apis/k8s/v1alpha1/sidecar_types.go index 5af136773..474d57f9d 100644 --- a/k8s/apis/k8s/v1alpha1/sidecar_types.go +++ b/k8s/apis/k8s/v1alpha1/sidecar_types.go @@ -34,7 +34,7 @@ type SidecarSpec struct { // EtcdReplicas describe the etcd replicas // +kubebuilder:validation:Optional - // +kubebuilder:default:=3 + // +kubebuilder:default:=1 EtcdReplicas int `json:"etcdReplicas,omitempty"` // vineyard container configuration diff --git a/k8s/apis/k8s/v1alpha1/vineyardd_types.go b/k8s/apis/k8s/v1alpha1/vineyardd_types.go index 968a2644e..4d6585753 100644 --- a/k8s/apis/k8s/v1alpha1/vineyardd_types.go +++ b/k8s/apis/k8s/v1alpha1/vineyardd_types.go @@ -194,7 +194,7 @@ type VineyarddSpec struct { // EtcdReplicas describe the etcd replicas // +kubebuilder:validation:Optional - // +kubebuilder:default:=3 + // +kubebuilder:default:=1 EtcdReplicas int `json:"etcdReplicas,omitempty"` // vineyardd's service diff --git a/k8s/cmd/README.md b/k8s/cmd/README.md index 2916bd81e..54fae9bbe 100644 --- a/k8s/cmd/README.md +++ b/k8s/cmd/README.md @@ -715,7 +715,7 @@ vineyardctl deploy vineyard-deployment [flags] --pluginImage.distributedAssemblyImage string the distributed image of vineyard workflow (default "ghcr.io/v6d-io/v6d/distributed-assembly") --pluginImage.localAssemblyImage string the local assembly image of vineyardd workflow (default "ghcr.io/v6d-io/v6d/local-assembly") --pluginImage.recoverImage string the recover image of vineyardd (default "ghcr.io/v6d-io/v6d/recover-job") - --vineyard.etcd.replicas int the number of etcd replicas in a vineyard cluster (default 3) + --vineyard.etcd.replicas int the number of etcd replicas in a vineyard cluster (default 1) --vineyard.replicas int the number of vineyardd replicas (default 3) --vineyardd.envs strings The environment variables of vineyardd --vineyardd.image string the image of vineyardd (default "vineyardcloudnative/vineyardd:latest") @@ -845,7 +845,7 @@ vineyardctl deploy vineyardd [flags] --pluginImage.distributedAssemblyImage string the distributed image of vineyard workflow (default "ghcr.io/v6d-io/v6d/distributed-assembly") --pluginImage.localAssemblyImage string the local assembly image of vineyardd workflow (default "ghcr.io/v6d-io/v6d/local-assembly") --pluginImage.recoverImage string the recover image of vineyardd (default "ghcr.io/v6d-io/v6d/recover-job") - --vineyard.etcd.replicas int the number of etcd replicas in a vineyard cluster (default 3) + --vineyard.etcd.replicas int the number of etcd replicas in a vineyard cluster (default 1) --vineyard.replicas int the number of vineyardd replicas (default 3) --vineyardd.envs strings The environment variables of vineyardd --vineyardd.image string the image of vineyardd (default "vineyardcloudnative/vineyardd:latest") @@ -875,7 +875,7 @@ Inject the vineyard sidecar container into a workload ### Synopsis Inject the vineyard sidecar container into a workload. You can -input a workload yaml or a workload json and then get the injected +input a workload yaml or a workload json and then get the injected workload and some etcd manifests from the output. The output is a set of manifests that includes the injected workload, @@ -884,6 +884,7 @@ pods and services). Next, we will introduce a simple example to show the injection. Assume you have the following workload yaml: + ```yaml apiVersion: apps/v1 kind: Deployment @@ -906,12 +907,14 @@ spec: ports: - containerPort: 80 ``` + Then, you can use the following command to inject the vineyard sidecar $ vineyardctl inject -f workload.yaml --apply-resources After running the command, the main output(removed some unnecessary fields) is as follows: + ```yaml apiVersion: apps/v1 kind: Deployment @@ -968,6 +971,7 @@ template: - emptyDir: {} name: vineyard-socket ``` + The sidecar template can be accessed from the following link: https://github.com/v6d-io/v6d/blob/main/k8s/pkg/templates/sidecar/injection-template.yaml also you can get some inspiration from the doc link: @@ -1007,7 +1011,7 @@ vineyardctl inject [flags] # inject the default vineyard sidecar container into a workload # output all injected manifests and then deploy them vineyardctl inject -f workload.yaml | kubectl apply -f - - + # if you only want to get the injected workload yaml rather than # all manifests that includes the etcd cluster and the rpc service, # you can enable the apply-resources and then the manifests will be diff --git a/k8s/cmd/commands/flags/vineyardd_flags.go b/k8s/cmd/commands/flags/vineyardd_flags.go index bd62789fb..9de088709 100644 --- a/k8s/cmd/commands/flags/vineyardd_flags.go +++ b/k8s/cmd/commands/flags/vineyardd_flags.go @@ -139,7 +139,7 @@ func ApplyVineyarddOpts(cmd *cobra.Command) { cmd.Flags().IntVarP(&VineyarddOpts.Replicas, "vineyard.replicas", "", 3, "the number of vineyardd replicas") cmd.Flags().IntVarP(&VineyarddOpts.EtcdReplicas, "vineyard.etcd.replicas", - "", 3, "the number of etcd replicas in a vineyard cluster") + "", 1, "the number of etcd replicas in a vineyard cluster") cmd.Flags().StringVarP(&VineyarddFile, "file", "f", "", "the path of vineyardd") // setup the vineyardd name ApplyVineyarddNameOpts(cmd) diff --git a/k8s/cmd/commands/sidecar/inject.go b/k8s/cmd/commands/sidecar/inject.go index e8fc9add7..322cf71c3 100644 --- a/k8s/cmd/commands/sidecar/inject.go +++ b/k8s/cmd/commands/sidecar/inject.go @@ -71,16 +71,16 @@ var ( injectLong = util.LongDesc(` Inject the vineyard sidecar container into a workload. You can - input a workload yaml or a workload json and then get the injected + input a workload yaml or a workload json and then get the injected workload and some etcd manifests from the output. - + The output is a set of manifests that includes the injected workload, the rpc service, the etcd service and the etcd cluster(e.g. several pods and services). Next, we will introduce a simple example to show the injection. Assume you have the following workload yaml:` + - "\n```yaml" + ` + "\n\n```yaml" + ` apiVersion: apps/v1 kind: Deployment metadata: @@ -102,13 +102,14 @@ var ( ports: - containerPort: 80` + "\n```" + ` + Then, you can use the following command to inject the vineyard sidecar - + $ vineyardctl inject -f workload.yaml --apply-resources - + After running the command, the main output(removed some unnecessary fields) is as follows:` + - "\n```yaml" + ` + "\n\n```yaml" + ` apiVersion: apps/v1 kind: Deployment metadata: @@ -164,6 +165,7 @@ var ( - emptyDir: {} name: vineyard-socket` + "\n```" + ` + The sidecar template can be accessed from the following link: https://github.com/v6d-io/v6d/blob/main/k8s/pkg/templates/sidecar/injection-template.yaml also you can get some inspiration from the doc link: @@ -193,7 +195,7 @@ var ( # inject the default vineyard sidecar container into a workload # output all injected manifests and then deploy them vineyardctl inject -f workload.yaml | kubectl apply -f - - + # if you only want to get the injected workload yaml rather than # all manifests that includes the etcd cluster and the rpc service, # you can enable the apply-resources and then the manifests will be diff --git a/k8s/cmd/commands/util/template.go b/k8s/cmd/commands/util/template.go index 83677371e..da4a3ba66 100644 --- a/k8s/cmd/commands/util/template.go +++ b/k8s/cmd/commands/util/template.go @@ -59,7 +59,7 @@ func BuildObjsFromEtcdManifests(EtcdConfig *k8s.EtcdConfig, name string, return podObjs, svcObjs, errors.Wrap(err, "failed to get etcd manifests") } // set up the etcd config - *EtcdConfig = k8s.BuildEtcdConfig(name, namespace, replicas, image) + *EtcdConfig = k8s.NewEtcdConfig(name, namespace, replicas, image) for i := 0; i < replicas; i++ { EtcdConfig.Rank = i diff --git a/k8s/config/certmanager/certificate.yaml b/k8s/config/certmanager/certificate.yaml index d361f623c..9ae9e45b3 100644 --- a/k8s/config/certmanager/certificate.yaml +++ b/k8s/config/certmanager/certificate.yaml @@ -23,4 +23,4 @@ spec: issuerRef: kind: Issuer name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize + secretName: vineyard-operator-webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml b/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml index 03bd80e4b..3155ce9be 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml @@ -35,7 +35,7 @@ spec: spec: properties: etcdReplicas: - default: 3 + default: 1 type: integer metric: default: diff --git a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml index acadd8fbd..bcf721fcb 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml @@ -35,7 +35,7 @@ spec: spec: properties: etcdReplicas: - default: 3 + default: 1 type: integer metric: default: diff --git a/k8s/config/default/manager_webhook_patch.yaml b/k8s/config/default/manager_webhook_patch.yaml index 738de350b..e18c6bb0d 100644 --- a/k8s/config/default/manager_webhook_patch.yaml +++ b/k8s/config/default/manager_webhook_patch.yaml @@ -20,4 +20,4 @@ spec: - name: cert secret: defaultMode: 420 - secretName: webhook-server-cert + secretName: vineyard-operator-webhook-server-cert diff --git a/k8s/controllers/k8s/backup_controller.go b/k8s/controllers/k8s/backup_controller.go index 057fbe3e5..6801d3408 100644 --- a/k8s/controllers/k8s/backup_controller.go +++ b/k8s/controllers/k8s/backup_controller.go @@ -47,7 +47,7 @@ type FailoverConfig struct { VineyarddName string Endpoint string VineyardSockPath string - Allinstances string + AllInstances string } // BackupConfig holds all configuration about backup @@ -57,18 +57,6 @@ type BackupConfig struct { FailoverConfig } -// Backup contains the configuration about backup -var Backup BackupConfig - -// GetBackupConfig get backup configuratiin from Backup -func getBackupConfig() BackupConfig { - return Backup -} - -func getResourceStorage(q resource.Quantity) string { - return q.String() -} - // BackupReconciler reconciles a Backup object type BackupReconciler struct { client.Client @@ -96,27 +84,32 @@ func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } logger.Info("Reconciling Backup", "backup", backup) + backupCfg := BackupConfig{} + backupCfg.Name = "backup-" + backup.Spec.VineyarddName + "-" + backup.Spec.VineyarddNamespace + backupCfg.Limit = strconv.Itoa(backup.Spec.Limit) + failoverCfg, err := newFailoverConfig(r.Client, &backup) + if err != nil { + logger.Error(err, "unable to build failover config") + return ctrl.Result{}, err + } + backupCfg.FailoverConfig = failoverCfg + app := kubernetes.Application{ Client: r.Client, FileRepo: templates.Repo, CR: &backup, GVK: k8sv1alpha1.GroupVersion.WithKind("Backup"), TmplFunc: map[string]interface{}{ - "getResourceStorage": getResourceStorage, - "getBackupConfig": getBackupConfig, + "getResourceStorage": func(q resource.Quantity) string { + return q.String() + }, + "getBackupConfig": func() BackupConfig { + return backupCfg + }, }, Recorder: r.EventRecorder, } - Backup.Name = "backup-" + backup.Spec.VineyarddName + "-" + backup.Spec.VineyarddNamespace - Backup.Limit = strconv.Itoa(backup.Spec.Limit) - config, err := BuildFailoverConfig(r.Client, &backup) - if err != nil { - logger.Error(err, "unable to build failover config") - return ctrl.Result{}, err - } - Backup.FailoverConfig = config - if backup.Status.State == "" || backup.Status.State == RunningState { if _, err := app.Apply(ctx, "backup/job.yaml", logger, false); err != nil { logger.Error(err, "failed to apply backup job") @@ -183,15 +176,17 @@ func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -// BuildFailoverConfig builds the failover config from the backup -func BuildFailoverConfig(c client.Client, backup *k8sv1alpha1.Backup) (FailoverConfig, error) { +// newFailoverConfig builds the failover config from the backup +func newFailoverConfig(c client.Client, backup *k8sv1alpha1.Backup) (FailoverConfig, error) { failoverConfig := FailoverConfig{} // get vineyardd vineyarddNamespace := backup.Spec.VineyarddNamespace vineyarddName := backup.Spec.VineyarddName vineyardd := &k8sv1alpha1.Vineyardd{} - if err := c.Get(context.Background(), client.ObjectKey{Namespace: vineyarddNamespace, - Name: vineyarddName}, vineyardd); err != nil { + if err := c.Get(context.Background(), client.ObjectKey{ + Namespace: vineyarddNamespace, + Name: vineyarddName, + }, vineyardd); err != nil { return failoverConfig, errors.Wrap(err, "unable to fetch Vineyardd") } @@ -212,7 +207,7 @@ func BuildFailoverConfig(c client.Client, backup *k8sv1alpha1.Backup) (FailoverC return failoverConfig, errors.Wrap(err, "unable to resolve vineyardd socket") } failoverConfig.VineyardSockPath = socket - failoverConfig.Allinstances = strconv.Itoa(vineyardd.Spec.Replicas) + failoverConfig.AllInstances = strconv.Itoa(vineyardd.Spec.Replicas) return failoverConfig, nil } diff --git a/k8s/controllers/k8s/operation_controller.go b/k8s/controllers/k8s/operation_controller.go index e1ee77ec2..c16b3edf1 100644 --- a/k8s/controllers/k8s/operation_controller.go +++ b/k8s/controllers/k8s/operation_controller.go @@ -80,7 +80,8 @@ func (r *OperationReconciler) Reconcile( GVK: k8sv1alpha1.GroupVersion.WithKind("Operation"), TmplFunc: map[string]interface{}{ "getDistributedAssemblyConfig": operation.GetDistributedAssemblyConfig, - "getAssemblyConfig": operation.GetAssemblyConfig, "getDaskRepartitionConfig": operation.GetDaskRepartitionConfig, + "getAssemblyConfig": operation.GetAssemblyConfig, + "getDaskRepartitionConfig": operation.GetDaskRepartitionConfig, }, Recorder: r.EventRecorder, } diff --git a/k8s/controllers/k8s/recover_controller.go b/k8s/controllers/k8s/recover_controller.go index 93fbf3b3b..c56de877c 100644 --- a/k8s/controllers/k8s/recover_controller.go +++ b/k8s/controllers/k8s/recover_controller.go @@ -62,14 +62,6 @@ type RecoverConfig struct { FailoverConfig } -// Recover contains the configuration about recover -var Recover RecoverConfig - -// getRecoverConfig get recover configuratiin from Recover -func getRecoverConfig() RecoverConfig { - return Recover -} - // +kubebuilder:rbac:groups=k8s.v6d.io,resources=recovers,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8s.v6d.io,resources=recovers/status,verbs=get;update;patch // +kubebuilder:rbac:groups=k8s.v6d.io,resources=vineyardds,verbs=get;list;watch;create;update;patch;delete @@ -93,29 +85,33 @@ func (r *RecoverReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } logger.Info("Reconciling Recover", "recover", recover) - app := swckkube.Application{ - Client: r.Client, - FileRepo: templates.Repo, - CR: &recover, - GVK: k8sv1alpha1.GroupVersion.WithKind("Recover"), - TmplFunc: map[string]interface{}{"getRecoverConfig": getRecoverConfig}, - } - backup := k8sv1alpha1.Backup{} - if err := r.Get(ctx, client.ObjectKey{Namespace: recover.Spec.BackupNamespace, Name: recover.Spec.BackupName}, &backup); err != nil { + if err := r.Get(ctx, client.ObjectKey{ + Namespace: recover.Spec.BackupNamespace, + Name: recover.Spec.BackupName, + }, &backup); err != nil { logger.Error(err, "unable to get Backup") return ctrl.Result{}, client.IgnoreNotFound(err) } // setup the recover configuration - Recover.Name = "recover-" + backup.Name - Recover.BackupPVCName = backup.Name - config, err := BuildFailoverConfig(r.Client, &backup) + recoverCfg := RecoverConfig{} + recoverCfg.Name = "recover-" + backup.Name + recoverCfg.BackupPVCName = backup.Name + failoverCfg, err := newFailoverConfig(r.Client, &backup) if err != nil { logger.Error(err, "unable to build failover configuration") return ctrl.Result{}, err } - Recover.FailoverConfig = config + recoverCfg.FailoverConfig = failoverCfg + + app := swckkube.Application{ + Client: r.Client, + FileRepo: templates.Repo, + CR: &recover, + GVK: k8sv1alpha1.GroupVersion.WithKind("Recover"), + TmplFunc: map[string]interface{}{"getRecoverConfig": func() RecoverConfig { return recoverCfg }}, + } if recover.Status.State == "" || recover.Status.State == RunningState { if _, err := app.Apply(ctx, "recover/job.yaml", logger, false); err != nil { diff --git a/k8s/controllers/k8s/sidecar_controller.go b/k8s/controllers/k8s/sidecar_controller.go index 3f9541f44..a6b5762db 100644 --- a/k8s/controllers/k8s/sidecar_controller.go +++ b/k8s/controllers/k8s/sidecar_controller.go @@ -36,14 +36,6 @@ import ( "github.com/v6d-io/v6d/k8s/pkg/templates" ) -// SidecarEtcd contains the configuration about etcd of sidecar container -var SidecarEtcd EtcdConfig - -// getSidecarEtcdConfig get etcd configuratiin from Etcd -func getSidecarEtcdConfig() EtcdConfig { - return SidecarEtcd -} - // SidecarReconciler reconciles a Sidecar object type SidecarReconciler struct { client.Client @@ -74,17 +66,17 @@ func (r *SidecarReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct CR: sidecar, GVK: k8sv1alpha1.GroupVersion.WithKind("Sidecar"), Recorder: r.EventRecorder, - TmplFunc: map[string]interface{}{ - "getEtcdConfig": getSidecarEtcdConfig, - }, + TmplFunc: map[string]interface{}{"getEtcdConfig": nil}, } - // setup the etcd configuration - etcdReplicas := sidecar.Spec.EtcdReplicas - SidecarEtcd = BuildEtcdConfig(sidecar.Name, sidecar.Namespace, - etcdReplicas, sidecar.Spec.Vineyard.Image) - - for i := 0; i < etcdReplicas; i++ { - SidecarEtcd.Rank = i + // setup the etcdCfg configuration + etcdCfg := NewEtcdConfig(sidecar.Name, sidecar.Namespace, + sidecar.Spec.EtcdReplicas, sidecar.Spec.Vineyard.Image) + + for rank := 0; rank < sidecar.Spec.EtcdReplicas; rank++ { + sidecarApp.TmplFunc["getEtcdConfig"] = func() EtcdConfig { + etcdCfg.Rank = rank + return etcdCfg + } if _, err := sidecarApp.Apply(ctx, "etcd/etcd.yaml", logger, false); err != nil { logger.Error(err, "failed to apply etcd pod") return ctrl.Result{}, err diff --git a/k8s/controllers/k8s/utils.go b/k8s/controllers/k8s/utils.go index 0c9961ec4..c4b1c9bd0 100644 --- a/k8s/controllers/k8s/utils.go +++ b/k8s/controllers/k8s/utils.go @@ -18,6 +18,8 @@ package k8s import ( "crypto/rand" "fmt" + "strconv" + "strings" "github.com/v6d-io/v6d/k8s/pkg/log" ) @@ -29,3 +31,33 @@ func GenerateRandomName(length int) string { } return fmt.Sprintf("%x", bs)[:length] } + +// EtcdConfig holds all configuration about etcd +type EtcdConfig struct { + Name string + Namespace string + Rank int + Endpoints string + Image string +} + +// NewEtcdConfig builds the etcd config. +func NewEtcdConfig(name string, namespace string, + replicas int, image string, +) EtcdConfig { + etcdConfig := EtcdConfig{} + // the etcd is built in the vineyardd image + etcdConfig.Name = name + etcdConfig.Image = image + etcdConfig.Namespace = namespace + etcdEndpoints := make([]string, 0, replicas) + for i := 0; i < replicas; i++ { + etcdName := fmt.Sprintf("%v-etcd-%v", name, strconv.Itoa(i)) + etcdEndpoints = append( + etcdEndpoints, + fmt.Sprintf("%v=http://%v:2380", etcdName, etcdName), + ) + } + etcdConfig.Endpoints = strings.Join(etcdEndpoints, ",") + return etcdConfig +} diff --git a/k8s/controllers/k8s/vineyardd_controller.go b/k8s/controllers/k8s/vineyardd_controller.go index 47fbb8329..613617c99 100644 --- a/k8s/controllers/k8s/vineyardd_controller.go +++ b/k8s/controllers/k8s/vineyardd_controller.go @@ -18,9 +18,6 @@ package k8s import ( "context" - "fmt" - "strconv" - "strings" "time" "github.com/pkg/errors" @@ -46,27 +43,6 @@ type VineyarddReconciler struct { Scheme *runtime.Scheme } -// EtcdConfig holds all configuration about etcd -type EtcdConfig struct { - Name string - Namespace string - Rank int - Endpoints string - Image string -} - -// Etcd contains the configuration about etcd -var Etcd EtcdConfig - -// GetEtcdConfig get etcd configuration from Etcd -func getEtcdConfig() EtcdConfig { - return Etcd -} - -func getStorage(q resource.Quantity) string { - return q.String() -} - // +kubebuilder:rbac:groups=k8s.v6d.io,resources=vineyardds,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8s.v6d.io,resources=vineyardds/status,verbs=get;update;patch // +kubebuilder:rbac:groups=k8s.v6d.io,resources=vineyardds/finalizers,verbs=update @@ -109,7 +85,9 @@ func (r *VineyarddReconciler) Reconcile( GVK: k8sv1alpha1.GroupVersion.WithKind("Vineyardd"), Recorder: r.EventRecorder, TmplFunc: map[string]interface{}{ - "getStorage": getStorage, + "getStorage": func(q resource.Quantity) string { + return q.String() + }, }, } etcdApp := kubernetes.Application{ @@ -117,16 +95,18 @@ func (r *VineyarddReconciler) Reconcile( FileRepo: templates.Repo, CR: &vineyardd, GVK: k8sv1alpha1.GroupVersion.WithKind("Vineyardd"), - TmplFunc: map[string]interface{}{"getEtcdConfig": getEtcdConfig}, + TmplFunc: map[string]interface{}{"getEtcdConfig": nil}, } - // set up the etcd configuration - etcdReplicas := vineyardd.Spec.EtcdReplicas - Etcd = BuildEtcdConfig(vineyardd.Name, vineyardd.Namespace, - etcdReplicas, vineyardd.Spec.Vineyard.Image) + // set up the etcdCfg configuration + etcdCfg := NewEtcdConfig(vineyardd.Name, vineyardd.Namespace, + vineyardd.Spec.EtcdReplicas, vineyardd.Spec.Vineyard.Image) - for i := 0; i < etcdReplicas; i++ { - Etcd.Rank = i + for rank := 0; rank < vineyardd.Spec.EtcdReplicas; rank++ { + etcdApp.TmplFunc["getEtcdConfig"] = func() EtcdConfig { + etcdCfg.Rank = rank + return etcdCfg + } if _, err := etcdApp.Apply(ctx, "etcd/etcd.yaml", logger, true); err != nil { logger.Error(err, "failed to apply etcd pod") return ctrl.Result{}, err @@ -191,22 +171,3 @@ func (r *VineyarddReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&k8sv1alpha1.Vineyardd{}). Complete(r) } - -// BuildEtcdConfig builds the etcd config. -func BuildEtcdConfig(name string, namespace string, - replicas int, image string) EtcdConfig { - etcdConfig := EtcdConfig{} - // the etcd is built in the vineyardd image - etcdConfig.Name = name - etcdConfig.Image = image - etcdConfig.Namespace = namespace - etcdEndpoints := make([]string, 0, replicas) - for i := 0; i < replicas; i++ { - etcdEndpoints = append( - etcdEndpoints, - fmt.Sprintf("etcd%v=http://etcd%v:2380", strconv.Itoa(i), strconv.Itoa(i)), - ) - } - etcdConfig.Endpoints = strings.Join(etcdEndpoints, ",") - return etcdConfig -} diff --git a/k8s/pkg/templates/backup/job.yaml b/k8s/pkg/templates/backup/job.yaml index b3b5d7705..7307a94d5 100644 --- a/k8s/pkg/templates/backup/job.yaml +++ b/k8s/pkg/templates/backup/job.yaml @@ -46,7 +46,7 @@ spec: - name: SELECTOR value: {{ .Name }} - name: ALLINSTANCES - value: "{{ $failoverConfig.Allinstances }}" + value: "{{ $failoverConfig.AllInstances }}" - name: POD_NAME valueFrom: fieldRef: diff --git a/k8s/pkg/templates/etcd/etcd.yaml b/k8s/pkg/templates/etcd/etcd.yaml index adfcbef15..ba05c0011 100644 --- a/k8s/pkg/templates/etcd/etcd.yaml +++ b/k8s/pkg/templates/etcd/etcd.yaml @@ -18,30 +18,31 @@ apiVersion: v1 kind: Pod metadata: labels: + app.vineyard.io/role: etcd app.vineyard.io/name: {{ $etcd.Name }} - etcd_node: etcd {{- $etcd.Rank }} - name: etcd {{- $etcd.Rank }} + etcd_node: {{ $etcd.Name }}-etcd-{{ $etcd.Rank }} + name: {{ $etcd.Name }}-etcd-{{ $etcd.Rank }} namespace: {{ $etcd.Namespace }} spec: containers: - - command: + - name: etcd + image: {{ $etcd.Image }} + command: - etcd - --name - - etcd {{- $etcd.Rank}} + - {{ $etcd.Name }}-etcd-{{ $etcd.Rank }} - --initial-advertise-peer-urls - - http://etcd{{- $etcd.Rank -}} :2380 + - http://{{ $etcd.Name }}-etcd-{{ $etcd.Rank }}:2380 + - --advertise-client-urls + - http://{{ $etcd.Name }}-etcd-{{ $etcd.Rank }}:2379 - --listen-peer-urls - http://0.0.0.0:2380 - --listen-client-urls - http://0.0.0.0:2379 - - --advertise-client-urls - - http://etcd{{- $etcd.Rank}}:2379 - --initial-cluster - {{ $etcd.Endpoints }} - --initial-cluster-state - new - image: {{ $etcd.Image }} - name: etcd ports: - containerPort: 2379 name: client diff --git a/k8s/pkg/templates/etcd/service.yaml b/k8s/pkg/templates/etcd/service.yaml index be2680a4a..86c999b1a 100644 --- a/k8s/pkg/templates/etcd/service.yaml +++ b/k8s/pkg/templates/etcd/service.yaml @@ -18,8 +18,8 @@ apiVersion: v1 kind: Service metadata: labels: - etcd_node: etcd {{- $etcd.Rank }} - name: etcd {{- $etcd.Rank }} + etcd_node: {{ $etcd.Name }}-etcd-{{ $etcd.Rank }} + name: {{ $etcd.Name }}-etcd-{{ $etcd.Rank }} namespace: {{ $etcd.Namespace }} spec: ports: @@ -32,4 +32,5 @@ spec: protocol: TCP targetPort: 2380 selector: - etcd_node: etcd {{- $etcd.Rank }} + app.vineyard.io/role: etcd + etcd_node: {{ $etcd.Name }}-etcd-{{ $etcd.Rank }} diff --git a/k8s/pkg/templates/recover/job.yaml b/k8s/pkg/templates/recover/job.yaml index 9ddd64d08..7d935f19b 100644 --- a/k8s/pkg/templates/recover/job.yaml +++ b/k8s/pkg/templates/recover/job.yaml @@ -38,13 +38,13 @@ spec: imagePullPolicy: IfNotPresent env: - name: RECOVER_PATH - value: {{ $failoverConfig.Path }} + value: {{ $failoverConfig.Path }} - name: ENDPOINT value: {{ $failoverConfig.Endpoint }} - name: SELECTOR value: {{ .Name }} - name: ALLINSTANCES - value: "{{ $failoverConfig.Allinstances }}" + value: "{{ $failoverConfig.AllInstances }}" - name: POD_NAME valueFrom: fieldRef: diff --git a/k8s/pkg/templates/vineyardd/deployment.yaml b/k8s/pkg/templates/vineyardd/deployment.yaml index 4ae044332..e4da36bea 100644 --- a/k8s/pkg/templates/vineyardd/deployment.yaml +++ b/k8s/pkg/templates/vineyardd/deployment.yaml @@ -32,6 +32,7 @@ spec: template: metadata: labels: + app.vineyard.io/role: vineyardd app.vineyard.io/name: {{ .Name }} app.kubernetes.io/name: {{ .Name }} app.kubernetes.io/instance: {{ .Namespace -}} - {{- .Name }} diff --git a/k8s/pkg/templates/vineyardd/etcd-service.yaml b/k8s/pkg/templates/vineyardd/etcd-service.yaml index 5cee15484..f855c3095 100644 --- a/k8s/pkg/templates/vineyardd/etcd-service.yaml +++ b/k8s/pkg/templates/vineyardd/etcd-service.yaml @@ -20,9 +20,10 @@ metadata: namespace: {{ .Namespace }} spec: ports: - - name: etcd-for-vineyard-port + - name: {{ .Name }}-etcd-for-vineyard-port port: 2379 protocol: TCP targetPort: 2379 selector: - app.vineyard.io/name: {{ .Name }} \ No newline at end of file + app.vineyard.io/role: etcd + app.vineyard.io/name: {{ .Name }} diff --git a/k8s/pkg/templates/vineyardd/service.yaml b/k8s/pkg/templates/vineyardd/service.yaml index ad603355a..78bdb45ba 100644 --- a/k8s/pkg/templates/vineyardd/service.yaml +++ b/k8s/pkg/templates/vineyardd/service.yaml @@ -27,4 +27,5 @@ spec: protocol: TCP name: vineyard-rpc selector: + app.vineyard.io/role: vineyardd app.vineyard.io/name: {{ .Name }} diff --git a/src/server/memory/malloc.cc b/src/server/memory/malloc.cc index 8f379e7bc..90b79e90b 100644 --- a/src/server/memory/malloc.cc +++ b/src/server/memory/malloc.cc @@ -81,6 +81,11 @@ struct memfd_create_compat { if (handle_) { memfd_create_fn = (int (*)(const char*, unsigned int)) dlsym(handle_, "memfd_create"); + if (memfd_create_fn) { + LOG(INFO) << "Use memfd_create(2) for shared memory"; + } else { + LOG(INFO) << "Use mkstemp(3) for shared memory"; + } } } diff --git a/src/server/services/meta_service.h b/src/server/services/meta_service.h index 7593074ab..13bc125d9 100644 --- a/src/server/services/meta_service.h +++ b/src/server/services/meta_service.h @@ -706,7 +706,7 @@ class IMetaService : public std::enable_shared_from_this { // update instance status if (boost::algorithm::starts_with(op.kv.key, "/instances/")) { - instanceUpdate(op); + instanceUpdate(op, from_remote); } #ifndef NDEBUG @@ -822,7 +822,7 @@ class IMetaService : public std::enable_shared_from_this { VINEYARD_SUPPRESS(server_ptr_->ProcessDeferred(meta_)); } - void instanceUpdate(const op_t& op) { + void instanceUpdate(const op_t& op, const bool from_remote = true) { std::vector key_segments; boost::split(key_segments, op.kv.key, boost::is_any_of("/")); if (key_segments[0].empty()) { @@ -831,13 +831,19 @@ class IMetaService : public std::enable_shared_from_this { if (key_segments[2] == "hostid") { uint64_t instance_id = std::stoul(key_segments[1].substr(1)); if (op.op == op_t::op_type_t::kPut) { - LOG(INFO) << "Instance join: " << instance_id; + if (from_remote) { + LOG(INFO) << "Instance join: " << instance_id; + } instances_list_.emplace(instance_id); } else if (op.op == op_t::op_type_t::kDel) { - LOG(INFO) << "Instance exit: " << instance_id; + if (from_remote) { + LOG(INFO) << "Instance exit: " << instance_id; + } instances_list_.erase(instance_id); } else { - LOG(ERROR) << "Unknown op type: " << op.ToString(); + if (from_remote) { + LOG(ERROR) << "Unknown op type: " << op.ToString(); + } } LOG_SUMMARY("instances_total", "", instances_list_.size()); }