Add tests and docs

cybozu-go · Mar 20, 2024 · 065ee00 · 065ee00
1 parent 553d470
commit 065ee00
Show file tree

Hide file tree

Showing 16 changed files with 386 additions and 107 deletions.
diff --git a/charts/cattage/templates/generated.yaml b/charts/cattage/templates/generated.yaml
@@ -294,6 +294,26 @@ metadata:
     helm.sh/chart: '{{ include "cattage.chart" . }}'
   name: '{{ template "cattage.fullname" . }}-validating-webhook-configuration'
 webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ template "cattage.fullname" . }}-webhook-service'
+        namespace: '{{ .Release.Namespace }}'
+        path: /validate-argoproj-io-application
+    failurePolicy: Fail
+    name: vapplication.kb.io
+    rules:
+      - apiGroups:
+          - argoproj.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - applications
+    sideEffects: None
   - admissionReviewVersions:
       - v1
     clientConfig:

diff --git a/cmd/cattage-controller/sub/run.go b/cmd/cattage-controller/sub/run.go
@@ -82,6 +82,7 @@ func subMain(ns, addr string, port int) error {
 	}
 
 	hooks.SetupTenantWebhook(mgr, admission.NewDecoder(scheme), cfg)
+	hooks.SetupApplicationWebhook(mgr, admission.NewDecoder(scheme), cfg)
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

diff --git a/config/manager/configmap.yaml b/config/manager/configmap.yaml
@@ -65,3 +65,4 @@ data:
             {{- else }}
             - '*'
             {{- end }}
+      preventAppCreationInArgoCDNamespace: true
diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml
@@ -30,6 +30,26 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: validating-webhook-configuration
 webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-argoproj-io-application
+  failurePolicy: Fail
+  name: vapplication.kb.io
+  rules:
+  - apiGroups:
+    - argoproj.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - applications
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   clientConfig:

diff --git a/controllers/tenant_controller.go b/controllers/tenant_controller.go
@@ -613,7 +613,7 @@ func (r *TenantReconciler) updateConfigMap(ctx context.Context, controllerName s
 func (r *TenantReconciler) updateAllTenantNamespacesConfigMap(ctx context.Context) error {
 	logger := log.FromContext(ctx)
 
-	configMapName := "tenant-namespaces-cm"
+	configMapName := "all-tenant-namespaces-cm"
 	cm := &corev1.ConfigMap{}
 	cm.Name = configMapName
 	cm.Namespace = r.config.ArgoCD.Namespace

diff --git a/controllers/tenant_controller_test.go b/controllers/tenant_controller_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	_ "embed"
 	"errors"
+	"strings"
 	"time"
 
 	cattagev1beta1 "github.com/cybozu-go/cattage/api/v1beta1"
@@ -30,7 +31,7 @@ var appProjectTemplate string
 //go:embed testdata/rolebindingtemplate.yaml
 var roleBindingTemplate string
 
-var _ = Describe("Tenant controller", func() {
+var _ = Describe("Tenant controller", Ordered, func() {
 	ctx := context.Background()
 	var stopFunc func()
 	var config *tenantconfig.Config
@@ -61,8 +62,9 @@ var _ = Describe("Tenant controller", func() {
 				RoleBindingTemplate: roleBindingTemplate,
 			},
 			ArgoCD: tenantconfig.ArgoCDConfig{
-				Namespace:          "argocd",
-				AppProjectTemplate: appProjectTemplate,
+				Namespace:                           "argocd",
+				AppProjectTemplate:                  appProjectTemplate,
+				PreventAppCreationInArgoCDNamespace: true,
 			},
 		}
 		tr := NewTenantReconciler(mgr.GetClient(), config)
@@ -205,6 +207,60 @@ var _ = Describe("Tenant controller", func() {
 		}))
 	})
 
+	It("should create configmaps for sharding", func() {
+		allNsCm := &corev1.ConfigMap{}
+		defaultCm := &corev1.ConfigMap{}
+		Eventually(func(g Gomega) {
+			err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "all-tenant-namespaces-cm"}, allNsCm)
+			g.Expect(err).NotTo(HaveOccurred())
+			allNs := strings.Split(allNsCm.Data["application.namespaces"], ",")
+			g.Expect(allNs).Should(ConsistOf("app-x", "sub-4"))
+		}).Should(Succeed())
+
+		Eventually(func(g Gomega) {
+			err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "default-application-controller-cm"}, defaultCm)
+			g.Expect(err).NotTo(HaveOccurred())
+			defaultNs := strings.Split(defaultCm.Data["application.namespaces"], ",")
+			g.Expect(defaultNs).Should(ConsistOf("app-x", "sub-4"))
+		}).Should(Succeed())
+
+		tenantS := &cattagev1beta1.Tenant{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "a-team",
+			},
+			Spec: cattagev1beta1.TenantSpec{
+				RootNamespaces: []cattagev1beta1.RootNamespaceSpec{
+					{
+						Name: "app-a",
+					},
+				},
+				ControllerName: "second",
+			},
+		}
+		err := k8sClient.Create(ctx, tenantS)
+		Expect(err).ToNot(HaveOccurred())
+
+		secondCm := &corev1.ConfigMap{}
+		Eventually(func(g Gomega) {
+			err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "all-tenant-namespaces-cm"}, allNsCm)
+			g.Expect(err).NotTo(HaveOccurred())
+			allNs := strings.Split(allNsCm.Data["application.namespaces"], ",")
+			g.Expect(allNs).Should(ConsistOf("app-x", "sub-4", "app-a", "sub-1", "sub-2", "sub-3"))
+		}).Should(Succeed())
+		Eventually(func(g Gomega) {
+			err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "default-application-controller-cm"}, defaultCm)
+			g.Expect(err).NotTo(HaveOccurred())
+			defaultNs := strings.Split(defaultCm.Data["application.namespaces"], ",")
+			g.Expect(defaultNs).Should(ConsistOf("app-x", "sub-4"))
+		}).Should(Succeed())
+		Eventually(func(g Gomega) {
+			err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "second-application-controller-cm"}, secondCm)
+			g.Expect(err).NotTo(HaveOccurred())
+			secondNs := strings.Split(secondCm.Data["application.namespaces"], ",")
+			g.Expect(secondNs).Should(ConsistOf("app-a", "sub-1", "sub-2", "sub-3"))
+		}).Should(Succeed())
+	})
+
 	It("should disown root namespace", func() {
 		tenant := &cattagev1beta1.Tenant{
 			ObjectMeta: metav1.ObjectMeta{

diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
@@ -7,6 +7,7 @@
 - [Overview](overview.md)
 - [Setup](setup.md)
 - [Usage](usage.md)
+- [Sharding](sharding.md)
 
 # References
 

diff --git a/docs/config.md b/docs/config.md
@@ -7,13 +7,14 @@ The location can be changed with `--config-file` flag.
 
 The configuration file should be a JSON or YAML file having the following keys:
 
-| Key                             | Type                | Description                                                                                                                                      |
-|---------------------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|
-| `namespace.commonLabels`        | `map[string]string` | Labels to be added to all namespaces belonging to all tenants. This may be overridden by `rootNamespaces.labels` of a tenant resource.           |
-| `namespace.commonAnnotations`   | `map[string]string` | Annotations to be added to all namespaces belonging to all tenants. This may be overridden by `rootNamespaces.annotations` of a tenant resource. |
-| `namespace.roleBindingTemplate` | `string`            | Template for RoleBinding resource that is created on all namespaces belonging to a tenant.                                                       |
-| `argocd.namepsace`              | `string`            | The name of namespace where Argo CD is running.                                                                                                  |
-| `argocd.appProjectTemplate`     | `string`            | Template for AppProject resources that is created for each tenant.                                                                               |
+| Key                                          | Type                | Description                                                                                                                                      |
+|----------------------------------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|
+| `namespace.commonLabels`                     | `map[string]string` | Labels to be added to all namespaces belonging to all tenants. This may be overridden by `rootNamespaces.labels` of a tenant resource.           |
+| `namespace.commonAnnotations`                | `map[string]string` | Annotations to be added to all namespaces belonging to all tenants. This may be overridden by `rootNamespaces.annotations` of a tenant resource. |
+| `namespace.roleBindingTemplate`              | `string`            | Template for RoleBinding resource that is created on all namespaces belonging to a tenant.                                                       |
+| `argocd.namepsace`                           | `string`            | The name of namespace where Argo CD is running.                                                                                                  |
+| `argocd.appProjectTemplate`                  | `string`            | Template for AppProject resources that is created for each tenant.                                                                               |
+| `argocd.preventAppCreationInArgoCDNamespace` | `bool`              | If true, prevent creating applications in the Argo CD namespace. This is used to enable sharding.                                                |
 
 The repository includes an example as follows:
 

diff --git a/docs/crd_tenant.md b/docs/crd_tenant.md
@@ -1,93 +0,0 @@
-
-### Custom Resources
-
-* [Tenant](#tenant)
-
-### Sub Resources
-
-* [ArgoCDSpec](#argocdspec)
-* [DelegateSpec](#delegatespec)
-* [RootNamespaceSpec](#rootnamespacespec)
-* [TenantList](#tenantlist)
-* [TenantSpec](#tenantspec)
-* [TenantStatus](#tenantstatus)
-
-#### ArgoCDSpec
-
-ArgoCDSpec defines the desired state of the settings for Argo CD.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| repositories | Repositories contains list of repository URLs which can be used by the tenant. | []string | false |
-
-[Back to Custom Resources](#custom-resources)
-
-#### DelegateSpec
-
-DelegateSpec defines a tenant that is delegated access to a tenant.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| name | Name is the name of a delegated tenant. | string | true |
-| roles | Roles is a list of roles that the tenant has. | []string | true |
-
-[Back to Custom Resources](#custom-resources)
-
-#### RootNamespaceSpec
-
-RootNamespaceSpec defines the desired state of Namespace.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| name | Name is the name of namespace to be generated. | string | true |
-| labels | Labels are the labels to add to the namespace. This supersedes `namespace.commonLabels` in the configuration. | map[string]string | false |
-| annotations | Annotations are the annotations to add to the namespace. This supersedes `namespace.commonAnnotations` in the configuration. | map[string]string | false |
-
-[Back to Custom Resources](#custom-resources)
-
-#### Tenant
-
-Tenant is the Schema for the tenants API.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| metadata |  | metav1.ObjectMeta | false |
-| spec |  | [TenantSpec](#tenantspec) | false |
-| status |  | [TenantStatus](#tenantstatus) | false |
-
-[Back to Custom Resources](#custom-resources)
-
-#### TenantList
-
-TenantList contains a list of Tenant.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| metadata |  | metav1.ListMeta | false |
-| items |  | [][Tenant](#tenant) | true |
-
-[Back to Custom Resources](#custom-resources)
-
-#### TenantSpec
-
-TenantSpec defines the desired state of Tenant.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| rootNamespaces | RootNamespaces are the list of root namespaces that belong to this tenant. | [][RootNamespaceSpec](#rootnamespacespec) | true |
-| argocd | ArgoCD is the settings of Argo CD for this tenant. | [ArgoCDSpec](#argocdspec) | false |
-| delegates | Delegates is a list of other tenants that are delegated access to this tenant. | [][DelegateSpec](#delegatespec) | false |
-| controllerName | ControllerName is the name of the application-controller that manages this tenant's applications. If not specified, the default controller is used. | string | false |
-
-[Back to Custom Resources](#custom-resources)
-
-#### TenantStatus
-
-TenantStatus defines the observed state of Tenant.
-
-| Field | Description | Scheme | Required |
-| ----- | ----------- | ------ | -------- |
-| health | Health is the health of Tenant. | TenantHealth | false |
-| conditions | Conditions is an array of conditions. | []metav1.Condition | false |
-
-[Back to Custom Resources](#custom-resources)

diff --git a/docs/overview.md b/docs/overview.md
@@ -21,6 +21,11 @@ Cattage is a Kubernetes controller that enhances the multi-tenancy of [Argo CD][
     Sometimes users may want to move the ownership of an application to another tenant.
     When the parent of a sub-namespace is changed, Cattage will automatically update the permissions.
 
+- Sharding application-controller instances
+
+    Cattage can shard application-controller instances by the tenant.
+    This feature is useful when you have a large number of tenants and want to avoid a single application-controller instance from being overloaded.
+
 [Accurate]: https://github.com/cybozu-go/accurate
 [Argo CD]: https://argo-cd.readthedocs.io/en/stable/
 [App Of Apps Pattern]: https://argo-cd.readthedocs.io/en/stable/operator-manual/cluster-bootstrapping/#app-of-apps-pattern