From 98833897de1369d17073c0585bb4ec65565c77f0 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Fri, 22 Nov 2024 13:44:11 -0800 Subject: [PATCH 1/4] Ensure CRDs are created on boot --- cmd/main.go | 9 +++++++++ pkg/crds/crds.go | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/cmd/main.go b/cmd/main.go index c13c18c932..577f108749 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -345,6 +345,15 @@ func main() { } setupLog.WithValues("provider", provider).Info("Checking type of cluster") + // If configured to managed CRDs, do a preliminary install of them here. The Installation controller + // will reconcile them as well, but we need to make sure they are installed before we start the rest of the controllers. + if manageCRDs { + if err := crds.Ensure(mgr.GetClient()); err != nil { + setupLog.Error(err, "Failed to ensure CRDs are created") + os.Exit(1) + } + } + // Determine if we're running in single or multi-tenant mode. multiTenant, err := utils.MultiTenant(ctx, clientset) if err != nil { diff --git a/pkg/crds/crds.go b/pkg/crds/crds.go index efabdf0d39..0afd49b2f3 100644 --- a/pkg/crds/crds.go +++ b/pkg/crds/crds.go @@ -15,14 +15,17 @@ package crds import ( + "context" "embed" "fmt" "path" "regexp" "strings" "sync" + "time" apiextenv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" // gopkg.in/yaml.v2 didn't parse all the fields but this package did @@ -184,3 +187,21 @@ func ToRuntimeObjects(crds ...*apiextenv1.CustomResourceDefinition) []client.Obj } return objs } + +// Ensure ensures that the CRDs necessary for bootstrapping exist in the cluster. +// Further reconciliation of the CRDs is handled by the core controller. +func Ensure(c client.Client) error { + // Ensure Calico CRDs exist, which will allow us to bootstrap. + for _, crd := range GetCRDs(opv1.Calico) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := c.Create(ctx, crd); err != nil { + // Ignore if the CRD already exists + if !errors.IsAlreadyExists(err) { + cancel() + return fmt.Errorf("failed to create CustomResourceDefinition %s: %s", crd.Name, err) + } + } + cancel() + } + return nil +} From 8c5783cde35283f76c2bb052f8c8c3876ea77505 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Mon, 25 Nov 2024 13:41:46 -0800 Subject: [PATCH 2/4] Add option to init CRDs and exit --- cmd/main.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 577f108749..3049291e69 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -105,6 +105,10 @@ func main() { var manageCRDs bool var preDelete bool + // bootstrapCRDs is a flag that can be used to install the CRDs and exit. This is useful for + // workflows that use an init container to install CustomResources prior to the operator starting. + var bootstrapCRDs bool + flag.BoolVar(&enableLeaderElection, "enable-leader-election", true, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -124,6 +128,7 @@ func main() { "Operator should manage the projectcalico.org and operator.tigera.io CRDs.") flag.BoolVar(&preDelete, "pre-delete", false, "Run helm pre-deletion hook logic, then exit.") + flag.BoolVar(&bootstrapCRDs, "bootstrap-crds", false, "Install CRDs and exist") opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -269,6 +274,20 @@ func main() { os.Exit(1) } + // If configured to managed CRDs, do a preliminary install of them here. The Installation controller + // will reconcile them as well, but we need to make sure they are installed before we start the rest of the controllers. + if bootstrapCRDs || manageCRDs { + if err := crds.Ensure(mgr.GetClient()); err != nil { + setupLog.Error(err, "Failed to ensure CRDs are created") + os.Exit(1) + } + + if bootstrapCRDs { + setupLog.Info("CRDs installed successfully") + os.Exit(0) + } + } + // Start a goroutine to handle termination. go func() { // Cancel the main context when we are done. @@ -345,15 +364,6 @@ func main() { } setupLog.WithValues("provider", provider).Info("Checking type of cluster") - // If configured to managed CRDs, do a preliminary install of them here. The Installation controller - // will reconcile them as well, but we need to make sure they are installed before we start the rest of the controllers. - if manageCRDs { - if err := crds.Ensure(mgr.GetClient()); err != nil { - setupLog.Error(err, "Failed to ensure CRDs are created") - os.Exit(1) - } - } - // Determine if we're running in single or multi-tenant mode. multiTenant, err := utils.MultiTenant(ctx, clientset) if err != nil { From 8f6b54c147c2292dea2a4a34b67dac15a06db216 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 26 Nov 2024 14:55:45 -0800 Subject: [PATCH 3/4] Add config option for default product variant --- cmd/main.go | 47 +++++++++++++++++++++++++---------------------- pkg/crds/crds.go | 9 +++++++-- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 3049291e69..27829ff115 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -104,31 +104,34 @@ func main() { var sgSetup bool var manageCRDs bool var preDelete bool + var variant string // bootstrapCRDs is a flag that can be used to install the CRDs and exit. This is useful for // workflows that use an init container to install CustomResources prior to the operator starting. var bootstrapCRDs bool - flag.BoolVar(&enableLeaderElection, "enable-leader-election", true, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&urlOnlyKubeconfig, "url-only-kubeconfig", "", - "Path to a kubeconfig, but only for the apiserver url.") - flag.BoolVar(&showVersion, "version", false, - "Show version information") - flag.StringVar(&printImages, "print-images", "", - "Print the default images the operator could deploy and exit. Possible values: list") - flag.StringVar(&printCalicoCRDs, "print-calico-crds", "", - "Print the Calico CRDs the operator has bundled then exit. Possible values: all, . If a value other than 'all' is specified, the first CRD with a prefix of the specified value will be printed.") - flag.StringVar(&printEnterpriseCRDs, "print-enterprise-crds", "", - "Print the Enterprise CRDs the operator has bundled then exit. Possible values: all, . If a value other than 'all' is specified, the first CRD with a prefix of the specified value will be printed.") - flag.BoolVar(&sgSetup, "aws-sg-setup", false, - "Setup Security Groups in AWS (should only be used on OpenShift).") - flag.BoolVar(&manageCRDs, "manage-crds", false, - "Operator should manage the projectcalico.org and operator.tigera.io CRDs.") - flag.BoolVar(&preDelete, "pre-delete", false, - "Run helm pre-deletion hook logic, then exit.") - flag.BoolVar(&bootstrapCRDs, "bootstrap-crds", false, "Install CRDs and exist") + flag.BoolVar( + &enableLeaderElection, "enable-leader-election", true, + "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.", + ) + flag.StringVar( + &printCalicoCRDs, "print-calico-crds", "", + `Print the Calico CRDs the operator has bundled then exit. Possible values: all, . +If a value other than 'all' is specified, the first CRD with a prefix of the specified value will be printed.`, + ) + flag.StringVar( + &printEnterpriseCRDs, "print-enterprise-crds", "", + `Print the Enterprise CRDs the operator has bundled then exit. Possible values: all, . +If a value other than 'all' is specified, the first CRD with a prefix of the specified value will be printed.`, + ) + flag.StringVar(&urlOnlyKubeconfig, "url-only-kubeconfig", "", "Path to a kubeconfig, but only for the apiserver url.") + flag.BoolVar(&showVersion, "version", false, "Show version information") + flag.StringVar(&printImages, "print-images", "", "Print the default images the operator could deploy and exit. Possible values: list") + flag.BoolVar(&sgSetup, "aws-sg-setup", false, "Setup Security Groups in AWS (should only be used on OpenShift).") + flag.BoolVar(&manageCRDs, "manage-crds", false, "Operator should manage the projectcalico.org and operator.tigera.io CRDs.") + flag.BoolVar(&preDelete, "pre-delete", false, "Run helm pre-deletion hook logic, then exit.") + flag.BoolVar(&bootstrapCRDs, "bootstrap-crds", false, "Install CRDs and exit") + flag.StringVar(&variant, "variant", string(operatortigeraiov1.Calico), "Default product variant to assume during boostrapping.") opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -274,10 +277,10 @@ func main() { os.Exit(1) } - // If configured to managed CRDs, do a preliminary install of them here. The Installation controller + // If configured to manage CRDs, do a preliminary install of them here. The Installation controller // will reconcile them as well, but we need to make sure they are installed before we start the rest of the controllers. if bootstrapCRDs || manageCRDs { - if err := crds.Ensure(mgr.GetClient()); err != nil { + if err := crds.Ensure(mgr.GetClient(), variant); err != nil { setupLog.Error(err, "Failed to ensure CRDs are created") os.Exit(1) } diff --git a/pkg/crds/crds.go b/pkg/crds/crds.go index 0afd49b2f3..4eed86aa16 100644 --- a/pkg/crds/crds.go +++ b/pkg/crds/crds.go @@ -190,9 +190,14 @@ func ToRuntimeObjects(crds ...*apiextenv1.CustomResourceDefinition) []client.Obj // Ensure ensures that the CRDs necessary for bootstrapping exist in the cluster. // Further reconciliation of the CRDs is handled by the core controller. -func Ensure(c client.Client) error { +func Ensure(c client.Client, variant string) error { // Ensure Calico CRDs exist, which will allow us to bootstrap. - for _, crd := range GetCRDs(opv1.Calico) { + for _, crd := range GetCRDs(opv1.ProductVariant(variant)) { + // Skip the Tenant CRD - this is only used in Calico Cloud. + if crd.Name == "tenants.projectcalico.org" { + continue + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) if err := c.Create(ctx, crd); err != nil { // Ignore if the CRD already exists From 827e2b92987aeca7aa2e08a17992510d634a7d59 Mon Sep 17 00:00:00 2001 From: Casey Davenport Date: Tue, 26 Nov 2024 14:56:05 -0800 Subject: [PATCH 4/4] Fix tenant name --- pkg/crds/crds.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/crds/crds.go b/pkg/crds/crds.go index 4eed86aa16..08c0df72f7 100644 --- a/pkg/crds/crds.go +++ b/pkg/crds/crds.go @@ -194,7 +194,7 @@ func Ensure(c client.Client, variant string) error { // Ensure Calico CRDs exist, which will allow us to bootstrap. for _, crd := range GetCRDs(opv1.ProductVariant(variant)) { // Skip the Tenant CRD - this is only used in Calico Cloud. - if crd.Name == "tenants.projectcalico.org" { + if crd.Name == "tenants.operator.tigera.io" { continue }