diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..2df3063 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,19 @@ +name: CI +on: + push: + branches: + - master + - 'release/**' + pull_request: +jobs: + main: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: actions/setup-go@v4 + with: + go-version: 1.21.x + - run: go test -v ./... + - run: go install . diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5dcdce3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/kubetest2-kindinv +/_* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c0e2c7e --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# kubetest2 driver for Kubernetes in (Rootless) Docker in (GCE) VM + +`kubetest2-kindinv` provides a "kind in VM" driver for [kubetest2](https://github.com/kubernetes-sigs/kubetest2). + +This driver was written for the sake of running the tests with [rootless kind](https://kind.sigs.k8s.io/docs/user/rootless/). + + +## Requirements +- [kubetest2](https://github.com/kubernetes-sigs/kubetest2). + +- Docker and [`kind`](https://kind.sigs.k8s.io/) have to be installed on the local host, + as the driver executes `kind build node-image` on the local host (currently). + The local host can be macOS. + +- `gcloud` command has to be configured with the permissions for creating and removing the following resources: + - GCE Instances + - VPCs + - Firewall rules + +## Usage + +```bash +go install github.com/rootless-containers/kubetest2-kindinv@master + +cd ${GOPATH}/src/github.com/kubernetes/kubernetes +make WHAT=test/e2e/e2e.test +make ginkgo +make kubectl + +kubetest2 kindinv \ + --run-id=foo \ + --gcp-project=${CLOUDSDK_CORE_PROJECT} \ + --gcp-zone=us-west1-a \ + --instance-image=ubuntu-os-cloud/ubuntu-2204-lts \ + --instance-type=n2-standard-4 \ + --kind-rootless \ + --kube-root=${GOPATH}/src/github.com/kubernetes/kubernetes \ + --build \ + --up \ + --down \ + --test=ginkgo \ + -- \ + --use-built-binaries \ + --focus-regex='\[NodeConformance\]' \ + --skip-regex='Sysctl .*|\[Slow\]' \ + --parallel=8 +``` + +The example command above usually takes more than 30 minutes in total. diff --git a/deployer/deployer.go b/deployer/deployer.go new file mode 100644 index 0000000..aa67a14 --- /dev/null +++ b/deployer/deployer.go @@ -0,0 +1,561 @@ +/* +Copyright The kubetest2-kindinv Authors. +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package deployer implements the kubetest2 kind deployer +package deployer + +import ( + "bytes" + "context" + _ "embed" // for provisionShContent + "errors" + "flag" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/octago/sflags/gen/gpflag" + "github.com/rootless-containers/kubetest2-kindinv/version" + "github.com/spf13/pflag" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/artifacts" + "sigs.k8s.io/kubetest2/pkg/types" +) + +//go:embed kubetest2-kindinv-provision.sh +var provisionShContent []byte + +// Name is the name of the deployer +const Name = "kindinv" + +// local commands +// TODO: support injecting additional options via env +const ( + gcloud = "gcloud" + ssh = "ssh" + rsync = "rsync" + kind = "kind" + docker = "docker" +) + +// the name of the files in runDir +const ( + runDirSSHKeys = "gce-ssh-keys" + runDirProvisionSh = "kubetest2-kindinv-provision.sh" + runDirKindImageTar = "kind-image.tar" + runDirKubeconfig = "kubeconfig" +) + +// the name of the files in artifactsDir +const ( + artifactsDirLogs = "logs" +) + +// gopath returns GOPATH or an empty string. +func gopath() string { + if goBinary, err := exec.LookPath("go"); err == nil { + cmd := exec.Command(goBinary, "env", "GOPATH") + if b, err := cmd.Output(); err == nil { + return strings.TrimSpace(string(b)) + } + } + return os.Getenv("GOPATH") +} + +// New implements deployer.New for kind +func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { + kubeRoot := "." + if s := gopath(); s != "" { + kubeRoot = filepath.Join(s, "src", "github.com", "kubernetes", "kubernetes") + } + d := &deployer{ + commonOptions: opts, + KubeRoot: kubeRoot, + GCPProject: os.Getenv("CLOUDSDK_CORE_PROJECT"), + GCPZone: os.Getenv("CLOUDSDK_COMPUTE_ZONE"), + InstanceImage: "ubuntu-os-cloud/ubuntu-2204-lts", + InstanceType: "n2-standard-4", + DiskGiB: 200, + KindRootless: false, + } + // assertions + var ( + _ types.DeployerWithKubeconfig = d + _ types.DeployerWithVersion = d + ) + return d, bindFlags(d) +} + +type deployer struct { + commonOptions types.Options + + KubeRoot string `desc:"${GOPATH}/src/github.com/kubernetes/kubernetes"` + + GCPProject string `desc:"GCP project, defaults to $CLOUDSDK_CORE_PROJECT"` + GCPZone string `desc:"GCP zone, defaults to $CLOUDSDK_COMPUTE_ZONE"` + InstanceImage string `desc:"Instance image"` + InstanceType string `desc:"Instance type"` + DiskGiB int `flag:"~disk-gib" desc:"Disk size in GiB"` + + KindRootless bool `desc:"Run kind in rootless mode"` + + isUp bool +} + +func (d *deployer) shortRunID() string { + id := d.commonOptions.RunID() + if len(id) > 8 { + id = id[:8] + } + return id +} + +func (d *deployer) kindImageRef() string { + return "kindest/node:runid-" + d.shortRunID() +} + +func (d *deployer) instanceName() string { + userName := os.Getenv("USER") + if userName == "" { + userName = "unknown" + } + return fmt.Sprintf("kt2-%s-%s-%s", Name, userName, d.shortRunID()) +} + +func (d *deployer) networkName() string { + return d.instanceName() +} + +func (d *deployer) firewallRuleName() string { + return d.instanceName() +} + +func (d *deployer) sshAddr() (string, error) { + if d.GCPProject == "" { + return "", errors.New("gcp-project is unset") + } + if d.GCPZone == "" { + return "", errors.New("gcp-zone is unset") + } + return fmt.Sprintf("%s.%s.%s", d.instanceName(), d.GCPZone, d.GCPProject), nil +} + +func (d *deployer) gcloud(ctx context.Context, args ...string) (*exec.Cmd, error) { + if d.GCPProject == "" { + return nil, errors.New("gcp-project is unset") + } + cmd := exec.CommandContext(ctx, gcloud, append([]string{"--project=" + d.GCPProject}, args...)...) + return cmd, nil +} + +func (d *deployer) sshOptionPairs() []string { + return []string{"StrictHostKeyChecking=no"} +} + +func (d *deployer) ssh(ctx context.Context, args ...string) *exec.Cmd { + var a []string + for _, o := range d.sshOptionPairs() { + a = append(a, "-o", o) + } + return exec.CommandContext(ctx, ssh, append(a, args...)...) +} + +func (d *deployer) rsync(ctx context.Context, args ...string) *exec.Cmd { + e := ssh + for _, o := range d.sshOptionPairs() { + e += " -o " + o + } + return exec.CommandContext(ctx, rsync, append([]string{"-e", e}, args...)...) +} + +func stringifyCmdArgs(ss []string) string { + s := "[" + for i, f := range ss { + s += fmt.Sprintf("%q", f) + if i < len(ss)-1 { + s += " " + } + } + s += "]" + return s +} + +func execCmd(cmd *exec.Cmd) error { + s := stringifyCmdArgs(cmd.Args) + klog.V(0).Infof("Executing: %s", s) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to run %s: %w", s, err) + } + return nil +} + +func execCmdWithRetry(cmdFn func() *exec.Cmd, retry int) error { + var err error + for i := 0; i < retry; i++ { + // cmdFn is called every time, as cmd cannot be reused after its failure + if err = execCmd(cmdFn()); err == nil { + return nil + } + time.Sleep(10 * time.Second) + } + return fmt.Errorf("%w (retried %d times)", err, retry) +} + +func (d *deployer) remoteHome(ctx context.Context) (string, error) { + sshAddr, err := d.sshAddr() + if err != nil { + return "", err + } + cmd := d.ssh(ctx, sshAddr, "--", "echo", "$HOME") + var stderr bytes.Buffer + cmd.Stderr = &stderr + b, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to run %s: %w (stdout=%q, stderr=%q)", + stringifyCmdArgs(cmd.Args), err, string(b), stderr.String()) + } + return strings.TrimSpace(string(b)), nil +} + +func (d *deployer) kubeAPIServerPort(ctx context.Context) (int, error) { + sshAddr, err := d.sshAddr() + if err != nil { + return 0, err + } + const kindControlPlaneContainerName = "kind-control-plane" // FIXME: avoid hard coding + cmd := d.ssh(ctx, sshAddr, "--", "docker", "container", "port", kindControlPlaneContainerName) + var stderr bytes.Buffer + cmd.Stderr = &stderr + b, err := cmd.Output() + if err != nil { + return 0, fmt.Errorf("failed to run %s: %w (stdout=%q, stderr=%q)", + stringifyCmdArgs(cmd.Args), err, string(b), stderr.String()) + } + s := strings.TrimSpace(string(b)) // like "127.0.0.1:45825" + _, portStr, err := net.SplitHostPort(s) + if err != nil { + return 0, err + } + port, err := strconv.Atoi(portStr) + if err != nil { + return 0, fmt.Errorf("failed to atoi %q (split from %q): %w", portStr, s, err) + } + return port, nil +} + +func (d *deployer) upVM(ctx context.Context) error { + if d.GCPZone == "" { + return errors.New("gcp-zone is unset") + } + + home, err := os.UserHomeDir() + if err != nil { + return err + } + gcePubPath := filepath.Join(home, ".ssh/google_compute_engine.pub") + gcePubContent, err := os.ReadFile(gcePubPath) + if err != nil { + return err + } + userName := os.Getenv("USER") + if userName == "" { + return errors.New("needs $USER to be set") + } + sshKeysContent := []byte(userName + ":" + string(gcePubContent)) + runDir := d.commonOptions.RunDir() + sshKeysPath := filepath.Join(runDir, runDirSSHKeys) + _ = os.RemoveAll(sshKeysPath) + if err = os.WriteFile(sshKeysPath, sshKeysContent, 0400); err != nil { + return err + } + + nwName, fwRuleName, instName := d.networkName(), d.firewallRuleName(), d.instanceName() + description := fmt.Sprintf("kubetest2-%s instance (for user %q)", Name, userName) + instImgPair := strings.SplitN(d.InstanceImage, "/", 2) + + gcloudCmds := [][]string{ + {"compute", "networks", "create", nwName}, + {"compute", "firewall-rules", "create", fwRuleName, "--network=" + nwName, "--allow=tcp:22"}, + {"compute", "instances", "create", + "--zone=" + d.GCPZone, + "--description=" + description, + "--labels=owner=" + userName, + "--network=" + nwName, + "--image-project=" + instImgPair[0], + "--image-family=" + instImgPair[1], + "--machine-type=" + d.InstanceType, + fmt.Sprintf("--boot-disk-size=%dGiB", d.DiskGiB), + "--metadata=block-project-ssh-keys=TRUE", + instName, + }, + {"compute", "instances", "add-metadata", instName, "--zone=" + d.GCPZone, "--metadata-from-file=ssh-keys=" + sshKeysPath}, + {"compute", "config-ssh"}, + } + for _, f := range gcloudCmds { + cmd, err := d.gcloud(ctx, f...) + if err != nil { + return err + } + if err = execCmd(cmd); err != nil { + klog.Error(err) + // continue, to allow "already exists" errors + } + } + + // Check SSH connectivity + sshAddr, err := d.sshAddr() + if err != nil { + return err + } + cmdFn := func() *exec.Cmd { return d.ssh(ctx, sshAddr, "--", "uname", "-a") } + if err = execCmdWithRetry(cmdFn, 10); err != nil { + return err + } + cmd := d.ssh(ctx, sshAddr, "--", "grep", "^PRETTY_NAME=", "/etc/os-release") + if err = execCmd(cmd); err != nil { + return err + } + + // Prepare the provisioning script + provisionShLocal := filepath.Join(runDir, runDirProvisionSh) + _ = os.RemoveAll(provisionShLocal) + if err = os.WriteFile(provisionShLocal, provisionShContent, 0755); err != nil { + return err + } + provisionShRemote := "~/kubetest2-kindinv-provision.sh" + cmd = d.rsync(ctx, "-av", "--progress", provisionShLocal, sshAddr+":"+provisionShRemote) + if err = execCmd(cmd); err != nil { + return err + } + + // Execute the provisioning script to install Docker and kind + cmd = d.ssh(ctx, sshAddr, "--", "sudo", provisionShRemote) + if err = execCmd(cmd); err != nil { + return err + } + + // Allow the current user to use Docker + if d.KindRootless { + cmd = d.ssh(ctx, sshAddr, "--", "dockerd-rootless-setuptool.sh", "install", "-f") + } else { + cmd = d.ssh(ctx, sshAddr, "--", "sudo", "usermod -aG", "docker", userName) + } + if err = execCmd(cmd); err != nil { + return err + } + + return nil +} + +func (d *deployer) Up() error { + ctx := context.TODO() + // Start VM, and install Docker in it + if err := d.upVM(ctx); err != nil { + return err + } + sshAddr, err := d.sshAddr() + if err != nil { + return err + } + + // Run `kind delete cluster` (for idempotence) + cmd := d.ssh(ctx, sshAddr, "--", "kind", "delete", "cluster") + if err = execCmd(cmd); err != nil { + klog.V(2).Info(err) // negligible + } + + // Run `kind create cluster` + var kindArgs []string + if d.commonOptions.ShouldBuild() { + kindArgs = append(kindArgs, "--image="+d.kindImageRef()) + } + cmd = d.ssh(ctx, append([]string{sshAddr, "--", "kind", "create", "cluster"}, kindArgs...)...) + if err = execCmd(cmd); err != nil { + return err + } + + // Get kube-apiserver port + kubeAPIPort, err := d.kubeAPIServerPort(ctx) + if err != nil { + return fmt.Errorf("failed to get kube-apiserver port: %w", err) + } + + // Foward kube-apiserver port + cmd = d.ssh(ctx, "-o", "ExitOnForwardFailure=yes", "-f", "-N", + "-L", fmt.Sprintf("%d:127.0.0.1:%d", kubeAPIPort, kubeAPIPort), sshAddr) + if err = execCmd(cmd); err != nil { + return err + } + + // Copy kubeconfig + runDir := d.commonOptions.RunDir() + kubeconfigRemote := "~/.kube/config" + kubeconfigLocal := filepath.Join(runDir, runDirKubeconfig) + cmd = d.rsync(ctx, "-av", "--progress", sshAddr+":"+kubeconfigRemote, kubeconfigLocal) + if err = execCmd(cmd); err != nil { + return err + } + klog.Infof("KUBECONFIG=%q", kubeconfigLocal) + + d.isUp = true + return nil +} + +func (d *deployer) Down() error { + if d.GCPZone == "" { + return errors.New("gcp-zone is unset") + } + + ctx := context.TODO() + instName, fwRuleName, nwName := d.instanceName(), d.firewallRuleName(), d.networkName() + gcloudCmds := [][]string{ + {"--quiet", "compute", "instances", "delete", "--zone=" + d.GCPZone, instName}, + {"--quiet", "compute", "firewall-rules", "delete", fwRuleName}, + {"--quiet", "compute", "networks", "delete", nwName}, + } + for _, f := range gcloudCmds { + cmd, err := d.gcloud(ctx, f...) + if err != nil { + return err + } + if err := execCmd(cmd); err != nil { + klog.Error(err) + // continue, to allow "not found" errors + } + } + return nil +} + +func (d *deployer) IsUp() (up bool, err error) { + return d.isUp, nil +} + +func (d *deployer) DumpClusterLogs() error { + ctx := context.TODO() + sshAddr, err := d.sshAddr() + if err != nil { + return err + } + logsRemote := "~/logs" + cmd := d.ssh(ctx, sshAddr, "--", "kind", "export", "logs", logsRemote) + if err := execCmd(cmd); err != nil { + return err + } + artifactsDir := artifacts.BaseDir() + logsLocal := filepath.Join(artifactsDir, artifactsDirLogs) + cmd = d.rsync(ctx, "-av", "--progress", sshAddr+":"+logsRemote, logsLocal) + if err = execCmd(cmd); err != nil { + return err + } + return nil +} + +func (d *deployer) Build() error { + ctx := context.TODO() + runDir := d.commonOptions.RunDir() + + // Prepare `runDir/{e2e.test, ginkgo}` + for _, f := range []string{"e2e.test", "ginkgo", "kubectl"} { + src := filepath.Join(d.KubeRoot, "_output", "bin", f) + if _, err := os.Stat(src); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + switch f { + case "e2e.test": + return fmt.Errorf("%w (Hint: `make WHAT=test/e2e/e2e.test -C $(go env GOPATH)/src/github.com/kubernetes/kubernetes`)", err) + case "ginkgo": + return fmt.Errorf("%w (Hint: `make ginkgo -C $(go env GOPATH)/src/github.com/kubernetes/kubernetes`)", err) + default: + klog.Warning(err) + continue + } + } + dst := filepath.Join(runDir, f) + cmd := exec.CommandContext(ctx, "cp", "-af", src, dst) + if err := execCmd(cmd); err != nil { + return err + } + } + + // Build the kind image (on the local host, currently) + cmd := exec.CommandContext(ctx, kind, "build", "node-image", "--image="+d.kindImageRef(), d.KubeRoot) + if err := execCmd(cmd); err != nil { + return err + } + kindImageLocal := filepath.Join(runDir, runDirKindImageTar) + cmd = exec.CommandContext(ctx, docker, "image", "save", "--output="+kindImageLocal, d.kindImageRef()) + if err := execCmd(cmd); err != nil { + return err + } + isUp, err := d.IsUp() + if err != nil { + return err + } + if !isUp { + if err := d.upVM(ctx); err != nil { + return err + } + } + // Load the image archive to into the remote docker + sshAddr, err := d.sshAddr() + if err != nil { + return err + } + remoteHome, err := d.remoteHome(ctx) + if err != nil { + return err + } + kindImageRemote := filepath.Join(remoteHome, "kind-image.tar") // docker CLI cannot parse "~" + cmd = d.rsync(ctx, "-av", "--progress", "--compress", kindImageLocal, sshAddr+":"+kindImageRemote) + if err = execCmd(cmd); err != nil { + return err + } + cmd = d.ssh(ctx, sshAddr, "--", "docker", "image", "load", "--input="+kindImageRemote) + if err = execCmd(cmd); err != nil { + return err + } + return nil +} + +func (d *deployer) Kubeconfig() (string, error) { + runDir := d.commonOptions.RunDir() + return filepath.Join(runDir, runDirKubeconfig), nil +} + +func (d *deployer) Version() string { + return version.GetVersion() +} + +func bindFlags(d *deployer) *pflag.FlagSet { + flags, err := gpflag.Parse(d) + if err != nil { + klog.Fatalf("unable to generate flags from deployer") + return nil + } + + flags.AddGoFlagSet(flag.CommandLine) + + return flags +} diff --git a/deployer/kubetest2-kindinv-provision.sh b/deployer/kubetest2-kindinv-provision.sh new file mode 100644 index 0000000..d34551e --- /dev/null +++ b/deployer/kubetest2-kindinv-provision.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# The provisioning script for kubetest2-kindinv. +# Expected to be used with Ubuntu 22.04. +# TODO: support non-Ubuntu too +set -eux -o pipefail + +# Install dependencies (apt-get) +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y uidmap + +# Install dependencies (snap) +for f in go kubectl; do + snap install "$f" --classic +done + +# Set up cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/ +mkdir -p "/etc/systemd/system/user@.service.d" +cat >"/etc/systemd/system/user@.service.d/rootless.conf" <