From bf7af39688f43952955f09a1b4b458c6fa7cbf32 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 20 Oct 2023 19:27:40 +0900 Subject: [PATCH] Execute `kind build node-image` on the remote Docker and `kind` no longer need to be present on the local host. Fix issue 1 Signed-off-by: Akihiro Suda --- README.md | 4 -- deployer/deployer.go | 74 +++++++++++++++++++------ deployer/kubetest2-kindinv-provision.sh | 4 +- 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 60c023b..cbab356 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,6 @@ This driver was written for the sake of running the tests with [rootless kind](h ## Requirements - [kubetest2](https://github.com/kubernetes-sigs/kubetest2). -- Docker and [`kind`](https://kind.sigs.k8s.io/) have to be installed on the local host, - as the driver executes `kind build node-image` on the local host (currently). - The local host can be macOS. - - `gcloud` command has to be configured with the permissions for creating and removing the following resources: - GCE Instances - VPCs diff --git a/deployer/deployer.go b/deployer/deployer.go index ec5d1a7..3865518 100644 --- a/deployer/deployer.go +++ b/deployer/deployer.go @@ -53,8 +53,7 @@ const ( gcloud = "gcloud" ssh = "ssh" rsync = "rsync" - kind = "kind" - docker = "docker" + git = "git" ) // the name of the files in runDir @@ -197,6 +196,29 @@ func (d *deployer) rsync(ctx context.Context, args ...string) *exec.Cmd { return exec.CommandContext(ctx, rsync, append([]string{"-e", e}, args...)...) } +func (d *deployer) git(ctx context.Context, args ...string) *exec.Cmd { + e := ssh + for _, o := range d.sshOptionPairs() { + e += " -o " + o + } + cmd := exec.CommandContext(ctx, git, args...) + cmd.Env = append(os.Environ(), "GIT_SSH_COMMAND="+e) + return cmd +} + +func (d *deployer) gitDescribeWithRepoDir(ctx context.Context, repoDir string) (string, error) { + cmd := d.git(ctx, "describe") + cmd.Dir = repoDir + var stderr bytes.Buffer + cmd.Stderr = &stderr + b, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to run %s: %w (stdout=%q, stderr=%q)", + stringifyCmdArgs(cmd.Args), err, string(b), stderr.String()) + } + return strings.TrimSpace(string(b)), nil +} + func stringifyCmdArgs(ss []string) string { s := "[" for i, f := range ss { @@ -505,16 +527,6 @@ func (d *deployer) Build() error { } } - // Build the kind image (on the local host, currently) - cmd := exec.CommandContext(ctx, kind, "build", "node-image", "--image="+d.kindImageRef(), d.KubeRoot) - if err := execCmd(cmd); err != nil { - return err - } - kindImageLocal := filepath.Join(runDir, runDirKindImageTar) - cmd = exec.CommandContext(ctx, docker, "image", "save", "--output="+kindImageLocal, d.kindImageRef()) - if err := execCmd(cmd); err != nil { - return err - } isUp, err := d.IsUp() if err != nil { return err @@ -524,7 +536,6 @@ func (d *deployer) Build() error { return err } } - // Load the image archive to into the remote docker sshAddr, err := d.sshAddr() if err != nil { return err @@ -533,15 +544,46 @@ func (d *deployer) Build() error { if err != nil { return err } - kindImageRemote := filepath.Join(remoteHome, "kind-image.tar") // docker CLI cannot parse "~" - cmd = d.rsync(ctx, "-av", "--progress", "--compress", kindImageLocal, sshAddr+":"+kindImageRemote) + gopathKK := filepath.Join(remoteHome, "go", "src", "github.com", "kubernetes", "kubernetes") + + // Clone k/k from github, assuming that cloning the repo from github + // is faster than pushing it from the local host. + cmd := d.ssh(ctx, sshAddr, "--", + fmt.Sprintf(`/bin/sh -euxc "[ -e %s ] || git clone --progress https://github.com/kubernetes/kubernetes.git %s"`, + gopathKK, gopathKK)) + if err = execCmd(cmd); err != nil { + return err + } + cmd = d.ssh(ctx, sshAddr, "--", + fmt.Sprintf(`/bin/sh -euxc "cd %s && git config --add receive.denyCurrentBranch warn"`, + gopathKK)) + if err = execCmd(cmd); err != nil { + return err + } + + // Push the local changes to the remote + headCommit, err := d.gitDescribeWithRepoDir(ctx, d.KubeRoot) + if err != nil { + return fmt.Errorf("failed to get the head of %q: %w", d.KubeRoot, err) + } + cmd = d.git(ctx, "push", "--progress", "-f", + fmt.Sprintf("ssh://%s@%s:%s", d.User, sshAddr, gopathKK)) + cmd.Dir = d.KubeRoot if err = execCmd(cmd); err != nil { return err } - cmd = d.ssh(ctx, sshAddr, "--", "docker", "image", "load", "--input="+kindImageRemote) + cmd = d.ssh(ctx, sshAddr, "--", + fmt.Sprintf(`/bin/sh -euxc "cd %s && git checkout %s"`, gopathKK, headCommit)) if err = execCmd(cmd); err != nil { return err } + + // Build the kind image + cmd = d.ssh(ctx, sshAddr, "--", "kind", "build", "node-image", + "--image="+d.kindImageRef(), gopathKK) + if err := execCmd(cmd); err != nil { + return err + } return nil } diff --git a/deployer/kubetest2-kindinv-provision.sh b/deployer/kubetest2-kindinv-provision.sh index d34551e..d9f37ca 100644 --- a/deployer/kubetest2-kindinv-provision.sh +++ b/deployer/kubetest2-kindinv-provision.sh @@ -5,9 +5,11 @@ set -eux -o pipefail # Install dependencies (apt-get) +# - make: for `kind build node-image` +# - uidmap: for rootless export DEBIAN_FRONTEND=noninteractive apt-get update -apt-get install -y uidmap +apt-get install -y make uidmap # Install dependencies (snap) for f in go kubectl; do