Skip to content

Commit

Permalink
plugincontainer: Support mlock
Browse files Browse the repository at this point in the history
  • Loading branch information
tomhjp committed Sep 21, 2023
1 parent 7a5e901 commit e7cbc21
Show file tree
Hide file tree
Showing 8 changed files with 486 additions and 271 deletions.
51 changes: 43 additions & 8 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,24 +42,23 @@ jobs:
run: |
(
set -e
ARCH=$(uname -m)
URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}
wget --quiet ${URL}/runsc ${URL}/runsc.sha512 \
${URL}/containerd-shim-runsc-v1 ${URL}/containerd-shim-runsc-v1.sha512
ARCH="$(uname -m)"
URL="https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}"
wget --quiet "${URL}/runsc" "${URL}/runsc.sha512" \
"${URL}/containerd-shim-runsc-v1" "${URL}/containerd-shim-runsc-v1.sha512"
sha512sum -c runsc.sha512 \
-c containerd-shim-runsc-v1.sha512
rm -f *.sha512
rm -f -- *.sha512
chmod a+rx runsc containerd-shim-runsc-v1
sudo mv runsc containerd-shim-runsc-v1 /usr/local/bin
)
cat | sudo tee /etc/docker/daemon.json <<EOF
sudo tee /etc/docker/daemon.json <<EOF
{
"runtimes": {
"runsc": {
"path": "/usr/local/bin/runsc",
"runtimeArgs": [
"--host-uds=all",
"--host-fifo=open"
"--host-uds=all"
]
}
}
Expand All @@ -68,6 +67,42 @@ jobs:
sudo systemctl reload docker
- name: Install rootless docker
if: ${{ matrix.module == 'plugincontainer' }}
run: |
sudo apt-get install -y uidmap dbus-user-session
cat /etc/subuid /etc/subgid
export FORCE_ROOTLESS_INSTALL=1
curl -fsSL https://get.docker.com/rootless | sh
mkdir -p ~/.config/docker/
tee ~/.config/docker/daemon.json <<EOF
{
"runtimes": {
"runsc": {
"path": "/usr/local/bin/runsc",
"runtimeArgs": [
"--host-uds=all",
"--ignore-cgroups"
]
}
}
}
EOF
systemctl --user restart docker
- name: Install rootless podman
if: ${{ matrix.module == 'plugincontainer' }}
run: |
sudo apt-get install -y podman slirp4netns fuse-overlayfs
mkdir -p ~/local/bin
RUNSC_SCRIPT=~/local/bin/runsc.podman
tee "${RUNSC_SCRIPT}" <<EOF
#!/bin/bash
/usr/local/bin/runsc --host-uds=all --ignore-cgroups "\$@"
EOF
chmod u+x "${RUNSC_SCRIPT}"
podman --runtime "${RUNSC_SCRIPT}" system service -t 0 &
- name: Test
run: cd ${{ matrix.module }} && go test ./...

Expand Down
133 changes: 133 additions & 0 deletions plugincontainer/compatibility_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package plugincontainer_test

import (
"fmt"
"os"
"testing"

"github.com/hashicorp/go-secure-stdlib/plugincontainer"
)

const (
engineDocker = "docker"
enginePodman = "podman"
runtimeRunc = "runc"
runtimeRunsc = "runsc"
)

type matrixInput struct {
containerEngine string
containerRuntime string
rootlessEngine bool
rootlessUser bool
mlock bool
}

func (m matrixInput) String() string {
var s string
if m.rootlessEngine {
s = "rootless " + m.containerEngine
} else {
s = m.containerEngine
}
s += ":" + m.containerRuntime
if m.rootlessUser {
s += ":" + "nonroot"
}
if m.mlock {
s += ":" + "mlock"
}
return s
}

func TestCompatibilityMatrix(t *testing.T) {
runCmd(t, "go", "build", "-o=examples/container/go-plugin-counter", "./examples/container/plugin-counter")

for _, engine := range []string{engineDocker, enginePodman} {
for _, runtime := range []string{runtimeRunc, runtimeRunsc} {
for _, rootlessEngine := range []bool{true, false} {
for _, rootlessUser := range []bool{true, false} {
for _, mlock := range []bool{true, false} {
i := matrixInput{
containerEngine: engine,
containerRuntime: runtime,
rootlessEngine: rootlessEngine,
rootlessUser: rootlessUser,
mlock: mlock,
}
t.Run(i.String(), func(t *testing.T) {
runExamplePlugin(t, i)
})
}
}
}
}
}
}

func skipIfUnsupported(t *testing.T, i matrixInput) {
switch {
case i.rootlessEngine && i.rootlessUser:
t.Skip("Unix socket permissions not yet working for rootless engine + nonroot container user")
case i.containerEngine == enginePodman && !i.rootlessEngine:
t.Skip("TODO: These tests would pass but CI doesn't have the environment set up yet")
case i.mlock && i.rootlessEngine:
if i.containerEngine == engineDocker && i.containerRuntime == runtimeRunsc {
// runsc works in rootless because it has its own implementation of mlockall(2)
} else {
t.Skip("TODO: These tests should work if the rootless engine is given the IPC_LOCK capability")
}
}
}

func setDockerHost(t *testing.T, containerEngine string, rootlessEngine bool) {
var socketFile string
switch {
case containerEngine == engineDocker && !rootlessEngine:
socketFile = "/var/run/docker.sock"
case containerEngine == engineDocker && rootlessEngine:
socketFile = fmt.Sprintf("/run/user/%d/docker.sock", os.Getuid())
case containerEngine == enginePodman && !rootlessEngine:
socketFile = "/var/run/podman/podman.sock"
case containerEngine == enginePodman && rootlessEngine:
socketFile = fmt.Sprintf("/run/user/%d/podman/podman.sock", os.Getuid())
default:
t.Fatalf("Unsupported combination: %s, %v", containerEngine, rootlessEngine)
}
if _, err := os.Stat(socketFile); err != nil {
t.Fatal("Did not find expected socket file:", err)
}
t.Setenv("DOCKER_HOST", "unix://"+socketFile)
}

func runExamplePlugin(t *testing.T, i matrixInput) {
skipIfUnsupported(t, i)
setDockerHost(t, i.containerEngine, i.rootlessEngine)
tag := goPluginCounterImage
target := "root"
if i.rootlessUser {
tag += ":nonroot"
target = "nonroot"
}
runCmd(t, i.containerEngine, "build", "--tag="+tag, "--target="+target, "--file=examples/container/Dockerfile", "examples/container")

// TODO: Install rootless and podman on CI
cfg := &plugincontainer.Config{
Image: goPluginCounterImage,
GroupAdd: os.Getgid(),

// Test inputs
Runtime: i.containerRuntime,
CapIPCLock: i.mlock,
}
if i.mlock {
cfg.Env = append(cfg.Env, "MLOCK=true")
}
if i.rootlessUser {
cfg.Tag = "nonroot"
}
exerciseExamplePlugin(t, cfg)
}
3 changes: 2 additions & 1 deletion plugincontainer/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,11 @@ type Config struct {
Labels map[string]string // Arbitrary metadata to facilitate querying containers.

// container.HostConfig options
Runtime string // OCI runtime.
Runtime string // OCI runtime. NOTE: Has no effect if using podman's system service API
CgroupParent string // Parent Cgroup for the container
NanoCpus int64 // CPU quota in billionths of a CPU core
Memory int64 // Memory quota in bytes
CapIPCLock bool // Whether to add the capability IPC_LOCK, to allow the mlockall(2) syscall

// network.NetworkConfig options
EndpointsConfig map[string]*network.EndpointSettings // Endpoint configs for each connecting network
Expand Down
14 changes: 9 additions & 5 deletions plugincontainer/container_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ var (
_ runner.Runner = (*containerRunner)(nil)

errUnsupportedOS = errors.New("plugincontainer currently only supports Linux")
errSHA256Mismatch = errors.New("SHA256 mismatch")
ErrSHA256Mismatch = errors.New("SHA256 mismatch")
)

const pluginSocketDir = "/tmp/go-plugin-container"
Expand Down Expand Up @@ -142,6 +142,10 @@ func (cfg *Config) NewContainerRunner(logger hclog.Logger, cmd *exec.Cmd, hostSo
hostConfig.GroupAdd = append(hostConfig.GroupAdd, strconv.Itoa(cfg.GroupAdd))
}

if cfg.CapIPCLock {
hostConfig.CapAdd = append(hostConfig.CapAdd, "IPC_LOCK")
}

// Network config.
networkConfig := &network.NetworkingConfig{
EndpointsConfig: cfg.EndpointsConfig,
Expand Down Expand Up @@ -186,19 +190,19 @@ func (c *containerRunner) Start(ctx context.Context) error {
}
}
if !imageFound {
return fmt.Errorf("could not find any locally available images named %s that match with the provided SHA256 hash %s: %w", ref, c.sha256, errSHA256Mismatch)
return fmt.Errorf("could not find any locally available images named %s that match with the provided SHA256 hash %s: %w", ref, c.sha256, ErrSHA256Mismatch)
}
}

resp, err := c.dockerClient.ContainerCreate(ctx, c.containerConfig, c.hostConfig, c.networkConfig, nil, "")
if err != nil {
return err
return fmt.Errorf("error creating container: %w", err)
}
c.id = resp.ID
c.logger.Trace("created container", "image", c.image, "id", c.id)

if err := c.dockerClient.ContainerStart(ctx, c.id, types.ContainerStartOptions{}); err != nil {
return err
return fmt.Errorf("error starting container: %w", err)
}

// ContainerLogs combines stdout and stderr.
Expand Down Expand Up @@ -296,7 +300,7 @@ func (c *containerRunner) Stderr() io.ReadCloser {
func (c *containerRunner) PluginToHost(pluginNet, pluginAddr string) (hostNet string, hostAddr string, err error) {
if path.Dir(pluginAddr) != pluginSocketDir {
return "", "", fmt.Errorf("expected address to be in directory %s, but was %s; "+
"the plugin may need to be recompiled with the latest go-plugin version", c.hostSocketDir, pluginAddr)
"the plugin may need to be recompiled with the latest go-plugin version", pluginSocketDir, pluginAddr)
}
return pluginNet, path.Join(c.hostSocketDir, path.Base(pluginAddr)), nil
}
Expand Down
Loading

0 comments on commit e7cbc21

Please sign in to comment.