diff --git a/Dockerfile b/Dockerfile index b79b9b1..043f589 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:jammy +FROM ubuntu:noble # xxx switch to debian:bookworm ENV FQDN hostname-default @@ -30,7 +30,7 @@ EXPOSE 80 443 RUN apt-get -yqq update && \ apt-get -yqq --no-install-recommends install \ zsh sudo rsync dnsutils supervisor curl wget iproute2 \ - apt-transport-https ca-certificates software-properties-common gpgv2 gpg-agent \ + apt-transport-https ca-certificates software-properties-common gpg-agent \ podman unzip && \ # # install binaries and service files diff --git a/bin/bootstrap.sh b/bin/bootstrap.sh index efea150..3e9a1ae 100755 --- a/bin/bootstrap.sh +++ b/bin/bootstrap.sh @@ -44,19 +44,25 @@ else sleep 5 - if [ "$HOST_UNAME" = Darwin ]; then - apt-get install -yqq fuse-overlayfs - echo; echo - echo -n 'echo -n ' - grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' \n' - echo ' | podman secret create NOMAD_TOKEN -' - echo; echo - else - consul keygen |tr -d '^\n' | podman -r secret create HIND_C - - nomad operator gossip keyring generate |tr -d '^\n' | podman -r secret create HIND_N - - grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' ' | podman -r secret create NOMAD_TOKEN - - fi + consul keygen |tr -d '^\n' | podman -r secret create HIND_C - + nomad operator gossip keyring generate |tr -d '^\n' | podman -r secret create HIND_N - + grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' ' | podman -r secret create NOMAD_TOKEN - rm -f /tmp/* + if [ $HOST_UNAME = Darwin ]; then + echo ' +client { + # https://github.com/hashicorp/nomad/issues/11046 + cpu_total_compute = 1000 +}' >> $NOMAD_HCL + + echo ' +plugin "nomad-driver-podman" { + config { + socket_path = "unix:///run/podman/podman.sock" # xxx check if works *and better* w/ non-mac too + } +}' >> $NOMAD_HCL + fi + fi diff --git a/etc/hello-world.hcl b/etc/hello-world.hcl index cf28ded..3ff166c 100644 --- a/etc/hello-world.hcl +++ b/etc/hello-world.hcl @@ -30,7 +30,7 @@ job "hello-world" { group "group" { network { port "http" { - to = 5000 + to = 5555 } } service { diff --git a/etc/mac.md b/etc/mac.md new file mode 100644 index 0000000..0f8f502 --- /dev/null +++ b/etc/mac.md @@ -0,0 +1,34 @@ +# mac with HinD notes + +## run locally +```sh +perl -i -pe 's/podman pull/#podman pull/' install.sh +export VERBOSE=1 + +podman build --tag ghcr.io/internetarchive/hind:main . + +./install.sh +``` + +## research & development +```sh +# podman run --rm --privileged hind zsh -c 'podman run hello-world' +# helpful https://forums.docker.com/t/cgroup-v2-the-saga-continues/139329 + +# podman and nomad! +podman run --privileged --secret NOMAD_TOKEN,type=env -it --rm localhost/hind zsh -c 'echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control; echo +cpuset > /sys/fs/cgroup/cgroup.controllers; nomad agent -config /etc/nomad.d & sleep 20; echo;echo;echo;nomad status; podman run hello-world' + +podman run --cgroups disabled --privileged --secret NOMAD_TOKEN,type=env -it --rm localhost/hind zsh -c 'echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control; echo +cpuset > /sys/fs/cgroup/cgroup.controllers; nomad agent -config /etc/nomad.d & sleep 20; echo;echo;echo;nomad status; podman run --cgroups disabled hello-world' + +podman run --rm --privileged -v $SOCK:/run/podman/podman.sock podman podman -r ps -a +``` + + +### other init/run args to try +- https://serverfault.com/questions/1053187/systemd-fails-to-run-in-a-docker-container-when-using-cgroupv2-cgroupns-priva +```sh +-v /sys/fs/cgroup:/sys/fs/cgroup:ro +--cgroupns=host +--cgroups disabled +``` +previously had also tried: `-v /sys/fs/cgroup:/sys/fs/cgroup:rw` diff --git a/install.sh b/install.sh index ab48a92..2ad088f 100755 --- a/install.sh +++ b/install.sh @@ -21,23 +21,31 @@ export FQDN=$(hostname -f) podman -v > /dev/null || echo 'please install the podman package first' podman -v > /dev/null || exit 1 -if [ "$HOST_UNAME" = Darwin ]; then - export FQDN=http://$FQDN +# NOTE: we use `podman.sock`, since we want HinD containers to create secrets and +# `podman run` nomad jobs on the outside/VM, not inside itself +SOCK=$(podman info |grep -F podman.sock |rev |cut -f1 -d ' ' |rev) +ARGS_SOCK="-v $SOCK:/run/podman/podman.sock" +ARGS_RUN="$ARGS_SOCK -v /opt/nomad/data/alloc:/opt/nomad/data/alloc --secret HIND_C,type=env --secret HIND_N,type=env" + +if [ $HOST_UNAME = Darwin ]; then + # setup socket so podman remote will work + # https://github.com/containers/podman/blob/main/docs/tutorials/mac_win_client.md + podman machine ssh 'systemctl --user enable --now podman.socket' + podman machine ssh 'sudo loginctl enable-linger $USER' + podman machine ssh 'sudo mkdir -p -m777 /opt/nomad/data/alloc' + PV=$HOME/pv + export FQDN=http://$FQDN - ARGS_INIT='' - ARGS_RUN='-p 8000:80 -p 4000:443 --secret NOMAD_TOKEN,type=env' - # previously had also added above: '-v /sys/fs/cgroup:/sys/fs/cgroup:rw' + ARGS_SEC="--cap-add SYS_ADMIN --security-opt seccomp=unconfined" + ARGS_INIT="$ARGS_SEC" + ARGS_RUN="$ARGS_SEC $ARGS_RUN -p 8000:80 -p 4000:443" else - SOCK=$(podman info |grep -F podman.sock |rev |cut -f1 -d ' ' |rev) PV=/pv - - # NOTE: we use `podman.sock`, since we want HinD containers to create secrets and - # `podman run` nomad jobs on the outside/VM, not inside itself - ARGS_INIT="--net=host --cgroupns=host -v $SOCK:$SOCK" - ARGS_RUN="$ARGS_INIT -v /opt/nomad/data/alloc:/opt/nomad/data/alloc --secret HIND_C,type=env --secret HIND_N,type=env" + ARGS_INIT="--net=host --cgroupns=host" fi + ( # clear any prior run (likely fail?) set +e @@ -64,22 +72,12 @@ fi mkdir -p -m777 /opt/nomad/data/alloc podman pull $QUIET $IMG > $OUT - podman run --privileged $ARGS_INIT -e FQDN -e HOST_UNAME --name hind-init $QUIET "$@" $IMG + podman run --privileged $ARGS_INIT $ARGS_SOCK -e FQDN -e HOST_UNAME --name hind-init $QUIET "$@" $IMG podman commit $QUIET hind-init localhost/hind > $OUT 2>&1 podman rm -v hind-init > $OUT 2>&1 ) -if [ "$HOST_UNAME" = Darwin ]; then - set +x - echo ' - -COPY/PASTE THE NOMAD_TOKEN secret create ABOVE NOW - -' - read cont -fi - # Now run the new docker image in the background. (