upgrade to latest ubuntu. macos use podman remote now (so secrets set…

…up the std way)

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:jammy
+FROM ubuntu:noble
 # xxx switch to debian:bookworm
 
 ENV FQDN hostname-default
@@ -30,7 +30,7 @@ EXPOSE 80 443
 RUN apt-get -yqq update  && \
     apt-get -yqq --no-install-recommends install  \
     zsh  sudo  rsync  dnsutils  supervisor  curl  wget  iproute2  \
-    apt-transport-https  ca-certificates  software-properties-common  gpgv2  gpg-agent  \
+    apt-transport-https  ca-certificates  software-properties-common  gpg-agent  \
     podman  unzip && \
     #
     # install binaries and service files

bin/bootstrap.sh

@@ -44,19 +44,25 @@ else
   sleep 5
 
 
-  if [ "$HOST_UNAME" = Darwin ]; then
-    apt-get install -yqq fuse-overlayfs
-    echo; echo
-    echo -n 'echo -n '
-    grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' \n'
-    echo  ' | podman secret create NOMAD_TOKEN -'
-    echo; echo
-  else
-    consul keygen                                  |tr -d '^\n' | podman -r secret create HIND_C -
-    nomad operator gossip keyring generate         |tr -d '^\n' | podman -r secret create HIND_N -
-    grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' ' | podman -r secret create NOMAD_TOKEN -
-  fi
+  consul keygen                                  |tr -d '^\n' | podman -r secret create HIND_C -
+  nomad operator gossip keyring generate         |tr -d '^\n' | podman -r secret create HIND_N -
+  grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' ' | podman -r secret create NOMAD_TOKEN -
 
   rm -f /tmp/*
 
+  if [ $HOST_UNAME = Darwin ]; then
+    echo '
+client {
+  # https://github.com/hashicorp/nomad/issues/11046
+  cpu_total_compute = 1000
+}' >> $NOMAD_HCL
+
+    echo '
+plugin "nomad-driver-podman" {
+  config {
+    socket_path = "unix:///run/podman/podman.sock" # xxx check if works *and better* w/ non-mac too
+  }
+}' >> $NOMAD_HCL
+  fi
+
 fi

etc/hello-world.hcl

@@ -30,7 +30,7 @@ job "hello-world" {
   group "group" {
     network {
       port "http" {
-        to = 5000
+        to = 5555
       }
     }
     service {

etc/mac.md

@@ -0,0 +1,34 @@
+# mac with HinD notes
+
+## run locally
+```sh
+perl -i -pe 's/podman pull/#podman pull/' install.sh
+export VERBOSE=1
+
+podman build --tag ghcr.io/internetarchive/hind:main .
+
+./install.sh
+```
+
+## research & development
+```sh
+# podman run --rm --privileged hind zsh -c 'podman run hello-world'
+# helpful https://forums.docker.com/t/cgroup-v2-the-saga-continues/139329
+
+# podman and nomad!
+podman run --privileged --secret NOMAD_TOKEN,type=env -it --rm localhost/hind zsh -c 'echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control; echo +cpuset > /sys/fs/cgroup/cgroup.controllers; nomad agent -config /etc/nomad.d & sleep 20; echo;echo;echo;nomad status; podman run hello-world'
+
+podman run --cgroups disabled --privileged --secret NOMAD_TOKEN,type=env -it --rm localhost/hind zsh -c 'echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control; echo +cpuset > /sys/fs/cgroup/cgroup.controllers; nomad agent -config /etc/nomad.d & sleep 20; echo;echo;echo;nomad status; podman run --cgroups disabled hello-world'
+
+podman run --rm --privileged  -v $SOCK:/run/podman/podman.sock  podman podman -r ps -a
+```
+
+
+### other init/run args to try
+- https://serverfault.com/questions/1053187/systemd-fails-to-run-in-a-docker-container-when-using-cgroupv2-cgroupns-priva
+```sh
+-v /sys/fs/cgroup:/sys/fs/cgroup:ro
+--cgroupns=host
+--cgroups disabled
+```
+previously had also tried: `-v /sys/fs/cgroup:/sys/fs/cgroup:rw`

install.sh

@@ -21,23 +21,31 @@ export FQDN=$(hostname -f)
 podman -v > /dev/null || echo 'please install the podman package first'
 podman -v > /dev/null || exit 1
 
-if [ "$HOST_UNAME" = Darwin ]; then
-  export FQDN=http://$FQDN
+# NOTE: we use `podman.sock`, since we want HinD containers to create secrets and
+# `podman run` nomad jobs on the outside/VM, not inside itself
+SOCK=$(podman info |grep -F podman.sock |rev |cut -f1 -d ' ' |rev)
+ARGS_SOCK="-v $SOCK:/run/podman/podman.sock"
+ARGS_RUN="$ARGS_SOCK -v /opt/nomad/data/alloc:/opt/nomad/data/alloc --secret HIND_C,type=env --secret HIND_N,type=env"
+
+if [ $HOST_UNAME = Darwin ]; then
+  # setup socket so podman remote will work
+  # https://github.com/containers/podman/blob/main/docs/tutorials/mac_win_client.md
+  podman machine ssh 'systemctl --user enable --now podman.socket'
+  podman machine ssh 'sudo loginctl enable-linger $USER'
+  podman machine ssh 'sudo mkdir -p -m777 /opt/nomad/data/alloc'
+
   PV=$HOME/pv
+  export FQDN=http://$FQDN
 
-  ARGS_INIT=''
-  ARGS_RUN='-p 8000:80 -p 4000:443 --secret NOMAD_TOKEN,type=env'
-  # previously had also added above: '-v /sys/fs/cgroup:/sys/fs/cgroup:rw'
+  ARGS_SEC="--cap-add SYS_ADMIN --security-opt seccomp=unconfined"
+  ARGS_INIT="$ARGS_SEC"
+  ARGS_RUN="$ARGS_SEC $ARGS_RUN -p 8000:80 -p 4000:443"
 else
-  SOCK=$(podman info |grep -F podman.sock |rev |cut -f1 -d ' ' |rev)
   PV=/pv
-
-  # NOTE: we use `podman.sock`, since we want HinD containers to create secrets and
-  # `podman run` nomad jobs on the outside/VM, not inside itself
-  ARGS_INIT="--net=host --cgroupns=host -v $SOCK:$SOCK"
-  ARGS_RUN="$ARGS_INIT -v /opt/nomad/data/alloc:/opt/nomad/data/alloc --secret HIND_C,type=env --secret HIND_N,type=env"
+  ARGS_INIT="--net=host --cgroupns=host"
 fi
 
+
 (
   # clear any prior run (likely fail?)
   set +e
@@ -64,22 +72,12 @@ fi
   mkdir -p -m777 /opt/nomad/data/alloc
 
   podman pull $QUIET $IMG > $OUT
-  podman run --privileged $ARGS_INIT -e FQDN -e HOST_UNAME --name hind-init $QUIET "$@" $IMG
+  podman run --privileged $ARGS_INIT $ARGS_SOCK -e FQDN -e HOST_UNAME --name hind-init $QUIET "$@" $IMG
   podman commit $QUIET hind-init localhost/hind > $OUT 2>&1
   podman rm  -v        hind-init > $OUT 2>&1
 )
 
 
-if [ "$HOST_UNAME" = Darwin ]; then
-  set +x
-  echo '
-
-COPY/PASTE THE NOMAD_TOKEN secret create ABOVE NOW
-
-'
-  read cont
-fi
-
 
 # Now run the new docker image in the background.
 (