WIP with ingress

README.md

@@ -67,7 +67,7 @@ spec:
 Service resources are ignored and it's implicit that a pod has a service with url: `<labels.name>.<metadata.namespace>.cluster.skate`
 
 Plan:
-- Nginx container mounts /etc/skate/ingress.conf
+- Nginx container mounts /var/lib/skate/ingress/nginx.conf
 - nginx reloads on file change
 - skatelet updates the file on ingress resource change
 - use letsencrypt and http verification

hack/test-deployment.yaml

@@ -7,7 +7,7 @@ metadata:
   labels:
     app: nginx
 spec:
-  replicas: 3
+  replicas: 1
   selector:
     matchLabels:
       app: nginx

images/coredns/Makefile

@@ -1,7 +1,7 @@
 .PHONY: build-and-push
 build-and-push:
-	GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=0 go build -ldflags="-w -s" -o build/armv6/coredns .
-	GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=0 go build -ldflags="-w -s" -o build/armv7/coredns .
+#	GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=0 go build -ldflags="-w -s" -o build/armv6/coredns .
+#	GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=0 go build -ldflags="-w -s" -o build/armv7/coredns .
 	GOOS=linux GOARCH=arm64  CGO_ENABLED=0 go build -ldflags="-w -s" -o build/arm64/coredns .
 	GOOS=linux GOARCH=amd64  CGO_ENABLED=0 go build -ldflags="-w -s" -o build/amd64/coredns .
-	docker buildx build --platform linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64  -t ghcr.io/skateco/coredns:latest --push .
+	docker buildx build --platform linux/arm64,linux/amd64  -t ghcr.io/skateco/coredns:latest --push .

images/nginx-ingress/Dockerfile

@@ -14,6 +14,7 @@ RUN DEBIAN_FRONTEND=noninteractive \
     curl \
     unzip \
     make \
+    python3 \
     bsdmainutils && \
     rm -rf /var/lib/apt/lists/* && \
     curl -L https://luarocks.org/releases/luarocks-3.11.1.tar.gz --output /tmp/luarocks-3.11.1.tar.gz && \
@@ -38,9 +39,9 @@ RUN DEBIAN_FRONTEND=noninteractive \
 
 COPY docker_entrypoint.sh /docker_entrypoint.sh
 
-COPY error.html /etc/skate/ingress/
-#COPY nginx.conf.tmpl /etc/skate/ingress/
-#COPY test/ /etc/skate/ingress/test
+COPY error.html /etc/nginx-ingress/
+#COPY nginx.conf.tmpl /etc/nginx-ingress
+#COPY test/ /etc/nginx-ingress/test
 
 
 COPY lua $RESTY_ROOT/nginx/lua

images/nginx-ingress/Makefile

@@ -0,0 +1,3 @@
+.PHONY: build-and-push
+build-and-push:
+	docker buildx build --platform linux/arm64,linux/amd64  -t ghcr.io/skateco/nginx-ingress:latest --push .

images/nginx-ingress/docker_entrypoint.sh

@@ -3,7 +3,11 @@ set -eu
 
 
 # the path openresty will look for the nginx config
-CONF_PATH="/etc/skate/ingress/nginx.conf"
+CONF_PATH="/var/lib/skate/ingress/nginx.conf"
+
+if [ ! -f "$CONF_PATH" ]; then
+    cp /etc/openresty/nginx.conf $CONF_PATH
+fi
 
 pidfile=/usr/local/openresty/nginx/logs/nginx.pid
 

images/nginx-ingress/lua/error_page.lua

@@ -32,8 +32,8 @@ local function getVars(code)
 end
 
 local template = require "resty.template".new({
-    root     = "/etc/skate/ingress",
-    location = "/etc/skate/ingress"
+    root     = "/etc/nginx-ingress/",
+    location = "/etc/nginx-ingress"
 })
 
 function _M.go(err_code)

manifests/ingress.yaml

@@ -2,27 +2,27 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: nginx
+  name: nginx-ingress
   namespace: skate
   labels:
-    app: nginx
+    app: nginx-ingress
 spec:
   selector:
     matchLabels:
-      app: nginx
+      app: nginx-ingress
   template:
     metadata:
       labels:
-        app: nginx
+        app: nginx-ingress
     spec:
       hostNetwork: true
       volumes:
-      - name: cni
+      - name: ingress
         hostPath:
           path: /var/lib/skate/ingress
       containers:
-      - name: coredns
-        image: ghcr.io/skateco/coredns
+      - name: nginx
+        image: ghcr.io/skateco/nginx-ingress
         volumeMounts:
         - mountPath: /var/lib/skate/ingress
           name: ingress

src/create.rs

@@ -19,6 +19,7 @@ use crate::ssh::{cluster_connections, node_connection, SshClient, SshClients};
 use crate::util::{CHECKBOX_EMOJI, CROSS_EMOJI};
 
 const COREDNS_MANIFEST: &str = include_str!("../manifests/coredns.yaml");
+const INGRESS_MANIFEST: &str = include_str!("../manifests/ingress.yaml");
 
 #[derive(Debug, Args)]
 pub struct CreateArgs {
@@ -176,6 +177,7 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
     let all_conns = &all_conns.unwrap_or(SshClients { clients: vec!() });
 
 
+    _ = conn.execute("sudo mkdir -p /var/lib/skate/ingress").await?;
     _ = conn.execute("sudo podman rm -fa").await;
 
     setup_networking(&conn, &all_conns, &cluster, &node).await?;
@@ -192,19 +194,18 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
 }
 
 async fn install_manifests(args: &CreateNodeArgs, config: &Cluster, node: &Node) -> Result<(), Box<dyn Error>> {
-
     /// COREDNS
     /// coredns listens on port 53 and 5533
     /// port 53 serves .cluster.skate by forwarding to all coredns instances on port 5553
     /// uses fanout plugin
 
-    let coredns_yaml_path = format!("/tmp/skate-coredns-{}.yaml", node.name);
-    let mut file = File::create(&coredns_yaml_path)?;
     // replace forward list in coredns config with that of other hosts
     let fanout_list = config.nodes.iter().map(|n| n.host.clone() + ":5553").join(" ");
 
     let coredns_yaml = COREDNS_MANIFEST.replace("%%fanout_list%%", &fanout_list);
 
+    let coredns_yaml_path = format!("/tmp/skate-coredns-{}.yaml", node.name);
+    let mut file = File::create(&coredns_yaml_path)?;
     file.write_all(coredns_yaml.as_bytes())?;
 
 
@@ -214,6 +215,19 @@ async fn install_manifests(args: &CreateNodeArgs, config: &Cluster, node: &Node)
         config: args.config.clone(),
     }).await?;
 
+    // nginx ingress
+
+    let nginx_yaml_path = format!("/tmp/skate-nginx-ingress-{}.yaml", node.name);
+    let mut file = File::create(&nginx_yaml_path)?;
+    file.write_all(INGRESS_MANIFEST.as_bytes())?;
+
+
+    apply(ApplyArgs {
+        filename: vec![nginx_yaml_path],
+        grace_period: 0,
+        config: args.config.clone(),
+    }).await?;
+
     Ok(())
 }