lots of changes

skateco · Nov 24, 2023 · 085f475 · 085f475
1 parent 2c51fe0
commit 085f475
Show file tree

Hide file tree

Showing 8 changed files with 137 additions and 57 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -0,0 +1,50 @@
+name: Release
+
+permissions:
+  contents: write
+
+on:
+  push:
+    tags:
+    - v[0-9]+.*
+
+jobs:
+  create-release:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: taiki-e/create-gh-release-action@v1
+      with:
+        # (optional) Path to changelog.
+        changelog: CHANGELOG.md
+        # (required) GitHub token for creating GitHub Releases.
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+  upload-assets:
+    needs: create-release
+    strategy:
+      matrix:
+        include:
+        - target: aarch64-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: aarch64-apple-darwin
+          os: macos-latest
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-latest
+        - target: x86_64-apple-darwin
+          os: macos-latest
+        # Universal macOS binary is supported as universal-apple-darwin.
+        - target: universal-apple-darwin
+          os: macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: taiki-e/upload-rust-binary-action@v1
+      with:
+        # (required) Comma-separated list of binary names (non-extension portion of filename) to build and upload.
+        # Note that glob pattern is not supported yet.
+        bin: skate,skatelet
+        # (optional) Target triple, default is host triple.
+        target: ${{ matrix.target }}
+        # (required) GitHub token for uploading assets to GitHub Releases.
+        token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/images/coredns/go.mod b/images/coredns/go.mod
@@ -4,6 +4,7 @@ go 1.21.1
 
 require (
 	github.com/coredns/coredns v1.11.1
+	github.com/networkservicemesh/fanout v1.9.1
 	github.com/openshift/coredns-mdns v0.0.0-20210625150643-8c0b6474833f
 )
 

diff --git a/images/coredns/go.sum b/images/coredns/go.sum
@@ -549,6 +549,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
 github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
+github.com/networkservicemesh/fanout v1.9.1 h1:hdTJBMky3VSGJU/GuzV/NTpLRPDNUJnQWmr7BuOBW7c=
+github.com/networkservicemesh/fanout v1.9.1/go.mod h1:EM8dDilQja7KTATYkS6En1OIdxyy19/n0ivm+ft6tDs=
 github.com/nrdcg/auroradns v1.0.0/go.mod h1:6JPXKzIRzZzMqtTDgueIhTi6rFf1QvYE/HzqidhOhjw=
 github.com/nrdcg/goinwx v0.6.1/go.mod h1:XPiut7enlbEdntAqalBIqcYcTEVhpv/dKWgDCX2SwKQ=
 github.com/nrdcg/namesilo v0.2.1/go.mod h1:lwMvfQTyYq+BbjJd30ylEG4GPSS6PII0Tia4rRpRiyw=
@@ -777,6 +779,8 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
+go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
@@ -843,6 +847,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=

diff --git a/images/coredns/main.go b/images/coredns/main.go
@@ -7,11 +7,12 @@ import (
 	"github.com/coredns/coredns/core/dnsserver"
 	_ "github.com/coredns/coredns/core/plugin"
 	"github.com/coredns/coredns/coremain"
+	_ "github.com/networkservicemesh/fanout"
 	_ "github.com/openshift/coredns-mdns"
 )
 
 func init() {
-	dnsserver.Directives = append(dnsserver.Directives, "mdns")
+	dnsserver.Directives = append(dnsserver.Directives, "mdns", "fanout")
 }
 
 func main() {

diff --git a/manifests/coredns.yaml b/manifests/coredns.yaml
@@ -17,21 +17,26 @@ spec:
     spec:
       hostNetwork: true
       volumes:
-      - name: dbus
+      - name: skate
         hostPath:
-          path: /var/run/dbus
-          type: Directory
+          path: /etc/skate
       containers:
       - name: coredns
         image: ghcr.io/skateco/coredns
         volumeMounts:
-        - mountPath: /var/run/dbus
-          name: dbus
+        - mountPath: /etc/skate
+          name: skate
         env:
         - name: CORE_FILE
           value: |
-            cluster:53 {
-              mdns cluster
+            cluster.skate:53 {
+                hosts /run/containers/cni/dnsname/podman/addnhosts {
+                    fallthrough
+                }
+            
+                errors {
+                  consolidate 5m ".*" warning
+                }
             }
             .:53 {
                 forward . 8.8.8.8

diff --git a/play/clusterplz b/play/clusterplz
@@ -33,7 +33,7 @@ users:
     - $(cat ~/.ssh/id_rsa.pub)
 EOF
         for f in $(seq $nodes); do
-            multipass launch -c 1 -m 1G -d 4G -n node-$f --cloud-init /tmp/cluster-plz.yaml
+            multipass launch -c 1 -m 1G -d 4G -n node-$f --cloud-init /tmp/cluster-plz.yaml --mount ../:/home/ubuntu/skate
         done
 
         for f in $(seq $nodes); do

diff --git a/src/create.rs b/src/create.rs
@@ -100,6 +100,10 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
         }
     };
 
+
+    config.clusters[cluster_index] = cluster.clone();
+    config.persist(Some(args.config.skateconfig.clone()))?;
+
     let conn = node_connection(&cluster, &node).await.map_err(|e| -> Box<dyn Error> { anyhow!("{}", e).into() })?;
     let info = conn.get_node_system_info().await?;
 
@@ -168,10 +172,6 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
 
     setup_networking(&conn, &cluster, &node, &info, &args).await?;
 
-    state.reconcile_node(&info)?;
-
-
-    config.clusters[cluster_index] = cluster;
 
     config.persist(Some(args.config.skateconfig))?;
 
@@ -181,13 +181,15 @@ async fn create_node(args: CreateNodeArgs) -> Result<(), Box<dyn Error>> {
 }
 
 async fn execute(conn: &SshClient, cmd: &str) -> Result<String, Box<dyn Error>> {
-    println!(">>> {}", cmd);
+    println!("{} >>> {}", conn.node_name, cmd);
     let result = conn.client.execute(cmd).await.
         map_err(|e| anyhow!("{} failed", cmd).context(e))?;
     if result.exit_status > 0 {
         return Err(anyhow!("{} failed", cmd).context(result.stderr).into());
     }
-    println!("{}", result.stdout);
+    if result.stdout.len() > 0 {
+        println!("{}", result.stdout);
+    }
     Ok(result.stdout)
 }
 
@@ -202,46 +204,9 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node,
     execute(conn, &cmd).await?;
 
     let gateway = node.subnet_cidr.split(".").take(3).join(".") + ".1";
-    let cni = "
-{
-  \"cniVersion\": \"0.4.0\",
-  \"name\": \"podman\",
-  \"plugins\": [
-    {
-      \"type\": \"bridge\",
-      \"bridge\": \"cni-podman0\",
-      \"isGateway\": true,
-      \"ipMasq\": true,
-      \"hairpinMode\": true,
-      \"ipam\": {
-        \"type\": \"host-local\",
-        \"routes\": [
-                { \"dst\": \"0.0.0.0/0\" }
-        ],
-        \"ranges\": [
-          [
-            {
-              \"subnet\": \"%%subnet%%\",
-              \"gateway\": \"%%gateway%%\"
-            }
-          ]
-        ]
-      }
-    },
-    {
-      \"type\": \"portmap\",
-      \"capabilities\": {
-        \"portMappings\": true
-      }
-    },
-    {
-      \"type\": \"firewall\"
-    },
-    {
-      \"type\": \"tuning\"
-    }
-  ]
-}\n".replace("%%subnet%%", &node.subnet_cidr).replace("%%gateway%%", &gateway);
+    // only allocate from ip 10 onwards, reserves 1-9 for other stuff
+    let cni = include_str!("./resources/podman-network.json").replace("%%subnet%%", &node.subnet_cidr)
+        .replace("%%gateway%%", &gateway);
 
     let cni = general_purpose::STANDARD.encode(cni.as_bytes());
 
@@ -278,13 +243,14 @@ async fn setup_networking(conn: &SshClient, cluster_conf: &Cluster, node: &Node,
         _ => {}
     }
 
+    let cmd = "sudo podman pull ghcr.io/skateco/coredns";
+    execute(conn, cmd).await?;
+
 
     let coredns_yaml_path = "/tmp/skate-coredns.yaml";
     let mut file = File::create(coredns_yaml_path)?;
     file.write_all(coredns_manifest_bytes)?;
 
-    let cmd = "sudo podman pull ghcr.io/skateco/coredns";
-    execute(conn, cmd).await?;
 
     apply(ApplyArgs {
         filename: vec![coredns_yaml_path.to_string()],
@@ -341,8 +307,11 @@ async fn create_replace_routes_file(conn: &SshClient, cluster_conf: &Cluster) ->
     }
 
     route_file = route_file + "sysctl -w net.ipv4.ip_forward=1\n";
+    route_file = route_file + "sysctl fs.inotify.max_user_instances=1280\n";
+    route_file = route_file + "sysctl fs.inotify.max_user_watches=655360\n";
     route_file = route_file + "sysctl -p\n";
 
+
     let route_file = general_purpose::STANDARD.encode(route_file.as_bytes());
     let cmd = format!("sudo bash -c -eu \"echo {}| base64 --decode > /etc/skate/routes.sh; chmod +x /etc/skate/routes.sh; /etc/skate/routes.sh\"", route_file);
     match execute(conn, &cmd).await {

diff --git a/src/resources/podman-network.json b/src/resources/podman-network.json
@@ -0,0 +1,48 @@
+{
+  "cniVersion": "0.4.0",
+  "name": "podman",
+  "plugins": [
+    {
+      "type": "bridge",
+      "bridge": "cni-podman0",
+      "isGateway": true,
+      "ipMasq": true,
+      "hairpinMode": true,
+      "ipam": {
+        "type": "host-local",
+        "routes": [
+          {
+            "dst": "0.0.0.0/0"
+          }
+        ],
+        "ranges": [
+          [
+            {
+              "subnet": "%%subnet%%",
+              "gateway": "%%gateway%%"
+            }
+          ]
+        ]
+      }
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+        "portMappings": true
+      }
+    },
+    {
+      "type": "firewall"
+    },
+    {
+      "type": "tuning"
+    },
+    {
+      "type": "dnsname",
+      "domainName": "cluster.skate",
+      "capabilities": {
+        "aliases": true
+      }
+    }
+  ]
+}