Add Chip Tool snap tests (#36)

* Add tests files

Moved from:
https://github.com/canonical/matter-snap-testing/tree/0f3c7fc001768aa541f8543bef1045d056aff033/test/suites/chip-tool

Co-authored-by: Mengyi Wang <[email protected]>

* Add workflow to run tests

* Pass snap name from build to test job

* Add step to setup latest go

* Remove commented code, fix Go version and caching

* Update go mod

---------

Co-authored-by: Mengyi Wang <[email protected]>

.gitattributes

@@ -0,0 +1 @@
+tests/bin/chip-all-clusters-minimal-app-* filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-and-test-snap.yml

@@ -1,4 +1,4 @@
-name: Snap Builder
+name: Snap Tester
 
 on:
   push:
@@ -10,6 +10,8 @@ on:
 
 jobs:
   build:
+    outputs:
+      snap: ${{ steps.snapcraft.outputs.snap }}
     runs-on: ubuntu-latest
     steps:
 
@@ -18,21 +20,48 @@ jobs:
 
       - name: Build snap
         uses: snapcore/action-build@v1
-        id: build
+        id: snapcraft
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: ${{ steps.build.outputs.snap }}
-          path: ${{ steps.build.outputs.snap }}
+          name: ${{ steps.snapcraft.outputs.snap }}
+          path: ${{ steps.snapcraft.outputs.snap }}
+          if-no-files-found: error
 
   test:
     needs: build
     runs-on: ubuntu-latest
     steps:
-      - name: Download and test snap
-        uses: canonical/matter-snap-testing/test@main
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Download locally built snap
+        uses: actions/download-artifact@v4
         with:
-          name: chip-tool
-          snap: ${{needs.build.outputs.snap}}
+          name: ${{ needs.build.outputs.snap }}
 
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.*'
+          cache: false
+
+      - name: Run tests
+        shell: bash
+        working-directory: tests
+        env:
+          SKIP_TEARDOWN_REMOVAL: true
+          LOCAL_SERVICE_SNAP: ../${{ needs.build.outputs.snap }}
+        run: |
+          go test -failfast -p 1 -timeout 30m -v
+
+      - name: Upload snap logs
+        if: always() 
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs
+          path: tests/*.log

.gitignore

@@ -1,2 +1,3 @@
 *.snap
 squashfs-root*
+*.log

README.md

@@ -98,3 +98,7 @@ Build remotely for all supported architectures:
 ```
 snapcraft remote-build
 ```
+
+## Test
+
+Refer to [tests](./tests).

tests/README.md

@@ -0,0 +1,11 @@
+# Run Tests
+
+```bash
+go test -v -failfast -count 1
+```
+
+where:
+- `-v` is to enable verbose output
+- `-failfast` makes the test stop after first failure
+- `-count 1` is to avoid Go test caching for example when testing a rebuilt snap
+

tests/go.mod

@@ -0,0 +1,13 @@
+module chip-tool-snap-tests
+
+go 1.21.6
+
+require github.com/canonical/matter-snap-testing v1.0.0-beta
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.8.3 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

tests/go.sum

@@ -0,0 +1,18 @@
+github.com/canonical/matter-snap-testing v1.0.0-beta h1:wEVJduO9Crz8gfp/SzGL9OkAmJmNahC3yQ/2o7s9J5c=
+github.com/canonical/matter-snap-testing v1.0.0-beta/go.mod h1:Ku1D0Dl99nWbxK/CGPfjOSBmIZ09NqkO14p1gjwTg5o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tests/snap_test.go

@@ -0,0 +1,153 @@
+package tests
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/canonical/matter-snap-testing/utils"
+)
+
+var start = time.Now()
+
+func TestMain(m *testing.M) {
+	teardown, err := setup()
+	if err != nil {
+		log.Fatalf("Failed to setup tests: %s", err)
+	}
+
+	code := m.Run()
+	teardown()
+
+	os.Exit(code)
+}
+
+func TestMatterDeviceOperations(t *testing.T) {
+	const (
+		chipAllClusterMinimalAppFile = "bin/chip-all-clusters-minimal-app-commit-1536ca2"
+		chipAllClusterMinimalAppLog  = "chip-all-clusters-minimal-app.log"
+	)
+
+	// Setup: remove exisiting log files
+	if err := os.Remove("./" + chipAllClusterMinimalAppLog); err != nil && !os.IsNotExist(err) {
+		t.Fatalf("Error deleting log file: %s\n", err)
+	}
+	if err := os.Remove("./chip-tool.log"); err != nil && !os.IsNotExist(err) {
+		t.Fatalf("Error deleting log file: %s\n", err)
+	}
+
+	// Setup: run and log chip-all-clusters-minimal-app in the background
+	logFile, err := os.Create(chipAllClusterMinimalAppLog)
+	if err != nil {
+		t.Fatalf("Error creating log file: %s\n", err)
+	}
+
+	cmd := exec.Command("./" + chipAllClusterMinimalAppFile)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+
+	err = cmd.Start()
+	if err != nil {
+		t.Fatalf("Error starting application: %s\n", err)
+	}
+
+	t.Cleanup(func() {
+		matches, err := filepath.Glob("/tmp/chip_*")
+		if err != nil {
+			t.Fatalf("Error finding tmp chip files: %s\n", err)
+		}
+
+		for _, match := range matches {
+			if err := os.Remove(match); err != nil {
+				t.Fatalf("Error removing tmp chip file %s: %s\n", match, err)
+			}
+		}
+
+		if err := cmd.Process.Kill(); err != nil {
+			t.Fatalf("Error killing process: %s\n", err)
+		}
+
+		if logFile != nil {
+			logFile.Close()
+		}
+	})
+
+	t.Run("Commission", func(t *testing.T) {
+		utils.Exec(t, "sudo chip-tool pairing onnetwork 110 20202021")
+	})
+
+	t.Run("Control", func(t *testing.T) {
+		utils.Exec(t, "sudo chip-tool onoff toggle 110 1")
+		waitForAppMessage(t, "./"+chipAllClusterMinimalAppLog, "CHIP:ZCL: Toggle ep1 on/off", start)
+	})
+}
+
+func setup() (teardown func(), err error) {
+	const chipToolSnap = "chip-tool"
+
+	log.Println("[CLEAN]")
+	utils.SnapRemove(nil, chipToolSnap)
+
+	log.Println("[SETUP]")
+
+	teardown = func() {
+		log.Println("[TEARDOWN]")
+		utils.SnapDumpLogs(nil, start, chipToolSnap)
+
+		log.Println("Removing installed snap:", !utils.SkipTeardownRemoval)
+		if !utils.SkipTeardownRemoval {
+			utils.SnapRemove(nil, chipToolSnap)
+		}
+	}
+
+	if utils.LocalServiceSnap() {
+		err = utils.SnapInstallFromFile(nil, utils.LocalServiceSnapPath)
+	} else {
+		err = utils.SnapInstallFromStore(nil, chipToolSnap, utils.ServiceChannel)
+	}
+	if err != nil {
+		teardown()
+		return
+	}
+
+	// connect interfaces
+	utils.SnapConnect(nil, chipToolSnap+":avahi-observe", "")
+	utils.SnapConnect(nil, chipToolSnap+":bluez", "")
+	utils.SnapConnect(nil, chipToolSnap+":process-control", "")
+
+	return
+}
+
+func waitForAppMessage(t *testing.T, appLogPath, expectedLog string, since time.Time) {
+	const maxRetry = 10
+
+	for i := 1; i <= maxRetry; i++ {
+		time.Sleep(1 * time.Second)
+		t.Logf("Retry %d/%d: Waiting for expected content in logs: %s", i, maxRetry, expectedLog)
+
+		logs, err := readLogFile(appLogPath)
+		if err != nil {
+			t.Fatalf("Error reading log file: %s\n", err)
+			continue
+		}
+
+		if strings.Contains(logs, expectedLog) {
+			t.Logf("Found expected content in logs: %s", expectedLog)
+			return
+		}
+	}
+
+	t.Fatalf("Time out: reached max %d retries.", maxRetry)
+}
+
+func readLogFile(filePath string) (string, error) {
+	text, err := os.ReadFile(filePath)
+	if err != nil {
+		return "", err
+	}
+	return string(text), nil
+}