Merge pull request #5 from openmeterio/ci

ci: add dagger ci and release pipeline

.github/workflows/ci.yaml

@@ -9,6 +9,20 @@ permissions:
   contents: read
 
 jobs:
+  ci:
+    name: CI
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Run pipeline
+        uses: dagger/dagger-for-github@e86a41a730841597c2b728134e83596e4e9c7804 # v5.1.0
+        with:
+          verb: call
+          module: github.com/openmeterio/benthos-openmeter/ci@${{ github.event.pull_request.head.sha }}
+          args: --checkout --ref ${{ github.ref }} ci
+          cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
+          version: "0.9.5"
+
   build:
     name: Build
     runs-on: ubuntu-latest

ci/.gitignore

@@ -0,0 +1,3 @@
+/dagger.gen.go
+/internal/querybuilder/
+/querybuilder/

ci/build.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"time"
+)
+
+// Build individual artifacts. (Useful for testing and development)
+func (m *Ci) Build() *Build {
+	return &Build{
+		Source: m.Source,
+	}
+}
+
+type Build struct {
+	// +private
+	Source *Directory
+}
+
+func (m *Build) containerImages(version string) []*Container {
+	platforms := []Platform{
+		"linux/amd64",
+		"linux/arm64",
+	}
+
+	variants := make([]*Container, 0, len(platforms))
+
+	for _, platform := range platforms {
+		variants = append(variants, m.containerImage(platform, Opt(version)))
+	}
+
+	return variants
+}
+
+// Build a container image.
+func (m *Build) ContainerImage(
+	// Platform in the format of OS/ARCH[/VARIANT] (eg. "darwin/arm64/v7")
+	platform Optional[Platform],
+) *Container {
+	return m.containerImage(platform.GetOr(""), OptEmpty[string]())
+}
+
+func (m *Build) containerImage(platform Platform, version Optional[string]) *Container {
+	return dag.Container(ContainerOpts{Platform: platform}).
+		From(alpineBaseImage).
+		WithLabel("org.opencontainers.image.title", "benthos-openmeter").
+		WithLabel("org.opencontainers.image.description", "Ingest events into OpenMeter from everywhere").
+		WithLabel("org.opencontainers.image.url", "https://github.com/openmeterio/benthos-openmeter").
+		WithLabel("org.opencontainers.image.created", time.Now().String()). // TODO: embed commit timestamp
+		WithLabel("org.opencontainers.image.source", "https://github.com/openmeterio/benthos-openmeter").
+		WithLabel("org.opencontainers.image.licenses", "Apache-2.0").
+		With(func(c *Container) *Container {
+			if v, ok := version.Get(); ok {
+				c = c.WithLabel("org.opencontainers.image.version", v)
+			}
+
+			return c
+		}).
+		WithExec([]string{"apk", "add", "--update", "--no-cache", "ca-certificates", "tzdata", "bash"}).
+		WithFile("/usr/local/bin/benthos", m.binary(platform, version))
+}
+
+// Build a binary.
+func (m *Build) Binary(
+	// Platform in the format of OS/ARCH[/VARIANT] (eg. "darwin/arm64/v7")
+	platform Optional[Platform],
+) *File {
+	return m.binary(platform.GetOr(""), OptEmpty[string]())
+}
+
+func (m *Build) binary(platform Platform, version Optional[string]) *File {
+	return dag.Go(GoOpts{
+		Version: goVersion,
+	}).
+		WithPlatform(string(platform)).
+		WithCgoDisabled().
+		WithSource(m.Source).
+		Build(GoWithSourceBuildOpts{
+			Trimpath: true,
+			RawArgs: []string{
+				"-ldflags",
+				"-s -w -X main.version=" + version.GetOr("unknown"),
+			},
+		})
+}

ci/dagger.json

@@ -0,0 +1,15 @@
+{
+  "name": "ci",
+  "root": "..",
+  "sdk": "go",
+  "exclude": [
+    ".devenv",
+    ".direnv",
+    "api/client/node/node_modules"
+  ],
+  "dependencies": [
+    "github.com/sagikazarmark/daggerverse/go@7eca84be945d4f3932e7f5c2c1ceb6d04e703830",
+    "github.com/sagikazarmark/daggerverse/golangci-lint@d5da86877cae930fa6a6e2e6377cea565e5e0c62",
+    "github.com/shykes/daggerverse/supergit@4113b803fcf4ba83b39cd464856af94656197cbf"
+  ]
+}

ci/go.mod

@@ -0,0 +1,15 @@
+module main
+
+go 1.21.3
+
+require (
+	github.com/99designs/gqlgen v0.17.31
+	github.com/Khan/genqlient v0.6.0
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/sync v0.4.0
+)
+
+require (
+	github.com/stretchr/testify v1.8.3 // indirect
+	github.com/vektah/gqlparser/v2 v2.5.6 // indirect
+)

ci/go.sum

@@ -0,0 +1,35 @@
+github.com/99designs/gqlgen v0.17.31 h1:VncSQ82VxieHkea8tz11p7h/zSbvHSxSDZfywqWt158=
+github.com/99designs/gqlgen v0.17.31/go.mod h1:i4rEatMrzzu6RXaHydq1nmEPZkb3bKQsnxNRHS4DQB4=
+github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk=
+github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM=
+github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
+github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
+github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/vektah/gqlparser/v2 v2.5.6 h1:Ou14T0N1s191eRMZ1gARVqohcbe1e8FrcONScsq8cRU=
+github.com/vektah/gqlparser/v2 v2.5.6/go.mod h1:z8xXUff237NntSuH8mLFijZ+1tjV1swDbpDqjJmk6ME=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

ci/main.go

@@ -0,0 +1,182 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	goVersion           = "1.21.5"
+	golangciLintVersion = "v1.54.2"
+
+	alpineBaseImage = "alpine:3.19.0@sha256:51b67269f354137895d43f3b3d810bfacd3945438e94dc5ac55fdac340352f48"
+)
+
+const imageRepo = "ghcr.io/openmeterio/benthos-openmeter"
+
+type Ci struct {
+	// +private
+	RegistryUser string
+
+	// +private
+	RegistryPassword *Secret
+
+	// Project source directory
+	// This will become useful once pulling from remote becomes available
+	//
+	// +private
+	Source *Directory
+}
+
+func New(
+	// Checkout the repository and use that as the source directory instead of the local one.
+	checkout Optional[bool],
+
+	// Commit hash to check out.
+	// commit Optional[string],
+
+	// Ref to check out.
+	ref Optional[string],
+
+	// Container registry user (required for pushing images).
+	registryUser Optional[string],
+
+	// Container registry password (required for pushing images).
+	registryPassword Optional[*Secret],
+) (*Ci, error) {
+	var source *Directory
+
+	if checkout.GetOr(false) {
+		// c, ok := commit.Get()
+		// if !ok {
+		// 	return nil, errors.New("commit is required when --checkout option is set")
+		// }
+
+		// source = dag.Git("https://github.com/openmeterio/benthos-openmeter.git", GitOpts{
+		// 	KeepGitDir: true,
+		// }).Commit(c).Tree()
+
+		r, ok := ref.Get()
+		if !ok {
+			return nil, errors.New("ref is required when --checkout option is set")
+		}
+
+		source = dag.Supergit().
+			Repository().
+			WithRemote("origin", "https://github.com/openmeterio/benthos-openmeter.git").
+			WithGitCommand([]string{"pull", "origin", r}).Worktree()
+		// Commit(c).Tree()
+
+	} else {
+		source = projectDir()
+	}
+
+	return &Ci{
+		RegistryUser:     registryUser.GetOr(""),
+		RegistryPassword: registryPassword.GetOr(nil),
+		Source:           source,
+	}, nil
+}
+
+// Run all checks and build all artifacts.
+func (m *Ci) Ci(ctx context.Context) error {
+	group, ctx := errgroup.WithContext(ctx)
+
+	group.Go(func() error {
+		_, err := m.Test().Sync(ctx)
+
+		return err
+	})
+
+	group.Go(func() error {
+		_, err := m.Lint().Sync(ctx)
+
+		return err
+	})
+
+	// TODO: run trivy scan on container(s?)
+	// TODO: version should be the commit hash (if any?)?
+	group.Go(func() error {
+		images := m.Build().containerImages("ci")
+
+		for _, image := range images {
+			_, err := image.Sync(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+
+	return group.Wait()
+}
+
+func (m *Ci) Test() *Container {
+	return dag.Go(GoOpts{
+		Version: goVersion,
+	}).
+		WithSource(m.Source).
+		Exec([]string{"go", "test", "-v", "./..."})
+}
+
+func (m *Ci) Lint() *Container {
+	return dag.GolangciLint().
+		Run(GolangciLintRunOpts{
+			Version:   golangciLintVersion,
+			GoVersion: goVersion,
+			Source:    m.Source,
+			Verbose:   true,
+		})
+}
+
+// Build and publish a snapshot of all artifacts from the current development version.
+func (m *Ci) Snapshot(ctx context.Context) error {
+	// TODO: capture branch name and push it as tag/version
+	// TODO: version should be a combination of branch name and build time?
+	return m.pushImages(ctx, "latest", []string{"latest", "main"})
+}
+
+// Build and publish all release artifacts.
+func (m *Ci) Release(ctx context.Context, version string) error {
+	// TODO: refuse to publish release artifacts in a dirty git dir or when there is no tag pointing to the current ref
+	return m.pushImages(ctx, version, []string{version})
+}
+
+func (m *Ci) pushImages(ctx context.Context, version string, tags []string) error {
+	username, password := m.RegistryUser, m.RegistryPassword
+
+	if username == "" {
+		return errors.New("registry user is required to push images to ghcr.io")
+	}
+
+	if password == nil {
+		return errors.New("registry password is required to push images to ghcr.io")
+	}
+
+	images := m.Build().containerImages(version)
+
+	var group errgroup.Group
+
+	for _, tag := range tags {
+		tag := tag
+
+		group.Go(func() error {
+			_, err := dag.Container().
+				WithRegistryAuth("ghcr.io", username, password).
+				Publish(ctx, fmt.Sprintf("%s:%s", imageRepo, tag), ContainerPublishOpts{
+					PlatformVariants: images,
+				})
+			if err != nil {
+				return err
+			}
+
+			return nil
+		})
+	}
+
+	return group.Wait()
+}

ci/utils.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+)
+
+func root() string {
+	wd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	return filepath.Join(wd, "..")
+}
+
+// paths to exclude from all contexts
+var excludes = []string{
+	".direnv",
+	".devenv",
+	"ci",
+}
+
+func exclude(paths ...string) []string {
+	return append(slices.Clone(excludes), paths...)
+}
+
+func projectDir() *Directory {
+	return dag.Host().Directory(root(), HostDirectoryOpts{
+		Exclude: exclude(),
+	})
+}