diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 313ca3a..6006421 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,6 +9,20 @@ permissions: contents: read jobs: + ci: + name: CI + runs-on: ubuntu-latest + + steps: + - name: Run pipeline + uses: dagger/dagger-for-github@e86a41a730841597c2b728134e83596e4e9c7804 # v5.1.0 + with: + verb: call + module: github.com/openmeterio/benthos-openmeter/ci@${{ github.event.pull_request.head.sha }} + args: --checkout --ref ${{ github.ref }} ci + cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + version: "0.9.5" + build: name: Build runs-on: ubuntu-latest diff --git a/ci/.gitignore b/ci/.gitignore new file mode 100644 index 0000000..d10f24a --- /dev/null +++ b/ci/.gitignore @@ -0,0 +1,3 @@ +/dagger.gen.go +/internal/querybuilder/ +/querybuilder/ diff --git a/ci/build.go b/ci/build.go new file mode 100644 index 0000000..0bea12c --- /dev/null +++ b/ci/build.go @@ -0,0 +1,84 @@ +package main + +import ( + "time" +) + +// Build individual artifacts. (Useful for testing and development) +func (m *Ci) Build() *Build { + return &Build{ + Source: m.Source, + } +} + +type Build struct { + // +private + Source *Directory +} + +func (m *Build) containerImages(version string) []*Container { + platforms := []Platform{ + "linux/amd64", + "linux/arm64", + } + + variants := make([]*Container, 0, len(platforms)) + + for _, platform := range platforms { + variants = append(variants, m.containerImage(platform, Opt(version))) + } + + return variants +} + +// Build a container image. +func (m *Build) ContainerImage( + // Platform in the format of OS/ARCH[/VARIANT] (eg. "darwin/arm64/v7") + platform Optional[Platform], +) *Container { + return m.containerImage(platform.GetOr(""), OptEmpty[string]()) +} + +func (m *Build) containerImage(platform Platform, version Optional[string]) *Container { + return dag.Container(ContainerOpts{Platform: platform}). + From(alpineBaseImage). + WithLabel("org.opencontainers.image.title", "benthos-openmeter"). + WithLabel("org.opencontainers.image.description", "Ingest events into OpenMeter from everywhere"). + WithLabel("org.opencontainers.image.url", "https://github.com/openmeterio/benthos-openmeter"). + WithLabel("org.opencontainers.image.created", time.Now().String()). // TODO: embed commit timestamp + WithLabel("org.opencontainers.image.source", "https://github.com/openmeterio/benthos-openmeter"). + WithLabel("org.opencontainers.image.licenses", "Apache-2.0"). + With(func(c *Container) *Container { + if v, ok := version.Get(); ok { + c = c.WithLabel("org.opencontainers.image.version", v) + } + + return c + }). + WithExec([]string{"apk", "add", "--update", "--no-cache", "ca-certificates", "tzdata", "bash"}). + WithFile("/usr/local/bin/benthos", m.binary(platform, version)) +} + +// Build a binary. +func (m *Build) Binary( + // Platform in the format of OS/ARCH[/VARIANT] (eg. "darwin/arm64/v7") + platform Optional[Platform], +) *File { + return m.binary(platform.GetOr(""), OptEmpty[string]()) +} + +func (m *Build) binary(platform Platform, version Optional[string]) *File { + return dag.Go(GoOpts{ + Version: goVersion, + }). + WithPlatform(string(platform)). + WithCgoDisabled(). + WithSource(m.Source). + Build(GoWithSourceBuildOpts{ + Trimpath: true, + RawArgs: []string{ + "-ldflags", + "-s -w -X main.version=" + version.GetOr("unknown"), + }, + }) +} diff --git a/ci/dagger.json b/ci/dagger.json new file mode 100644 index 0000000..574e4be --- /dev/null +++ b/ci/dagger.json @@ -0,0 +1,15 @@ +{ + "name": "ci", + "root": "..", + "sdk": "go", + "exclude": [ + ".devenv", + ".direnv", + "api/client/node/node_modules" + ], + "dependencies": [ + "github.com/sagikazarmark/daggerverse/go@7eca84be945d4f3932e7f5c2c1ceb6d04e703830", + "github.com/sagikazarmark/daggerverse/golangci-lint@d5da86877cae930fa6a6e2e6377cea565e5e0c62", + "github.com/shykes/daggerverse/supergit@4113b803fcf4ba83b39cd464856af94656197cbf" + ] +} diff --git a/ci/go.mod b/ci/go.mod new file mode 100644 index 0000000..af4c13e --- /dev/null +++ b/ci/go.mod @@ -0,0 +1,15 @@ +module main + +go 1.21.3 + +require ( + github.com/99designs/gqlgen v0.17.31 + github.com/Khan/genqlient v0.6.0 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/sync v0.4.0 +) + +require ( + github.com/stretchr/testify v1.8.3 // indirect + github.com/vektah/gqlparser/v2 v2.5.6 // indirect +) diff --git a/ci/go.sum b/ci/go.sum new file mode 100644 index 0000000..7d9944b --- /dev/null +++ b/ci/go.sum @@ -0,0 +1,35 @@ +github.com/99designs/gqlgen v0.17.31 h1:VncSQ82VxieHkea8tz11p7h/zSbvHSxSDZfywqWt158= +github.com/99designs/gqlgen v0.17.31/go.mod h1:i4rEatMrzzu6RXaHydq1nmEPZkb3bKQsnxNRHS4DQB4= +github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= +github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM= +github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/vektah/gqlparser/v2 v2.5.6 h1:Ou14T0N1s191eRMZ1gARVqohcbe1e8FrcONScsq8cRU= +github.com/vektah/gqlparser/v2 v2.5.6/go.mod h1:z8xXUff237NntSuH8mLFijZ+1tjV1swDbpDqjJmk6ME= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ci/main.go b/ci/main.go new file mode 100644 index 0000000..531b869 --- /dev/null +++ b/ci/main.go @@ -0,0 +1,182 @@ +package main + +import ( + "context" + "errors" + "fmt" + + "golang.org/x/sync/errgroup" +) + +const ( + goVersion = "1.21.5" + golangciLintVersion = "v1.54.2" + + alpineBaseImage = "alpine:3.19.0@sha256:51b67269f354137895d43f3b3d810bfacd3945438e94dc5ac55fdac340352f48" +) + +const imageRepo = "ghcr.io/openmeterio/benthos-openmeter" + +type Ci struct { + // +private + RegistryUser string + + // +private + RegistryPassword *Secret + + // Project source directory + // This will become useful once pulling from remote becomes available + // + // +private + Source *Directory +} + +func New( + // Checkout the repository and use that as the source directory instead of the local one. + checkout Optional[bool], + + // Commit hash to check out. + // commit Optional[string], + + // Ref to check out. + ref Optional[string], + + // Container registry user (required for pushing images). + registryUser Optional[string], + + // Container registry password (required for pushing images). + registryPassword Optional[*Secret], +) (*Ci, error) { + var source *Directory + + if checkout.GetOr(false) { + // c, ok := commit.Get() + // if !ok { + // return nil, errors.New("commit is required when --checkout option is set") + // } + + // source = dag.Git("https://github.com/openmeterio/benthos-openmeter.git", GitOpts{ + // KeepGitDir: true, + // }).Commit(c).Tree() + + r, ok := ref.Get() + if !ok { + return nil, errors.New("ref is required when --checkout option is set") + } + + source = dag.Supergit(). + Repository(). + WithRemote("origin", "https://github.com/openmeterio/benthos-openmeter.git"). + WithGitCommand([]string{"pull", "origin", r}).Worktree() + // Commit(c).Tree() + + } else { + source = projectDir() + } + + return &Ci{ + RegistryUser: registryUser.GetOr(""), + RegistryPassword: registryPassword.GetOr(nil), + Source: source, + }, nil +} + +// Run all checks and build all artifacts. +func (m *Ci) Ci(ctx context.Context) error { + group, ctx := errgroup.WithContext(ctx) + + group.Go(func() error { + _, err := m.Test().Sync(ctx) + + return err + }) + + group.Go(func() error { + _, err := m.Lint().Sync(ctx) + + return err + }) + + // TODO: run trivy scan on container(s?) + // TODO: version should be the commit hash (if any?)? + group.Go(func() error { + images := m.Build().containerImages("ci") + + for _, image := range images { + _, err := image.Sync(ctx) + if err != nil { + return err + } + } + + return nil + }) + + return group.Wait() +} + +func (m *Ci) Test() *Container { + return dag.Go(GoOpts{ + Version: goVersion, + }). + WithSource(m.Source). + Exec([]string{"go", "test", "-v", "./..."}) +} + +func (m *Ci) Lint() *Container { + return dag.GolangciLint(). + Run(GolangciLintRunOpts{ + Version: golangciLintVersion, + GoVersion: goVersion, + Source: m.Source, + Verbose: true, + }) +} + +// Build and publish a snapshot of all artifacts from the current development version. +func (m *Ci) Snapshot(ctx context.Context) error { + // TODO: capture branch name and push it as tag/version + // TODO: version should be a combination of branch name and build time? + return m.pushImages(ctx, "latest", []string{"latest", "main"}) +} + +// Build and publish all release artifacts. +func (m *Ci) Release(ctx context.Context, version string) error { + // TODO: refuse to publish release artifacts in a dirty git dir or when there is no tag pointing to the current ref + return m.pushImages(ctx, version, []string{version}) +} + +func (m *Ci) pushImages(ctx context.Context, version string, tags []string) error { + username, password := m.RegistryUser, m.RegistryPassword + + if username == "" { + return errors.New("registry user is required to push images to ghcr.io") + } + + if password == nil { + return errors.New("registry password is required to push images to ghcr.io") + } + + images := m.Build().containerImages(version) + + var group errgroup.Group + + for _, tag := range tags { + tag := tag + + group.Go(func() error { + _, err := dag.Container(). + WithRegistryAuth("ghcr.io", username, password). + Publish(ctx, fmt.Sprintf("%s:%s", imageRepo, tag), ContainerPublishOpts{ + PlatformVariants: images, + }) + if err != nil { + return err + } + + return nil + }) + } + + return group.Wait() +} diff --git a/ci/utils.go b/ci/utils.go new file mode 100644 index 0000000..340f7d8 --- /dev/null +++ b/ci/utils.go @@ -0,0 +1,32 @@ +package main + +import ( + "os" + "path/filepath" + "slices" +) + +func root() string { + wd, err := os.Getwd() + if err != nil { + panic(err) + } + return filepath.Join(wd, "..") +} + +// paths to exclude from all contexts +var excludes = []string{ + ".direnv", + ".devenv", + "ci", +} + +func exclude(paths ...string) []string { + return append(slices.Clone(excludes), paths...) +} + +func projectDir() *Directory { + return dag.Host().Directory(root(), HostDirectoryOpts{ + Exclude: exclude(), + }) +} diff --git a/flake.lock b/flake.lock index 34e16c3..cf7ada5 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,25 @@ { "nodes": { + "dagger": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1704294979, + "narHash": "sha256-66B+UtFsv9Uq2RS0v9/+oLJAQ6rOwzRRXf6cXeiABX8=", + "owner": "dagger", + "repo": "nix", + "rev": "5bc932eb359181053d42f6d73cc173966b8ff258", + "type": "github" + }, + "original": { + "owner": "dagger", + "repo": "nix", + "type": "github" + } + }, "devenv": { "inputs": { "flake-compat": "flake-compat", @@ -247,6 +267,7 @@ }, "root": { "inputs": { + "dagger": "dagger", "devenv": "devenv", "flake-parts": "flake-parts", "nixpkgs": "nixpkgs_2" diff --git a/flake.nix b/flake.nix index 47588b5..82e936b 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,8 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-parts.url = "github:hercules-ci/flake-parts"; devenv.url = "github:cachix/devenv"; + dagger.url = "github:dagger/nix"; + dagger.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = inputs@{ flake-parts, ... }: @@ -14,6 +16,16 @@ systems = [ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" ]; perSystem = { config, self', inputs', pkgs, system, ... }: rec { + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + + overlays = [ + (final: prev: { + dagger = inputs'.dagger.packages.dagger; + }) + ]; + }; + devenv.shells = { default = { languages = { @@ -38,10 +50,15 @@ packages = with pkgs; [ golangci-lint + dagger benthos ]; + env = { + DAGGER_MODULE = "ci"; + }; + # https://github.com/cachix/devenv/issues/528#issuecomment-1556108767 containers = pkgs.lib.mkForce { }; };