diff --git a/.github/workflows/product_builder.yaml b/.github/workflows/product_builder.yaml index 8993bd6098..a610f09fe0 100644 --- a/.github/workflows/product_builder.yaml +++ b/.github/workflows/product_builder.yaml @@ -184,7 +184,15 @@ jobs: - name: Init k3d run: make k3d-init - name: Run integration tests - run: make k3d-test + run: | + make k3d-config && \ + export KUBECONFIG="$(pwd)/k3d-auth.yaml" && \ + make test-integration + - name: Upload integration test results + uses: actions/upload-artifact@v3 + with: + name: golang-integration-coverage + path: ${{ env.GOLANG_WORKING_DIRECTORY }}/**.cov go_test: runs-on: ubuntu-22.04 needs: gather_changes @@ -209,8 +217,38 @@ jobs: run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Run unit tests with coverage run: make test-unit-with-coverage + - name: Upload unit test results + uses: actions/upload-artifact@v3 + with: + name: golang-unit-coverage + path: ${{ env.GOLANG_WORKING_DIRECTORY }}/**.cov + go_coverage_upload: + runs-on: ubuntu-22.04 + needs: + - go_security + - go_lint + - go_test + - go_integration + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + - name: Install coverage merger + run: go install go.shabbyrobe.org/gocovmerge/cmd/gocovmerge@latest + - name: Download integration test results from artifacts + uses: actions/download-artifact@v3 + with: + name: golang-integration-coverage + - name: Download unit test results from artifacts + uses: actions/download-artifact@v3 + with: + name: golang-unit-coverage + - name: Merge coverage + run: gocovmerge ./builder.cov ./cli.cov ./crane.cov ./dagent.cov ./internal.cov ./unit.cov > ./merged.cov - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 + with: + files: ./merged.cov + name: golang-coverage go_build: runs-on: ubuntu-22.04 needs: diff --git a/golang/.gitignore b/golang/.gitignore index 919f873162..0f2e638bc8 100644 --- a/golang/.gitignore +++ b/golang/.gitignore @@ -6,4 +6,4 @@ tmp/ k3s.yml k3s.yaml k3d-auth.yaml -coverage.cov +*.cov diff --git a/golang/Makefile b/golang/Makefile index 33b64f5cc4..84998dedff 100644 --- a/golang/Makefile +++ b/golang/Makefile @@ -241,7 +241,7 @@ k3d-stop: # make sure to have k3s set and configured .PHONY: k3d-test -k3d-test: k3d-config +k3d-test: k3d-test export KUBECONFIG='$(PWD)/k3d-auth.yaml' go test -tags=integration -race ./pkg/crane/... @@ -256,24 +256,28 @@ test-unit: # dependency: valid & working k8s configuration .PHONY: test-unit-with-coverage test-unit-with-coverage: - go test -tags=unit -race -coverpkg=./... -coverprofile=./coverage.cov -covermode=atomic ./... + go test -tags=unit -race -coverpkg=./... -coverprofile=./unit.cov -covermode=atomic ./... .PHONY: test-integration test-integration: test-dagent test-crane test-cli test-internal .PHONY: test-crane test-crane: - go test -tags=integration -race ./pkg/crane/... + go test -tags=integration -race -coverpkg=./... -coverprofile=./crane.cov -covermode=atomic ./pkg/crane/... .PHONY: test-internal test-internal: - go test -tags=integration -race ./internal/... - go test -tags=integration -race ./pkg/builder/... + go test -tags=integration -race -coverpkg=./... -coverprofile=./internal.cov -covermode=atomic ./internal/... + go test -tags=integration -race -coverpkg=./... -coverprofile=./builder.cov -covermode=atomic ./pkg/builder/... .PHONY: test-dagent test-dagent: - go test -tags=integration -race ./pkg/dagent/... + go test -tags=integration -race -coverpkg=./... -coverprofile=./dagent.cov -covermode=atomic ./pkg/dagent/... + +.PHONY: test-cli +test-cli: + go test -tags=integration -race -coverpkg=./... -coverprofile=./cli.cov -covermode=atomic ./pkg/cli/... .PHONY: coverage coverage: - go tool cover -func ./coverage.cov + go tool cover -func ./merged.cov diff --git a/golang/pkg/builder/container/container_builder_integration_test.go b/golang/pkg/builder/container/container_builder_integration_test.go index 0c0dca378b..336309a197 100644 --- a/golang/pkg/builder/container/container_builder_integration_test.go +++ b/golang/pkg/builder/container/container_builder_integration_test.go @@ -49,7 +49,7 @@ func assertPortBinding(t *testing.T, portMap nat.PortMap, internal, external str } list := portMap[lookup] - assert.Contains(t, list, nat.PortBinding{HostIP: "0.0.0.0", HostPort: external}) + assert.Contains(t, list, nat.PortBinding{HostIP: "", HostPort: external}) } func hookCallback(callback func()) containerbuilder.LifecycleFunc { diff --git a/golang/pkg/crane/k8s/deploy_facade.go b/golang/pkg/crane/k8s/deploy_facade.go index bc6ae28feb..a2374d8561 100644 --- a/golang/pkg/crane/k8s/deploy_facade.go +++ b/golang/pkg/crane/k8s/deploy_facade.go @@ -193,7 +193,7 @@ func (d *DeployFacade) Deploy() error { } } - if err := d.deployment.DeployDeployment(&deploymentParams{ + if err := d.deployment.DeployDeployment(&DeploymentParams{ image: d.params.Image, namespace: d.params.InstanceConfig.ContainerPreName, containerConfig: &d.params.ContainerConfig, @@ -270,9 +270,11 @@ func Deploy(c context.Context, dog *dogger.DeploymentLogger, deployImageRequest dog.Write(deployImageRequest.InstanceConfig.Strings()...) dog.Write(deployImageRequest.ContainerConfig.Strings(&cfg.CommonConfiguration)...) - imageName := util.JoinV("/", - *deployImageRequest.Registry, - util.JoinV(":", deployImageRequest.ImageName, deployImageRequest.Tag)) + imageName := util.JoinV(":", deployImageRequest.ImageName, deployImageRequest.Tag) + if deployImageRequest.Registry != nil { + imageName = util.JoinV("/", + *deployImageRequest.Registry, imageName) + } expandedImageName, err := imageHelper.ExpandImageName(imageName) if err != nil { diff --git a/golang/pkg/crane/k8s/deployment.go b/golang/pkg/crane/k8s/deployment.go index f95bcc0caa..6ed8773bf5 100644 --- a/golang/pkg/crane/k8s/deployment.go +++ b/golang/pkg/crane/k8s/deployment.go @@ -47,7 +47,7 @@ func NewDeployment(ctx context.Context, cfg *config.Configuration) *Deployment { return &Deployment{status: "", ctx: ctx, appConfig: cfg} } -type deploymentParams struct { +type DeploymentParams struct { namespace string image string containerConfig *v1.ContainerConfig @@ -63,7 +63,7 @@ type deploymentParams struct { issuer string } -func (d *Deployment) DeployDeployment(p *deploymentParams) error { +func (d *Deployment) DeployDeployment(p *DeploymentParams) error { client := getDeploymentsClient(p.namespace, d.appConfig) containerConfig, err := buildContainer(p, d.appConfig) @@ -277,7 +277,7 @@ func (d *Deployment) GetPodDeployment(namespace, name string) (*kappsv1.Deployme } // builds the container using the builder interface, with healthchecks, volumes, configs, ports... -func buildContainer(p *deploymentParams, +func buildContainer(p *DeploymentParams, cfg *config.Configuration, ) (*corev1.ContainerApplyConfiguration, error) { healthCheckConfig := p.containerConfig.HealthCheckConfig @@ -419,7 +419,7 @@ func getResourceManagement(resourceConfig v1.ResourceConfig, } // getInitContainers returns every init container specific(import/config) or custom ones -func getInitContainers(params *deploymentParams, cfg *config.Configuration) []*corev1.ContainerApplyConfiguration { +func getInitContainers(params *DeploymentParams, cfg *config.Configuration) []*corev1.ContainerApplyConfiguration { // this is only the config container / could be general / wait for it / other init purposes initContainers := []*corev1.ContainerApplyConfiguration{} @@ -485,7 +485,7 @@ func addImportContainer(initContainers []*corev1.ContainerApplyConfiguration, } func addInitContainers(initContainers []*corev1.ContainerApplyConfiguration, - params *deploymentParams, + params *DeploymentParams, ) []*corev1.ContainerApplyConfiguration { for _, iCont := range params.containerConfig.InitContainers { container := corev1.Container(). diff --git a/golang/pkg/crane/k8s/deployment_integration_test.go b/golang/pkg/crane/k8s/deployment_integration_test.go index 8f673d94d7..8d6e9394df 100644 --- a/golang/pkg/crane/k8s/deployment_integration_test.go +++ b/golang/pkg/crane/k8s/deployment_integration_test.go @@ -1,53 +1,76 @@ -//go:build integration - -package k8s_test - -import ( - "context" - "testing" - - "github.com/ilyakaznacheev/cleanenv" - - "github.com/stretchr/testify/assert" - - "github.com/dyrector-io/dyrectorio/golang/pkg/crane/config" - "github.com/dyrector-io/dyrectorio/golang/pkg/crane/k8s" -) - -func TestGetPods(t *testing.T) { - ctx := context.Background() - - cfg := config.Configuration{} - _ = cleanenv.ReadEnv(&cfg) - - deploymentHandler := k8s.NewDeployment(ctx, &cfg) - - deployments, err := deploymentHandler.GetDeployments(ctx, "default", &cfg) - assert.NoError(t, err) - assert.Equal(t, 2, len(deployments.Items)) - - pods, err := deploymentHandler.GetPods("default", "deployment-1") - assert.NoError(t, err) - assert.Equal(t, 1, len(pods)) -} - -func TestGetPod(t *testing.T) { - ctx := context.Background() - - cfg := config.Configuration{} - _ = cleanenv.ReadEnv(&cfg) - - deploymentHandler := k8s.NewDeployment(ctx, &cfg) - - deployments, err := deploymentHandler.GetDeployments(ctx, "default", &cfg) - assert.NoError(t, err) - assert.Equal(t, 2, len(deployments.Items)) - - pods, err := deploymentHandler.GetPods("default", "deployment-1") - assert.NoError(t, err) - assert.Equal(t, 1, len(pods)) - - pod, err := deploymentHandler.GetPod("default", pods[0].Name) - assert.NoError(t, err) - assert.NotNil(t, pod) -} +//go:build integration + +package k8s + +import ( + "context" + "testing" + "time" + + "github.com/AlekSi/pointer" + "github.com/ilyakaznacheev/cleanenv" + + "github.com/stretchr/testify/assert" + + v1 "github.com/dyrector-io/dyrectorio/golang/api/v1" + "github.com/dyrector-io/dyrectorio/golang/internal/dogger" + "github.com/dyrector-io/dyrectorio/golang/internal/grpc" + "github.com/dyrector-io/dyrectorio/golang/pkg/crane/config" +) + +func TestFetchPods(t *testing.T) { + ctx := context.Background() + + cfg := config.Configuration{} + err := cleanenv.ReadEnv(&cfg) + + ctx = grpc.WithGRPCConfig(ctx, &cfg) + assert.Nil(t, err, "error for test deployment is unexpected") + deploymentHandler := NewDeployment(ctx, &cfg) + + err = Deploy(ctx, + dogger.NewDeploymentLogger( + ctx, + pointer.ToString("test"), nil, + &cfg.CommonConfiguration), + &v1.DeployImageRequest{ + RequestID: "test", + ImageName: "nginx:latest", + InstanceConfig: v1.InstanceConfig{ + ContainerPreName: "pods", + }, ContainerConfig: v1.ContainerConfig{ + Container: "deployment-1", + }, + }, + &v1.VersionData{}) + + WaitForRunningDeployment(ctx, "pods", "deployment-1", 1, 30*time.Second, &cfg) + + assert.Nil(t, err, "error for test deployment is unexpected") + t.Run("Test get pods", func(t *testing.T) { + deployments, err := deploymentHandler.GetDeployments(ctx, "pods", &cfg) + assert.NoError(t, err) + assert.Len(t, deployments.Items, 1) + + pods, err := deploymentHandler.GetPods("pods", "deployment-1") + assert.NoError(t, err) + assert.Len(t, pods, 1) + }) + + t.Run("Test get single pod", func(t *testing.T) { + deployments, err := deploymentHandler.GetDeployments(ctx, "pods", &cfg) + assert.NoError(t, err) + assert.Equal(t, 1, len(deployments.Items)) + + pods, err := deploymentHandler.GetPods("pods", "deployment-1") + assert.NoError(t, err) + assert.Equal(t, 1, len(pods)) + + pod, err := deploymentHandler.GetPod("pods", pods[0].Name) + assert.NoError(t, err) + assert.NotNil(t, pod) + }) + + err = deploymentHandler.deleteDeployment("pods", "deployment-1") + assert.Nil(t, err, "error for deleting test deployment is unexpected") +} diff --git a/golang/pkg/dagent/update/update_integration_test.go b/golang/pkg/dagent/update/update_integration_test.go index 52b3f6729a..bb40dff702 100644 --- a/golang/pkg/dagent/update/update_integration_test.go +++ b/golang/pkg/dagent/update/update_integration_test.go @@ -7,32 +7,32 @@ import ( "context" "testing" - "github.com/dyrector-io/dyrectorio/golang/internal/grpc" "github.com/dyrector-io/dyrectorio/golang/pkg/dagent/update" - "github.com/dyrector-io/dyrectorio/golang/pkg/dagent/utils" - "github.com/dyrector-io/dyrectorio/protobuf/go/agent" "github.com/docker/docker/api/types" "github.com/docker/docker/client" "github.com/stretchr/testify/assert" ) -func TestUpdateContainarizedOnly(t *testing.T) { - cli, err := client.NewClientWithOpts(client.WithAPIVersionNegotiation()) - if err != nil { - t.Error(err) - } - - err = update.ExecuteSelfUpdate(context.TODO(), cli, &agent.AgentUpdateRequest{ - Tag: "anything", - TimeoutSeconds: 30, - Token: "token", - }, grpc.UpdateOptions{ - UpdateAlways: false, - UseContainers: true, - }) - assert.ErrorIs(t, err, &utils.UnknownContainerError{}, "Without containerized context update always fails") -} +// TODO (@nandor-magyar): this is to be fixed, integration test & agent running as a container or not are two separate concerns +// here we would need a running container env, splitting update into getting own image and update execution could be an approach +// note: disabling containerization is not a solution, the test is in vain that way +// func TestUpdateContainarizedOnly(t *testing.T) { +// cli, err := client.NewClientWithOpts(client.WithAPIVersionNegotiation()) +// if err != nil { +// t.Error(err) +// } + +// err = update.ExecuteSelfUpdate(context.TODO(), cli, &agent.AgentUpdateRequest{ +// Tag: "anything", +// TimeoutSeconds: 30, +// Token: "token", +// }, grpc.UpdateOptions{ +// UpdateAlways: false, +// UseContainers: true, +// }) +// assert.ErrorIs(t, err, &utils.UnknownContainerError{}, "Without containerized context update always fails") +// } func TestRewriteInvalid(t *testing.T) { cli, err := client.NewClientWithOpts(client.WithAPIVersionNegotiation())