From 9834d184f444588c037b7bfbcac4fa2e8738c347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20H=C3=B6gqvist=20Tabor?= <111101+mkhq@users.noreply.github.com> Date: Tue, 19 Sep 2023 14:50:39 +0200 Subject: [PATCH] chore: bump conftest to 0.45.0 (#5107) * chore: bump conftest to 0.45.0 * fix: handle changes in conftest output format * chore: updated doc strings to reference open-policy-agent * docs: re-generate docs --------- Co-authored-by: Vladimir Vagaytsev --- .../action-types/Test/conftest-helm.md | 2 +- docs/reference/action-types/Test/conftest.md | 2 +- docs/reference/module-types/conftest.md | 2 +- docs/reference/providers/conftest.md | 2 +- examples/conftest/README.md | 8 +- plugins/conftest/index.ts | 122 +++++++++++------- plugins/conftest/test/conftest.ts | 4 +- 7 files changed, 83 insertions(+), 59 deletions(-) diff --git a/docs/reference/action-types/Test/conftest-helm.md b/docs/reference/action-types/Test/conftest-helm.md index 114de22ca5..b4bf675aed 100644 --- a/docs/reference/action-types/Test/conftest-helm.md +++ b/docs/reference/action-types/Test/conftest-helm.md @@ -13,7 +13,7 @@ If the helm Deploy requires runtime outputs from other actions, you must list th > Note: In most cases, you'll let the [`conftest-kubernetes`](../../providers/conftest-kubernetes.md) provider create this Test automatically, but you may in some cases want or need to manually specify files to test. -See the [conftest docs](https://github.com/instrumenta/conftest) for details on how to configure policies. +See the [conftest docs](https://github.com/open-policy-agent/conftest) for details on how to configure policies. Below is the full schema reference for the action. For an introduction to configuring Garden, please look at our [Configuration guide](../../../using-garden/configuration-overview.md). diff --git a/docs/reference/action-types/Test/conftest.md b/docs/reference/action-types/Test/conftest.md index 4b27b15256..f66b0cb24c 100644 --- a/docs/reference/action-types/Test/conftest.md +++ b/docs/reference/action-types/Test/conftest.md @@ -11,7 +11,7 @@ Creates a test that runs `conftest` on the specified files, with the specified ( > Note: In many cases, you'll let specific conftest providers (e.g. [`conftest-container`](../../providers/conftest-container.md) and [`conftest-kubernetes`](../../providers/conftest-kubernetes.md) create this automatically, but you may in some cases want or need to manually specify files to test. -See the [conftest docs](https://github.com/instrumenta/conftest) for details on how to configure policies. +See the [conftest docs](https://github.com/open-policy-agent/conftest) for details on how to configure policies. Below is the full schema reference for the action. For an introduction to configuring Garden, please look at our [Configuration guide](../../../using-garden/configuration-overview.md). diff --git a/docs/reference/module-types/conftest.md b/docs/reference/module-types/conftest.md index 4a8e316af8..478a2d6af0 100644 --- a/docs/reference/module-types/conftest.md +++ b/docs/reference/module-types/conftest.md @@ -16,7 +16,7 @@ namespace. > Note: In many cases, you'll let specific conftest providers (e.g. [`conftest-container`](../providers/conftest-container.md) and [`conftest-kubernetes`](../providers/conftest-kubernetes.md) create this action type automatically, but you may in some cases want or need to manually specify files to test. -See the [conftest docs](https://github.com/instrumenta/conftest) for details on how to configure policies. +See the [conftest docs](https://github.com/open-policy-agent/conftest) for details on how to configure policies. Below is the full schema reference. For an introduction to configuring Garden modules, please look at our [Configuration guide](../../using-garden/configuration-overview.md). diff --git a/docs/reference/providers/conftest.md b/docs/reference/providers/conftest.md index a1e2a98aa9..3955b96176 100644 --- a/docs/reference/providers/conftest.md +++ b/docs/reference/providers/conftest.md @@ -7,7 +7,7 @@ tocTitle: "`conftest`" ## Description -This provider allows you to validate your configuration files against policies that you specify, using the [conftest tool](https://github.com/instrumenta/conftest) and Open Policy Agent rego query files. The provider creates Test action types of the same name, which allow you to specify files to validate. +This provider allows you to validate your configuration files against policies that you specify, using the [conftest tool](https://github.com/open-policy-agent/conftest) and Open Policy Agent rego query files. The provider creates Test action types of the same name, which allow you to specify files to validate. Note that, in many cases, you'll actually want to use more specific providers that can automatically configure your `conftest` actions, e.g. the [`conftest-container`](./conftest-container.md) and/or [`conftest-kubernetes`](./conftest-kubernetes.md) providers. See the [conftest example project](https://github.com/garden-io/garden/tree/0.13.13/examples/conftest) for a simple usage example of the latter. diff --git a/examples/conftest/README.md b/examples/conftest/README.md index cb5de95eb9..f4fd3c4758 100644 --- a/examples/conftest/README.md +++ b/examples/conftest/README.md @@ -1,6 +1,6 @@ # conftest example -This simple example shows you how you can easily drop [conftest](https://github.com/instrumenta/conftest) into your project to validate your Kubernetes manifests. +This simple example shows you how you can easily drop [conftest](https://github.com/open-policy-agent/conftest) into your project to validate your Kubernetes manifests. The [project config](./garden.yml) contains a single line that automatically creates a `conftest` test for each `kubernetes` and `helm` module in your project: @@ -12,11 +12,11 @@ environments: - name: local providers: - name: local-kubernetes - - name: conftest-kubernetes # <------ + - name: conftest-kubernetes # <------ ``` -For the example, we've copied the [kubernetes example](https://github.com/instrumenta/conftest/tree/master/examples/kubernetes) from the conftest repository, and added a `helm` module type for good measure. +For the example, we've copied the [kubernetes example](https://github.com/open-policy-agent/conftest/tree/master/examples/kubernetes) from the conftest repository, and added a `helm` module type for good measure. To test this, simply run `garden test` in this directory. You should quickly see a few tests failing because resources don't match the policies defined under the `policy` directory. -Note that you could also manually specify tests using the [conftest module type](https://docs.garden.io/reference/module-types/conftest). +Note that you could also manually specify tests using the [conftest Test action type](https://docs.garden.io/reference/action-types/test/conftest). diff --git a/plugins/conftest/index.ts b/plugins/conftest/index.ts index 8ec3d0397a..c064399434 100644 --- a/plugins/conftest/index.ts +++ b/plugins/conftest/index.ts @@ -120,7 +120,7 @@ export const gardenPlugin = () => createGardenPlugin({ name: "conftest", docs: dedent` - This provider allows you to validate your configuration files against policies that you specify, using the [conftest tool](https://github.com/instrumenta/conftest) and Open Policy Agent rego query files. The provider creates Test action types of the same name, which allow you to specify files to validate. + This provider allows you to validate your configuration files against policies that you specify, using the [conftest tool](https://github.com/open-policy-agent/conftest) and Open Policy Agent rego query files. The provider creates Test action types of the same name, which allow you to specify files to validate. Note that, in many cases, you'll actually want to use more specific providers that can automatically configure your \`conftest\` actions, e.g. the [\`conftest-container\`](./conftest-container.md) and/or [\`conftest-kubernetes\`](./conftest-kubernetes.md) providers. See the [conftest example project](${gitHubUrl}) for a simple usage example of the latter. @@ -138,7 +138,7 @@ export const gardenPlugin = () => > Note: In many cases, you'll let specific conftest providers (e.g. [\`conftest-container\`](../../providers/conftest-container.md) and [\`conftest-kubernetes\`](../../providers/conftest-kubernetes.md) create this automatically, but you may in some cases want or need to manually specify files to test. - See the [conftest docs](https://github.com/instrumenta/conftest) for details on how to configure policies. + See the [conftest docs](https://github.com/open-policy-agent/conftest) for details on how to configure policies. `, schema: testActionSchema(), handlers: >>{ @@ -206,7 +206,7 @@ export const gardenPlugin = () => > Note: In most cases, you'll let the [\`conftest-kubernetes\`](../../providers/conftest-kubernetes.md) provider create this Test automatically, but you may in some cases want or need to manually specify files to test. - See the [conftest docs](https://github.com/instrumenta/conftest) for details on how to configure policies. + See the [conftest docs](https://github.com/open-policy-agent/conftest) for details on how to configure policies. `, schema: testActionSchema().keys({ helmDeploy: joi @@ -272,7 +272,6 @@ export const gardenPlugin = () => const templates = await renderTemplates({ ctx: k8sCtx, action: sourceAction, - log, }) @@ -320,7 +319,7 @@ export const gardenPlugin = () => > Note: In many cases, you'll let specific conftest providers (e.g. [\`conftest-container\`](../providers/conftest-container.md) and [\`conftest-kubernetes\`](../providers/conftest-kubernetes.md) create this action type automatically, but you may in some cases want or need to manually specify files to test. - See the [conftest docs](https://github.com/instrumenta/conftest) for details on how to configure policies. + See the [conftest docs](https://github.com/open-policy-agent/conftest) for details on how to configure policies. `, schema: commonModuleSchema(), needsBuild: false, @@ -370,7 +369,7 @@ export const gardenPlugin = () => > Note: In most cases, you'll let the [\`conftest-kubernetes\`](../providers/conftest-kubernetes.md) provider create this action type automatically, but you may in some cases want or need to manually specify files to test. - See the [conftest docs](https://github.com/instrumenta/conftest) for details on how to configure policies. + See the [conftest docs](https://github.com/open-policy-agent/conftest) for details on how to configure policies. `, schema: commonModuleSchema().keys({ sourceModule: joiIdentifier().required().description("Specify a helm module whose chart we want to test."), @@ -401,40 +400,59 @@ export const gardenPlugin = () => tools: [ { name: "conftest", - version: "0.17.1", + version: "0.45.0", description: "A rego-based configuration validator.", type: "binary", _includeInGardenImage: true, builds: [ - // this version has no arm support yet. If you add a later release, please add the "arm64" architecture. { platform: "darwin", architecture: "amd64", - url: "https://github.com/open-policy-agent/conftest/releases/download/v0.17.1/conftest_0.17.1_Darwin_x86_64.tar.gz", - sha256: "1c97f0e43fab99c94593696d362fc1e00e8e80bd0321729412de51d83ecbfb73", + url: "https://github.com/open-policy-agent/conftest/releases/download/v0.45.0/conftest_0.45.0_Darwin_x86_64.tar.gz", + sha256: "cd199c00fb634242e9062fb6b68692040198b1a2fee88537add7a719485a9839", + extract: { + format: "tar", + targetPath: "conftest", + }, + }, + { + platform: "darwin", + architecture: "arm64", + url: "https://github.com/open-policy-agent/conftest/releases/download/v0.45.0/conftest_0.45.0_Darwin_arm64.tar.gz", + sha256: "3c4e2d7fd01e7a2a17558e4e5f8086bc92312a8e8773747e2d4a067ca20127b4", extract: { format: "tar", targetPath: "conftest", }, }, - // this version has no arm support yet. If you add a later release, please add the "arm64" architecture. { platform: "linux", architecture: "amd64", - url: "https://github.com/open-policy-agent/conftest/releases/download/v0.17.1/conftest_0.17.1_Linux_x86_64.tar.gz", - sha256: "d18c95a4b04e87bfd59e06cc980801d2df5dabb371b495506ef03f70a0a40624", + url: "https://github.com/open-policy-agent/conftest/releases/download/v0.45.0/conftest_0.45.0_Linux_x86_64.tar.gz", + sha256: "65edcf630f5cd2142138555542f10f8cbc99588e5dfcefbfa1e8074c7cc82c23", + extract: { + format: "tar", + targetPath: "conftest", + }, + }, + { + platform: "linux", + architecture: "arm64", + url: "https://github.com/open-policy-agent/conftest/releases/download/v0.45.0/conftest_0.45.0_Linux_arm64.tar.gz", + sha256: "9851d4c2a6488fbaab6af34223ed77425bc6fb5a4b349a53e6e1410cdf4798f0", extract: { format: "tar", targetPath: "conftest", }, }, + { platform: "windows", architecture: "amd64", url: - "https://github.com/open-policy-agent/conftest/releases/download/v0.17.1/" + - "conftest_0.17.1_Windows_x86_64.zip", - sha256: "4c2df80420f2f148ec085bb75a8c5b92e1c665c6a041768a79924c81082527c3", + "https://github.com/open-policy-agent/conftest/releases/download/v0.45.0/" + + "conftest_0.45.0_Windows_x86_64.zip", + sha256: "376135229a8ee5e4a1e77d10dad00dc907b04c4efb7d3857e542371902e309ce", extract: { format: "zip", targetPath: "conftest.exe", @@ -471,55 +489,61 @@ function parseConftestResult(provider: ConftestProvider, log: Log, result: Execa throw new PluginError({ message: `Error running conftest: ${result.all}` }) } - const allFailures = parsed.filter((p: any) => p.failures?.length > 0) - const allWarnings = parsed.filter((p: any) => p.warnings?.length > 0) - const resultCategories: string[] = [] let formattedResult = "OK" - if (allFailures.length > 0) { - resultCategories.push(`${allFailures.length} failure(s)`) - } + let countFailures = 0 + let countWarnings = 0 - if (allWarnings.length > 0) { - resultCategories.push(`${allWarnings.length} warning(s)`) - } + const lines: string[] = [] - let formattedHeader = `conftest reported ${naturalList(resultCategories)}` + // We let the format match the conftest output + for (const { filename, warnings, failures } of parsed) { + const failuresForFilename = failures || [] + for (const failure of failuresForFilename) { + lines.push( + chalk.redBright.bold("FAIL") + chalk.gray(" - ") + chalk.redBright(filename) + chalk.gray(" - ") + failure.msg + ) + countFailures += 1 + } - if (allFailures.length > 0 || allWarnings.length > 0) { - const lines = [`${formattedHeader}:\n`] - - // We let the format match the conftest output - for (const { filename, warnings, failures } of parsed) { - for (const failure of failures) { - lines.push( - chalk.redBright.bold("FAIL") + chalk.gray(" - ") + chalk.redBright(filename) + chalk.gray(" - ") + failure.msg - ) - } - for (const warning of warnings) { - lines.push( - chalk.yellowBright.bold("WARN") + - chalk.gray(" - ") + - chalk.yellowBright(filename) + - chalk.gray(" - ") + - warning.msg - ) - } + const warningsForFilename = warnings || [] + for (const warning of warningsForFilename) { + lines.push( + chalk.yellowBright.bold("WARN") + + chalk.gray(" - ") + + chalk.yellowBright(filename) + + chalk.gray(" - ") + + warning.msg + ) + + countWarnings += 1 } + } + + if (countFailures > 0) { + resultCategories.push(`${countFailures} failure(s)`) + } - formattedResult = lines.join("\n") + if (countWarnings > 0) { + resultCategories.push(`${countWarnings} warning(s)`) } + let formattedHeader = `conftest reported ${naturalList(resultCategories)}` + const threshold = provider.config.testFailureThreshold - if (allWarnings.length > 0 && threshold === "warn") { + if (countWarnings > 0 && threshold === "warn") { success = false - } else if (allFailures.length > 0 && threshold !== "none") { + } else if (countFailures > 0 && threshold !== "none") { success = false - } else if (allWarnings.length > 0) { + } else if (countWarnings > 0) { log.warn(chalk.yellow(formattedHeader)) } + if (!success) { + formattedResult = formattedHeader + ":\n\n" + lines.join("\n") + } + return { success, formattedResult } } diff --git a/plugins/conftest/test/conftest.ts b/plugins/conftest/test/conftest.ts index 28758fee8d..58467e635d 100644 --- a/plugins/conftest/test/conftest.ts +++ b/plugins/conftest/test/conftest.ts @@ -20,7 +20,7 @@ import { TestTask } from "@garden-io/core/build/src/tasks/test" import { defaultDotIgnoreFile } from "@garden-io/core/build/src/util/fs" import { GardenApiVersion } from "@garden-io/core/build/src/constants" -describe.skip("conftest provider", () => { +describe("conftest provider", () => { const projectRoot = join(__dirname, "test-project") const projectConfig: ProjectConfig = { @@ -35,7 +35,7 @@ describe.skip("conftest provider", () => { variables: {}, } - describe("testModule", () => { + describe.skip("testModule", () => { it("should format warnings and errors nicely", async () => { const garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin()],