From 8b91472fb20d03a6f7541afaaca147da0e0f95ca Mon Sep 17 00:00:00 2001 From: michaeljguarino Date: Fri, 13 Sep 2024 15:07:44 -0400 Subject: [PATCH] Prepare upcoming 0.10.25 release (#1374) --- AGENT_VERSION | 2 +- Dockerfile | 2 +- assets/src/generated/graphql.ts | 9 + ...deployments.plural.sh_customstackruns.yaml | 13 ++ .../deployments.plural.sh_prautomations.yaml | 13 ++ go/client/models_gen.go | 27 ++- .../api/v1alpha1/prautomation_types.go | 26 ++- .../api/v1alpha1/zz_generated.deepcopy.go | 30 +++ ...deployments.plural.sh_customstackruns.yaml | 13 ++ .../deployments.plural.sh_prautomations.yaml | 13 ++ go/controller/docs/api.md | 208 +++++++++++++++++- lib/console/deployments/git.ex | 9 +- lib/console/deployments/pr/impl/bitbucket.ex | 5 +- lib/console/deployments/pr/impl/github.ex | 2 +- lib/console/deployments/pr/impl/gitlab.ex | 2 +- lib/console/deployments/pr/utils.ex | 4 + lib/console/deployments/pr/validation.ex | 41 ++++ lib/console/deployments/pubsub/recurse.ex | 8 +- lib/console/deployments/stacks.ex | 30 ++- lib/console/graphql/deployments/git.ex | 7 + lib/console/graphql/helpers.ex | 11 +- lib/console/schema/embeds.ex | 11 + lib/console/schema/stack.ex | 7 + lib/console/services/base.ex | 19 +- mix.exs | 2 +- mix.lock | 2 +- ...deployments.plural.sh_customstackruns.yaml | 13 ++ .../deployments.plural.sh_prautomations.yaml | 13 ++ schema/schema.graphql | 10 + test/console/deployments/git_test.exs | 35 ++- test/console/deployments/stacks_test.exs | 27 +++ .../deployments/stack_mutations_test.exs | 7 +- 32 files changed, 575 insertions(+), 46 deletions(-) create mode 100644 lib/console/deployments/pr/validation.ex diff --git a/AGENT_VERSION b/AGENT_VERSION index 696a302384..1e8a26035d 100644 --- a/AGENT_VERSION +++ b/AGENT_VERSION @@ -1 +1 @@ -v0.4.43 \ No newline at end of file +v0.4.45 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ff705d5d00..dbc8b6a821 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,7 +73,7 @@ ENV HELM_VERSION=v3.10.3 ENV TERRAFORM_VERSION=v1.2.9 # renovate: datasource=github-releases depName=pluralsh/plural-cli -ENV CLI_VERSION=v0.9.14 +ENV CLI_VERSION=v0.9.15 # renovate: datasource=github-tags depName=kubernetes/kubernetes ENV KUBECTL_VERSION=v1.25.5 diff --git a/assets/src/generated/graphql.ts b/assets/src/generated/graphql.ts index e605a4d51f..cfe21e37a1 100644 --- a/assets/src/generated/graphql.ts +++ b/assets/src/generated/graphql.ts @@ -1337,6 +1337,14 @@ export type ConfigurationValidation = { type?: Maybe; }; +/** Validations to apply to this configuration entry prior to PR creation */ +export type ConfigurationValidationAttributes = { + /** whether the string is json encoded */ + json?: InputMaybe; + /** regex a string value should match */ + regex?: InputMaybe; +}; + export enum Conjunction { And = 'AND', Or = 'OR' @@ -4091,6 +4099,7 @@ export type PrConfigurationAttributes = { optional?: InputMaybe; placeholder?: InputMaybe; type: ConfigurationType; + validation?: InputMaybe; values?: InputMaybe>>; }; diff --git a/charts/controller/crds/deployments.plural.sh_customstackruns.yaml b/charts/controller/crds/deployments.plural.sh_customstackruns.yaml index 6baa31b489..ec2244dc22 100644 --- a/charts/controller/crds/deployments.plural.sh_customstackruns.yaml +++ b/charts/controller/crds/deployments.plural.sh_customstackruns.yaml @@ -114,6 +114,19 @@ spec: - PASSWORD - ENUM type: string + validation: + description: Any additional validations you want to apply to + this configuration item before generating a pr + properties: + json: + description: Whether the string value is supposed to be + json-encoded + type: boolean + regex: + description: A regex to match string-valued configuration + items + type: string + type: object values: items: type: string diff --git a/charts/controller/crds/deployments.plural.sh_prautomations.yaml b/charts/controller/crds/deployments.plural.sh_prautomations.yaml index f740c2c72a..1f1cddbae9 100644 --- a/charts/controller/crds/deployments.plural.sh_prautomations.yaml +++ b/charts/controller/crds/deployments.plural.sh_prautomations.yaml @@ -185,6 +185,19 @@ spec: - PASSWORD - ENUM type: string + validation: + description: Any additional validations you want to apply to + this configuration item before generating a pr + properties: + json: + description: Whether the string value is supposed to be + json-encoded + type: boolean + regex: + description: A regex to match string-valued configuration + items + type: string + type: object values: items: type: string diff --git a/go/client/models_gen.go b/go/client/models_gen.go index f0ffbf0346..2471cf835d 100644 --- a/go/client/models_gen.go +++ b/go/client/models_gen.go @@ -1056,6 +1056,14 @@ type ConfigurationValidation struct { Message *string `json:"message,omitempty"` } +// Validations to apply to this configuration entry prior to PR creation +type ConfigurationValidationAttributes struct { + // regex a string value should match + Regex *string `json:"regex,omitempty"` + // whether the string is json encoded + JSON *bool `json:"json,omitempty"` +} + type ConsoleConfiguration struct { GitCommit *string `json:"gitCommit,omitempty"` IsDemoProject *bool `json:"isDemoProject,omitempty"` @@ -3405,15 +3413,16 @@ type PrConfiguration struct { // the a configuration item for creating a new pr type PrConfigurationAttributes struct { - Type ConfigurationType `json:"type"` - Name string `json:"name"` - Default *string `json:"default,omitempty"` - Documentation *string `json:"documentation,omitempty"` - Longform *string `json:"longform,omitempty"` - Placeholder *string `json:"placeholder,omitempty"` - Optional *bool `json:"optional,omitempty"` - Condition *ConditionAttributes `json:"condition,omitempty"` - Values []*string `json:"values,omitempty"` + Type ConfigurationType `json:"type"` + Name string `json:"name"` + Default *string `json:"default,omitempty"` + Documentation *string `json:"documentation,omitempty"` + Longform *string `json:"longform,omitempty"` + Placeholder *string `json:"placeholder,omitempty"` + Optional *bool `json:"optional,omitempty"` + Condition *ConditionAttributes `json:"condition,omitempty"` + Validation *ConfigurationValidationAttributes `json:"validation,omitempty"` + Values []*string `json:"values,omitempty"` } // declaritive spec for whether a config item is relevant given prior config diff --git a/go/controller/api/v1alpha1/prautomation_types.go b/go/controller/api/v1alpha1/prautomation_types.go index 06cfe88915..954f4033b6 100644 --- a/go/controller/api/v1alpha1/prautomation_types.go +++ b/go/controller/api/v1alpha1/prautomation_types.go @@ -358,12 +358,27 @@ type PrAutomationConfiguration struct { // +kubebuilder:validation:Optional Placeholder *string `json:"placeholder,omitempty"` + // Any additional validations you want to apply to this configuration item before generating a pr + // +kubebuilder:validation:Optional + Validation *PrAutomationConfigurationValidation `json:"validation,omitempty"` + // +kubebuilder:validation:Optional Values []*string `json:"values,omitempty"` } +// PrAutomationConfigurationValidation validations to apply to configuration items in a PR Automation +type PrAutomationConfigurationValidation struct { + // A regex to match string-valued configuration items + // +kubebuilder:validation:Optional + Regex *string `json:"regex,omitempty"` + + // Whether the string value is supposed to be json-encoded + // +kubebuilder:validation:Optional + Json *bool `json:"json,omitempty"` +} + func (in *PrAutomationConfiguration) Attributes() *console.PrConfigurationAttributes { - return &console.PrConfigurationAttributes{ + conf := &console.PrConfigurationAttributes{ Type: in.Type, Name: in.Name, Default: in.Default, @@ -374,6 +389,15 @@ func (in *PrAutomationConfiguration) Attributes() *console.PrConfigurationAttrib Condition: in.Condition.Attributes(), Values: in.Values, } + + if in.Validation != nil { + conf.Validation = &console.ConfigurationValidationAttributes{ + Regex: in.Validation.Regex, + JSON: in.Validation.Json, + } + } + + return conf } // Condition ... diff --git a/go/controller/api/v1alpha1/zz_generated.deepcopy.go b/go/controller/api/v1alpha1/zz_generated.deepcopy.go index fb1c3fa497..25d5552359 100644 --- a/go/controller/api/v1alpha1/zz_generated.deepcopy.go +++ b/go/controller/api/v1alpha1/zz_generated.deepcopy.go @@ -3074,6 +3074,11 @@ func (in *PrAutomationConfiguration) DeepCopyInto(out *PrAutomationConfiguration *out = new(string) **out = **in } + if in.Validation != nil { + in, out := &in.Validation, &out.Validation + *out = new(PrAutomationConfigurationValidation) + (*in).DeepCopyInto(*out) + } if in.Values != nil { in, out := &in.Values, &out.Values *out = make([]*string, len(*in)) @@ -3097,6 +3102,31 @@ func (in *PrAutomationConfiguration) DeepCopy() *PrAutomationConfiguration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrAutomationConfigurationValidation) DeepCopyInto(out *PrAutomationConfigurationValidation) { + *out = *in + if in.Regex != nil { + in, out := &in.Regex, &out.Regex + *out = new(string) + **out = **in + } + if in.Json != nil { + in, out := &in.Json, &out.Json + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrAutomationConfigurationValidation. +func (in *PrAutomationConfigurationValidation) DeepCopy() *PrAutomationConfigurationValidation { + if in == nil { + return nil + } + out := new(PrAutomationConfigurationValidation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PrAutomationCreateConfiguration) DeepCopyInto(out *PrAutomationCreateConfiguration) { *out = *in diff --git a/go/controller/config/crd/bases/deployments.plural.sh_customstackruns.yaml b/go/controller/config/crd/bases/deployments.plural.sh_customstackruns.yaml index 6baa31b489..ec2244dc22 100644 --- a/go/controller/config/crd/bases/deployments.plural.sh_customstackruns.yaml +++ b/go/controller/config/crd/bases/deployments.plural.sh_customstackruns.yaml @@ -114,6 +114,19 @@ spec: - PASSWORD - ENUM type: string + validation: + description: Any additional validations you want to apply to + this configuration item before generating a pr + properties: + json: + description: Whether the string value is supposed to be + json-encoded + type: boolean + regex: + description: A regex to match string-valued configuration + items + type: string + type: object values: items: type: string diff --git a/go/controller/config/crd/bases/deployments.plural.sh_prautomations.yaml b/go/controller/config/crd/bases/deployments.plural.sh_prautomations.yaml index f740c2c72a..1f1cddbae9 100644 --- a/go/controller/config/crd/bases/deployments.plural.sh_prautomations.yaml +++ b/go/controller/config/crd/bases/deployments.plural.sh_prautomations.yaml @@ -185,6 +185,19 @@ spec: - PASSWORD - ENUM type: string + validation: + description: Any additional validations you want to apply to + this configuration item before generating a pr + properties: + json: + description: Whether the string value is supposed to be + json-encoded + type: boolean + regex: + description: A regex to match string-valued configuration + items + type: string + type: object values: items: type: string diff --git a/go/controller/docs/api.md b/go/controller/docs/api.md index 79b150ded6..0350ce5473 100644 --- a/go/controller/docs/api.md +++ b/go/controller/docs/api.md @@ -23,6 +23,7 @@ Package v1alpha1 contains API Schema definitions for the deployments v1alpha1 AP - [NotificationRouter](#notificationrouter) - [NotificationSink](#notificationsink) - [ObservabilityProvider](#observabilityprovider) +- [Observer](#observer) - [Pipeline](#pipeline) - [PipelineContext](#pipelinecontext) - [PrAutomation](#prautomation) @@ -754,6 +755,7 @@ _Appears in:_ | `host` _string_ | | | | | `user` _string_ | user to connect with basic auth | | | | `password` _string_ | password to connect w/ for basic auth | | | +| `passwordSecretRef` _[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#secretkeyselector-v1-core)_ | PasswordSecretRef selects a key of a password Secret | | Optional: {}
| @@ -786,6 +788,8 @@ _Appears in:_ _Appears in:_ - [HelmRepositorySpec](#helmrepositoryspec) +- [ObserverHelm](#observerhelm) +- [ObserverOci](#observeroci) | Field | Description | Default | Validation | | --- | --- | --- | --- | @@ -1147,7 +1151,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `name` _string_ | Name the name of this service, if not provided NotificationSink's own name from NotificationSink.ObjectMeta will be used. | | Optional: {}
| -| `type` _[SinkType](#sinktype)_ | Type the channel type of this sink. | | Enum: [SLACK TEAMS]
Optional: {}
| +| `type` _[SinkType](#sinktype)_ | Type the channel type of this sink. | | Enum: [SLACK TEAMS PLURAL]
Optional: {}
| | `configuration` _[SinkConfiguration](#sinkconfiguration)_ | Configuration for the specific type | | Optional: {}
| | `bindings` _[Binding](#binding) array_ | Bindings to determine users/groups to be notified for PLURAL sync types | | Optional: {}
| @@ -1224,6 +1228,189 @@ _Appears in:_ | `observabilityProviderRef` _[ObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectreference-v1-core)_ | | | Required: {}
| +#### Observer + + + +Observer is the Schema for the observers API + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `deployments.plural.sh/v1alpha1` | | | +| `kind` _string_ | `Observer` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[ObserverSpec](#observerspec)_ | | | | + + +#### ObserverAction + + + + + + + +_Appears in:_ +- [ObserverSpec](#observerspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `type` _[ObserverActionType](#observeractiontype)_ | | | Enum: [PIPELINE PR]
Type: string
| +| `configuration` _[ObserverConfiguration](#observerconfiguration)_ | | | | + + +#### ObserverConfiguration + + + + + + + +_Appears in:_ +- [ObserverAction](#observeraction) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `pr` _[ObserverPrAction](#observerpraction)_ | | | | +| `pipeline` _[ObserverPipelineAction](#observerpipelineaction)_ | | | | + + +#### ObserverGit + + + + + + + +_Appears in:_ +- [ObserverTarget](#observertarget) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `gitRepositoryRef` _[ObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectreference-v1-core)_ | GitRepositoryRef references to Git repository. | | | +| `type` _[ObserverGitTargetType](#observergittargettype)_ | | | Enum: [TAGS]
Type: string
| + + +#### ObserverHelm + + + + + + + +_Appears in:_ +- [ObserverTarget](#observertarget) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `url` _string_ | URL of the Helm repository. | | Required: {}
| +| `chart` _string_ | Chart of the Helm repository. | | Required: {}
| +| `provider` _[HelmAuthProvider](#helmauthprovider)_ | Provider is the name of the Helm auth provider. | | Enum: [BASIC BEARER GCP AZURE AWS]
Type: string
| +| `auth` _[HelmRepositoryAuth](#helmrepositoryauth)_ | Auth contains authentication credentials for the Helm repository. | | Optional: {}
| + + +#### ObserverOci + + + + + + + +_Appears in:_ +- [ObserverTarget](#observertarget) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `url` _string_ | URL of the Helm repository. | | Required: {}
| +| `provider` _[HelmAuthProvider](#helmauthprovider)_ | Provider is the name of the Helm auth provider. | | Enum: [BASIC BEARER GCP AZURE AWS]
Type: string
| +| `auth` _[HelmRepositoryAuth](#helmrepositoryauth)_ | Auth contains authentication credentials for the Helm repository. | | Optional: {}
| + + +#### ObserverPipelineAction + + + + + + + +_Appears in:_ +- [ObserverConfiguration](#observerconfiguration) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `pipelineRef` _[ObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectreference-v1-core)_ | PipelineRef references to Pipeline. | | | +| `context` _[RawExtension](https://pkg.go.dev/k8s.io/apimachinery/pkg/runtime#RawExtension)_ | | | | + + +#### ObserverPrAction + + + + + + + +_Appears in:_ +- [ObserverConfiguration](#observerconfiguration) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `prAutomationRef` _[ObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectreference-v1-core)_ | PrAutomationRef references to PR automation. | | | +| `repository` _string_ | | | Optional: {}
| +| `branchTemplate` _string_ | BranchTemplate a template to use for the created branch, use $value to interject the observed value | | | +| `context` _[RawExtension](https://pkg.go.dev/k8s.io/apimachinery/pkg/runtime#RawExtension)_ | Context is a ObserverPrAction context | | | + + +#### ObserverSpec + + + +ObserverSpec defines the desired state of Observer + + + +_Appears in:_ +- [Observer](#observer) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | the name of this observer, if not provided Observer's own name from Observer.ObjectMeta will be used. | | Optional: {}
| +| `crontab` _string_ | | | | +| `target` _[ObserverTarget](#observertarget)_ | | | | +| `actions` _[ObserverAction](#observeraction) array_ | | | | +| `projectRef` _[ObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectreference-v1-core)_ | ProjectRef references project this observer belongs to.
If not provided, it will use the default project. | | Optional: {}
| + + +#### ObserverTarget + + + + + + + +_Appears in:_ +- [ObserverSpec](#observerspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `target` _[ObserverTargetType](#observertargettype)_ | | | Enum: [OCI HELM GIT]
Type: string
| +| `format` _string_ | | | Optional: {}
| +| `order` _[ObserverTargetOrder](#observertargetorder)_ | | | Enum: [SEMVER LATEST]
Type: string
| +| `helm` _[ObserverHelm](#observerhelm)_ | | | Optional: {}
| +| `oci` _[ObserverOci](#observeroci)_ | | | Optional: {}
| +| `git` _[ObserverGit](#observergit)_ | | | Optional: {}
| + + #### Pipeline @@ -1447,9 +1634,27 @@ _Appears in:_ | `longform` _string_ | | | Optional: {}
| | `optional` _boolean_ | | | Optional: {}
| | `placeholder` _string_ | | | Optional: {}
| +| `validation` _[PrAutomationConfigurationValidation](#prautomationconfigurationvalidation)_ | Any additional validations you want to apply to this configuration item before generating a pr | | Optional: {}
| | `values` _string array_ | | | Optional: {}
| +#### PrAutomationConfigurationValidation + + + +PrAutomationConfigurationValidation validations to apply to configuration items in a PR Automation + + + +_Appears in:_ +- [PrAutomationConfiguration](#prautomationconfiguration) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `regex` _string_ | A regex to match string-valued configuration items | | Optional: {}
| +| `json` _boolean_ | Whether the string value is supposed to be json-encoded | | Optional: {}
| + + #### PrAutomationCreateConfiguration @@ -1990,6 +2195,7 @@ _Appears in:_ | `kustomize` _[ServiceKustomize](#servicekustomize)_ | Kustomize settings for service kustomization | | Optional: {}
| | `syncConfig` _[SyncConfigAttributes](#syncconfigattributes)_ | SyncConfig attributes to configure sync settings for this service | | Optional: {}
| | `dependencies` _[ObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectreference-v1-core) array_ | Dependencies contain dependent services | | Optional: {}
| +| `configurationRef` _[SecretReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#secretreference-v1-core)_ | ConfigurationRef is a secret reference which should contain service configuration. | | Optional: {}
| #### SinkConfiguration diff --git a/lib/console/deployments/git.ex b/lib/console/deployments/git.ex index bdba9e514b..85061b6c4a 100644 --- a/lib/console/deployments/git.ex +++ b/lib/console/deployments/git.ex @@ -5,7 +5,7 @@ defmodule Console.Deployments.Git do alias Console.PubSub alias Console.Deployments.{Settings, Services, Clusters} alias Console.Services.Users - alias Console.Deployments.Pr.Dispatcher + alias Console.Deployments.Pr.{Dispatcher, Validation} alias Console.Schema.{ GitRepository, User, @@ -19,6 +19,8 @@ defmodule Console.Deployments.Git do Observer } + require Logger + @cache Console.conf(:cache_adapter) @ttl :timer.minutes(30) @@ -281,8 +283,11 @@ defmodule Console.Deployments.Git do def create_pull_request(attrs \\ %{}, ctx, id, branch, identifier \\ nil, %User{} = user) do pr = get_pr_automation!(id) |> Repo.preload([:write_bindings, :create_bindings, :connection]) - with {:ok, pr} <- allow(pr, user, :create), + with :ok <- Validation.validate(pr, ctx), + {:ok, pr} <- allow(pr, user, :create), {:ok, pr_attrs} <- Dispatcher.create(prep(pr, user, identifier), branch, ctx) do + Logger.info "creating pr #{pr_attrs[:url]}" + %PullRequest{} |> PullRequest.changeset( Map.merge(pr_attrs, Map.take(pr, ~w(cluster_id service_id)a)) diff --git a/lib/console/deployments/pr/impl/bitbucket.ex b/lib/console/deployments/pr/impl/bitbucket.ex index e10d5e4b56..e1711b23b8 100644 --- a/lib/console/deployments/pr/impl/bitbucket.ex +++ b/lib/console/deployments/pr/impl/bitbucket.ex @@ -52,7 +52,10 @@ defmodule Console.Deployments.Pr.Impl.BitBucket do with {:ok, group, number} <- get_pull_id(url), {:ok, conn} <- connection(conn) do case post(conn, Path.join(["/repositories", "#{URI.encode(group)}", "pullrequests", number, "comments"]), %{ - content: %{raw: body, markup: "markdown"} + content: %{ + raw: filter_ansi(body), + markup: "markdown" + } }) do {:ok, %{"id" => id}} -> {:ok, "#{id}"} err -> err diff --git a/lib/console/deployments/pr/impl/github.ex b/lib/console/deployments/pr/impl/github.ex index a18659059f..4dcffbcded 100644 --- a/lib/console/deployments/pr/impl/github.ex +++ b/lib/console/deployments/pr/impl/github.ex @@ -58,7 +58,7 @@ defmodule Console.Deployments.Pr.Impl.Github do with {:ok, owner, repo, number} <- get_pull_id(url), {:ok, client} <- client(conn) do case Tentacat.Pulls.Reviews.create(client, owner, repo, number, %{ - "body" => body, + "body" => filter_ansi(body), "event" => "COMMENT" }) do {_, %{"id" => id}, _} -> {:ok, "#{id}"} diff --git a/lib/console/deployments/pr/impl/gitlab.ex b/lib/console/deployments/pr/impl/gitlab.ex index b18eab203b..f9c7744314 100644 --- a/lib/console/deployments/pr/impl/gitlab.ex +++ b/lib/console/deployments/pr/impl/gitlab.ex @@ -63,7 +63,7 @@ defmodule Console.Deployments.Pr.Impl.Gitlab do with {:ok, group, number} <- get_pull_id(url), {:ok, conn} <- connection(conn) do case post(conn, Path.join(["/api/v4/projects", "#{URI.encode(group)}", "merge_requests", number]), %{ - body: body + body: filter_ansi(body) }) do {:ok, %{"id" => id}} -> {:ok, "#{id}"} err -> err diff --git a/lib/console/deployments/pr/utils.ex b/lib/console/deployments/pr/utils.ex index f56d6c93ce..531d1dd86f 100644 --- a/lib/console/deployments/pr/utils.ex +++ b/lib/console/deployments/pr/utils.ex @@ -5,10 +5,14 @@ defmodule Console.Deployments.Pr.Utils do @ttl :timer.hours(1) + @ansi_code ~r/\x1b\[[0-9;]*m/ + @stack_regex [~r/plrl\/stacks?\/([[:alnum:]_\-]+)\/?/, ~r/plrl\(stacks?:([[:alnum:]_\-]*)\)/, ~r/Plural Stacks?: ([[:alnum:]_\-]+)/] @svc_regex [~r/plrl\/svcs?\/([[:alnum:]_\-]+)\/?/, ~r/plrl\(services?:([[:alnum:]_\-\/]*)\)/, ~r/Plural Services?: ([[:alnum:]_\/\-]+)/] @cluster_regex [~r/plrl\/clusters?\/([[:alnum:]_\-]+)\/?/, ~r/plrl\(clusters?:([[:alnum:]_\-]*)\)/, ~r/Plural Clusters?: ([[:alnum:]_\-]+)/] + def filter_ansi(text), do: String.replace(text, @ansi_code, "") + def pr_associations(content) do Enum.reduce(~w(stack service cluster)a, %{}, &maybe_add(&2, :"#{&1}_id", scrape(&1, content))) end diff --git a/lib/console/deployments/pr/validation.ex b/lib/console/deployments/pr/validation.ex new file mode 100644 index 0000000000..2983a6b9f7 --- /dev/null +++ b/lib/console/deployments/pr/validation.ex @@ -0,0 +1,41 @@ +defmodule Console.Deployments.Pr.Validation do + alias Console.Schema.{PrAutomation, Configuration} + + def validate(%PrAutomation{configuration: [_ | _] = config}, ctx) do + Enum.reduce_while(config, :ok, fn %Configuration{name: name} = conf, _ -> + case do_validate(conf, ctx[name]) do + :ok -> {:cont, :ok} + {:error, _} = err -> {:halt, err} + end + end) + end + def validate(_, _), do: :ok + + defp do_validate(%Configuration{type: :int}, val) when is_integer(val), do: :ok + defp do_validate(%Configuration{type: :bool}, val) when is_boolean(val), do: :ok + + defp do_validate(%Configuration{type: :enum, values: vals}, val) do + case val in vals do + true -> :ok + false -> {:error, "#{inspect(val)} is not a member of {#{Enum.join(vals, ",")}}"} + end + end + + defp do_validate(%Configuration{type: :string, validation: %Configuration.Validation{json: true}}, val) when is_binary(val) do + case Jason.decode(val) do + {:ok, _} -> :ok + _ -> {:error, "value #{val} is not a json-encoded string"} + end + end + + defp do_validate(%Configuration{type: :string, validation: %Configuration.Validation{regex: r}}, val) when is_binary(r) and is_binary(val) do + case String.match?(val, ~r/#{r}/) do + true -> :ok + false -> {:error, "value #{val} does not match regex #{r}"} + end + end + + defp do_validate(%Configuration{type: :string}, val) when is_binary(val), do: :ok + defp do_validate(%Configuration{type: t}, val), + do: {:error, "value #{inspect(val)} does not match type #{String.upcase(to_string(t))}"} +end diff --git a/lib/console/deployments/pubsub/recurse.ex b/lib/console/deployments/pubsub/recurse.ex index 471170d9ea..17c3491e74 100644 --- a/lib/console/deployments/pubsub/recurse.ex +++ b/lib/console/deployments/pubsub/recurse.ex @@ -138,13 +138,15 @@ defimpl Console.PubSub.Recurse, for: Console.PubSub.StackRunUpdated do end defimpl Console.PubSub.Recurse, for: Console.PubSub.StackRunCreated do - alias Console.Schema.{Stack, PullRequest} + alias Console.Schema.{Stack, StackRun, PullRequest} alias Console.Deployments.Stacks def process(%{item: run}) do case Console.Repo.preload(run, [:stack, :pull_request]) do - %{pull_request: %PullRequest{} = pr} -> Stacks.dequeue(pr) - %{stack: %Stack{} = stack} -> Stacks.dequeue(stack) + %StackRun{pull_request: %PullRequest{} = pr} -> + Stacks.dequeue(pr) + %StackRun{stack: %Stack{} = stack} -> + Stacks.dequeue(stack) _ -> :ok end end diff --git a/lib/console/deployments/stacks.ex b/lib/console/deployments/stacks.ex index de41e6a77b..d1cc50d763 100644 --- a/lib/console/deployments/stacks.ex +++ b/lib/console/deployments/stacks.ex @@ -118,18 +118,27 @@ defmodule Console.Deployments.Stacks do end @doc """ - Updates an existing stack + Updates an existing stack and creates a new run if a runnable change occurred """ @spec update_stack(map, binary, User.t) :: stack_resp def update_stack(attrs, id, %User{} = user) do - get_stack!(id) - |> preloaded() - |> allow(user, :write) - |> when_ok(fn s -> - Stack.changeset(s, attrs) - |> Stack.update_changeset() + start_transaction() + |> add_operation(:stack, fn _ -> + get_stack!(id) + |> preloaded() + |> allow(user, :write) + |> when_ok(fn s -> + Stack.changeset(s, attrs) + |> Stack.update_changeset() + end) + |> when_ok(:update) end) - |> when_ok(:update) + |> add_operation(:run, fn + %{stack: %Stack{runnable: true} = stack} -> + trigger_run(stack.id, user) + _ -> {:ok, nil} + end) + |> execute(extract: :stack) |> notify(:update, user) end @@ -546,7 +555,8 @@ defmodule Console.Deployments.Stacks do end) |> add_operation(:run, fn %{stack: stack} -> case latest_run(stack.id) do - %StackRun{git: %{ref: sha}} -> create_run(stack, sha) + %StackRun{git: %{ref: sha}, message: msg} -> + create_run(stack, sha, %{message: msg}) _ -> poll(stack) end end) @@ -754,7 +764,7 @@ defmodule Console.Deployments.Stacks do defp notify(pass, _, _), do: pass defp notify({:ok, %StackRun{} = stack}, :create), - do: handle_notify(PubSub.StackRunCreated, stack) + do: notify_after(50, PubSub.StackRunCreated, stack) defp notify({:ok, %RunLog{} = log}, :create), do: handle_notify(PubSub.RunLogsCreated, log) defp notify({:ok, %StackRun{} = stack}, :update), diff --git a/lib/console/graphql/deployments/git.ex b/lib/console/graphql/deployments/git.ex index 14e8eb14ab..3d2e64968f 100644 --- a/lib/console/graphql/deployments/git.ex +++ b/lib/console/graphql/deployments/git.ex @@ -136,6 +136,7 @@ defmodule Console.GraphQl.Deployments.Git do field :placeholder, :string field :optional, :boolean field :condition, :condition_attributes + field :validation, :configuration_validation_attributes field :values, list_of(:string) end @@ -146,6 +147,12 @@ defmodule Console.GraphQl.Deployments.Git do field :value, :string end + @desc "Validations to apply to this configuration entry prior to PR creation" + input_object :configuration_validation_attributes do + field :regex, :string, description: "regex a string value should match" + field :json, :boolean, description: "whether the string is json encoded" + end + @desc "The operations to be performed on the files w/in the pr" input_object :pr_automation_update_spec_attributes do field :regexes, list_of(:string) diff --git a/lib/console/graphql/helpers.ex b/lib/console/graphql/helpers.ex index 38cb4251ed..3ef8956a02 100644 --- a/lib/console/graphql/helpers.ex +++ b/lib/console/graphql/helpers.ex @@ -5,7 +5,16 @@ defmodule Console.GraphQl.Helpers do end) |> Enum.concat(resolve_changeset(changes)) end - def resolve_changeset(%{} = changes), do: Enum.flat_map(changes, fn {_, cs} -> resolve_changeset(cs) end) + + def resolve_changeset(%{__struct__: _}), do: [] + + def resolve_changeset(%{} = changes) do + Enum.flat_map(changes, fn + {_, %Ecto.Changeset{} = cs} -> resolve_changeset(cs) + _ -> [] + end) + end + def resolve_changeset(l) when is_list(l), do: Enum.flat_map(l, &resolve_changeset/1) def resolve_changeset(_), do: [] end diff --git a/lib/console/schema/embeds.ex b/lib/console/schema/embeds.ex index fd45a5cafe..79fcd75e36 100644 --- a/lib/console/schema/embeds.ex +++ b/lib/console/schema/embeds.ex @@ -131,12 +131,23 @@ defmodule Console.Schema.Configuration do field :values, {:array, :string} embeds_one :condition, Condition + + embeds_one :validation, Validation, on_replace: :update do + field :regex, :string + field :json, :boolean + end end def changeset(model, attrs \\ %{}) do model |> cast(attrs, ~w(type name default values documentation longform placeholder optional)a) |> cast_embed(:condition) + |> cast_embed(:validation, with: &validation_changeset/2) |> validate_required([:type, :name]) end + + defp validation_changeset(model, attrs) do + model + |> cast(attrs, ~w(regex json)a) + end end diff --git a/lib/console/schema/stack.ex b/lib/console/schema/stack.ex index 2d92d93b1d..30bd25c1c1 100644 --- a/lib/console/schema/stack.ex +++ b/lib/console/schema/stack.ex @@ -78,6 +78,7 @@ defmodule Console.Schema.Stack do field :variables, :map field :actor_changed, :boolean, virtual: true + field :runnable, :boolean, virtual: true field :write_policy_id, :binary_id field :read_policy_id, :binary_id @@ -188,6 +189,7 @@ defmodule Console.Schema.Stack do |> put_new_change(:write_policy_id, &Ecto.UUID.generate/0) |> put_new_change(:read_policy_id, &Ecto.UUID.generate/0) |> change_markers(actor_id: :actor_changed) + |> determine_runnable() |> validate_required(~w(name type status project_id)a) end @@ -224,4 +226,9 @@ defmodule Console.Schema.Stack do model |> cast(attrs, ~w(locked_at)a) end + + defp determine_runnable(cs) do + significant = Enum.any?(~w(files environment variables git job_spec configuration)a, &get_change(cs, &1)) + put_change(cs, :runnable, significant) + end end diff --git a/lib/console/services/base.ex b/lib/console/services/base.ex index 81c9842e97..be2c974513 100644 --- a/lib/console/services/base.ex +++ b/lib/console/services/base.ex @@ -89,11 +89,14 @@ defmodule Console.Services.Base do end def when_ok(error, _), do: error + def notify_after(timeout, event_type, resource, additional \\ %{}) do + event = build_event(event_type, resource, additional) + :timer.apply_after(timeout, Console.PubSub.Broadcaster, :notify, [event]) + {:ok, resource} + end + def handle_notify(event_type, resource, additional \\ %{}) do - Map.new(additional) - |> Map.put(:item, resource) - |> Map.put(:context, Console.Services.Audits.context()) - |> event_type.__struct__() + build_event(event_type, resource, additional) |> Console.PubSub.Broadcaster.notify() |> case do :ok -> {:ok, resource} @@ -101,6 +104,14 @@ defmodule Console.Services.Base do end end + defp build_event(event, resource, additional) do + Map.new(additional) + |> Map.put(:item, resource) + |> Map.put(:context, Console.Services.Audits.context()) + |> Map.put(:source_pid, self()) + |> event.__struct__() + end + def timestamped(map) do map |> Map.put(:inserted_at, DateTime.utc_now()) diff --git a/mix.exs b/mix.exs index c4a636e9b9..6eacad9a3a 100644 --- a/mix.exs +++ b/mix.exs @@ -85,7 +85,7 @@ defmodule Console.MixProject do {:ecto_sql, "~> 3.9.0"}, {:yajwt, "~> 1.4"}, {:joken, "~> 2.6"}, - {:piazza_core, "~> 0.3.8", git: "https://github.com/michaeljguarino/piazza_core"}, + {:piazza_core, "~> 0.3.9", git: "https://github.com/michaeljguarino/piazza_core", branch: "master", override: true}, {:flow, "~> 0.15.0"}, {:bourne, "~> 1.1"}, {:phoenix_html, "~> 2.11"}, diff --git a/mix.lock b/mix.lock index 33cd468ce5..6dd6f66d0b 100644 --- a/mix.lock +++ b/mix.lock @@ -101,7 +101,7 @@ "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, "phoenix_template": {:hex, :phoenix_template, "1.0.3", "32de561eefcefa951aead30a1f94f1b5f0379bc9e340bb5c667f65f1edfa4326", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "16f4b6588a4152f3cc057b9d0c0ba7e82ee23afa65543da535313ad8d25d8e2c"}, "phoenix_view": {:hex, :phoenix_view, "2.0.2", "6bd4d2fd595ef80d33b439ede6a19326b78f0f1d8d62b9a318e3d9c1af351098", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "a929e7230ea5c7ee0e149ffcf44ce7cf7f4b6d2bfe1752dd7c084cdff152d36f"}, - "piazza_core": {:git, "https://github.com/michaeljguarino/piazza_core", "ebbda3cff4f49f5ec647a28c03a3dd3ce03766db", []}, + "piazza_core": {:git, "https://github.com/michaeljguarino/piazza_core", "b4b357be96ba698346900597ea5c4217f539942c", [branch: "master"]}, "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, diff --git a/plural/helm/console/crds/deployments.plural.sh_customstackruns.yaml b/plural/helm/console/crds/deployments.plural.sh_customstackruns.yaml index 6baa31b489..ec2244dc22 100644 --- a/plural/helm/console/crds/deployments.plural.sh_customstackruns.yaml +++ b/plural/helm/console/crds/deployments.plural.sh_customstackruns.yaml @@ -114,6 +114,19 @@ spec: - PASSWORD - ENUM type: string + validation: + description: Any additional validations you want to apply to + this configuration item before generating a pr + properties: + json: + description: Whether the string value is supposed to be + json-encoded + type: boolean + regex: + description: A regex to match string-valued configuration + items + type: string + type: object values: items: type: string diff --git a/plural/helm/console/crds/deployments.plural.sh_prautomations.yaml b/plural/helm/console/crds/deployments.plural.sh_prautomations.yaml index f740c2c72a..1f1cddbae9 100644 --- a/plural/helm/console/crds/deployments.plural.sh_prautomations.yaml +++ b/plural/helm/console/crds/deployments.plural.sh_prautomations.yaml @@ -185,6 +185,19 @@ spec: - PASSWORD - ENUM type: string + validation: + description: Any additional validations you want to apply to + this configuration item before generating a pr + properties: + json: + description: Whether the string value is supposed to be + json-encoded + type: boolean + regex: + description: A regex to match string-valued configuration + items + type: string + type: object values: items: type: string diff --git a/schema/schema.graphql b/schema/schema.graphql index ab136d1c90..e0db03f345 100644 --- a/schema/schema.graphql +++ b/schema/schema.graphql @@ -4583,6 +4583,7 @@ input PrConfigurationAttributes { placeholder: String optional: Boolean condition: ConditionAttributes + validation: ConfigurationValidationAttributes values: [String] } @@ -4593,6 +4594,15 @@ input ConditionAttributes { value: String } +"Validations to apply to this configuration entry prior to PR creation" +input ConfigurationValidationAttributes { + "regex a string value should match" + regex: String + + "whether the string is json encoded" + json: Boolean +} + "The operations to be performed on the files w\/in the pr" input PrAutomationUpdateSpecAttributes { regexes: [String] diff --git a/test/console/deployments/git_test.exs b/test/console/deployments/git_test.exs index 04d4b704d7..21f74c9f65 100644 --- a/test/console/deployments/git_test.exs +++ b/test/console/deployments/git_test.exs @@ -268,7 +268,11 @@ defmodule Console.Deployments.GitTest do connection: conn, updates: %{regexes: ["regex"], match_strategy: :any, files: ["file.yaml"], replace_template: "replace"}, write_bindings: [%{user_id: user.id}], - create_bindings: [%{user_id: user.id}] + create_bindings: [%{user_id: user.id}], + configuration: [ + %{name: "first", type: :int}, + %{name: "second", type: :string, validation: %{regex: "[a-z0-9]+:[a-z0-9]+(,[a-z0-9]+:[a-z0-9]+)*"}} + ] ) expect(Plural, :template, fn f, _, _ -> File.read(f) end) expect(Tentacat.Pulls, :create, fn _, "pluralsh", "console", %{head: "pr-test"} -> @@ -278,7 +282,10 @@ defmodule Console.Deployments.GitTest do expect(Console.Deployments.Pr.Git, :commit, fn _, _ -> {:ok, ""} end) expect(Console.Deployments.Pr.Git, :push, fn _, "pr-test" -> {:ok, ""} end) - {:ok, pr} = Git.create_pull_request(%{}, pra.id, "pr-test", user) + {:ok, pr} = Git.create_pull_request(%{ + "first" => 10, + "second" => "webapp:name1,cron:name2" + }, pra.id, "pr-test", user) assert pr.cluster_id == pra.cluster_id assert pr.url == "https://github.com/pr/url" @@ -287,6 +294,30 @@ defmodule Console.Deployments.GitTest do assert_receive {:event, %PubSub.PullRequestCreated{item: ^pr}} end + test "it will reject a pull request w/o valid configuration" do + user = insert(:user) + conn = insert(:scm_connection, token: "some-pat") + pra = insert(:pr_automation, + identifier: "pluralsh/console", + cluster: build(:cluster), + connection: conn, + updates: %{regexes: ["regex"], match_strategy: :any, files: ["file.yaml"], replace_template: "replace"}, + write_bindings: [%{user_id: user.id}], + create_bindings: [%{user_id: user.id}], + configuration: [ + %{name: "first", type: :int}, + %{name: "second", type: :string, validation: %{regex: "[a-z0-9]+:[a-z0-9]+(,[a-z0-9]+:[a-z0-9]+)*"}} + ] + ) + + {:error, err} = Git.create_pull_request(%{ + "first" => 10, + "second" => "bogus" + }, pra.id, "pr-test", user) + + assert err =~ "does not match regex" + end + test "it can create a pull request with a github app" do user = insert(:user) {:ok, pem_string, _} = Console.keypair("console@plural.sh") diff --git a/test/console/deployments/stacks_test.exs b/test/console/deployments/stacks_test.exs index c8f87fdbc5..a905e7886d 100644 --- a/test/console/deployments/stacks_test.exs +++ b/test/console/deployments/stacks_test.exs @@ -144,6 +144,30 @@ defmodule Console.Deployments.StacksTest do assert_receive {:event, %PubSub.StackUpdated{item: ^stack}} end + test "if it makes a meaningful change, a run will be auto-created" do + user = insert(:user) + stack = insert(:stack, write_bindings: [%{user_id: user.id}]) + expect(Discovery, :sha, fn _, _ -> {:ok, "new-sha"} end) + expect(Discovery, :changes, fn _, _, _, _ -> {:ok, ["new-folder/main.tf"], "a commit message"} end) + + {:ok, stack} = Stacks.update_stack(%{ + name: "my-stack", + type: :terraform, + approval: true, + git: %{ref: "main", folder: "new-folder"}, + }, stack.id, user) + + assert stack.name == "my-stack" + assert stack.type == :terraform + assert stack.approval + assert stack.git.ref == "main" + assert stack.git.folder == "new-folder" + + [_] = StackRun.for_stack(stack.id) |> Console.Repo.all() + + assert_receive {:event, %PubSub.StackUpdated{item: ^stack}} + end + test "you can update bindings" do user = admin_user() stack = insert(:stack) @@ -449,6 +473,9 @@ defmodule Console.Deployments.StacksTest do assert_receive {:event, %PubSub.StackRunCreated{item: ^run}} + %{pull_request: sideload} = Console.Repo.preload(run, [:pull_request]) + assert sideload.id == pr.id + assert refetch(pr).sha == "new-sha" end diff --git a/test/console/graphql/mutations/deployments/stack_mutations_test.exs b/test/console/graphql/mutations/deployments/stack_mutations_test.exs index f539621595..48945273ce 100644 --- a/test/console/graphql/mutations/deployments/stack_mutations_test.exs +++ b/test/console/graphql/mutations/deployments/stack_mutations_test.exs @@ -117,7 +117,6 @@ defmodule Console.GraphQl.Deployments.StackMutationsTest do cluster { id } repository { id } git { ref folder } - configuration { version } } } """, %{"id" => stack.id, "attrs" => %{ @@ -125,8 +124,7 @@ defmodule Console.GraphQl.Deployments.StackMutationsTest do "type" => "TERRAFORM", "repositoryId" => stack.repository.id, "clusterId" => stack.cluster.id, - "git" => %{"ref" => "main", "folder" => "terraform"}, - "configuration" => %{"version" => "1.7.0"} + "git" => %{"ref" => stack.git.ref, "folder" => stack.git.folder} }}, %{current_user: admin_user()}) assert found["id"] @@ -134,9 +132,6 @@ defmodule Console.GraphQl.Deployments.StackMutationsTest do assert found["type"] == "TERRAFORM" assert found["repository"]["id"] == stack.repository.id assert found["cluster"]["id"] == stack.cluster.id - assert found["git"]["ref"] == "main" - assert found["git"]["folder"] == "terraform" - assert found["configuration"]["version"] == "1.7.0" end end