diff --git a/assets/src/generated/graphql.ts b/assets/src/generated/graphql.ts index 10eea8e95f..faf7a6e520 100644 --- a/assets/src/generated/graphql.ts +++ b/assets/src/generated/graphql.ts @@ -654,6 +654,8 @@ export type Cluster = { updatedAt?: Maybe; /** desired k8s version for the cluster */ version?: Maybe; + /** Computes a list of statistics for OPA constraint violations w/in this cluster */ + violationStatistics?: Maybe>>; /** write policy for this cluster */ writeBindings?: Maybe>>; }; @@ -664,7 +666,10 @@ export type ClusterPolicyConstraintsArgs = { after?: InputMaybe; before?: InputMaybe; first?: InputMaybe; + kind?: InputMaybe; last?: InputMaybe; + namespace?: InputMaybe; + q?: InputMaybe; }; @@ -676,6 +681,12 @@ export type ClusterRevisionsArgs = { last?: InputMaybe; }; + +/** a representation of a cluster you can deploy to */ +export type ClusterViolationStatisticsArgs = { + field: ConstraintViolationField; +}; + /** A common kubernetes cluster add-on like cert-manager, istio, etc */ export type ClusterAddOn = { __typename?: 'ClusterAddOn'; @@ -1125,6 +1136,11 @@ export type ConstraintRefAttributes = { name: Scalars['String']['input']; }; +export enum ConstraintViolationField { + Kind = 'KIND', + Namespace = 'NAMESPACE' +} + export type Container = { __typename?: 'Container'; image?: Maybe; @@ -4333,6 +4349,7 @@ export type RootQueryType = { pod?: Maybe; pods?: Maybe; policyConstraint?: Maybe; + policyConstraints?: Maybe; postgresDatabase?: Maybe; postgresDatabases?: Maybe>>; prAutomation?: Maybe; @@ -4378,6 +4395,7 @@ export type RootQueryType = { upgradePolicies?: Maybe>>; user?: Maybe; users?: Maybe; + violationStatistics?: Maybe>>; webhooks?: Maybe; wireguardPeer?: Maybe; wireguardPeers?: Maybe>>; @@ -4858,6 +4876,17 @@ export type RootQueryTypePolicyConstraintArgs = { }; +export type RootQueryTypePolicyConstraintsArgs = { + after?: InputMaybe; + before?: InputMaybe; + first?: InputMaybe; + kind?: InputMaybe; + last?: InputMaybe; + namespace?: InputMaybe; + q?: InputMaybe; +}; + + export type RootQueryTypePostgresDatabaseArgs = { name: Scalars['String']['input']; namespace: Scalars['String']['input']; @@ -5107,6 +5136,11 @@ export type RootQueryTypeUsersArgs = { }; +export type RootQueryTypeViolationStatisticsArgs = { + field: ConstraintViolationField; +}; + + export type RootQueryTypeWebhooksArgs = { after?: InputMaybe; before?: InputMaybe; @@ -6043,6 +6077,17 @@ export type ViolationAttributes = { version?: InputMaybe; }; +/** A summary of statistics for violations w/in a specific column */ +export type ViolationStatistic = { + __typename?: 'ViolationStatistic'; + /** the total number of policy constraints */ + count?: Maybe; + /** the value of this field being aggregated */ + value: Scalars['String']['output']; + /** the total number of violations found */ + violations?: Maybe; +}; + export type WaitingState = { __typename?: 'WaitingState'; message?: Maybe; diff --git a/charts/controller/crds/deployments.plural.sh_servicedeployments.yaml b/charts/controller/crds/deployments.plural.sh_servicedeployments.yaml index 728335c0fa..196d85976c 100644 --- a/charts/controller/crds/deployments.plural.sh_servicedeployments.yaml +++ b/charts/controller/crds/deployments.plural.sh_servicedeployments.yaml @@ -289,9 +289,9 @@ spec: type: string type: array valuesFrom: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: Fetches the helm values from a secret in this cluster, + will consider any key with yaml data a values file and merge + them iteratively properties: name: description: name is unique within a namespace to reference diff --git a/controller/api/v1alpha1/servicedeployment_types.go b/controller/api/v1alpha1/servicedeployment_types.go index 3442b39fac..5d3e0f2ab4 100644 --- a/controller/api/v1alpha1/servicedeployment_types.go +++ b/controller/api/v1alpha1/servicedeployment_types.go @@ -24,7 +24,7 @@ type ServiceKustomize struct { } type ServiceHelm struct { - // Fetches the helm values from a secret in this cluster, defaults to using the "values.yaml" key + // Fetches the helm values from a secret in this cluster, will consider any key with yaml data a values file and merge them iteratively // +kubebuilder:validation:Optional ValuesFrom *corev1.SecretReference `json:"valuesFrom,omitempty"` // +kubebuilder:validation:Optional diff --git a/controller/config/crd/bases/deployments.plural.sh_servicedeployments.yaml b/controller/config/crd/bases/deployments.plural.sh_servicedeployments.yaml index 728335c0fa..196d85976c 100644 --- a/controller/config/crd/bases/deployments.plural.sh_servicedeployments.yaml +++ b/controller/config/crd/bases/deployments.plural.sh_servicedeployments.yaml @@ -289,9 +289,9 @@ spec: type: string type: array valuesFrom: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: Fetches the helm values from a secret in this cluster, + will consider any key with yaml data a values file and merge + them iteratively properties: name: description: name is unique within a namespace to reference diff --git a/controller/internal/controller/servicedeployment_controller.go b/controller/internal/controller/servicedeployment_controller.go index 98f9557b1e..706bbead6b 100644 --- a/controller/internal/controller/servicedeployment_controller.go +++ b/controller/internal/controller/servicedeployment_controller.go @@ -363,11 +363,12 @@ func (r *ServiceReconciler) MergeHelmValues(ctx context.Context, secretRef *core return nil, err } - // TODO: allow users to specify this key in another CRD field. - if vals, ok := valuesFromSecret.Data["values.yaml"]; ok { - if err := yaml.Unmarshal(vals, &valuesFromMap); err != nil { - return nil, err + for _, vals := range valuesFromSecret.Data { + current := map[string]interface{}{} + if err := yaml.Unmarshal(vals, ¤t); err != nil { + continue } + valuesFromMap = algorithms.Merge(valuesFromMap, current) } } diff --git a/lib/console/graphql/deployments/cluster.ex b/lib/console/graphql/deployments/cluster.ex index 04c6bbbc14..9861af7242 100644 --- a/lib/console/graphql/deployments/cluster.ex +++ b/lib/console/graphql/deployments/cluster.ex @@ -251,9 +251,20 @@ defmodule Console.GraphQl.Deployments.Cluster do @desc "lists OPA constraints registered in this cluster" connection field :policy_constraints, node_type: :policy_constraint do + arg :namespace, :string, description: "only show constraints with a violation for the given namespace" + arg :kind, :string, description: "only show constraints with a violation for the given kind" + arg :q, :string + resolve &Deployments.list_policy_constraints/3 end + @desc "Computes a list of statistics for OPA constraint violations w/in this cluster" + field :violation_statistics, list_of(:violation_statistic) do + arg :field, non_null(:constraint_violation_field) + + resolve &Deployments.violation_statistics/3 + end + @desc "fetches a list of runtime services found in this cluster, this is an expensive operation that should not be done in list queries" field :runtime_services, list_of(:runtime_service), resolve: &Deployments.runtime_services/3 diff --git a/lib/console/graphql/deployments/policy.ex b/lib/console/graphql/deployments/policy.ex index 2fbbef2764..227b5afa6a 100644 --- a/lib/console/graphql/deployments/policy.ex +++ b/lib/console/graphql/deployments/policy.ex @@ -2,6 +2,11 @@ defmodule Console.GraphQl.Deployments.Policy do use Console.GraphQl.Schema.Base alias Console.GraphQl.Resolvers.Deployments + enum :constraint_violation_field do + value :namespace + value :kind + end + @desc "inputs to add constraint data from an OPA gatekeeper constraint CRD" input_object :policy_constraint_attributes do field :name, non_null(:string) @@ -14,7 +19,7 @@ defmodule Console.GraphQl.Deployments.Policy do input_object :constraint_ref_attributes do field :kind, non_null(:string) - field :name, non_null(:string) + field :name, non_null(:string) end input_object :violation_attributes do @@ -47,7 +52,14 @@ defmodule Console.GraphQl.Deployments.Policy do object :constraint_ref do field :kind, non_null(:string) - field :name, non_null(:string) + field :name, non_null(:string) + end + + @desc "A summary of statistics for violations w/in a specific column" + object :violation_statistic do + field :value, non_null(:string), description: "the value of this field being aggregated" + field :violations, :integer, description: "the total number of violations found" + field :count, :integer, description: "the total number of policy constraints" end @desc "A violation of a given OPA Gatekeeper constraint" @@ -66,6 +78,22 @@ defmodule Console.GraphQl.Deployments.Policy do connection node_type: :policy_constraint object :policy_queries do + connection field :policy_constraints, node_type: :policy_constraint do + middleware Authenticated + arg :kind, :string + arg :namespace, :string + arg :q, :string + + resolve &Deployments.list_policy_constraints/2 + end + + field :violation_statistics, list_of(:violation_statistic) do + middleware Authenticated + arg :field, non_null(:constraint_violation_field) + + resolve &Deployments.violation_statistics/2 + end + field :policy_constraint, :policy_constraint do middleware Authenticated arg :id, non_null(:id) diff --git a/lib/console/graphql/resolvers/deployments/policy.ex b/lib/console/graphql/resolvers/deployments/policy.ex index 5c314845b5..c2cf5b8486 100644 --- a/lib/console/graphql/resolvers/deployments/policy.ex +++ b/lib/console/graphql/resolvers/deployments/policy.ex @@ -8,12 +8,44 @@ defmodule Console.GraphQl.Resolvers.Deployments.Policy do |> allow(user, :read) end + def list_policy_constraints(args, %{context: %{current_user: user}}) do + PolicyConstraint.for_user(user) + |> PolicyConstraint.globally_ordered() + |> maybe_search(PolicyConstraint, args) + |> apply_filters(args) + |> paginate(args) + end + def list_policy_constraints(cluster, args, _) do PolicyConstraint.for_cluster(cluster.id) |> PolicyConstraint.ordered() + |> maybe_search(PolicyConstraint, args) + |> apply_filters(args) |> paginate(args) end + defp apply_filters(query, args) do + Enum.reduce(args, query, fn + {:namespace, ns}, q -> PolicyConstraint.for_namespace(q, ns) + {:kind, k}, q -> PolicyConstraint.for_kind(q, k) + _, q -> q + end) + end + + def violation_statistics(%{field: f}, %{context: %{current_user: user}}) do + PolicyConstraint.for_user(user) + |> PolicyConstraint.statistics(f) + |> Console.Repo.all() + |> ok() + end + + def violation_statistics(cluster, %{field: f}, _) do + PolicyConstraint.for_cluster(cluster.id) + |> PolicyConstraint.statistics(f) + |> Console.Repo.all() + |> ok() + end + def fetch_constraint(%{ref: %{name: name, kind: kind}, cluster_id: cluster_id}, _, _) do path = Kube.Client.Base.path("constraints.gatekeeper.sh", "v1beta1", kind, nil, name) with %Cluster{} = cluster <- Clusters.get_cluster(cluster_id), diff --git a/lib/console/schema/policy_constraint.ex b/lib/console/schema/policy_constraint.ex index f1803edca3..11542a82da 100644 --- a/lib/console/schema/policy_constraint.ex +++ b/lib/console/schema/policy_constraint.ex @@ -16,6 +16,7 @@ defmodule Console.Schema.PolicyConstraint do has_many :violations, ConstraintViolation, on_replace: :delete, foreign_key: :constraint_id + belongs_to :cluster, Cluster timestamps() @@ -25,10 +26,49 @@ defmodule Console.Schema.PolicyConstraint do from(p in query, where: p.name not in ^names) end + def for_user(query \\ __MODULE__, user) do + clusters = Cluster.for_user(user) + from(p in query, + join: c in subquery(clusters), + as: :clusters, + on: c.id == p.cluster_id + ) + end + + def globally_ordered(query \\ __MODULE__) do + from([p, clusters: c] in query, order_by: [asc: c.name, asc: p.name]) + end + def for_cluster(query \\ __MODULE__, cluster_id) do from(p in query, where: p.cluster_id == ^cluster_id) end + def for_namespace(query \\ __MODULE__, ns) do + from(p in query, + join: v in assoc(p, :violations), + where: v.namespace == ^ns + ) + end + + def for_kind(query \\ __MODULE__, kind) do + from(p in query, + join: v in assoc(p, :violations), + where: v.kind == ^kind + ) + end + + def search(query \\ __MODULE__, q) do + from(p in query, where: ilike(p.name, ^"%#{q}%")) + end + + def statistics(query \\ __MODULE__, field) do + from(p in query, + join: v in assoc(p, :violations), + group_by: field(v, ^field), + select: %{value: field(v, ^field), violations: count(v.id, :distinct), count: count(p.id, :distinct)} + ) + end + def ordered(query \\ __MODULE__, order \\ [asc: :name]) do from(p in query, order_by: ^order) end diff --git a/plural/helm/console/crds/deployments.plural.sh_servicedeployments.yaml b/plural/helm/console/crds/deployments.plural.sh_servicedeployments.yaml index 728335c0fa..196d85976c 100644 --- a/plural/helm/console/crds/deployments.plural.sh_servicedeployments.yaml +++ b/plural/helm/console/crds/deployments.plural.sh_servicedeployments.yaml @@ -289,9 +289,9 @@ spec: type: string type: array valuesFrom: - description: |- - SecretReference represents a Secret Reference. It has enough information to retrieve secret - in any namespace + description: Fetches the helm values from a secret in this cluster, + will consider any key with yaml data a values file and merge + them iteratively properties: name: description: name is unique within a namespace to reference diff --git a/schema/schema.graphql b/schema/schema.graphql index 4894027820..10bab0b95b 100644 --- a/schema/schema.graphql +++ b/schema/schema.graphql @@ -318,6 +318,10 @@ type RootQueryType { notificationRouters(after: String, first: Int, before: String, last: Int, q: String): NotificationRouterConnection + policyConstraints(after: String, first: Int, before: String, last: Int, kind: String, namespace: String, q: String): PolicyConstraintConnection + + violationStatistics(field: ConstraintViolationField!): [ViolationStatistic] + policyConstraint(id: ID!): PolicyConstraint deploymentSettings: DeploymentSettings @@ -757,6 +761,11 @@ input RbacAttributes { writeBindings: [PolicyBindingAttributes] } +enum ConstraintViolationField { + NAMESPACE + KIND +} + "inputs to add constraint data from an OPA gatekeeper constraint CRD" input PolicyConstraintAttributes { name: String! @@ -817,6 +826,18 @@ type ConstraintRef { name: String! } +"A summary of statistics for violations w\/in a specific column" +type ViolationStatistic { + "the value of this field being aggregated" + value: String! + + "the total number of violations found" + violations: Int + + "the total number of policy constraints" + count: Int +} + "A violation of a given OPA Gatekeeper constraint" type Violation { id: ID! @@ -2380,7 +2401,26 @@ type Cluster { revisions(after: String, first: Int, before: String, last: Int): ClusterRevisionConnection "lists OPA constraints registered in this cluster" - policyConstraints(after: String, first: Int, before: String, last: Int): PolicyConstraintConnection + policyConstraints( + after: String + + first: Int + + before: String + + last: Int + + "only show constraints with a violation for the given namespace" + namespace: String + + "only show constraints with a violation for the given kind" + kind: String + + q: String + ): PolicyConstraintConnection + + "Computes a list of statistics for OPA constraint violations w\/in this cluster" + violationStatistics(field: ConstraintViolationField!): [ViolationStatistic] "fetches a list of runtime services found in this cluster, this is an expensive operation that should not be done in list queries" runtimeServices: [RuntimeService] diff --git a/test/console/graphql/queries/deployments/policy_queries_test.exs b/test/console/graphql/queries/deployments/policy_queries_test.exs index 52722f45bb..55f1510188 100644 --- a/test/console/graphql/queries/deployments/policy_queries_test.exs +++ b/test/console/graphql/queries/deployments/policy_queries_test.exs @@ -1,6 +1,44 @@ defmodule Console.GraphQl.Deployments.PolicyQueriesTest do use Console.DataCase, async: true + describe "cluster" do + test "it can fetch namespace constraint statistics for a cluster" do + cluster = insert(:cluster) + con1 = insert(:policy_constraint, violation_count: 2, cluster: cluster) + insert_list(2, :constraint_violation, constraint: con1, namespace: "test") + + {:ok, %{data: %{"cluster" => %{"violationStatistics" => [res]}}}} = run_query(""" + query cluster($id: ID!) { + cluster(id: $id) { + violationStatistics(field: NAMESPACE) { value count violations } + } + } + """, %{"id" => cluster.id}, %{current_user: admin_user()}) + + assert res["value"] == "test" + assert res["count"] == 1 + assert res["violations"] == 2 + end + + test "it can fetch namespace kind statistics for a cluster" do + cluster = insert(:cluster) + con1 = insert(:policy_constraint, cluster: cluster) + insert_list(2, :constraint_violation, constraint: con1, kind: "Service") + + {:ok, %{data: %{"cluster" => %{"violationStatistics" => [res]}}}} = run_query(""" + query cluster($id: ID!) { + cluster(id: $id) { + violationStatistics(field: KIND) { value count violations } + } + } + """, %{"id" => cluster.id}, %{current_user: admin_user()}) + + assert res["value"] == "Service" + assert res["count"] == 1 + assert res["violations"] == 2 + end + end + describe "policyConstraint" do test "admins can query a policy constraint by id" do constraint = insert(:policy_constraint) @@ -44,4 +82,53 @@ defmodule Console.GraphQl.Deployments.PolicyQueriesTest do """, %{"id" => constraint.id}, %{current_user: insert(:user)}) end end + + describe "policyConstraints" do + test "it can fetch constraints for all accessible clusters" do + [cluster1, cluster2] = insert_list(2, :cluster) + first = insert_list(2, :policy_constraint, cluster: cluster1) + second = insert_list(3, :policy_constraint, cluster: cluster2) + + {:ok, %{data: %{"policyConstraints" => found}}} = run_query(""" + query { + policyConstraints(first: 5) { + edges { node { id } } + } + } + """, %{}, %{current_user: admin_user()}) + + assert from_connection(found) + |> ids_equal(first ++ second) + end + end + + describe "violationStatistics" do + test "it can fetch statistics for violations globally" do + cluster = insert(:cluster) + con1 = insert(:policy_constraint, violation_count: 2, cluster: cluster) + insert_list(2, :constraint_violation, constraint: con1, namespace: "test") + + cluster2 = insert(:cluster) + con2 = insert(:policy_constraint, violation_count: 2, cluster: cluster2) + insert_list(2, :constraint_violation, constraint: con2, namespace: "stage") + + {:ok, %{data: %{"violationStatistics" => stats}}} = run_query(""" + query { + violationStatistics(field: NAMESPACE) { + value + count + violations + } + } + """, %{}, %{current_user: admin_user()}) + + %{"test" => test, "stage" => stage} = Map.new(stats, & {&1["value"], &1}) + + assert test["count"] == 1 + assert test["violations"] == 2 + + assert stage["count"] == 1 + assert stage["violations"] == 2 + end + end end