More advanced policy constraint queries (#763)

assets/src/generated/graphql.ts

@@ -654,6 +654,8 @@ export type Cluster = {
   updatedAt?: Maybe<Scalars['DateTime']['output']>;
   /** desired k8s version for the cluster */
   version?: Maybe<Scalars['String']['output']>;
+  /** Computes a list of statistics for OPA constraint violations w/in this cluster */
+  violationStatistics?: Maybe<Array<Maybe<ViolationStatistic>>>;
   /** write policy for this cluster */
   writeBindings?: Maybe<Array<Maybe<PolicyBinding>>>;
 };
@@ -664,7 +666,10 @@ export type ClusterPolicyConstraintsArgs = {
   after?: InputMaybe<Scalars['String']['input']>;
   before?: InputMaybe<Scalars['String']['input']>;
   first?: InputMaybe<Scalars['Int']['input']>;
+  kind?: InputMaybe<Scalars['String']['input']>;
   last?: InputMaybe<Scalars['Int']['input']>;
+  namespace?: InputMaybe<Scalars['String']['input']>;
+  q?: InputMaybe<Scalars['String']['input']>;
 };
 
 
@@ -676,6 +681,12 @@ export type ClusterRevisionsArgs = {
   last?: InputMaybe<Scalars['Int']['input']>;
 };
 
+
+/** a representation of a cluster you can deploy to */
+export type ClusterViolationStatisticsArgs = {
+  field: ConstraintViolationField;
+};
+
 /** A common kubernetes cluster add-on like cert-manager, istio, etc */
 export type ClusterAddOn = {
   __typename?: 'ClusterAddOn';
@@ -1125,6 +1136,11 @@ export type ConstraintRefAttributes = {
   name: Scalars['String']['input'];
 };
 
+export enum ConstraintViolationField {
+  Kind = 'KIND',
+  Namespace = 'NAMESPACE'
+}
+
 export type Container = {
   __typename?: 'Container';
   image?: Maybe<Scalars['String']['output']>;
@@ -4333,6 +4349,7 @@ export type RootQueryType = {
   pod?: Maybe<Pod>;
   pods?: Maybe<PodConnection>;
   policyConstraint?: Maybe<PolicyConstraint>;
+  policyConstraints?: Maybe<PolicyConstraintConnection>;
   postgresDatabase?: Maybe<Postgresql>;
   postgresDatabases?: Maybe<Array<Maybe<Postgresql>>>;
   prAutomation?: Maybe<PrAutomation>;
@@ -4378,6 +4395,7 @@ export type RootQueryType = {
   upgradePolicies?: Maybe<Array<Maybe<UpgradePolicy>>>;
   user?: Maybe<User>;
   users?: Maybe<UserConnection>;
+  violationStatistics?: Maybe<Array<Maybe<ViolationStatistic>>>;
   webhooks?: Maybe<WebhookConnection>;
   wireguardPeer?: Maybe<WireguardPeer>;
   wireguardPeers?: Maybe<Array<Maybe<WireguardPeer>>>;
@@ -4858,6 +4876,17 @@ export type RootQueryTypePolicyConstraintArgs = {
 };
 
 
+export type RootQueryTypePolicyConstraintsArgs = {
+  after?: InputMaybe<Scalars['String']['input']>;
+  before?: InputMaybe<Scalars['String']['input']>;
+  first?: InputMaybe<Scalars['Int']['input']>;
+  kind?: InputMaybe<Scalars['String']['input']>;
+  last?: InputMaybe<Scalars['Int']['input']>;
+  namespace?: InputMaybe<Scalars['String']['input']>;
+  q?: InputMaybe<Scalars['String']['input']>;
+};
+
+
 export type RootQueryTypePostgresDatabaseArgs = {
   name: Scalars['String']['input'];
   namespace: Scalars['String']['input'];
@@ -5107,6 +5136,11 @@ export type RootQueryTypeUsersArgs = {
 };
 
 
+export type RootQueryTypeViolationStatisticsArgs = {
+  field: ConstraintViolationField;
+};
+
+
 export type RootQueryTypeWebhooksArgs = {
   after?: InputMaybe<Scalars['String']['input']>;
   before?: InputMaybe<Scalars['String']['input']>;
@@ -6043,6 +6077,17 @@ export type ViolationAttributes = {
   version?: InputMaybe<Scalars['String']['input']>;
 };
 
+/** A summary of statistics for violations w/in a specific column */
+export type ViolationStatistic = {
+  __typename?: 'ViolationStatistic';
+  /** the total number of policy constraints */
+  count?: Maybe<Scalars['Int']['output']>;
+  /** the value of this field being aggregated */
+  value: Scalars['String']['output'];
+  /** the total number of violations found */
+  violations?: Maybe<Scalars['Int']['output']>;
+};
+
 export type WaitingState = {
   __typename?: 'WaitingState';
   message?: Maybe<Scalars['String']['output']>;

charts/controller/crds/deployments.plural.sh_servicedeployments.yaml

@@ -289,9 +289,9 @@ spec:
                       type: string
                     type: array
                   valuesFrom:
-                    description: |-
-                      SecretReference represents a Secret Reference. It has enough information to retrieve secret
-                      in any namespace
+                    description: Fetches the helm values from a secret in this cluster,
+                      will consider any key with yaml data a values file and merge
+                      them iteratively
                     properties:
                       name:
                         description: name is unique within a namespace to reference

controller/api/v1alpha1/servicedeployment_types.go

@@ -24,7 +24,7 @@ type ServiceKustomize struct {
 }
 
 type ServiceHelm struct {
-	// Fetches the helm values from a secret in this cluster, defaults to using the "values.yaml" key
+	// Fetches the helm values from a secret in this cluster, will consider any key with yaml data a values file and merge them iteratively
 	// +kubebuilder:validation:Optional
 	ValuesFrom *corev1.SecretReference `json:"valuesFrom,omitempty"`
 	// +kubebuilder:validation:Optional

controller/config/crd/bases/deployments.plural.sh_servicedeployments.yaml

@@ -289,9 +289,9 @@ spec:
                       type: string
                     type: array
                   valuesFrom:
-                    description: |-
-                      SecretReference represents a Secret Reference. It has enough information to retrieve secret
-                      in any namespace
+                    description: Fetches the helm values from a secret in this cluster,
+                      will consider any key with yaml data a values file and merge
+                      them iteratively
                     properties:
                       name:
                         description: name is unique within a namespace to reference

controller/internal/controller/servicedeployment_controller.go

@@ -363,11 +363,12 @@ func (r *ServiceReconciler) MergeHelmValues(ctx context.Context, secretRef *core
 			return nil, err
 		}
 
-		// TODO: allow users to specify this key in another CRD field.
-		if vals, ok := valuesFromSecret.Data["values.yaml"]; ok {
-			if err := yaml.Unmarshal(vals, &valuesFromMap); err != nil {
-				return nil, err
+		for _, vals := range valuesFromSecret.Data {
+			current := map[string]interface{}{}
+			if err := yaml.Unmarshal(vals, &current); err != nil {
+				continue
 			}
+			valuesFromMap = algorithms.Merge(valuesFromMap, current)
 		}
 	}
 

lib/console/graphql/deployments/cluster.ex

@@ -251,9 +251,20 @@ defmodule Console.GraphQl.Deployments.Cluster do
 
     @desc "lists OPA constraints registered in this cluster"
     connection field :policy_constraints, node_type: :policy_constraint do
+      arg :namespace, :string, description: "only show constraints with a violation for the given namespace"
+      arg :kind,      :string, description: "only show constraints with a violation for the given kind"
+      arg :q,         :string
+
       resolve &Deployments.list_policy_constraints/3
     end
 
+    @desc "Computes a list of statistics for OPA constraint violations w/in this cluster"
+    field :violation_statistics, list_of(:violation_statistic) do
+      arg :field, non_null(:constraint_violation_field)
+
+      resolve &Deployments.violation_statistics/3
+    end
+
     @desc "fetches a list of runtime services found in this cluster, this is an expensive operation that should not be done in list queries"
     field :runtime_services, list_of(:runtime_service), resolve: &Deployments.runtime_services/3
 

lib/console/graphql/deployments/policy.ex

@@ -2,6 +2,11 @@ defmodule Console.GraphQl.Deployments.Policy do
   use Console.GraphQl.Schema.Base
   alias Console.GraphQl.Resolvers.Deployments
 
+  enum :constraint_violation_field do
+    value :namespace
+    value :kind
+  end
+
   @desc "inputs to add constraint data from an OPA gatekeeper constraint CRD"
   input_object :policy_constraint_attributes do
     field :name,            non_null(:string)
@@ -14,7 +19,7 @@ defmodule Console.GraphQl.Deployments.Policy do
 
   input_object :constraint_ref_attributes do
     field :kind, non_null(:string)
-    field :name,  non_null(:string)
+    field :name, non_null(:string)
   end
 
   input_object :violation_attributes do
@@ -47,7 +52,14 @@ defmodule Console.GraphQl.Deployments.Policy do
 
   object :constraint_ref do
     field :kind, non_null(:string)
-    field :name,  non_null(:string)
+    field :name, non_null(:string)
+  end
+
+  @desc "A summary of statistics for violations w/in a specific column"
+  object :violation_statistic do
+    field :value,      non_null(:string), description: "the value of this field being aggregated"
+    field :violations, :integer, description: "the total number of violations found"
+    field :count,      :integer, description: "the total number of policy constraints"
   end
 
   @desc "A violation of a given OPA Gatekeeper constraint"
@@ -66,6 +78,22 @@ defmodule Console.GraphQl.Deployments.Policy do
   connection node_type: :policy_constraint
 
   object :policy_queries do
+    connection field :policy_constraints, node_type: :policy_constraint do
+      middleware Authenticated
+      arg :kind,      :string
+      arg :namespace, :string
+      arg :q,         :string
+
+      resolve &Deployments.list_policy_constraints/2
+    end
+
+    field :violation_statistics, list_of(:violation_statistic) do
+      middleware Authenticated
+      arg :field, non_null(:constraint_violation_field)
+
+      resolve &Deployments.violation_statistics/2
+    end
+
     field :policy_constraint, :policy_constraint do
       middleware Authenticated
       arg :id, non_null(:id)

lib/console/graphql/resolvers/deployments/policy.ex

@@ -8,12 +8,44 @@ defmodule Console.GraphQl.Resolvers.Deployments.Policy do
     |> allow(user, :read)
   end
 
+  def list_policy_constraints(args, %{context: %{current_user: user}}) do
+    PolicyConstraint.for_user(user)
+    |> PolicyConstraint.globally_ordered()
+    |> maybe_search(PolicyConstraint, args)
+    |> apply_filters(args)
+    |> paginate(args)
+  end
+
   def list_policy_constraints(cluster, args, _) do
     PolicyConstraint.for_cluster(cluster.id)
     |> PolicyConstraint.ordered()
+    |> maybe_search(PolicyConstraint, args)
+    |> apply_filters(args)
     |> paginate(args)
   end
 
+  defp apply_filters(query, args) do
+    Enum.reduce(args, query, fn
+      {:namespace, ns}, q -> PolicyConstraint.for_namespace(q, ns)
+      {:kind, k}, q -> PolicyConstraint.for_kind(q, k)
+      _, q -> q
+    end)
+  end
+
+  def violation_statistics(%{field: f}, %{context: %{current_user: user}}) do
+    PolicyConstraint.for_user(user)
+    |> PolicyConstraint.statistics(f)
+    |> Console.Repo.all()
+    |> ok()
+  end
+
+  def violation_statistics(cluster, %{field: f}, _) do
+    PolicyConstraint.for_cluster(cluster.id)
+    |> PolicyConstraint.statistics(f)
+    |> Console.Repo.all()
+    |> ok()
+  end
+
   def fetch_constraint(%{ref: %{name: name, kind: kind}, cluster_id: cluster_id}, _, _) do
     path = Kube.Client.Base.path("constraints.gatekeeper.sh", "v1beta1", kind, nil, name)
     with %Cluster{} = cluster <- Clusters.get_cluster(cluster_id),

lib/console/schema/policy_constraint.ex

@@ -16,6 +16,7 @@ defmodule Console.Schema.PolicyConstraint do
     has_many :violations, ConstraintViolation,
       on_replace: :delete,
       foreign_key: :constraint_id
+
     belongs_to :cluster, Cluster
 
     timestamps()
@@ -25,10 +26,49 @@ defmodule Console.Schema.PolicyConstraint do
     from(p in query, where: p.name not in ^names)
   end
 
+  def for_user(query \\ __MODULE__, user) do
+    clusters = Cluster.for_user(user)
+    from(p in query,
+      join: c in subquery(clusters),
+        as: :clusters,
+        on: c.id == p.cluster_id
+    )
+  end
+
+  def globally_ordered(query \\ __MODULE__) do
+    from([p, clusters: c] in query, order_by: [asc: c.name, asc: p.name])
+  end
+
   def for_cluster(query \\ __MODULE__, cluster_id) do
     from(p in query, where: p.cluster_id == ^cluster_id)
   end
 
+  def for_namespace(query \\ __MODULE__, ns) do
+    from(p in query,
+      join: v in assoc(p, :violations),
+      where: v.namespace == ^ns
+    )
+  end
+
+  def for_kind(query \\ __MODULE__, kind) do
+    from(p in query,
+      join: v in assoc(p, :violations),
+      where: v.kind == ^kind
+    )
+  end
+
+  def search(query \\ __MODULE__, q) do
+    from(p in query, where: ilike(p.name, ^"%#{q}%"))
+  end
+
+  def statistics(query \\ __MODULE__, field) do
+    from(p in query,
+      join: v in assoc(p, :violations),
+      group_by: field(v, ^field),
+      select: %{value: field(v, ^field), violations: count(v.id, :distinct), count: count(p.id, :distinct)}
+    )
+  end
+
   def ordered(query \\ __MODULE__, order \\ [asc: :name]) do
     from(p in query, order_by: ^order)
   end

plural/helm/console/crds/deployments.plural.sh_servicedeployments.yaml

@@ -289,9 +289,9 @@ spec:
                       type: string
                     type: array
                   valuesFrom:
-                    description: |-
-                      SecretReference represents a Secret Reference. It has enough information to retrieve secret
-                      in any namespace
+                    description: Fetches the helm values from a secret in this cluster,
+                      will consider any key with yaml data a values file and merge
+                      them iteratively
                     properties:
                       name:
                         description: name is unique within a namespace to reference