Merge branch 'TAN-449-data-for-reports' into

TAN-459-narrow-mode-and-report-context

back/Gemfile.lock

@@ -282,6 +282,7 @@ PATH
   remote: engines/commercial/report_builder
   specs:
     report_builder (0.1.0)
+      analytics
       content_builder
       rails (~> 7.0)
 

back/db/migrate/20231031175023_create_dimension_user_custom_field_values.analytics.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# This migration comes from analytics (originally 20231031174154)
+class CreateDimensionUserCustomFieldValues < ActiveRecord::Migration[7.0]
+  def change
+    create_view :analytics_dimension_user_custom_field_values
+  end
+end

back/db/migrate/20231103094549_create_published_graph_data_units.rb

@@ -0,0 +1,11 @@
+class CreatePublishedGraphDataUnits < ActiveRecord::Migration[7.0]
+  def change
+    create_table :report_builder_published_graph_data_units, id: :uuid do |t|
+      t.references :report_builder_report, null: false, foreign_key: true, index: { name: :report_builder_published_data_units_report_id_idx }, type: :uuid
+      t.string :graph_id, null: false
+      t.jsonb :data, null: false
+
+      t.timestamps
+    end
+  end
+end

back/db/structure.sql

@@ -110,6 +110,7 @@ ALTER TABLE IF EXISTS ONLY public.permissions_custom_fields DROP CONSTRAINT IF E
 ALTER TABLE IF EXISTS ONLY public.analytics_dimension_projects_fact_visits DROP CONSTRAINT IF EXISTS fk_rails_4ecebb6e8a;
 ALTER TABLE IF EXISTS ONLY public.initiative_images DROP CONSTRAINT IF EXISTS fk_rails_4df6f76970;
 ALTER TABLE IF EXISTS ONLY public.notifications DROP CONSTRAINT IF EXISTS fk_rails_4aea6afa11;
+ALTER TABLE IF EXISTS ONLY public.report_builder_published_graph_data_units DROP CONSTRAINT IF EXISTS fk_rails_4846b9a405;
 ALTER TABLE IF EXISTS ONLY public.notifications DROP CONSTRAINT IF EXISTS fk_rails_46dd2ccfd1;
 ALTER TABLE IF EXISTS ONLY public.email_campaigns_examples DROP CONSTRAINT IF EXISTS fk_rails_465d6356b2;
 ALTER TABLE IF EXISTS ONLY public.insights_text_network_analysis_tasks_views DROP CONSTRAINT IF EXISTS fk_rails_3e0e58a177;
@@ -141,6 +142,7 @@ DROP TRIGGER IF EXISTS que_state_notify ON public.que_jobs;
 DROP TRIGGER IF EXISTS que_job_notify ON public.que_jobs;
 DROP INDEX IF EXISTS public.users_unique_lower_email_idx;
 DROP INDEX IF EXISTS public.spam_reportable_index;
+DROP INDEX IF EXISTS public.report_builder_published_data_units_report_id_idx;
 DROP INDEX IF EXISTS public.que_poll_idx_with_job_schema_version;
 DROP INDEX IF EXISTS public.que_poll_idx;
 DROP INDEX IF EXISTS public.que_jobs_data_gin_idx;
@@ -431,6 +433,7 @@ ALTER TABLE IF EXISTS ONLY public.static_pages_topics DROP CONSTRAINT IF EXISTS
 ALTER TABLE IF EXISTS ONLY public.spam_reports DROP CONSTRAINT IF EXISTS spam_reports_pkey;
 ALTER TABLE IF EXISTS ONLY public.schema_migrations DROP CONSTRAINT IF EXISTS schema_migrations_pkey;
 ALTER TABLE IF EXISTS ONLY public.report_builder_reports DROP CONSTRAINT IF EXISTS report_builder_reports_pkey;
+ALTER TABLE IF EXISTS ONLY public.report_builder_published_graph_data_units DROP CONSTRAINT IF EXISTS report_builder_published_graph_data_units_pkey;
 ALTER TABLE IF EXISTS ONLY public.que_values DROP CONSTRAINT IF EXISTS que_values_pkey;
 ALTER TABLE IF EXISTS ONLY public.que_lockers DROP CONSTRAINT IF EXISTS que_lockers_pkey;
 ALTER TABLE IF EXISTS ONLY public.que_jobs DROP CONSTRAINT IF EXISTS que_jobs_pkey;
@@ -560,6 +563,7 @@ DROP TABLE IF EXISTS public.static_page_files;
 DROP TABLE IF EXISTS public.spam_reports;
 DROP TABLE IF EXISTS public.schema_migrations;
 DROP TABLE IF EXISTS public.report_builder_reports;
+DROP TABLE IF EXISTS public.report_builder_published_graph_data_units;
 DROP TABLE IF EXISTS public.que_values;
 DROP TABLE IF EXISTS public.que_lockers;
 DROP SEQUENCE IF EXISTS public.que_jobs_id_seq;
@@ -631,7 +635,6 @@ DROP TABLE IF EXISTS public.email_campaigns_consents;
 DROP TABLE IF EXISTS public.email_campaigns_campaigns_groups;
 DROP TABLE IF EXISTS public.email_campaigns_campaign_email_commands;
 DROP TABLE IF EXISTS public.custom_forms;
-DROP TABLE IF EXISTS public.custom_fields;
 DROP TABLE IF EXISTS public.custom_field_options;
 DROP TABLE IF EXISTS public.cosponsors_initiatives;
 DROP TABLE IF EXISTS public.content_builder_layouts;
@@ -669,7 +672,9 @@ DROP VIEW IF EXISTS public.analytics_fact_email_deliveries;
 DROP TABLE IF EXISTS public.email_campaigns_deliveries;
 DROP TABLE IF EXISTS public.email_campaigns_campaigns;
 DROP VIEW IF EXISTS public.analytics_dimension_users;
+DROP VIEW IF EXISTS public.analytics_dimension_user_custom_field_values;
 DROP TABLE IF EXISTS public.users;
+DROP TABLE IF EXISTS public.custom_fields;
 DROP TABLE IF EXISTS public.analytics_dimension_types;
 DROP VIEW IF EXISTS public.analytics_dimension_statuses;
 DROP TABLE IF EXISTS public.initiative_statuses;
@@ -1356,6 +1361,36 @@ CREATE TABLE public.analytics_dimension_types (
 );
 
 
+--
+-- Name: custom_fields; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.custom_fields (
+    id uuid DEFAULT shared_extensions.gen_random_uuid() NOT NULL,
+    resource_type character varying,
+    key character varying,
+    input_type character varying,
+    title_multiloc jsonb DEFAULT '{}'::jsonb,
+    description_multiloc jsonb DEFAULT '{}'::jsonb,
+    required boolean DEFAULT false,
+    ordering integer,
+    created_at timestamp without time zone NOT NULL,
+    updated_at timestamp without time zone NOT NULL,
+    enabled boolean DEFAULT true NOT NULL,
+    code character varying,
+    resource_id uuid,
+    hidden boolean DEFAULT false NOT NULL,
+    maximum integer,
+    minimum_label_multiloc jsonb DEFAULT '{}'::jsonb NOT NULL,
+    maximum_label_multiloc jsonb DEFAULT '{}'::jsonb NOT NULL,
+    logic jsonb DEFAULT '{}'::jsonb NOT NULL,
+    answer_visible_to character varying,
+    select_count_enabled boolean DEFAULT false NOT NULL,
+    maximum_select_count integer,
+    minimum_select_count integer
+);
+
+
 --
 -- Name: users; Type: TABLE; Schema: public; Owner: -
 --
@@ -1395,6 +1430,22 @@ CREATE TABLE public.users (
 );
 
 
+--
+-- Name: analytics_dimension_user_custom_field_values; Type: VIEW; Schema: public; Owner: -
+--
+
+CREATE VIEW public.analytics_dimension_user_custom_field_values AS
+ SELECT DISTINCT u.id AS dimension_user_id,
+    cf.key,
+    cf.value
+   FROM ((public.users u
+     LEFT JOIN LATERAL ( SELECT custom_fields.key,
+            (u.custom_field_values ->> (custom_fields.key)::text) AS value
+           FROM public.custom_fields
+          WHERE ((custom_fields.resource_type)::text = 'User'::text)) cf ON (true))
+     LEFT JOIN LATERAL ( SELECT jsonb_object_keys(u.custom_field_values) AS key) cfv ON (true));
+
+
 --
 -- Name: analytics_dimension_users; Type: VIEW; Schema: public; Owner: -
 --
@@ -2152,36 +2203,6 @@ CREATE TABLE public.custom_field_options (
 );
 
 
---
--- Name: custom_fields; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.custom_fields (
-    id uuid DEFAULT shared_extensions.gen_random_uuid() NOT NULL,
-    resource_type character varying,
-    key character varying,
-    input_type character varying,
-    title_multiloc jsonb DEFAULT '{}'::jsonb,
-    description_multiloc jsonb DEFAULT '{}'::jsonb,
-    required boolean DEFAULT false,
-    ordering integer,
-    created_at timestamp without time zone NOT NULL,
-    updated_at timestamp without time zone NOT NULL,
-    enabled boolean DEFAULT true NOT NULL,
-    code character varying,
-    resource_id uuid,
-    hidden boolean DEFAULT false NOT NULL,
-    maximum integer,
-    minimum_label_multiloc jsonb DEFAULT '{}'::jsonb NOT NULL,
-    maximum_label_multiloc jsonb DEFAULT '{}'::jsonb NOT NULL,
-    logic jsonb DEFAULT '{}'::jsonb NOT NULL,
-    answer_visible_to character varying,
-    select_count_enabled boolean DEFAULT false NOT NULL,
-    maximum_select_count integer,
-    minimum_select_count integer
-);
-
-
 --
 -- Name: custom_forms; Type: TABLE; Schema: public; Owner: -
 --
@@ -3316,6 +3337,20 @@ CREATE TABLE public.que_values (
 WITH (fillfactor='90');
 
 
+--
+-- Name: report_builder_published_graph_data_units; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.report_builder_published_graph_data_units (
+    id uuid DEFAULT shared_extensions.gen_random_uuid() NOT NULL,
+    report_builder_report_id uuid NOT NULL,
+    graph_id character varying NOT NULL,
+    data jsonb NOT NULL,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL
+);
+
+
 --
 -- Name: report_builder_reports; Type: TABLE; Schema: public; Owner: -
 --
@@ -4489,6 +4524,14 @@ ALTER TABLE ONLY public.que_values
     ADD CONSTRAINT que_values_pkey PRIMARY KEY (key);
 
 
+--
+-- Name: report_builder_published_graph_data_units report_builder_published_graph_data_units_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.report_builder_published_graph_data_units
+    ADD CONSTRAINT report_builder_published_graph_data_units_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: report_builder_reports report_builder_reports_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -6535,6 +6578,13 @@ CREATE INDEX que_poll_idx ON public.que_jobs USING btree (queue, priority, run_a
 CREATE INDEX que_poll_idx_with_job_schema_version ON public.que_jobs USING btree (job_schema_version, queue, priority, run_at, id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL));
 
 
+--
+-- Name: report_builder_published_data_units_report_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX report_builder_published_data_units_report_id_idx ON public.report_builder_published_graph_data_units USING btree (report_builder_report_id);
+
+
 --
 -- Name: spam_reportable_index; Type: INDEX; Schema: public; Owner: -
 --
@@ -6779,6 +6829,14 @@ ALTER TABLE ONLY public.notifications
     ADD CONSTRAINT fk_rails_46dd2ccfd1 FOREIGN KEY (phase_id) REFERENCES public.phases(id);
 
 
+--
+-- Name: report_builder_published_graph_data_units fk_rails_4846b9a405; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.report_builder_published_graph_data_units
+    ADD CONSTRAINT fk_rails_4846b9a405 FOREIGN KEY (report_builder_report_id) REFERENCES public.report_builder_reports(id);
+
+
 --
 -- Name: notifications fk_rails_4aea6afa11; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -7981,6 +8039,8 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20231003095622'),
 ('20231018083110'),
 ('20231024082513'),
+('20231031175023'),
+('20231103094549'),
 ('20231109101517'),
 ('20231110112415');
 

back/db/views/analytics_dimension_user_custom_field_values_v01.sql

@@ -0,0 +1,19 @@
+SELECT
+  DISTINCT u.id as dimension_user_id,
+  cf.key,
+  cf.value
+FROM
+  users u
+  LEFT JOIN LATERAL (
+    SELECT
+      key,
+      custom_field_values ->> key AS value
+    FROM
+      custom_fields
+    where
+      custom_fields.resource_type = 'User'
+  ) cf ON true
+  LEFT JOIN LATERAL (
+    SELECT
+      jsonb_object_keys(u.custom_field_values) AS key
+  ) cfv ON true;

back/engines/commercial/analytics/README.md

@@ -1,6 +1,6 @@
 # Analytics
 This is a separate analytics engine, which separates out the data for dashboard from the main operational data
-using both database views on the existing data tables and data copied from other sources. 
+using both database views on the existing data tables and data copied from other sources.
 The data is modelled following the conventions of a **star schema**.
 
 These views and tables sit in the same tenant schema. However, in the future it is intended
@@ -21,6 +21,35 @@ Views are copied across when the migration is run.
 Views and tables should be named as follows:
 
 * analytics_dimension_* - holds dimensions by which the facts can be filtered
-* analytics_fact_* - holds 'facts' - the actual data we're interested in 
+* analytics_fact_* - holds 'facts' - the actual data we're interested in
 such as posts, participations, visits
 
+### Testing queries in your dev env
+
+You can use this request in the browser developer console.
+```js
+fetch(window.location.origin + "/web_api/v1/analytics", {
+  headers: {
+    authorization: `Bearer ${document.cookie.split("; ").find((x) => x.startsWith("cl2_jwt")).replace("cl2_jwt=", "")}`,
+    "content-type": "application/json",
+  },
+  body: `{
+    "query": {
+      "fact": "participation",
+      "groups": "dimension_user_custom_field_values.value",
+      "filters": {
+        "dimension_user_custom_field_values.key": "gender"
+      },
+      "aggregations": {
+        "dimension_user_custom_field_values.dimension_user_id": "count"
+      }
+    }
+  }`,
+  method: "POST",
+})
+  .then((response) => response.json())
+  .then((data) => {
+    console.log(JSON.stringify(data.data, null, 2));
+    console.log(data);
+  });
+```

back/engines/commercial/analytics/app/controllers/analytics/web_api/v1/analytics_controller.rb

@@ -8,11 +8,11 @@ class AnalyticsController < ::ApplicationController
       skip_after_action :verify_policy_scoped, only: :index
       after_action :verify_authorized, only: :index
       def index
-        handle_request
+        handle_request(params[:query])
       end
 
       def create
-        handle_request
+        handle_request(params[:query])
       end
 
       def schema
@@ -27,12 +27,12 @@ def schema
 
       private
 
-      def handle_request
+      def handle_request(query)
         authorize :analytics, policy_class: AnalyticsPolicy
 
-        results, errors, paginations = handle_multiple(Array.wrap(params[:query]))
+        results, errors, paginations = handle_multiple(Array.wrap(query))
 
-        unless params[:query].instance_of?(Array)
+        unless query.instance_of?(Array)
           results = results.empty? ? results : results[0]
           paginations = paginations.empty? ? paginations : paginations[0]
           errors = errors.key?(0) ? errors[0] : errors

back/engines/commercial/analytics/app/models/analytics/dimension_user_custom_field_value.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: analytics_dimension_user_custom_field_values
+#
+#  dimension_user_id :uuid
+#  key               :string
+#  value             :text
+#
+module Analytics
+  class DimensionUserCustomFieldValue < Analytics::ApplicationRecordView
+  end
+end

back/engines/commercial/analytics/app/models/analytics/fact_participation.rb

@@ -17,6 +17,7 @@ module Analytics
   class FactParticipation < Analytics::ApplicationRecordView
     self.primary_key = :id
     belongs_to :dimension_user, class_name: 'Analytics::DimensionUser'
+    has_many :dimension_user_custom_field_values, class_name: 'Analytics::DimensionUserCustomFieldValue', foreign_key: :dimension_user_id, primary_key: :dimension_user_id
     belongs_to :dimension_type, class_name: 'Analytics::DimensionType'
     belongs_to :dimension_date_created, class_name: 'Analytics::DimensionDate', primary_key: 'date'
     belongs_to :dimension_project, class_name: 'Analytics::DimensionProject', optional: true

back/engines/commercial/analytics/app/services/analytics/query_runner_service.rb

@@ -159,7 +159,7 @@ def page(results)
     def pagination_query_params(number)
       return if number.nil?
 
-      json_query = @json_query.to_unsafe_hash
+      json_query = @json_query.try(:to_unsafe_hash) || @json_query
       if @json_query.key?(:page)
         json_query[:page][:number] = number
       else

back/engines/commercial/analytics/app/services/analytics/query_validator_service.rb

@@ -45,7 +45,8 @@ def add_error(messages)
     end
 
     def validate_json
-      json_errors = JSON::Validator.fully_validate(self.class.schema, @json_query.to_unsafe_hash)
+      unsafe_hash = @json_query.try(:to_unsafe_hash) || @json_query
+      json_errors = JSON::Validator.fully_validate(self.class.schema, unsafe_hash)
       return true if json_errors.empty?
 
       add_error(json_errors)

back/engines/commercial/analytics/db/migrate/20231031174154_create_dimension_user_custom_field_values.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class CreateDimensionUserCustomFieldValues < ActiveRecord::Migration[7.0]
+  def change
+    create_view :analytics_dimension_user_custom_field_values
+  end
+end