Merge branch 'master' into TAN-816-editable-survey-title-backend

# Conflicts:
#	back/db/structure.sql

.circleci/config.yml

@@ -60,6 +60,10 @@ parameters:
     type: boolean
     default: false
 
+  chromatic:
+    type: boolean
+    default: false
+
 executors:
   cl2-back:
     parameters:
@@ -668,6 +672,30 @@ jobs:
           path: /tmp/report.html
           destination: lighthouse
 
+  front-chromatic:
+    docker:
+      - image: citizenlabdotco/cl2-devops-front-buildenv
+    resource_class: medium
+    working_directory: ~/citizenlab/front
+    steps:
+      # Non-shallow clone (required for chromatic)
+      - run: mkdir -p ~/.ssh
+      - run: ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run: |
+          cd ~
+          rm -rf ~/$(echo $CIRCLE_REPOSITORY_URL | cut -d '/' -f 2 | cut -d '.' -f 1)
+          git clone -b "$CIRCLE_BRANCH" "$CIRCLE_REPOSITORY_URL"
+
+      - restore_cache:
+          keys:
+            - v2-npm-cache-{{ checksum "package-lock.json" }}
+      - <<: *install-front-dependencies
+      - save_cache:
+          paths:
+            - /root/.npm
+          key: v2-npm-cache-{{ checksum "package-lock.json" }}
+      - run: npm run chromatic
+
   # E2E TESTS
   e2e-tests:
     docker:
@@ -1293,3 +1321,25 @@ workflows:
             - "Canada"
             - "US West"
       #     - "UK"
+
+  manual-chromatic:
+    when: << pipeline.parameters.chromatic >>
+    jobs:
+      - front-chromatic:
+          context:
+            - docker-hub-access
+            - chromatic
+
+  weekly-chromatic:
+    triggers:
+      - schedule:
+          cron: 0 5 * * 1 # Every Monday at 5am: https://crontab.guru/
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - front-chromatic:
+          context:
+            - docker-hub-access
+            - chromatic

Makefile

@@ -25,7 +25,7 @@ reset-dev-env:
 	docker-compose run --rm -e RAILS_ENV=test web bin/rails db:drop db:create db:schema:load
 
 migrate:
-	docker-compose run --rm web bin/rails db:migrate
+	docker-compose run --rm web bin/rails db:migrate cl2back:clean_tenant_settings email_campaigns:assure_campaign_records fix_existing_tenants:update_permissions cl2back:clear_cache_store email_campaigns:remove_deprecated
 
 be-up:
 	docker-compose up
@@ -61,12 +61,12 @@ add-campaign-and-notification:
 blint back-lint-autocorrect:
 	docker compose run web bundle exec rubocop -P --format simple --autocorrect
 
-# Usage example: 
+# Usage example:
 # make r file=spec/models/idea_spec.rb
 r rspec:
 	docker-compose run --rm web bin/rspec ${file}
 
-# Usage example: 
+# Usage example:
 # make feature-toggle feature=initiative_cosponsors enabled=true
 feature-toggle:
 	docker-compose run web "bin/rails runner \"enabled = ${enabled}; feature = '${feature}'; Tenant.find_by(host: 'localhost').switch!; c = AppConfiguration.first; c.settings['${feature}'] ||= {}; c.settings['${feature}']['allowed'] = ${enabled}; c.settings['${feature}']['enabled'] = ${enabled}; c.save!\""

back/Gemfile

@@ -89,7 +89,7 @@ gem 'pundit', '~> 2.3.0'
 gem 'active_model_serializers', '~> 0.10.12'
 
 gem 'jwt', '~> 2.7.0'
-gem 'que', '~> 1.4.1'
+gem 'que', '~> 2.0.0'
 gem 'que-web', '~> 0.10.0'
 
 gem 'activerecord-import', '~> 1.4'

back/Gemfile.lock

@@ -913,7 +913,7 @@ GEM
     pundit (2.3.0)
       activesupport (>= 3.0.0)
     qonfig (0.28.0)
-    que (1.4.1)
+    que (2.0.0)
     que-web (0.10.0)
       que (>= 1)
       sinatra
@@ -1307,7 +1307,7 @@ DEPENDENCIES
   public_api!
   puma (~> 6.4.2)
   pundit (~> 2.3.0)
-  que (~> 1.4.1)
+  que (~> 2.0.0)
   que-web (~> 0.10.0)
   rack-attack (~> 6)
   rack-cors (= 2.0.0)

back/app/mailers/application_mailer.rb

@@ -134,15 +134,15 @@ def default_from_email
   end
 
   def raw_from_email
-    app_settings.core.from_email.presence || default_from_email
+    app_settings.core['from_email'].presence || default_from_email
   end
 
   def to_email
     email_address_with_name(recipient.email, "#{recipient.first_name} #{recipient.last_name}")
   end
 
   def reply_to_email
-    app_settings.core.reply_to_email.presence || default_from_email
+    app_settings.core['reply_to_email'].presence || default_from_email
   end
 
   def domain
@@ -203,9 +203,13 @@ def text_direction
 
   def to_deep_struct(obj)
     case obj
-    when Hash  then OpenStruct.new(obj.transform_values { |nested_object| to_deep_struct(nested_object) })
-    when Array then obj.map { |nested_object| to_deep_struct(nested_object) }
-    else            obj
+    when Hash
+      struct_obj = obj.transform_values { |nested_object| to_deep_struct(nested_object) }
+      WhinyOpenStruct.new(struct_obj, raise_exception: false)
+    when Array
+      obj.map { |nested_object| to_deep_struct(nested_object) }
+    else
+      obj
     end
   end
 end

back/app/models/que_job.rb

@@ -16,20 +16,21 @@
 #  expired_at           :timestamptz
 #  args                 :jsonb            not null
 #  data                 :jsonb            not null
-#  job_schema_version   :integer          default(1)
+#  job_schema_version   :integer          not null
+#  kwargs               :jsonb            not null
 #
 # Indexes
 #
-#  que_jobs_args_gin_idx                 (args) USING gin
-#  que_jobs_data_gin_idx                 (data) USING gin
-#  que_poll_idx                          (queue,priority,run_at,id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL))
-#  que_poll_idx_with_job_schema_version  (job_schema_version,queue,priority,run_at,id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL))
+#  que_jobs_args_gin_idx    (args) USING gin
+#  que_jobs_data_gin_idx    (data) USING gin
+#  que_jobs_kwargs_gin_idx  (kwargs) USING gin
+#  que_poll_idx             (job_schema_version,queue,priority,run_at,id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL))
 #
 require 'que/active_record/model'
 
 class QueJob < Que::ActiveRecord::Model
   def self.find(id)
-    by_args(job_id: id).sole
+    by_args({ job_id: id }, {}).sole
   end
 
   def args

back/app/services/timeline_service.rb

@@ -18,9 +18,7 @@ def past_phases(project, time = Time.now)
   def current_phase(project, time = Time.now)
     date = time.in_time_zone(AppConfiguration.instance.settings('core', 'timezone')).to_date
 
-    project.phases.find do |phase|
-      phase.start_at <= date && (phase.end_at.nil? || phase.end_at >= date)
-    end
+    project.phases.find { |phase| phase.start_at <= date && (phase.end_at.nil? || phase.end_at >= date) }
   end
 
   def phase_is_complete?(phase, time = Time.now)

back/app/services/whiny_open_struct.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/OpenStructUse
+
+# Latest OpenStruct implementation doesn't define `respond_to_missing?` method
+# https://github.com/ruby/ruby/blob/v3_2_0/lib/ostruct.rb
+# rubocop:disable Style/MissingRespondToMissing
+
+# Inspired by https://stackoverflow.com/a/16905766
+class WhinyOpenStruct < OpenStruct
+  def initialize(hash, raise_exception: true)
+    @raise_exception = raise_exception
+    super(hash)
+  end
+
+  # See https://github.com/ruby/ruby/blob/v2_7_6/lib/ostruct.rb#L99
+  def method_missing(meth, *args)
+    if !meth.to_s.end_with?('=') && !@table.key?(meth)
+      msg = "No '#{meth}' member set yet"
+      @raise_exception ? raise(NoMethodError, msg) : ErrorReporter.report_msg(msg)
+    end
+
+    super
+  end
+end
+# rubocop:enable Style/OpenStructUse
+# rubocop:enable Style/MissingRespondToMissing

back/db/migrate/20240228145938_update_que_tables_to_version6.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class UpdateQueTablesToVersion6 < ActiveRecord::Migration[6.1]
+  def change
+    reversible do |dir|
+      dir.up { Que.migrate!(version: 6) }
+      dir.down { Que.migrate!(version: 5) }
+    end
+  end
+end

back/db/structure.sql

@@ -133,8 +133,8 @@ DROP TRIGGER IF EXISTS que_job_notify ON public.que_jobs;
 DROP INDEX IF EXISTS public.users_unique_lower_email_idx;
 DROP INDEX IF EXISTS public.spam_reportable_index;
 DROP INDEX IF EXISTS public.report_builder_published_data_units_report_id_idx;
-DROP INDEX IF EXISTS public.que_poll_idx_with_job_schema_version;
 DROP INDEX IF EXISTS public.que_poll_idx;
+DROP INDEX IF EXISTS public.que_jobs_kwargs_gin_idx;
 DROP INDEX IF EXISTS public.que_jobs_data_gin_idx;
 DROP INDEX IF EXISTS public.que_jobs_args_gin_idx;
 DROP INDEX IF EXISTS public.moderation_statuses_moderatable;
@@ -743,7 +743,8 @@ CREATE TABLE public.que_jobs (
     expired_at timestamp with time zone,
     args jsonb DEFAULT '[]'::jsonb NOT NULL,
     data jsonb DEFAULT '{}'::jsonb NOT NULL,
-    job_schema_version integer DEFAULT 1,
+    job_schema_version integer NOT NULL,
+    kwargs jsonb DEFAULT '{}'::jsonb NOT NULL,
     CONSTRAINT error_length CHECK (((char_length(last_error_message) <= 500) AND (char_length(last_error_backtrace) <= 10000))),
     CONSTRAINT job_class_length CHECK ((char_length(
 CASE job_class
@@ -761,7 +762,7 @@ WITH (fillfactor='90');
 -- Name: TABLE que_jobs; Type: COMMENT; Schema: public; Owner: -
 --
 
-COMMENT ON TABLE public.que_jobs IS '5';
+COMMENT ON TABLE public.que_jobs IS '6';
 
 
 --
@@ -5976,17 +5977,17 @@ CREATE INDEX que_jobs_data_gin_idx ON public.que_jobs USING gin (data jsonb_path
 
 
 --
--- Name: que_poll_idx; Type: INDEX; Schema: public; Owner: -
+-- Name: que_jobs_kwargs_gin_idx; Type: INDEX; Schema: public; Owner: -
 --
 
-CREATE INDEX que_poll_idx ON public.que_jobs USING btree (queue, priority, run_at, id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL));
+CREATE INDEX que_jobs_kwargs_gin_idx ON public.que_jobs USING gin (kwargs jsonb_path_ops);
 
 
 --
--- Name: que_poll_idx_with_job_schema_version; Type: INDEX; Schema: public; Owner: -
+-- Name: que_poll_idx; Type: INDEX; Schema: public; Owner: -
 --
 
-CREATE INDEX que_poll_idx_with_job_schema_version ON public.que_jobs USING btree (job_schema_version, queue, priority, run_at, id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL));
+CREATE INDEX que_poll_idx ON public.que_jobs USING btree (job_schema_version, queue, priority, run_at, id) WHERE ((finished_at IS NULL) AND (expired_at IS NULL));
 
 
 --
@@ -7395,6 +7396,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20240214125557'),
 ('20240226170510'),
 ('20240227092300'),
+('20240228145938'),
 ('20240229195843'),
 ('20240301120023');
 

back/engines/commercial/flag_inappropriate_content/spec/mailers/inappropriate_content_flagged_mailer_spec.rb

@@ -4,13 +4,10 @@
 
 RSpec.describe FlagInappropriateContent::EmailCampaigns::InappropriateContentFlaggedMailer do
   describe 'campaign_mail' do
-    let!(:recipient) { create(:user, locale: 'en') }
-    let!(:campaign) { FlagInappropriateContent::EmailCampaigns::Campaigns::InappropriateContentFlagged.create! }
-    let(:mail) { described_class.with(command: command, campaign: campaign).campaign_mail.deliver_now }
-
-    let(:flaggable) { create(:idea) }
-
-    let(:command) do
+    let_it_be(:recipient) { create(:user, locale: 'en') }
+    let_it_be(:campaign) { FlagInappropriateContent::EmailCampaigns::Campaigns::InappropriateContentFlagged.create! }
+    let_it_be(:flaggable) { create(:idea) }
+    let_it_be(:command) do
       {
         recipient: recipient,
         event_payload: {
@@ -24,6 +21,8 @@
       }
     end
 
+    let_it_be(:mail) { described_class.with(command: command, campaign: campaign).campaign_mail.deliver_now }
+
     before do
       EmailCampaigns::UnsubscriptionToken.create!(user_id: recipient.id)
     end
@@ -58,6 +57,8 @@
         command[:event_payload][:flag_automatically_detected] = true
       end
 
+      let(:mail) { described_class.with(command: command, campaign: campaign).campaign_mail.deliver_now }
+
       it 'includes the explanation note about automatic flagging' do
         expect(mail.body.encoded).to include('This post was automatically detected')
       end

back/engines/commercial/report_builder/app/policies/report_builder/published_graph_data_unit_policy.rb

@@ -3,7 +3,7 @@
 module ReportBuilder
   class PublishedGraphDataUnitPolicy < ::ApplicationPolicy
     def published?
-      record.report.phase.started? && PhasePolicy.new(user, record.report.phase).show?
+      ReportPolicy.new(user, record.report).layout?
     end
   end
 end

back/engines/commercial/report_builder/app/policies/report_builder/report_policy.rb

@@ -64,7 +64,7 @@ def write?
         true
       elsif user.project_or_folder_moderator?
         if record.phase?
-          PhasePolicy.new(user, record.phase).show? && access_to_data?
+          PhasePolicy.new(user, record.phase).update? && access_to_data?
         else
           record.owner == user && access_to_data?
         end

back/engines/commercial/report_builder/app/services/report_builder/side_fx_report_service.rb

@@ -19,7 +19,7 @@ def before_update(report, user)
     def after_update(report, user)
       super(report, user)
       layout_side_fx_service.after_update(report.layout, user) if report.layout.previous_changes.present?
-      ReportPublisher.new(report, user).publish
+      ReportPublisher.new(report, user).publish if report.layout.previous_changes.include?('craftjs_json')
     end
 
     def before_destroy(report, user)