Merge branch 'master' into TAN-1184-method-switching

# Conflicts:
#	back/app/models/idea.rb

.circleci/config.yml

@@ -60,6 +60,10 @@ parameters:
     type: boolean
     default: false
 
+  chromatic:
+    type: boolean
+    default: false
+
 executors:
   cl2-back:
     parameters:
@@ -185,7 +189,11 @@ jobs:
       # Solutions: upgrade Nokogiri (requires newer Ruby) or
       # Compile and link Nokogiri against new libxml2
       # https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-xc9x-jj77-9p9j
-      - run: bundle exec bundle-audit check --update --ignore CVE-2015-9284 GHSA-xc9x-jj77-9p9j
+      #
+      # CVE-2023-51774
+      # It has been fixed by upgrading to json-jwt to 1.16.6, but the CVE data has not
+      # been updated yet. So we can safely ignore it.
+      - run: bundle exec bundle-audit check --update --ignore CVE-2015-9284 GHSA-xc9x-jj77-9p9j CVE-2023-51774
 
   back-license-check:
     resource_class: small
@@ -664,6 +672,30 @@ jobs:
           path: /tmp/report.html
           destination: lighthouse
 
+  front-chromatic:
+    docker:
+      - image: citizenlabdotco/cl2-devops-front-buildenv
+    resource_class: medium
+    working_directory: ~/citizenlab/front
+    steps:
+      # Non-shallow clone (required for chromatic)
+      - run: mkdir -p ~/.ssh
+      - run: ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run: |
+          cd ~
+          rm -rf ~/$(echo $CIRCLE_REPOSITORY_URL | cut -d '/' -f 2 | cut -d '.' -f 1)
+          git clone -b "$CIRCLE_BRANCH" "$CIRCLE_REPOSITORY_URL"
+
+      - restore_cache:
+          keys:
+            - v2-npm-cache-{{ checksum "package-lock.json" }}
+      - <<: *install-front-dependencies
+      - save_cache:
+          paths:
+            - /root/.npm
+          key: v2-npm-cache-{{ checksum "package-lock.json" }}
+      - run: npm run chromatic
+
   # E2E TESTS
   e2e-tests:
     docker:
@@ -1289,3 +1321,25 @@ workflows:
             - "Canada"
             - "US West"
       #     - "UK"
+
+  manual-chromatic:
+    when: << pipeline.parameters.chromatic >>
+    jobs:
+      - front-chromatic:
+          context:
+            - docker-hub-access
+            - chromatic
+
+  weekly-chromatic:
+    triggers:
+      - schedule:
+          cron: 0 5 * * 1 # Every Monday at 5am: https://crontab.guru/
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - front-chromatic:
+          context:
+            - docker-hub-access
+            - chromatic

Makefile

@@ -25,7 +25,7 @@ reset-dev-env:
 	docker-compose run --rm -e RAILS_ENV=test web bin/rails db:drop db:create db:schema:load
 
 migrate:
-	docker-compose run --rm web bin/rails db:migrate
+	docker-compose run --rm web bin/rails db:migrate cl2back:clean_tenant_settings email_campaigns:assure_campaign_records fix_existing_tenants:update_permissions cl2back:clear_cache_store email_campaigns:remove_deprecated
 
 be-up:
 	docker-compose up
@@ -61,12 +61,12 @@ add-campaign-and-notification:
 blint back-lint-autocorrect:
 	docker compose run web bundle exec rubocop -P --format simple --autocorrect
 
-# Usage example: 
+# Usage example:
 # make r file=spec/models/idea_spec.rb
 r rspec:
 	docker-compose run --rm web bin/rspec ${file}
 
-# Usage example: 
+# Usage example:
 # make feature-toggle feature=initiative_cosponsors enabled=true
 feature-toggle:
 	docker-compose run web "bin/rails runner \"enabled = ${enabled}; feature = '${feature}'; Tenant.find_by(host: 'localhost').switch!; c = AppConfiguration.first; c.settings['${feature}'] ||= {}; c.settings['${feature}']['allowed'] = ${enabled}; c.settings['${feature}']['enabled'] = ${enabled}; c.save!\""

back/Gemfile

@@ -89,7 +89,7 @@ gem 'pundit', '~> 2.3.0'
 gem 'active_model_serializers', '~> 0.10.12'
 
 gem 'jwt', '~> 2.7.0'
-gem 'que', '~> 1.4.1'
+gem 'que', '~> 2.0.0'
 gem 'que-web', '~> 0.10.0'
 
 gem 'activerecord-import', '~> 1.4'

back/Gemfile.lock

@@ -3,7 +3,7 @@ PATH
   specs:
     admin_api (0.1.0)
       active_model_serializers (~> 0.10.7)
-      graphql (~> 1.8.0)
+      graphql (~> 1.11.10)
       kaminari (~> 1.2)
       multi_tenancy
       rails (~> 7.0)
@@ -488,8 +488,9 @@ GEM
       mimemagic (~> 0.3)
       nokogiri (~> 1.8, >= 1.8.2)
       rubyzip (~> 1.2, >= 1.2.1)
+    base64 (0.2.0)
     bcrypt (3.1.18)
-    bindata (2.4.15)
+    bindata (2.5.0)
     binding_of_caller (1.0.0)
       debug_inspector (>= 0.0.1)
     bootsnap (1.16.0)
@@ -570,7 +571,8 @@ GEM
       railties (>= 5.0.0)
     faker (3.2.0)
       i18n (>= 1.8.11, < 2)
-    faraday (2.7.10)
+    faraday (2.8.1)
+      base64
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-follow_redirects (0.3.0)
@@ -672,7 +674,7 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    graphql (1.8.18)
+    graphql (1.11.10)
     groupdate (4.3.0)
       activesupport (>= 5)
     grpc (1.56.2)
@@ -721,9 +723,10 @@ GEM
     intercom (4.2.1)
     jmespath (1.6.2)
     json (2.6.3)
-    json-jwt (1.16.3)
+    json-jwt (1.16.6)
       activesupport (>= 4.2)
       aes_key_wrap
+      base64
       bindata
       faraday (~> 2.0)
       faraday-follow_redirects
@@ -910,7 +913,7 @@ GEM
     pundit (2.3.0)
       activesupport (>= 3.0.0)
     qonfig (0.28.0)
-    que (1.4.1)
+    que (2.0.0)
     que-web (0.10.0)
       que (>= 1)
       sinatra
@@ -1304,7 +1307,7 @@ DEPENDENCIES
   public_api!
   puma (~> 6.4.2)
   pundit (~> 2.3.0)
-  que (~> 1.4.1)
+  que (~> 2.0.0)
   que-web (~> 0.10.0)
   rack-attack (~> 6)
   rack-cors (= 2.0.0)

back/app/controllers/web_api/v1/ideas_controller.rb

@@ -114,6 +114,11 @@ def by_slug
     render_show Idea.find_by!(slug: params[:slug])
   end
 
+  # Return a single draft idea for a phase - for native survey autosave
+  def draft_by_phase
+    render_show Idea.find_by!(creation_phase_id: params[:phase_id], author: current_user, publication_status: 'draft')
+  end
+
   #   Normal users always post in an active phase. They should never provide a phase id.
   #   Users who can moderate projects post in an active phase if no phase id is given.
   #   Users who can moderate projects post in the given phase if a phase id is given.
@@ -139,7 +144,6 @@ def create
 
     extract_custom_field_values_from_params! participation_method.custom_form
     params_for_create = idea_params participation_method.custom_form, is_moderator
-    params_for_file_upload_fields = extract_params_for_file_upload_fields participation_method.custom_form, params_for_create
     input = Idea.new params_for_create
     input.creation_phase = (phase if participation_method.creation_phase?)
     input.phase_ids = [phase.id] if phase_ids.empty?
@@ -162,17 +166,7 @@ def create
     save_options[:context] = :publication if params.dig(:idea, :publication_status) == 'published'
     ActiveRecord::Base.transaction do
       if input.save save_options
-        params_for_file_upload_fields.each do |key, params_for_files_field|
-          idea_file = FileUpload.create!(
-            idea: input,
-            file_by_content: {
-              name: params_for_files_field['name'],
-              content: params_for_files_field['content']
-            }
-          )
-          input.custom_field_values[key] = idea_file.id
-        end
-        input.save!
+        update_file_upload_fields input, participation_method.custom_form, params_for_create
         service.after_create(input, current_user)
         render json: WebApi::V1::IdeaSerializer.new(
           input.reload,
@@ -185,13 +179,6 @@ def create
     end
   end
 
-  def extract_params_for_file_upload_fields(custom_form, params_for_create)
-    return {} if params_for_create['custom_field_values'].blank?
-
-    file_upload_field_keys = IdeaCustomFieldsService.new(custom_form).all_fields.select(&:file_upload?).map(&:key)
-    params_for_create['custom_field_values'].extract!(*file_upload_field_keys)
-  end
-
   def update
     input = Idea.find params[:id]
     project = input.project
@@ -227,6 +214,7 @@ def update
     ActiveRecord::Base.transaction do
       if input.save save_options
         service.after_update(input, current_user)
+        update_file_upload_fields input, input.custom_form, update_params
         render json: WebApi::V1::IdeaSerializer.new(
           input.reload,
           params: jsonapi_serializer_params,
@@ -279,6 +267,38 @@ def extract_custom_field_values_from_params!(custom_form)
     params[:idea][:custom_field_values] = extra_field_values
   end
 
+  def extract_params_for_file_upload_fields(custom_form, params)
+    return {} if params['custom_field_values'].blank?
+
+    file_upload_field_keys = IdeaCustomFieldsService.new(custom_form).all_fields.select(&:file_upload?).map(&:key)
+    params['custom_field_values'].extract!(*file_upload_field_keys)
+  end
+
+  def update_file_upload_fields(input, custom_form, params)
+    file_uploads_exist = false
+    params_for_file_upload_fields = extract_params_for_file_upload_fields custom_form, params
+    params_for_file_upload_fields.each do |key, params_for_files_field|
+      if params_for_files_field['id']
+        idea_file = FileUpload.find(params_for_files_field['id'])
+        if idea_file
+          input.custom_field_values[key] = { id: idea_file.id, name: idea_file.name }
+          file_uploads_exist = true
+        end
+      elsif params_for_files_field['content']
+        idea_file = FileUpload.create!(
+          idea: input,
+          file_by_content: {
+            name: params_for_files_field['name'],
+            content: params_for_files_field['content']
+          }
+        )
+        input.custom_field_values[key] = { id: idea_file.id, name: idea_file.name }
+        file_uploads_exist = true
+      end
+    end
+    input.save! if file_uploads_exist
+  end
+
   # Do not save any 'other' text values if the select field does not include 'other' as an option
   def reject_other_text_values(extra_field_values)
     extra_field_values.each do |key, _value|

back/app/controllers/web_api/v1/phases_controller.rb

@@ -6,7 +6,7 @@ class WebApi::V1::PhasesController < ApplicationController
 
   def index
     @phases = policy_scope(Phase)
-      .includes(:permissions, :report)
+      .includes(:permissions, :report, :custom_form)
       .where(project_id: params[:project_id])
       .order(:start_at)
     @phases = paginate @phases

back/app/mailers/application_mailer.rb

@@ -134,15 +134,15 @@ def default_from_email
   end
 
   def raw_from_email
-    app_settings.core.from_email.presence || default_from_email
+    app_settings.core['from_email'].presence || default_from_email
   end
 
   def to_email
     email_address_with_name(recipient.email, "#{recipient.first_name} #{recipient.last_name}")
   end
 
   def reply_to_email
-    app_settings.core.reply_to_email.presence || default_from_email
+    app_settings.core['reply_to_email'].presence || default_from_email
   end
 
   def domain
@@ -203,9 +203,13 @@ def text_direction
 
   def to_deep_struct(obj)
     case obj
-    when Hash  then OpenStruct.new(obj.transform_values { |nested_object| to_deep_struct(nested_object) })
-    when Array then obj.map { |nested_object| to_deep_struct(nested_object) }
-    else            obj
+    when Hash
+      struct_obj = obj.transform_values { |nested_object| to_deep_struct(nested_object) }
+      WhinyOpenStruct.new(struct_obj, raise_exception: false)
+    when Array
+      obj.map { |nested_object| to_deep_struct(nested_object) }
+    else
+      obj
     end
   end
 end

back/app/models/custom_field.rb

@@ -213,7 +213,7 @@ def accept(visitor)
   # Special behaviour for ideation section 1
   def title_multiloc
     if code == 'ideation_section1'
-      project = resource.participation_context
+      project = resource.participation_context.project
       phase = TimelineService.new.current_or_last_can_contain_ideas_phase project
       input_term = phase&.input_term || Phase::DEFAULT_INPUT_TERM
 

back/app/models/idea.rb

@@ -162,6 +162,11 @@ class Idea < ApplicationRecord
   scope :native_survey, -> { where.not creation_phase_id: nil }
   scope :ideation, -> { where creation_phase_id: nil }
 
+  scope :draft_surveys, lambda {
+    where(publication_status: 'draft')
+      .where.not(creation_phase_id: nil)
+  }
+
   def just_published?
     publication_status_previous_change == %w[draft published] || publication_status_previous_change == [nil, 'published']
   end