Merge pull request #389 from OpenSourcePolitics/develop

.env-example

@@ -46,4 +46,5 @@ ENABLE_LETTER_OPENER=0
 TRANSLATOR_ENABLED=0
 TRANSLATOR_API_KEY=
 TRANSLATOR_HOST=
-TRANSLATOR_DELAY=0
+TRANSLATOR_DELAY=0
+GALLERY_ANIMATION_ENABLE=0

.github/workflows/ci_cd.yml

@@ -88,11 +88,7 @@ jobs:
           key: asset-cache-${{ runner.os }}-${{ steps.cache-hash.outputs.hash }}
       - run: mkdir -p ./spec/tmp/screenshots
         name: Create the screenshots folder
-
-      # TODO: Use latest version
       - uses: nanasess/setup-chromedriver@v2
-        with:
-          chromedriver-version: "114.0.5735.90"
       - run:  bundle exec rake "test:run[exclude, spec/system/**/*_spec.rb, ${{ matrix.slice }}]"
         name: RSpec
       - run: ./.github/upload_coverage.sh decidim-app $GITHUB_EVENT_PATH
@@ -157,10 +153,7 @@ jobs:
           key: asset-cache-${{ runner.os }}-${{ steps.cache-hash.outputs.hash }}
       - run: mkdir -p ./spec/tmp/screenshots
         name: Create the screenshots folder
-      # TODO: Use latest version
       - uses: nanasess/setup-chromedriver@v2
-        with:
-          chromedriver-version: "114.0.5735.90"
       - run:  bundle exec rake "test:run[include, spec/system/**/*_spec.rb, ${{ matrix.slice }}]"
         name: RSpec
       - run: ./.github/upload_coverage.sh decidim-app $GITHUB_EVENT_PATH

Gemfile.lock

@@ -384,7 +384,7 @@ GEM
       decidim-core (= 0.26.7)
       wicked_pdf (~> 2.1)
       wkhtmltopdf-binary (~> 0.12)
-    decidim-gallery (0.0.2)
+    decidim-gallery (0.0.4)
       decidim-admin (~> 0.26.0)
       decidim-core (~> 0.26.0)
       deface (~> 1.9)
@@ -485,7 +485,7 @@ GEM
       railties (>= 3.0.0)
     faker (2.23.0)
       i18n (>= 1.8.11, < 2)
-    faraday (2.7.9)
+    faraday (2.7.10)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-follow_redirects (0.3.0)
@@ -711,7 +711,7 @@ GEM
       actionmailer (>= 3)
       net-smtp
       premailer (~> 1.7, >= 1.7.9)
-    public_suffix (5.0.1)
+    public_suffix (5.0.3)
     puma (5.6.5)
       nio4r (~> 2.0)
     raabro (1.4.0)

app/services/decidim/repair_translations_service.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Decidim
+  class RepairTranslationsService
+    def initialize(logger: nil)
+      @logger = logger || Rails.logger
+    end
+
+    def self.run(logger: nil)
+      new(logger: logger).run
+    end
+
+    def run
+      @logger.info("Found #{translatable_resources.size} translatable resources")
+      updated_resources = []
+      translatable_resources.each do |resources|
+        @logger.info("Checking #{resources}...")
+        @logger.info("Found #{resources.count} resources")
+        resources.find_each do |resource|
+          @logger.info("Checking #{resource}...")
+          updated_resources << [resource.class, resource.id] if repair_translations(resource)
+        end
+      end
+
+      updated_resources
+    end
+
+    private
+
+    # Translations is based on a diff between last changes and current changes
+    # So we need to create a fake diff with the previous changes
+    def translatable_previous_changes(resource)
+      resource.slice(*resource.class.translatable_fields_list).transform_values { |value| [nil, value] }
+    end
+
+    def repair_translations(resource)
+      Decidim::MachineTranslationResourceJob.perform_later(
+        resource,
+        translatable_previous_changes(resource),
+        default_locale(resource)
+      )
+    end
+
+    def translatable_resources
+      @translatable_resources ||= Decidim.resource_manifests.map(&:model_class).select do |resource|
+        resource.respond_to?(:translatable_fields_list)
+      end
+    end
+
+    def default_locale(resource)
+      resource.try(:organization).try(:default_locale) || Decidim.default_locale
+    end
+  end
+end

app/services/decidim/repair_url_in_content_service.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "decidim/content_fixer"
+
+module Decidim
+  # Looks for any occurence of "@deprecated_endpoint" in every database columns of type COLUMN_TYPES
+  # For each field containing @deprecated_endpoint:
+  #   - Looks for the current ActiveStorage::Blob with the given filename
+  #   - Find the blob's service_url
+  #   - Replace the @deprecated_endpoint with the blob's service_url in text
+  #   - Update the column
+  # Context:
+  # After S3 assets migration with rake task "bundle exec rake scaleway:storage:migrate_from_local", every linked documents URL were well updated.
+  # However every links added to text fields redirecting to an uploaded file were outdated and still redirects to the old S3 bucket
+  class RepairUrlInContentService
+    COLUMN_TYPES = [:string, :jsonb, :text].freeze
+
+    # @param [String] deprecated_endpoint
+    # @param [Logger] logger
+    def self.run(deprecated_endpoint, logger = nil)
+      new(deprecated_endpoint, logger).run
+    end
+
+    # @param [String] deprecated_endpoint
+    # @param [Logger] logger
+    def initialize(deprecated_endpoint, logger = nil)
+      @logger = logger || Rails.logger
+      @deprecated_endpoint = deprecated_endpoint&.gsub(%r{https?://}, "")
+    end
+
+    def run
+      # Find all models that have a column of type string jsonb or text
+      # For each model, find all records that have a column of type string jsonb or text
+      # For each record, replace all urls contained in content with the new url
+      # Save the record
+      return false if @deprecated_endpoint.blank?
+
+      models.each do |model|
+        next unless model.respond_to?(:columns)
+
+        @logger.info("Checking model #{model} for deprecated endpoints #{@deprecated_endpoint}")
+        records = records_for model
+        next if records.blank?
+
+        @logger.info "Found #{records.count} records to update for #{model}"
+        records.each do |record|
+          columns = model.columns.select { |column| column.type.in? COLUMN_TYPES }
+          record = update_each_column(record, columns)
+
+          save_record!(record)
+        end
+      end
+    end
+
+    def save_record!(record)
+      if record.invalid?
+        @logger.warn "Invalid record #{record.class}##{record.id}: #{record.errors.full_messages.join(", ")}"
+        return
+      end
+
+      if record.has_changes_to_save?
+        record.class.transaction do
+          record.save!
+        end
+      else
+        @logger.info "No changes to save for #{record.class}##{record.id}"
+      end
+    end
+
+    # In some cases, the column returns a settings object,
+    # therefore we need to update each of its attributes before saving the column
+    # @param [Object] record
+    # @param [[ActiveRecord::ConnectionAdapters::PostgreSQL::Column]] columns
+    # @return record | nil
+    def update_each_column(record, columns)
+      columns.each do |column|
+        current_content = current_content_for(record, column)
+        next if current_content.blank?
+
+        column_name = column.try(:name) ? column.name : column
+
+        @logger.info "Updating ##{[record.class, record.try(:id), column_name].compact.join("# ")}"
+
+        if current_content.is_a?(Hash) || current_content.is_a?(Array) || current_content.is_a?(String)
+          next unless current_content.to_s.include?(@deprecated_endpoint)
+
+          new_content = Decidim::ContentFixer.repair(current_content, @deprecated_endpoint, @logger)
+
+          @logger.info "Old content: #{current_content}"
+          @logger.info "New content: #{new_content}"
+
+          write_attribute(record, column, new_content)
+        else
+          # If the column is a settings object, we need to update each of its attributes using a recursive call
+          write_attribute(record, column, update_each_column(current_content, current_content.instance_variables))
+        end
+      end
+
+      record
+    end
+
+    def write_attribute(record, column, value)
+      if column.try(:name)
+        record.write_attribute(:"#{column.name}", value)
+      else
+        record.instance_variable_set(column, value)
+      end
+    end
+
+    def current_content_for(record, column)
+      if column.try(:name)
+        record.send(column.name)
+      else
+        record.instance_variable_get(column)
+      end
+    end
+
+    def records_for(model)
+      model.columns.map do |col|
+        next unless col.type.in?(COLUMN_TYPES)
+
+        model.where("#{col.name}::text LIKE ?", "%#{@deprecated_endpoint}%")
+      end.compact.reduce(&:or)
+    rescue StandardError => e
+      @logger.warn "Error while fetching records from #{model}: #{e.message}"
+      []
+    end
+
+    def models
+      ActiveRecord::Base.connection.tables.map do |table|
+        next unless table.starts_with?("decidim_")
+
+        classify_model(table)
+      end.compact
+    end
+
+    # Because of the way decidim models are named, we need to try to find the model by subbing _ with / and then classify it
+    # For example "decidim_comments_comments" becomes "Decidim::CommentsComment", then "Decidim::Comments::Comment"
+    # This helps us find models that are namespaced
+    # @param [String] table
+    def classify_model(table)
+      if table.include?("_")
+        new_table = table.sub("_", "/")
+        model = new_table.classify.safe_constantize
+
+        return model if model
+
+        classify_model(new_table)
+      else
+        @logger.warn "Could not find model for table #{table}"
+
+        nil
+      end
+    end
+  end
+end