Merge pull request #348 from OpenSourcePolitics/develop

app/services/decidim/action_log_service.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Decidim
+  class ActionLogService < DatabaseService
+    private
+
+    def resource_types
+      @resource_types ||= Decidim::ActionLog.distinct.pluck(:resource_type)
+    end
+
+    def orphans_for(klass)
+      Decidim::ActionLog
+        .where(resource_type: klass)
+        .where.not(resource_id: [klass.constantize.ids])
+        .pluck(:action, :resource_id, :extra)
+    rescue StandardError => e
+      @logger.warn "Error found : #{e.message}"
+      @logger.warn "Skipping class : #{klass}"
+      []
+    end
+
+    def clear_data_for(klass)
+      actions = Decidim::ActionLog
+                .where(resource_type: klass)
+                .where.not(resource_id: [klass.constantize.ids])
+
+      actions.delete_all
+    end
+  end
+end

app/services/decidim/database_service.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Decidim
+  class DatabaseService
+    def initialize(**args)
+      @verbose = args[:verbose] || false
+      @logger = args[:logger] || Logger.new($stdout)
+    end
+
+    def orphans
+      if resource_types.blank?
+        @logger.info "No resource_types found, terminating..." if @verbose
+        return
+      end
+
+      @logger.info "Finding orphans rows in database for #{resource_types.join(", ")} ..." if @verbose
+
+      orphans = {}
+      resource_types.each do |klass|
+        current_orphans_h = { klass => orphans_count_for(klass) }
+        orphans.merge!(current_orphans_h)
+        @logger.info current_orphans_h if @verbose
+      end
+
+      orphans
+    end
+
+    def clear
+      @logger.info "Removing orphans rows in database for #{resource_types.join(", ")} ..." if @verbose
+
+      resource_types.each do |klass|
+        removed = clear_data_for(klass)
+        @logger.info({ klass => removed.size }) if @verbose
+      end
+    end
+
+    private
+
+    def resource_types
+      raise "Method resource_types isn't defined for #{self.class}"
+    end
+
+    def orphans_for(_klass)
+      raise "Method orphans_for isn't defined for #{self.class}"
+    end
+
+    def clear_data_for(_klass)
+      raise "Method clear_data_for isn't defined for #{self.class}"
+    end
+
+    def orphans_count_for(klass)
+      orphans_for(klass).count
+    end
+  end
+end

app/services/decidim/notification_service.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Decidim
+  class NotificationService < DatabaseService
+    private
+
+    def resource_types
+      @resource_types ||= Decidim::Notification.distinct.pluck(:decidim_resource_type)
+    end
+
+    def orphans_for(klass)
+      Decidim::Notification
+        .where(decidim_resource_type: klass)
+        .where.not(decidim_resource_id: [klass.constantize.ids])
+        .pluck(:event_name, :decidim_resource_id, :extra)
+    end
+
+    def clear_data_for(klass)
+      Decidim::Notification.where(decidim_resource_type: klass).where.not(decidim_resource_id: [klass.constantize.ids]).destroy_all
+    end
+  end
+end

app/services/decidim/repair_nickname_service.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Decidim
+  class RepairNicknameService
+    def self.run
+      new.execute
+    end
+
+    def execute
+      return [] if ok?
+
+      update_nicknames!
+    end
+
+    def ok?
+      invalid_users.empty?
+    end
+
+    def invalid_users
+      @invalid_users ||= Decidim::User.all.map do |user|
+        new_nickname = valid_nickname_for(user)
+        next if user.nickname == new_nickname
+
+        [user, new_nickname]
+      end.compact
+    end
+
+    private
+
+    # Update each users with new nickname
+    # Returns Array of updated User ID
+    def update_nicknames!
+      invalid_users.map do |user, new_nickname|
+        user.nickname = if Decidim::User.exists?(nickname: new_nickname)
+                          "#{new_nickname}#{user.id}"
+                        else
+                          new_nickname
+                        end
+
+        user.id if user.save!
+      end.compact
+    end
+
+    # Remove invalid chars from nickname and concatenate unique ID of user
+    def valid_nickname_for(user)
+      I18n.locale = user.locale
+      I18n.transliterate(user.nickname).codepoints.map { |ascii_code| ascii_to_valid_char(ascii_code) }.join
+    end
+
+    # Check for a given ascii code if it is included in valid_ascii_code list
+    # If true
+    #   Returns the corresponding char
+    # Else returns nil
+    def ascii_to_valid_char(id)
+      letters = ("A".."Z").to_a.join("").codepoints
+      letters += ("a".."z").to_a.join("").codepoints
+      digits = ("0".."9").to_a.join("").codepoints
+      special_chars = %w(- _).join("").codepoints
+
+      valid_ascii_code = letters + digits + special_chars
+
+      id.chr if valid_ascii_code.include?(id)
+    end
+  end
+end

app/services/decidim/surveys_service.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Decidim
+  class SurveysService < DatabaseService
+    def orphans
+      Decidim::Surveys::Survey
+        .where.not(decidim_component_id: [Decidim::Component.ids])
+        .pluck(:id, :decidim_component_id).each do |s|
+        @logger.info s.inspect if @verbose
+      end
+    end
+
+    def clear
+      @logger.info "Removing orphans rows in database for Decidim::SurveysService ..." if @verbose
+
+      removed = Decidim::Surveys::Survey
+                .where.not(decidim_component_id: [Decidim::Component.ids])
+                .destroy_all
+
+      @logger.info({ "Decidim::Surveys::Survey" => removed.size }) if @verbose
+    end
+  end
+end

config/environments/production.rb

@@ -62,13 +62,7 @@
   config.log_tags = [:request_id]
 
   # Use a different cache store in production.
-  config.cache_store = if ENV["MEMCACHEDCLOUD_SERVERS"].present?
-                         [:dalli_store, ENV["MEMCACHEDCLOUD_SERVERS"].split(","), {
-                           username: ENV["MEMCACHEDCLOUD_USERNAME"], password: ENV["MEMCACHEDCLOUD_PASSWORD"]
-                         }]
-                       else
-                         :mem_cache_store
-                       end
+  config.cache_store = :mem_cache_store, ENV.fetch("MEMCACHE_SERVERS", "localhost:11211")
 
   # Use a real queuing backend for Active Job (and separate queues per environment)
   config.active_job.queue_adapter = :sidekiq
@@ -150,4 +144,10 @@
   # are known to cause issue with moderation due to expiration
   # Setting this to 100 years should be enough
   config.global_id.expires_in = 100.years
+
+  config.ssl_options = {
+    redirect: {
+      exclude: ->(request) { /health_check|sidekiq_alive/.match?(request.path) }
+    }
+  }
 end

config/initializers/decidim.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "decidim-app/config"
+require "decidim_app/config"
 require "decidim/dev/dummy_translator"
 
 Decidim.configure do |config|

config/initializers/rack_attack.rb

@@ -1,59 +1,9 @@
 # frozen_string_literal: true
 
-require "decidim-app/rack_attack"
-require "decidim-app/rack_attack/throttling"
-require "decidim-app/rack_attack/fail2ban"
+require "decidim_app/rack_attack"
+require "decidim_app/rack_attack/throttling"
+require "decidim_app/rack_attack/fail2ban"
 
 # Enabled by default in production
 # Can be deactivated with 'ENABLE_RACK_ATTACK=0'
-Rack::Attack.enabled = DecidimApp::RackAttack.rack_enabled?
-return unless Rack::Attack.enabled
-
-# Remove the original throttle from decidim-core
-# see https://github.com/decidim/decidim/blob/release/0.26-stable/decidim-core/config/initializers/rack_attack.rb#L19
-DecidimApp::RackAttack::Throttling.deactivate_decidim_throttling! do
-  Rails.logger.info("Deactivating 'requests by ip' from Decidim Core")
-  Rack::Attack.throttles.delete("requests by ip")
-end
-
-Rack::Attack.throttled_response_retry_after_header = true
-# By default use the memory store for inspecting requests
-# Better to use MemCached or Redis in production mode
-Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new if !ENV["MEMCACHEDCLOUD_SERVERS"] || Rails.env.test?
-
-Rack::Attack.throttled_responder = lambda do |request|
-  rack_logger = Logger.new(Rails.root.join("log/rack_attack.log"))
-  throttling_limit = DecidimApp::RackAttack::Throttling.time_limit_for(request.env["rack.attack.match_data"])
-
-  request_uuid = request.env["action_dispatch.request_id"]
-  params = {
-    "ip" => request.ip,
-    "path" => request.path,
-    "get" => request.GET,
-    "host" => request.host,
-    "referer" => request.referer
-  }
-
-  rack_logger.warn("[#{request_uuid}] #{params}")
-
-  [429, { "Content-Type" => "text/html" }, [DecidimApp::RackAttack::Throttling.html_template(throttling_limit, request.env["decidim.current_organization"]&.name)]]
-end
-
-Rack::Attack.throttle(DecidimApp::RackAttack::Throttling.name,
-                      limit: DecidimApp::RackAttack::Throttling.max_requests,
-                      period: DecidimApp::RackAttack::Throttling.period) do |req|
-  req.ip unless DecidimApp::RackAttack::Throttling.authorized_path?(req.path)
-end
-
-if DecidimApp::RackAttack::Fail2ban.enabled?
-  # Block suspicious requests made for pentesting
-  # After 1 forbidden request, block all requests from that IP for 1 hour.
-  Rack::Attack.blocklist("fail2ban pentesters") do |req|
-    # `filter` returns truthy value if request fails, or if it's from a previously banned IP
-    # so the request is blocked
-    Rack::Attack::Fail2Ban.filter("pentesters-#{req.ip}", maxretry: 0, findtime: 10.minutes, bantime: 1.hour) do
-      # The count for the IP is incremented if the return value is truthy
-      DecidimApp::RackAttack::Fail2ban.unauthorized_path?(req.path)
-    end
-  end
-end
+DecidimApp::RackAttack.apply_configuration if DecidimApp::RackAttack.rack_enabled?

config/secrets.yml

@@ -15,7 +15,7 @@ default: &default
   decidim:
     currency: <%= ENV["CURRENCY"] || "€" %>
     rack_attack:
-      enabled: <%= ENV["ENABLE_RACK_ATTACK"]&.to_i || 1 %>
+      enabled: <%= ENV["ENABLE_RACK_ATTACK"] %>
       fail2ban:
         enabled: <%= ENV["RACK_ATTACK_FAIL2BAN"]&.to_i || 1 %>
       throttle:

docker-compose.yml

@@ -6,18 +6,23 @@ services:
       - pg-data:/var/lib/postgresql/data
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
+  memcached:
+    image: memcached
+    ports:
+      - "11211:11211"
   redis:
     image: redis
     ports:
-      - 6379:6379
+      - "6379:6379"
     volumes:
       - redis-data:/var/lib/redis/data
   sidekiq:
     build:
       context: .
-    command: ["bundle", "exec", "sidekiq", "-C", "config/sidekiq.yml"]
+    command: [ "bundle", "exec", "sidekiq", "-C", "config/sidekiq.yml" ]
     environment:
       - REDIS_URL=redis://redis:6379
+      - MEMCACHE_SERVERS=memcached:11211
     depends_on:
       - app
     links:
@@ -34,13 +39,15 @@ services:
       - DATABASE_USERNAME=postgres
       - DECIDIM_HOST=0.0.0.0
       - REDIS_URL=redis://redis:6379
+      - MEMCACHE_SERVERS=memcached:11211
     ports:
       - 3000:3000
     depends_on:
       - database
       - redis
+      - memcached
 
 volumes:
-  node_modules: {}
-  pg-data: {}
-  redis-data: {}
+  node_modules: { }
+  pg-data: { }
+  redis-data: { }