Fixing conflicts

.github/workflows/ci-tests.yml

@@ -7,9 +7,12 @@ on:
     branches:
     - master
     - develop
+    - epic*
+    - cv2*
   pull_request:
     branches:
     - develop
+
 
 env:
   CC_TEST_REPORTER_ID: "${{ secrets.CC_TEST_REPORTER_ID }}"

.gitlab-ci.yml

@@ -36,9 +36,10 @@ deploy_qa:
     AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
     GITHUB_TOKEN: $GITHUB_TOKEN
   script:
-    - pip install botocore==1.33.13
-    - pip install boto3==1.33.13
-    - pip install ecs-deploy==1.14.0
+    - pip install urllib3==2.0.6
+    - pip install botocore==1.31.62
+    - pip install boto3==1.28.62
+    - pip install ecs-deploy==1.15.0
     - pip install awscli==1.31.13
     - alias aws='docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION --rm amazon/aws-cli'
     - aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /qa/check-api/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/qa/check-api/##' > env.qa.names
@@ -106,9 +107,10 @@ deploy_live:
     AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
     GITHUB_TOKEN: $GITHUB_TOKEN
   script:
-    - pip install botocore==1.33.13
-    - pip install boto3==1.33.13
-    - pip install ecs-deploy==1.14.0
+    - pip install urllib3==2.0.6
+    - pip install botocore==1.31.62
+    - pip install boto3==1.28.62
+    - pip install ecs-deploy==1.15.0
     - pip install awscli==1.31.13
     - alias aws='docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION --rm amazon/aws-cli'
     - aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /live/check-api/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/live/check-api/##' > env.live.names

.rubocop.yml

@@ -224,7 +224,7 @@ Metrics/CyclomaticComplexity:
                  A complexity metric that is strongly correlated to the number
                  of test cases needed to validate a method.
   Enabled: true
-  Max: 12
+  Max: 13
 
 Metrics/LineLength:
   Description: 'Limit lines to 80 characters.'

app/graph/mutations/export_mutations.rb

@@ -0,0 +1,24 @@
+module ExportMutations
+  class ExportList < Mutations::BaseMutation
+    argument :query, GraphQL::Types::String, required: true # JSON
+    argument :type, GraphQL::Types::String, required: true # 'media', 'feed', 'fact-check' or 'explainer'
+
+    field :success, GraphQL::Types::Boolean, null: true
+
+    def resolve(query:, type:)
+      ability = context[:ability]
+      team = Team.find_if_can(Team.current.id, ability)
+      if ability.cannot?(:export_list, team)
+        { success: false }
+      else
+        export = ListExport.new(type.to_sym, query, team.id)
+        if export.number_of_rows > CheckConfig.get(:export_csv_maximum_number_of_results, 10000, :integer)
+          { success: false }
+        else
+          export.generate_csv_and_send_email_in_background(User.current)
+          { success: true }
+        end
+      end
+    end
+  end
+end

app/graph/types/mutation_type.rb

@@ -152,4 +152,6 @@ class MutationType < BaseObject
 
   field :createExplainerItem, mutation: ExplainerItemMutations::Create
   field :destroyExplainerItem, mutation: ExplainerItemMutations::Destroy
+
+  field :exportList, mutation: ExportMutations::ExportList
 end

app/graph/types/project_group_type.rb

@@ -8,7 +8,6 @@ class ProjectGroupType < DefaultObject
   field :description, GraphQL::Types::String, null: true
   field :team_id, GraphQL::Types::Int, null: true
   field :team, PublicTeamType, null: true
-  field :medias_count, GraphQL::Types::Int, null: true
 
   field :projects, ProjectType.connection_type, null: true
 end

app/graph/types/project_type.rb

@@ -9,7 +9,6 @@ class ProjectType < DefaultObject
   field :dbid, GraphQL::Types::Int, null: true
   field :permissions, GraphQL::Types::String, null: true
   field :pusher_channel, GraphQL::Types::String, null: true
-  field :medias_count, GraphQL::Types::Int, null: true
   field :search_id, GraphQL::Types::String, null: true
   field :url, GraphQL::Types::String, null: true
   field :search, CheckSearchType, null: true

app/lib/check_config.rb

@@ -3,6 +3,7 @@
 
 class CheckConfig
   def self.get(key, default = nil, type = nil)
+    key = key.to_s
     value = ENV[key]
     value ||= CONFIG[key] if CONFIG.has_key?(key)
     return default if value.nil?

app/mailers/export_list_mailer.rb

@@ -0,0 +1,13 @@
+class ExportListMailer < ApplicationMailer
+  layout nil
+
+  def send_csv(csv_file_url, user)
+    @csv_file_url = csv_file_url
+    @user = user
+    expire_in = Time.now.to_i + CheckConfig.get('export_csv_expire', 7.days.to_i, :integer)
+    @expire_in = I18n.l(Time.at(expire_in), format: :email)
+    subject = I18n.t('mails_notifications.export_list.subject')
+    Rails.logger.info "Sending export e-mail to #{@user.email}"
+    mail(to: @user.email, email_type: 'export_list', subject: subject)
+  end
+end

app/models/ability.rb

@@ -57,7 +57,7 @@ def admin_perms
     can :destroy, Team, :id => @context_team.id
     can :create, TeamUser, :team_id => @context_team.id, role: ['admin']
     can [:update, :destroy], TeamUser, team_id: @context_team.id
-    can :duplicate, Team, :id => @context_team.id
+    can [:duplicate, :export_list], Team, :id => @context_team.id
     can :set_privacy, Project, :team_id => @context_team.id
     can :read_feed_invitations, Feed, :team_id => @context_team.id
     can :destroy, Feed, :team_id => @context_team.id

app/models/annotations/annotation.rb

@@ -17,9 +17,11 @@ def destroy
     dec = self.disable_es_callbacks
     skip_ability = self.skip_check_ability
     a = self.load
-    a.disable_es_callbacks = dec
-    a.skip_check_ability = skip_ability
-    a.destroy
+    unless a.nil?
+      a.disable_es_callbacks = dec
+      a.skip_check_ability = skip_ability
+      a.destroy
+    end
   end
 
   private

app/models/bot/smooch.rb

@@ -542,7 +542,11 @@ def self.process_menu_option_value(value, option, message, language, workflow, a
   end
 
   def self.is_a_shortcut_for_submission?(state, message)
-    self.is_v2? && (state == 'main' || state == 'waiting_for_message') && (!message['mediaUrl'].blank? || ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer))
+    self.is_v2? && (state == 'main' || state == 'waiting_for_message') && (
+      !message['mediaUrl'].blank? ||
+      ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer) ||
+      !Twitter::TwitterText::Extractor.extract_urls(message['text'].to_s).blank? # URL in message?
+      )
   end
 
   def self.process_menu_option(message, state, app_id)

app/models/explainer.rb

@@ -48,6 +48,14 @@ def update_paragraphs_in_alegre
     self.class.delay_for(5.seconds).update_paragraphs_in_alegre(self.id, previous_paragraphs_count, Time.now.to_f)
   end
 
+  def self.get_exported_data(query, team)
+    data = [['ID', 'Title', 'Description', 'URL', 'Language']]
+    team.filtered_explainers(query).find_each do |exp|
+      data << [exp.id, exp.title, exp.description, exp.url, exp.language]
+    end
+    data
+  end
+
   def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp)
     explainer = Explainer.find(id)
 

app/models/fact_check.rb

@@ -47,6 +47,14 @@ def update_item_status
     end
   end
 
+  def self.get_exported_data(query, team)
+    data = [['ID', 'Title', 'Summary', 'URL', 'Language', 'Report Status', 'Imported?']]
+    team.filtered_fact_checks(query).find_each do |fc|
+      data << [fc.id, fc.title, fc.summary, fc.url, fc.language, fc.report_status, fc.imported.to_s]
+    end
+    data
+  end
+
   private
 
   def set_language

app/models/feed.rb

@@ -172,6 +172,14 @@ def saved_search_was
     SavedSearch.find_by_id(self.saved_search_id_before_last_save)
   end
 
+  def get_exported_data(filters)
+    data = [['Title', 'Number of media', 'Number of requests', 'Number of fact-checks']]
+    self.filtered_clusters(filters).find_each do |cluster|
+      data << [cluster.title, cluster.media_count, cluster.requests_count, cluster.fact_checks_count]
+    end
+    data
+  end
+
   # This takes some time to run because it involves external HTTP requests and writes to the database:
   # 1) If the query contains a media URL, it will be downloaded... if it contains some other URL, it will be sent to Pender
   # 2) Requests will be made to Alegre in order to index the request media and to look for similar requests

app/models/project.rb

@@ -40,41 +40,6 @@ class Project < ApplicationRecord
 
   check_settings
 
-  cached_field :medias_count,
-    start_as: 0,
-    recalculate: :recalculate_medias_count,
-    update_on: [
-      {
-        model: Relationship,
-        affected_ids: proc { |r| ProjectMedia.where(id: r.target_id).map(&:project_id) },
-        events: {
-          save: :recalculate,
-          destroy: :recalculate
-        }
-      },
-      {
-        model: ProjectMedia,
-        if: proc { |pm| !pm.project_id.nil? },
-        affected_ids: proc { |pm| [pm.project_id] },
-        events: {
-          create: :recalculate,
-          destroy: :recalculate
-        }
-      },
-      {
-        model: ProjectMedia,
-        if: proc { |pm| pm.saved_change_to_archived? || pm.saved_change_to_project_id? },
-        affected_ids: proc { |pm| [pm.project_id, pm.project_id_before_last_save] },
-        events: {
-          update: :recalculate,
-        }
-      },
-    ]
-
-  def recalculate_medias_count
-    self.team.medias_count(self)
-  end
-
   def check_search_team
     self.team.check_search_team
   end

app/models/project_group.rb

@@ -7,10 +7,6 @@ class ProjectGroup < ApplicationRecord
   belongs_to :team, optional: true
   has_many :projects, dependent: :nullify
 
-  def medias_count
-    self.projects.map(&:medias_count).sum
-  end
-
   def project_medias
     ProjectMedia.joins(:project).where('projects.project_group_id' => self.id)
   end

app/models/team.rb

@@ -532,7 +532,7 @@ def filtered_fact_checks(filters = {})
     query = query.where('fact_checks.imported' => !!filters[:imported]) unless filters[:imported].nil?
 
     # Filter by report status
-    query = query.where('fact_checks.report_status' => filters[:report_status].to_a.map(&:to_s)) unless filters[:report_status].blank?
+    query = query.where('fact_checks.report_status' => [filters[:report_status]].flatten.map(&:to_s)) unless filters[:report_status].blank?
 
     # Filter by text
     query = self.filter_by_keywords(query, filters) if filters[:text].to_s.size > 2
@@ -545,11 +545,11 @@ def filtered_fact_checks(filters = {})
   end
 
   def filter_by_keywords(query, filters, type = 'FactCheck')
-    tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text]]) # FIXME: May not work for all languages
+    tsquery = Team.sanitize_sql_array(["websearch_to_tsquery(?)", filters[:text]])
     if type == 'FactCheck'
-      tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(summary, '') || coalesce(url, ''))"
+      tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(summary, '') || ' ' || coalesce(url, '') || ' ' || coalesce(claim_descriptions.description, '') || ' ' || coalesce(claim_descriptions.context, ''))"
     else
-      tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(description, '') || coalesce(url, ''))"
+      tsvector = "to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(description, '') || ' ' || coalesce(url, ''))"
     end
     query.where(Arel.sql("#{tsvector} @@ #{tsquery}"))
   end