Some fixes and improvements for the articles feature (#1976)

- Don't include blank media in search queries (lists and counts) - Update item title when fact-check is updated - Filter out explainers already applied to the item - Index explainer titles in addition to summary paragraphs Reference: CV2-5000.
meedan · Aug 1, 2024 · f3c18a0 · f3c18a0
1 parent 0e6a3d7
commit f3c18a0
Show file tree

Hide file tree

Showing 10 changed files with 64 additions and 14 deletions.
diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb
@@ -305,6 +305,7 @@ def tipline_messages(uid:)
     argument :report_status, [GraphQL::Types::String, null: true], required: false, camelize: false
     argument :rating, [GraphQL::Types::String, null: true], required: false, camelize: false
     argument :imported, GraphQL::Types::Boolean, required: false, camelize: false # Only for fact-checks
+    argument :target_id, GraphQL::Types::Int, required: false, camelize: false # Exclude articles already applied to the `ProjectMedia` with this ID
   end
 
   def articles(**args)
@@ -334,6 +335,7 @@ def articles(**args)
     argument :report_status, [GraphQL::Types::String, null: true], required: false, camelize: false
     argument :rating, [GraphQL::Types::String, null: true], required: false, camelize: false
     argument :imported, GraphQL::Types::Boolean, required: false, camelize: false # Only for fact-checks
+    argument :target_id, GraphQL::Types::Int, required: false, camelize: false # Exclude articles already applied to the `ProjectMedia` with this ID
   end
 
   def articles_count(**args)

diff --git a/app/models/concerns/project_media_cached_fields.rb b/app/models/concerns/project_media_cached_fields.rb
@@ -13,16 +13,17 @@ def title_or_description_update
       [
         {
           model: ClaimDescription,
-          affected_ids: proc { |cd| [cd.project_media] },
+          affected_ids: proc { |cd| [cd.project_media_id, cd.project_media_id_before_last_save] },
           events: {
             save: :recalculate
           }
         },
         {
           model: FactCheck,
-          affected_ids: proc { |fc| [fc.claim_description.project_media] },
+          affected_ids: proc { |fc| [fc.claim_description.project_media_id] },
           events: {
-            save: :recalculate
+            save: :recalculate,
+            destroy: :recalculate
           }
         },
         {
@@ -71,7 +72,7 @@ def title_or_description_update
 
     FACT_CHECK_EVENT = {
       model: FactCheck,
-      affected_ids: proc { |fc| [fc.claim_description.project_media] },
+      affected_ids: proc { |fc| [fc.claim_description.project_media_id] },
       events: {
         save: :recalculate,
         destroy: :recalculate

diff --git a/app/models/concerns/project_media_getters.rb b/app/models/concerns/project_media_getters.rb
@@ -153,7 +153,7 @@ def get_title
       return self.send(title_mapping[title_field]).to_s
     end
     title = self.original_title
-    [self.analysis['file_title'], self.analysis['title'], self.fact_check_title, self.claim_description_content].each do |value|
+    [self.analysis['file_title'], self.analysis['title'], self.fact_check_title(true), self.claim_description_content].each do |value|
       title = value if !value.blank? && value != '-' && value != ''
     end
     title.to_s
@@ -174,7 +174,7 @@ def media_slug
   end
 
   def get_description
-    return self.fact_check_summary if self.get_main_channel == CheckChannels::ChannelCodes::FETCH
+    return self.fact_check_summary(true) if self.get_main_channel == CheckChannels::ChannelCodes::FETCH
     analysis_description = self.has_analysis_description? ? self.analysis_description : nil
     self.claim_description_content || analysis_description || self.original_description
   end

diff --git a/app/models/concerns/team_associations.rb b/app/models/concerns/team_associations.rb
@@ -93,6 +93,7 @@ def medias_count(obj = nil)
     conditions[key] = obj.id
     relationship_type = Team.sanitize_sql(Relationship.confirmed_type.to_yaml)
     ProjectMedia.where(conditions)
+    .joins(:media).where('medias.type != ?', 'Blank')
     .joins("LEFT JOIN relationships r ON r.target_id = project_medias.id AND r.relationship_type = '#{relationship_type}'")
     .where('r.id IS NULL').count
   end

diff --git a/app/models/explainer.rb b/app/models/explainer.rb
@@ -61,6 +61,15 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp)
       explainer_id: explainer.id
     }
 
+    # Index title
+    params = {
+      doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'title'].join(':')),
+      text: explainer.title,
+      models: ALEGRE_MODELS_AND_THRESHOLDS.keys,
+      context: base_context.merge({ field: 'title' })
+    }
+    Bot::Alegre.request('post', '/text/similarity/', params)
+
     # Index paragraphs
     count = 0
     explainer.description.to_s.gsub(/\r\n?/, "\n").split(/\n+/).reject{ |paragraph| paragraph.strip.blank? }.each do |paragraph|

diff --git a/app/models/team.rb b/app/models/team.rb
@@ -497,6 +497,10 @@ def filtered_explainers(filters = {})
     # Filter by text
     query = query.where('(title ILIKE ? OR url ILIKE ? OR description ILIKE ?)', *["%#{filters[:text]}%"]*3) if filters[:text].to_s.size > 2
 
+    # Exclude the ones already applied to a target item
+    target = ProjectMedia.find_by_id(filters[:target_id].to_i)
+    query = query.where.not(id: target.explainer_ids) unless target.nil?
+
     query
   end
 
@@ -533,6 +537,10 @@ def filtered_fact_checks(filters = {})
     # Filter by text
     query = query.where('(fact_checks.title ILIKE ? OR fact_checks.url ILIKE ? OR fact_checks.summary ILIKE ?)', *["%#{filters[:text]}%"]*3) if filters[:text].to_s.size > 2
 
+    # Exclude the ones already applied to a target item
+    target = ProjectMedia.find_by_id(filters[:target_id].to_i)
+    query = query.where.not('fact_checks.id' => target.fact_check_id) unless target.nil?
+
     query
   end
 

diff --git a/lib/check_search.rb b/lib/check_search.rb
@@ -46,7 +46,7 @@ def initialize(options, file = nil, team_id = Team.current&.id)
     @file = file
   end
 
-  MEDIA_TYPES = %w[claims links twitter youtube tiktok instagram facebook telegram weblink images videos audios blank]
+  MEDIA_TYPES = %w[claims links twitter youtube tiktok instagram facebook telegram weblink images videos audios]
   SORT_MAPPING = {
     'recent_activity' => 'updated_at', 'recent_added' => 'created_at', 'demand' => 'demand',
     'related' => 'linked_items_count', 'last_seen' => 'last_seen', 'share_count' => 'share_count',
@@ -147,21 +147,24 @@ def number_of_items(collection)
     collection.limit(nil).reorder(nil).offset(nil).count
   end
 
+  def query_all_types?
+    MEDIA_TYPES.size == media_types_filter.size
+  end
+
   def should_hit_elasticsearch?
     return true if feed_query?
     status_blank = true
     status_search_fields.each do |field|
       status_blank = false unless @options[field].blank?
     end
-    query_all_types = (MEDIA_TYPES.size == media_types_filter.size)
     filters_blank = true
     ['tags', 'keyword', 'rules', 'language', 'fc_language', 'request_language', 'report_language', 'team_tasks', 'assigned_to', 'report_status', 'range_numeric',
       'has_claim', 'cluster_teams', 'published_by', 'annotated_by', 'channels', 'cluster_published_reports'
     ].each do |filter|
       filters_blank = false unless @options[filter].blank?
     end
     range_filter = hit_es_for_range_filter
-    !(query_all_types && status_blank && filters_blank && !range_filter && ['recent_activity', 'recent_added', 'last_seen'].include?(@options['sort']))
+    !(query_all_types? && status_blank && filters_blank && !range_filter && ['recent_activity', 'recent_added', 'last_seen'].include?(@options['sort']))
   end
 
   def media_types_filter
@@ -190,7 +193,7 @@ def item_navigation_offset
       query = { bool: { must: conditions, must_not: must_not } }
       $repository.count(query: query)
     else
-      condition = sort_type == :asc ? "#{sort_key} < ?" : "#{sort_key} > ?"
+      condition = sort_type == :asc ? "project_medias.#{sort_key} < ?" : "project_medias.#{sort_key} > ?"
       get_pg_results_for_media.where(condition, pm.send(sort_key)).count
     end
   end
@@ -249,6 +252,7 @@ def get_pg_results_for_media
       core_conditions.merge!({ 'project_medias.id' => ids })
     end
     relation = relation.distinct('project_medias.id').includes(:media).includes(:project).where(core_conditions)
+    relation = relation.joins(:media).where('medias.type != ?', 'Blank') if query_all_types?
     relation
   end
 

diff --git a/lib/relay.idl b/lib/relay.idl
@@ -13103,11 +13103,12 @@ type Team implements Node {
     sort_type: String = "ASC"
     standalone: Boolean
     tags: [String]
+    target_id: Int
     text: String
     updated_at: String
     user_ids: [Int]
   ): ArticleUnionConnection
-  articles_count(article_type: String, imported: Boolean, language: [String], publisher_ids: [Int], rating: [String], report_status: [String], standalone: Boolean, tags: [String], text: String, updated_at: String, user_ids: [Int]): Int
+  articles_count(article_type: String, imported: Boolean, language: [String], publisher_ids: [Int], rating: [String], report_status: [String], standalone: Boolean, tags: [String], target_id: Int, text: String, updated_at: String, user_ids: [Int]): Int
   available_newsletter_header_types: JsonStringType
   avatar: String
   check_search_spam: CheckSearch

diff --git a/public/relay.json b/public/relay.json
@@ -68789,6 +68789,18 @@
                   "defaultValue": null,
                   "isDeprecated": false,
                   "deprecationReason": null
+                },
+                {
+                  "name": "target_id",
+                  "description": null,
+                  "type": {
+                    "kind": "SCALAR",
+                    "name": "Int",
+                    "ofType": null
+                  },
+                  "defaultValue": null,
+                  "isDeprecated": false,
+                  "deprecationReason": null
                 }
               ],
               "type": {
@@ -68958,6 +68970,18 @@
                   "defaultValue": null,
                   "isDeprecated": false,
                   "deprecationReason": null
+                },
+                {
+                  "name": "target_id",
+                  "description": null,
+                  "type": {
+                    "kind": "SCALAR",
+                    "name": "Int",
+                    "ofType": null
+                  },
+                  "defaultValue": null,
+                  "isDeprecated": false,
+                  "deprecationReason": null
                 }
               ],
               "type": {

diff --git a/test/models/explainer_test.rb b/test/models/explainer_test.rb
@@ -113,13 +113,13 @@ def setup
       This is the second paragraph.
     }
 
-    # Index two paragraphs when the explainer is created
-    Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(2)
+    # Index two paragraphs and title when the explainer is created
+    Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(3)
     Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).never
     ex = create_explainer description: description
 
     # Update the index when paragraphs change
-    Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).once
+    Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(2)
     Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).once
     ex = Explainer.find(ex.id)
     ex.description = 'Now this is the only paragraph'