diff --git a/Gemfile.lock b/Gemfile.lock index ff4d48122e..160a12b73b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -243,7 +243,7 @@ GEM encryptor (3.0.0) equalizer (0.0.11) erubi (1.12.0) - et-orbi (1.2.7) + et-orbi (1.2.11) tzinfo ethon (0.16.0) ffi (>= 1.15.0) @@ -302,8 +302,8 @@ GEM fog-core nokogiri (>= 1.5.11, < 2.0.0) formatador (1.1.0) - fugit (1.5.2) - et-orbi (~> 1.1, >= 1.1.8) + fugit (1.11.1) + et-orbi (~> 1, >= 1.2.11) raabro (~> 1.4) fx (0.8.0) activerecord (>= 6.0.0) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index d93be886b7..66619134ed 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -1,7 +1,7 @@ require 'active_support/concern' class AlegreTimeoutError < StandardError; end class TemporaryProjectMedia - attr_accessor :team_id, :id, :url, :text, :type + attr_accessor :team_id, :id, :url, :text, :type, :field def media media_type_map = { "claim" => "Claim", @@ -36,6 +36,10 @@ def is_video? def is_audio? self.type == "audio" end + + def is_uploaded_media? + self.is_image? || self.is_audio? || self.is_video? + end end module AlegreV2 @@ -160,31 +164,31 @@ def get_type(project_media) end def content_hash_for_value(value) - Digest::MD5.hexdigest(value) + value.nil? ? nil : Digest::MD5.hexdigest(value) end def content_hash(project_media, field) if Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS.include?(field) content_hash_for_value(project_media.send(field)) + elsif project_media.is_link? + return content_hash_for_value(project_media.media.url) + elsif project_media.is_a?(TemporaryProjectMedia) + return Rails.cache.read("url_sha:#{project_media.url}") + elsif project_media.is_uploaded_media? + return project_media.media.file.filename.split(".").first else - if project_media.is_link? - return content_hash_for_value(project_media.media.url) - elsif project_media.is_a?(TemporaryProjectMedia) - return Rails.cache.read("url_sha:#{project_media.url}") - elsif !project_media.is_text? - return project_media.media.file.filename.split(".").first - else - return content_hash_for_value(project_media.send(field).to_s) - end + return content_hash_for_value(project_media.send(field).to_s) end end def generic_package(project_media, field) - { - content_hash: content_hash(project_media, field), + content_hash_value = content_hash(project_media, field) + params = { doc_id: item_doc_id(project_media, field), context: get_context(project_media, field) } + params[:content_hash] = content_hash_value if !content_hash_value.nil? + params end def delete_package(project_media, field, params={}, quiet=false) @@ -282,11 +286,15 @@ def store_package_text(project_media, field, params) generic_package_text(project_media, field, params) end - def get_sync_raw_params(params, type) + def index_async_with_params(params, type, suppress_search_response=true) + request("post", async_path_for_type(type), params.merge(suppress_search_response: suppress_search_response)) + end + + def get_sync_with_params(params, type) request("post", sync_path_for_type(type), params) end - def get_async_raw_params(params, type) + def get_async_with_params(params, type) request("post", async_path_for_type(type), params) end @@ -309,6 +317,10 @@ def delete(project_media, field=nil, params={}) delete_package(project_media, field, params), project_media ) + rescue StandardError => e + error = Error.new(e) + Rails.logger.error("[AutoTagger Bot] Exception for event `#{body['event']}`: #{error.class} - #{error.message}") + CheckSentry.notify(error, bot: "alegre", project_media: project_media, params: params, field: field) end def get_per_model_threshold(project_media, threshold) @@ -508,25 +520,27 @@ def wait_for_results(project_media, args) end def get_items_with_similar_media_v2(args={}) + text = args[:text] + field = args[:field] media_url = args[:media_url] project_media = args[:project_media] threshold = args[:threshold] team_ids = args[:team_ids] type = args[:type] - if ['audio', 'image', 'video'].include?(type) - if project_media.nil? - project_media = TemporaryProjectMedia.new - project_media.url = media_url - project_media.id = Digest::MD5.hexdigest(project_media.url).to_i(16) - project_media.team_id = team_ids - project_media.type = type - end - get_similar_items_v2_async(project_media, nil, threshold) - wait_for_results(project_media, args) - response = get_similar_items_v2_callback(project_media, nil) - delete(project_media, nil) if project_media.is_a?(TemporaryProjectMedia) - return response + if project_media.nil? + project_media = TemporaryProjectMedia.new + project_media.text = text + project_media.field = field + project_media.url = media_url + project_media.id = Digest::MD5.hexdigest(project_media.url).to_i(16) + project_media.team_id = team_ids + project_media.type = type end + get_similar_items_v2_async(project_media, nil, threshold) + wait_for_results(project_media, args) + response = get_similar_items_v2_callback(project_media, nil) + delete(project_media, nil) if project_media.is_a?(TemporaryProjectMedia) + return response end def process_alegre_callback(params) @@ -535,9 +549,11 @@ def process_alegre_callback(params) should_relate = true if project_media.nil? project_media = TemporaryProjectMedia.new + project_media.text = params.dig('data', 'item', 'raw', 'text') project_media.url = params.dig('data', 'item', 'raw', 'url') project_media.id = params.dig('data', 'item', 'raw', 'context', 'project_media_id') project_media.team_id = params.dig('data', 'item', 'raw', 'context', 'team_id') + project_media.field = params.dig('data', 'item', 'raw', 'context', 'field') project_media.type = params['model_type'] should_relate = false end diff --git a/app/models/concerns/project_media_getters.rb b/app/models/concerns/project_media_getters.rb index b8662224e7..56040e69c7 100644 --- a/app/models/concerns/project_media_getters.rb +++ b/app/models/concerns/project_media_getters.rb @@ -31,6 +31,10 @@ def is_image? self.is_uploaded_image? end + def is_uploaded_media? + self.is_image? || self.is_audio? || self.is_video? + end + def is_text? self.is_claim? || self.is_link? end diff --git a/app/models/explainer.rb b/app/models/explainer.rb index a4319e718a..f1599f4617 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -63,24 +63,26 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) # Index title params = { + content_hash: Bot::Alegre.content_hash_for_value(explainer.title), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'title'].join(':')), + context: base_context.merge({ field: 'title' }), text: explainer.title, models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - context: base_context.merge({ field: 'title' }) } - Bot::Alegre.request('post', '/text/similarity/', params) + Bot::Alegre.index_async_with_params(params, "text") # Index paragraphs count = 0 explainer.description.to_s.gsub(/\r\n?/, "\n").split(/\n+/).reject{ |paragraph| paragraph.strip.blank? }.each do |paragraph| count += 1 params = { + content_hash: Bot::Alegre.content_hash_for_value(paragraph.strip), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'paragraph', count].join(':')), + context: base_context.merge({ paragraph: count }), text: paragraph.strip, models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - context: base_context.merge({ paragraph: count }) } - Bot::Alegre.request('post', '/text/similarity/', params) + Bot::Alegre.index_async_with_params(params, "text") end # Remove paragraphs that don't exist anymore (we delete after updating in order to avoid race conditions) @@ -91,7 +93,7 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) quiet: true, context: base_context.merge({ paragraph: count }) } - Bot::Alegre.request('delete', '/text/similarity/', params) + Bot::Alegre.request_delete_from_raw(params, "text") end end @@ -106,7 +108,7 @@ def self.search_by_similarity(text, language, team_id) language: language } } - response = Bot::Alegre.request('post', '/text/similarity/search/', params) + response = Bot::Alegre.get_async_with_params(params, "text") results = response['result'].to_a.sort_by{ |result| result['_score'] } explainer_ids = results.collect{ |result| result.dig('_source', 'context', 'explainer_id').to_i }.uniq.first(3) explainer_ids.empty? ? Explainer.none : Explainer.where(team_id: team_id, id: explainer_ids) diff --git a/config/initializers/report_designer.rb b/config/initializers/report_designer.rb index 08cd3dcbf8..ffbf4d7d2b 100644 --- a/config/initializers/report_designer.rb +++ b/config/initializers/report_designer.rb @@ -50,9 +50,11 @@ if fc.nil? FactCheck.create({ claim_description: pm.claim_description }.merge(fields)) else - fields.each { |field, value| fc.send("#{field}=", value) } - fc.skip_check_ability = true - fc.save! + PaperTrail.request(enabled: false) do + fields.each { |field, value| fc.send("#{field}=", value) } + fc.skip_check_ability = true + fc.save! + end end end @@ -66,16 +68,18 @@ # Update report fields fc = pm&.claim_description&.fact_check unless fc.nil? - state = self.data['state'] - fields = { - skip_report_update: true, - publisher_id: nil, - report_status: state, - rating: pm.status - } - fields.each { |field, value| fc.send("#{field}=", value) } - fc.skip_check_ability = true - fc.save! + PaperTrail.request(enabled: false) do + state = self.data['state'] + fields = { + skip_report_update: true, + publisher_id: nil, + report_status: state, + rating: pm.status + } + fields.each { |field, value| fc.send("#{field}=", value) } + fc.skip_check_ability = true + fc.save! + end end end end diff --git a/lib/tasks/check_khousheh.rake b/lib/tasks/check_khousheh.rake index ee1f299c65..098d9f3c06 100644 --- a/lib/tasks/check_khousheh.rake +++ b/lib/tasks/check_khousheh.rake @@ -192,6 +192,8 @@ namespace :check do Cluster.transaction do # Create clusters mapping = {} # Media ID => Cluster ID + # Cluster to delete in case there is no center (project_media_id) + cluster_to_delete = [] # Bulk-insert clusters c_inserted_items = [] clusters.length.times.each_slice(2500) do |rows| @@ -278,6 +280,7 @@ namespace :check do updated_cluster_attributes[:title] = cluster_title # Update cluster if updated_cluster_attributes[:project_media_id].blank? + cluster_to_delete << cluster.id error_logs << {Cluster: "Failed to update Cluster with id #{cluster.id}"} else cluster_items[cluster.id] = updated_cluster_attributes @@ -303,6 +306,8 @@ namespace :check do end search_after = [pm_ids.max] end + # Delete cluster with no project_media_id + Cluster.where(id: cluster_to_delete).delete_all Team.current = nil end puts "\nRebuilding clusters for feed #{feed.name} took #{Time.now.to_f - started_at} seconds." diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index 7a1adddc5b..a11f769b68 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -139,6 +139,7 @@ def send_message_outside_24_hours_window(template, pm = nil) test "should submit query without details on tipline bot v2" do WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, random_string, claim, random_string, random_string, '1' assert_saved_query_type 'default_requests' @@ -208,6 +209,7 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query with details on tipline bot v2" do + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, '2', random_string, claim, '1' @@ -285,7 +287,7 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query and handle search error on tipline bot v2" do - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers CheckSearch.any_instance.stubs(:medias).raises(StandardError) Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' @@ -384,7 +386,7 @@ def send_message_outside_24_hours_window(template, pm = nil) ProjectMedia.any_instance.stubs(:report_status).returns('published') ProjectMedia.any_instance.stubs(:analysis_published_article_url).returns(random_url) Bot::Alegre.stubs(:get_merged_similar_items).returns({ create_project_media.id => { score: 0.9 } }) - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do send_message 'hello', '1', '1', "Foo bar foo bar #{url} foo bar", '1' end @@ -693,7 +695,7 @@ def send_message_outside_24_hours_window(template, pm = nil) pm = create_project_media team: @team publish_report(pm, {}, nil, { language: 'pt', use_visual_card: false }) Bot::Smooch.stubs(:get_search_results).returns([pm]) - WebMock.stub_request(:post, /\/text\/similarity\/search/).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' end diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 4fd46ac40e..9a7cb345c0 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -600,6 +600,7 @@ def teardown end test "should include claim_description_content in smooch search" do + WebMock.stub_request(:post, 'http://alegre:3100/similarity/async/image').to_return(body: {}.to_json) WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) RequestStore.store[:skip_cached_field_update] = false t = create_team diff --git a/test/models/explainer_test.rb b/test/models/explainer_test.rb index a902379b6c..a87665421e 100644 --- a/test/models/explainer_test.rb +++ b/test/models/explainer_test.rb @@ -99,12 +99,12 @@ def setup } # Index two paragraphs and title when the explainer is created - Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(3) + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).times(3) Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).never ex = create_explainer description: description # Update the index when paragraphs change - Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(2) + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).times(2) Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).once ex = Explainer.find(ex.id) ex.description = 'Now this is the only paragraph'