Cv2 5085 move get items with similar text to presto (#2023)

* Cv2 5082 article indexing to presto (#1994)

* CV2-5087 move Articles side effecting saves to to it via presto

* CV2-5082 move article indexing to presto

* resolve test errors

* updates for broken tests

* small tweak

* set to sync

* more fixes

* rename function and revert request

* add response suppression and move to specific path for side effecting requests

* extend similar media to allow for temporary texts

* fix broken test fixture

* revert back to async

* fix another test

* fixes per PR review

* fixes per PR review

* more fixes after review

* Cv2 5080 request model to presto (#2015)

* CV2-5080 update request model alegre calls to use presto-based alegre querying

* move to sync

* update for bypassing async calls in tests

* CV2-5085 move get_items_from_similar_text calls to use sync endpoint

* review and resolve broken tests

* update stub

app/models/bot/alegre.rb

@@ -41,7 +41,7 @@ def similar_items_ids_and_scores(team_ids, thresholds = {})
         ALL_TEXT_SIMILARITY_FIELDS.each do |field|
           text = self.send(field)
           next if text.blank?
-          threads << Thread.new { ids_and_scores.merge!(Bot::Alegre.get_similar_texts(team_ids, text, Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, thresholds[:text]).to_h) }
+          threads << Thread.new { ids_and_scores.merge!(Bot::Alegre.get_items_from_similar_text(team_ids, text, Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, thresholds[:text]).to_h) }
         end
         threads.map(&:join)
       end
@@ -206,7 +206,7 @@ def self.get_items_from_similar_text(team_id, text, fields = nil, threshold = ni
     threshold ||= self.get_threshold_for_query('text', nil, true)
     models ||= [self.matching_model_to_use(team_ids)].flatten
     Hash[self.get_similar_items_from_api(
-      '/text/similarity/search/',
+      '/similarity/sync/text',
       self.similar_texts_from_api_conditions(text, models, fuzzy, team_ids, fields, threshold),
       threshold
     ).collect{|k,v| [k, v.merge(model: v[:model]||Bot::Alegre.default_matching_model)]}]
@@ -719,8 +719,4 @@ def self.is_text_too_short?(pm, length_threshold)
     is_short
   end
 
-  class <<self
-    alias_method :get_similar_texts, :get_items_from_similar_text
-  end
-
 end

app/resources/api/v2/report_resource.rb

@@ -57,7 +57,7 @@ def self.apply_text_similarity_filter(organization_ids, threshold, filters)
         ids = nil
         unless text.blank?
           fields = filters[:similarity_fields].blank? ? nil : filters[:similarity_fields].to_a.flatten
-          ids_and_scores = Bot::Alegre.get_similar_texts(organization_ids, text[0], fields, [{ value: threshold }], nil, filters.dig(:fuzzy, 0))
+          ids_and_scores = Bot::Alegre.get_items_from_similar_text(organization_ids, text[0], fields, [{ value: threshold }], nil, filters.dig(:fuzzy, 0))
           RequestStore.store[:scores] = ids_and_scores # Store the scores so we can return them
           ids = ids_and_scores.keys.uniq
           ids = [0] if ids.blank?

test/controllers/elastic_search_9_test.rb

@@ -60,7 +60,7 @@ def setup
       WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'result' => { 'language' => 'es' }}.to_json)
       WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success')
       WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json)
-      WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json)
+      WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: {success: true}.to_json)
       WebMock.stub_request(:post, 'http://alegre/image/classification/').with({ body: { uri: 'some/path' } }).to_return(body: {
         "result": valid_flags_data
       }.to_json)

test/controllers/graphql_controller_5_test.rb

@@ -57,14 +57,14 @@ def setup
     pm = create_project_media project: p, media: m
     pm2 = create_project_media project: p, media: m2
     create_claim_description project_media: pm2
-    Bot::Alegre.stubs(:get_similar_texts).returns({ pm2.id => 0.9, pm.id => 0.8 })
+    Bot::Alegre.stubs(:get_items_from_similar_text).returns({ pm2.id => 0.9, pm.id => 0.8 })
 
     query = 'query { project_media(ids: "' + [pm.id, p.id, t.id].join(',') + '") { similar_items(first: 10000) { edges { node { dbid, claim_description { id, fact_check { id } } } } } } }'
     post :create, params: { query: query, team: t.slug }
     assert_response :success
     assert_equal pm2.id, JSON.parse(@response.body)['data']['project_media']['similar_items']['edges'][0]['node']['dbid']
 
-    Bot::Alegre.unstub(:get_similar_texts)
+    Bot::Alegre.unstub(:get_items_from_similar_text)
   end
 
   test "should create and update flags and content warning" do

test/models/bot/alegre_2_test.rb

@@ -315,7 +315,7 @@ def teardown
       WebMock.stub_request(:post, 'http://alegre.test/text/similarity/').to_return(body: 'success')
       WebMock.stub_request(:delete, 'http://alegre.test/text/similarity/').to_return(body: { success: true }.to_json)
       WebMock.stub_request(:delete, 'http://alegre.test/image/similarity/').to_return(body: { success: true }.to_json)
-      WebMock.stub_request(:post, 'http://alegre.test/text/similarity/search/').to_return(body: { success: true }.to_json)
+      WebMock.stub_request(:post, 'http://alegre.test/similarity/sync/text').to_return(body: { success: true }.to_json)
       WebMock.stub_request(:post, 'http://alegre.test/image/ocr/').to_return(body: { text: 'Foo bar' }.to_json)
       WebMock.stub_request(:post, 'http://alegre.test/similarity/sync/image').to_return(body: {
         result: [

test/models/bot/alegre_3_test.rb

@@ -149,7 +149,7 @@ def teardown
       WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/
       WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success')
       WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json)
-      WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json)
+      WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: {success: true}.to_json)
       WebMock.stub_request(:post, 'http://alegre/audio/similarity/').to_return(body: {
         "success": true
       }.to_json)

test/models/bot/alegre_test.rb

@@ -44,7 +44,7 @@ def teardown
        WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response')
        WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response')
        WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success')
-       WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: 'success')
+       WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: 'success')
        WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/
        Bot::Alegre.any_instance.stubs(:get_language).raises(RuntimeError)
        assert_nothing_raised do

test/models/bot/smooch_6_test.rb

@@ -140,6 +140,7 @@ def send_message_outside_24_hours_window(template, pm = nil)
   test "should submit query without details on tipline bot v2" do
     WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers
     WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers
+    WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers
     claim = 'This is a test claim'
     send_message 'hello', '1', '1', random_string, random_string, claim, random_string, random_string, '1'
     assert_saved_query_type 'default_requests'
@@ -209,6 +210,7 @@ def send_message_outside_24_hours_window(template, pm = nil)
   end
 
   test "should submit query with details on tipline bot v2" do
+    WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers
     WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers
     WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers
     claim = 'This is a test claim'

test/models/explainer_test.rb

@@ -83,6 +83,7 @@ def setup
   end
 
   test "should create tag texts when setting tags" do
+    WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers
     Sidekiq::Testing.inline! do
       assert_difference 'TagText.count' do
         create_explainer tags: ['foo']