Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cv2 5085 move get items with similar text to presto #2023

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions app/models/bot/alegre.rb
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def similar_items_ids_and_scores(team_ids, thresholds = {})
ALL_TEXT_SIMILARITY_FIELDS.each do |field|
text = self.send(field)
next if text.blank?
threads << Thread.new { ids_and_scores.merge!(Bot::Alegre.get_similar_texts(team_ids, text, Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, thresholds[:text]).to_h) }
threads << Thread.new { ids_and_scores.merge!(Bot::Alegre.get_items_from_similar_text(team_ids, text, Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, thresholds[:text]).to_h) }
end
threads.map(&:join)
end
Expand Down Expand Up @@ -206,7 +206,7 @@ def self.get_items_from_similar_text(team_id, text, fields = nil, threshold = ni
threshold ||= self.get_threshold_for_query('text', nil, true)
models ||= [self.matching_model_to_use(team_ids)].flatten
Hash[self.get_similar_items_from_api(
'/text/similarity/search/',
'/similarity/sync/text',
self.similar_texts_from_api_conditions(text, models, fuzzy, team_ids, fields, threshold),
threshold
).collect{|k,v| [k, v.merge(model: v[:model]||Bot::Alegre.default_matching_model)]}]
Expand Down Expand Up @@ -719,8 +719,4 @@ def self.is_text_too_short?(pm, length_threshold)
is_short
end

class <<self
alias_method :get_similar_texts, :get_items_from_similar_text
end

end
2 changes: 1 addition & 1 deletion app/resources/api/v2/report_resource.rb
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def self.apply_text_similarity_filter(organization_ids, threshold, filters)
ids = nil
unless text.blank?
fields = filters[:similarity_fields].blank? ? nil : filters[:similarity_fields].to_a.flatten
ids_and_scores = Bot::Alegre.get_similar_texts(organization_ids, text[0], fields, [{ value: threshold }], nil, filters.dig(:fuzzy, 0))
ids_and_scores = Bot::Alegre.get_items_from_similar_text(organization_ids, text[0], fields, [{ value: threshold }], nil, filters.dig(:fuzzy, 0))
RequestStore.store[:scores] = ids_and_scores # Store the scores so we can return them
ids = ids_and_scores.keys.uniq
ids = [0] if ids.blank?
Expand Down
2 changes: 1 addition & 1 deletion test/controllers/elastic_search_9_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def setup
WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'result' => { 'language' => 'es' }}.to_json)
WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success')
WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json)
WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json)
WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: {success: true}.to_json)
WebMock.stub_request(:post, 'http://alegre/image/classification/').with({ body: { uri: 'some/path' } }).to_return(body: {
"result": valid_flags_data
}.to_json)
Expand Down
4 changes: 2 additions & 2 deletions test/controllers/graphql_controller_5_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,14 @@ def setup
pm = create_project_media project: p, media: m
pm2 = create_project_media project: p, media: m2
create_claim_description project_media: pm2
Bot::Alegre.stubs(:get_similar_texts).returns({ pm2.id => 0.9, pm.id => 0.8 })
Bot::Alegre.stubs(:get_items_from_similar_text).returns({ pm2.id => 0.9, pm.id => 0.8 })

query = 'query { project_media(ids: "' + [pm.id, p.id, t.id].join(',') + '") { similar_items(first: 10000) { edges { node { dbid, claim_description { id, fact_check { id } } } } } } }'
post :create, params: { query: query, team: t.slug }
assert_response :success
assert_equal pm2.id, JSON.parse(@response.body)['data']['project_media']['similar_items']['edges'][0]['node']['dbid']

Bot::Alegre.unstub(:get_similar_texts)
Bot::Alegre.unstub(:get_items_from_similar_text)
end

test "should create and update flags and content warning" do
Expand Down
2 changes: 1 addition & 1 deletion test/models/bot/alegre_2_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ def teardown
WebMock.stub_request(:post, 'http://alegre.test/text/similarity/').to_return(body: 'success')
WebMock.stub_request(:delete, 'http://alegre.test/text/similarity/').to_return(body: { success: true }.to_json)
WebMock.stub_request(:delete, 'http://alegre.test/image/similarity/').to_return(body: { success: true }.to_json)
WebMock.stub_request(:post, 'http://alegre.test/text/similarity/search/').to_return(body: { success: true }.to_json)
WebMock.stub_request(:post, 'http://alegre.test/similarity/sync/text').to_return(body: { success: true }.to_json)
WebMock.stub_request(:post, 'http://alegre.test/image/ocr/').to_return(body: { text: 'Foo bar' }.to_json)
WebMock.stub_request(:post, 'http://alegre.test/similarity/sync/image').to_return(body: {
result: [
Expand Down
2 changes: 1 addition & 1 deletion test/models/bot/alegre_3_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ def teardown
WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/
WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success')
WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json)
WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json)
WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: {success: true}.to_json)
WebMock.stub_request(:post, 'http://alegre/audio/similarity/').to_return(body: {
"success": true
}.to_json)
Expand Down
2 changes: 1 addition & 1 deletion test/models/bot/alegre_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def teardown
WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response')
WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response')
WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success')
WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: 'success')
WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: 'success')
WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/
Bot::Alegre.any_instance.stubs(:get_language).raises(RuntimeError)
assert_nothing_raised do
Expand Down
2 changes: 2 additions & 0 deletions test/models/bot/smooch_6_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ def send_message_outside_24_hours_window(template, pm = nil)
test "should submit query without details on tipline bot v2" do
WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers
WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers
WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers
claim = 'This is a test claim'
send_message 'hello', '1', '1', random_string, random_string, claim, random_string, random_string, '1'
assert_saved_query_type 'default_requests'
Expand Down Expand Up @@ -209,6 +210,7 @@ def send_message_outside_24_hours_window(template, pm = nil)
end

test "should submit query with details on tipline bot v2" do
WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers
WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers
WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers
claim = 'This is a test claim'
Expand Down
1 change: 1 addition & 0 deletions test/models/explainer_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ def setup
end

test "should create tag texts when setting tags" do
WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers
Sidekiq::Testing.inline! do
assert_difference 'TagText.count' do
create_explainer tags: ['foo']
Expand Down
Loading