From ff95711a09b9065424956b466969d538e3d0305d Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Tue, 27 Aug 2024 05:58:51 -0700 Subject: [PATCH 01/12] Cv2 5082 article indexing to presto (#1994) * CV2-5087 move Articles side effecting saves to to it via presto * CV2-5082 move article indexing to presto * resolve test errors * updates for broken tests * small tweak * set to sync * more fixes * rename function and revert request * add response suppression and move to specific path for side effecting requests * extend similar media to allow for temporary texts * fix broken test fixture * revert back to async * fix another test * fixes per PR review * fixes per PR review * more fixes after review --- app/models/concerns/alegre_v2.rb | 95 ++++++++++++++------ app/models/concerns/project_media_getters.rb | 4 + app/models/explainer.rb | 14 +-- test/models/bot/smooch_6_test.rb | 8 +- test/models/bot/smooch_7_test.rb | 1 + test/models/explainer_test.rb | 4 +- 6 files changed, 87 insertions(+), 39 deletions(-) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index 2dde5fc159..46db6ea13c 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -1,7 +1,7 @@ require 'active_support/concern' class AlegreTimeoutError < StandardError; end class TemporaryProjectMedia - attr_accessor :team_id, :id, :url, :type + attr_accessor :team_id, :id, :url, :text, :type, :field def media media_type_map = { "claim" => "Claim", @@ -36,6 +36,10 @@ def is_video? def is_audio? self.type == "audio" end + + def is_uploaded_media? + self.is_image? || self.is_audio? || self.is_video? + end end module AlegreV2 @@ -55,11 +59,18 @@ def sync_path_for_type(type) end def async_path(project_media) - "/similarity/async/#{get_type(project_media)}" + self.async_path_for_type(get_type(project_media)) + end + + def async_path_for_type(type) + "/similarity/async/#{type}" end def delete_path(project_media) - type = get_type(project_media) + self.delete_path_for_type(get_type(project_media)) + end + + def delete_path_for_type(type) "/#{type}/similarity/" end @@ -122,6 +133,10 @@ def request(method, path, params, retries=3) end end + def request_delete_from_raw(params, type) + request("delete", delete_path_for_type(type), params) + end + def request_delete(data, project_media) request("delete", delete_path(project_media), data) end @@ -148,28 +163,32 @@ def get_type(project_media) type end + def content_hash_for_value(value) + value.nil? ? nil : Digest::MD5.hexdigest(value) + end + def content_hash(project_media, field) if Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS.include?(field) - Digest::MD5.hexdigest(project_media.send(field)) + content_hash_for_value(project_media.send(field)) + elsif project_media.is_link? + return content_hash_for_value(project_media.media.url) + elsif project_media.is_a?(TemporaryProjectMedia) + return Rails.cache.read("url_sha:#{project_media.url}") + elsif project_media.is_uploaded_media? + return project_media.media.file.filename.split(".").first else - if project_media.is_link? - return Digest::MD5.hexdigest(project_media.media.url) - elsif project_media.is_a?(TemporaryProjectMedia) - return Rails.cache.read("url_sha:#{project_media.url}") - elsif !project_media.is_text? - return project_media.media.file.filename.split(".").first - else - return Digest::MD5.hexdigest(project_media.send(field).to_s) - end + return content_hash_for_value(project_media.send(field).to_s) end end def generic_package(project_media, field) - { - content_hash: content_hash(project_media, field), + content_hash_value = content_hash(project_media, field) + params = { doc_id: item_doc_id(project_media, field), context: get_context(project_media, field) } + params[:content_hash] = content_hash_value if !content_hash_value.nil? + params end def delete_package(project_media, field, params={}, quiet=false) @@ -267,6 +286,18 @@ def store_package_text(project_media, field, params) generic_package_text(project_media, field, params) end + def index_async_with_params(params, type, suppress_search_response=true) + request("post", async_path_for_type(type), params.merge(suppress_search_response: suppress_search_response)) + end + + def get_sync_with_params(params, type) + request("post", sync_path_for_type(type), params) + end + + def get_async_with_params(params, type) + request("post", async_path_for_type(type), params) + end + def get_sync(project_media, field=nil, params={}) request_sync( store_package(project_media, field, params), @@ -286,6 +317,10 @@ def delete(project_media, field=nil, params={}) delete_package(project_media, field, params), project_media ) + rescue StandardError => e + error = Error.new(e) + Rails.logger.error("[AutoTagger Bot] Exception for event `#{body['event']}`: #{error.class} - #{error.message}") + CheckSentry.notify(error, bot: "alegre", project_media: project_media, params: params, field: field) end def get_per_model_threshold(project_media, threshold) @@ -485,25 +520,27 @@ def wait_for_results(project_media, args) end def get_items_with_similar_media_v2(args={}) + text = args[:text] + field = args[:field] media_url = args[:media_url] project_media = args[:project_media] threshold = args[:threshold] team_ids = args[:team_ids] type = args[:type] - if ['audio', 'image', 'video'].include?(type) - if project_media.nil? - project_media = TemporaryProjectMedia.new - project_media.url = media_url - project_media.id = Digest::MD5.hexdigest(project_media.url).to_i(16) - project_media.team_id = team_ids - project_media.type = type - end - get_similar_items_v2_async(project_media, nil, threshold) - wait_for_results(project_media, args) - response = get_similar_items_v2_callback(project_media, nil) - delete(project_media, nil) if project_media.is_a?(TemporaryProjectMedia) - return response + if project_media.nil? + project_media = TemporaryProjectMedia.new + project_media.text = text + project_media.field = field + project_media.url = media_url + project_media.id = Digest::MD5.hexdigest(project_media.url).to_i(16) + project_media.team_id = team_ids + project_media.type = type end + get_similar_items_v2_async(project_media, nil, threshold) + wait_for_results(project_media, args) + response = get_similar_items_v2_callback(project_media, nil) + delete(project_media, nil) if project_media.is_a?(TemporaryProjectMedia) + return response end def process_alegre_callback(params) @@ -512,9 +549,11 @@ def process_alegre_callback(params) should_relate = true if project_media.nil? project_media = TemporaryProjectMedia.new + project_media.text = params.dig('data', 'item', 'raw', 'text') project_media.url = params.dig('data', 'item', 'raw', 'url') project_media.id = params.dig('data', 'item', 'raw', 'context', 'project_media_id') project_media.team_id = params.dig('data', 'item', 'raw', 'context', 'team_id') + project_media.field = params.dig('data', 'item', 'raw', 'context', 'field') project_media.type = params['model_type'] should_relate = false end diff --git a/app/models/concerns/project_media_getters.rb b/app/models/concerns/project_media_getters.rb index b8662224e7..56040e69c7 100644 --- a/app/models/concerns/project_media_getters.rb +++ b/app/models/concerns/project_media_getters.rb @@ -31,6 +31,10 @@ def is_image? self.is_uploaded_image? end + def is_uploaded_media? + self.is_image? || self.is_audio? || self.is_video? + end + def is_text? self.is_claim? || self.is_link? end diff --git a/app/models/explainer.rb b/app/models/explainer.rb index 5b55a57694..35b1bf2887 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -71,24 +71,26 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) # Index title params = { + content_hash: Bot::Alegre.content_hash_for_value(explainer.title), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'title'].join(':')), + context: base_context.merge({ field: 'title' }), text: explainer.title, models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - context: base_context.merge({ field: 'title' }) } - Bot::Alegre.request('post', '/text/similarity/', params) + Bot::Alegre.index_async_with_params(params, "text") # Index paragraphs count = 0 explainer.description.to_s.gsub(/\r\n?/, "\n").split(/\n+/).reject{ |paragraph| paragraph.strip.blank? }.each do |paragraph| count += 1 params = { + content_hash: Bot::Alegre.content_hash_for_value(paragraph.strip), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'paragraph', count].join(':')), + context: base_context.merge({ paragraph: count }), text: paragraph.strip, models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - context: base_context.merge({ paragraph: count }) } - Bot::Alegre.request('post', '/text/similarity/', params) + Bot::Alegre.index_async_with_params(params, "text") end # Remove paragraphs that don't exist anymore (we delete after updating in order to avoid race conditions) @@ -99,7 +101,7 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) quiet: true, context: base_context.merge({ paragraph: count }) } - Bot::Alegre.request('delete', '/text/similarity/', params) + Bot::Alegre.request_delete_from_raw(params, "text") end end @@ -114,7 +116,7 @@ def self.search_by_similarity(text, language, team_id) language: language } } - response = Bot::Alegre.request('post', '/text/similarity/search/', params) + response = Bot::Alegre.get_async_with_params(params, "text") results = response['result'].to_a.sort_by{ |result| result['_score'] } explainer_ids = results.collect{ |result| result.dig('_source', 'context', 'explainer_id').to_i }.uniq.first(3) explainer_ids.empty? ? Explainer.none : Explainer.where(team_id: team_id, id: explainer_ids) diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index de5b67ff21..3586980cac 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -139,6 +139,7 @@ def send_message_outside_24_hours_window(template, pm = nil) test "should submit query without details on tipline bot v2" do WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, random_string, claim, random_string, random_string, '1' assert_saved_query_type 'default_requests' @@ -208,6 +209,7 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query with details on tipline bot v2" do + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, '2', random_string, claim, '1' @@ -285,7 +287,7 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query and handle search error on tipline bot v2" do - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers CheckSearch.any_instance.stubs(:medias).raises(StandardError) Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' @@ -384,7 +386,7 @@ def send_message_outside_24_hours_window(template, pm = nil) ProjectMedia.any_instance.stubs(:report_status).returns('published') ProjectMedia.any_instance.stubs(:analysis_published_article_url).returns(random_url) Bot::Alegre.stubs(:get_merged_similar_items).returns({ create_project_media.id => { score: 0.9 } }) - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do send_message 'hello', '1', '1', "Foo bar foo bar #{url} foo bar", '1' end @@ -693,7 +695,7 @@ def send_message_outside_24_hours_window(template, pm = nil) pm = create_project_media team: @team publish_report(pm, {}, nil, { language: 'pt', use_visual_card: false }) Bot::Smooch.stubs(:get_search_results).returns([pm]) - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' end diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 4fd46ac40e..9a7cb345c0 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -600,6 +600,7 @@ def teardown end test "should include claim_description_content in smooch search" do + WebMock.stub_request(:post, 'http://alegre:3100/similarity/async/image').to_return(body: {}.to_json) WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) RequestStore.store[:skip_cached_field_update] = false t = create_team diff --git a/test/models/explainer_test.rb b/test/models/explainer_test.rb index a902379b6c..a87665421e 100644 --- a/test/models/explainer_test.rb +++ b/test/models/explainer_test.rb @@ -99,12 +99,12 @@ def setup } # Index two paragraphs and title when the explainer is created - Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(3) + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).times(3) Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).never ex = create_explainer description: description # Update the index when paragraphs change - Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(2) + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).times(2) Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).once ex = Explainer.find(ex.id) ex.description = 'Now this is the only paragraph' From 23d9b0f7ad0ee547558af042b6147cdb2146594b Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Tue, 3 Sep 2024 08:36:20 -0700 Subject: [PATCH 02/12] CV2-5086 second attempt on clean Smooch NLU to Presto branch --- app/lib/smooch_nlu.rb | 10 +++------- test/lib/smooch_nlu_test.rb | 10 +++++----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index 01e15fb32f..f264dc96ae 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -44,15 +44,11 @@ def update_keywords(language, keywords, keyword, operation, doc_id, context) } if operation == 'add' && !keywords.include?(keyword) keywords << keyword - alegre_operation = 'post' - alegre_params = common_alegre_params.merge({ text: keyword, models: ALEGRE_MODELS_AND_THRESHOLDS.keys }) + Bot::Alegre.get_sync_with_params(common_alegre_params.merge({ text: keyword, models: ALEGRE_MODELS_AND_THRESHOLDS.keys }), "text") elsif operation == 'remove' keywords -= [keyword] - alegre_operation = 'delete' - alegre_params = common_alegre_params.merge({ quiet: true }) + Bot::Alegre.request_delete_from_raw(common_alegre_params.merge({ quiet: true }), "text") end - # FIXME: Add error handling and better logging - Bot::Alegre.request(alegre_operation, '/text/similarity/', alegre_params) if alegre_operation && alegre_params keywords end @@ -91,7 +87,7 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k language: language, }.merge(context) } - response = Bot::Alegre.request('post', '/text/similarity/search/', params) + response = Bot::Alegre.get_sync_raw_params(params, "text") # One approach would be to take the option that has the most matches # Unfortunately this approach is influenced by the number of keywords per option diff --git a/test/lib/smooch_nlu_test.rb b/test/lib/smooch_nlu_test.rb index 86e01015f8..588701d93c 100644 --- a/test/lib/smooch_nlu_test.rb +++ b/test/lib/smooch_nlu_test.rb @@ -64,7 +64,7 @@ def create_team_with_smooch_bot_installed team = create_team_with_smooch_bot_installed nlu = SmoochNlu.new(team.slug) nlu.enable! - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe') expected_output = { 'en' => { @@ -85,7 +85,7 @@ def create_team_with_smooch_bot_installed end test 'should add keyword if it does not exist' do - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') end @@ -93,9 +93,9 @@ def create_team_with_smooch_bot_installed test 'should not add keyword if it exists' do team = create_team_with_smooch_bot_installed nlu = SmoochNlu.new(team.slug) - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.never + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.never nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') end @@ -114,7 +114,7 @@ def create_team_with_smooch_bot_installed end test 'should return a menu option if NLU is enabled' do - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => 'test' } } }, ]}) team = create_team_with_smooch_bot_installed From e4ad3cf6f706a37601bbb7b2991d4323487e5dc1 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Tue, 3 Sep 2024 12:49:36 -0700 Subject: [PATCH 03/12] fix broken test stubs --- test/models/bot/smooch_6_test.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index 3586980cac..fb922dfe3c 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -809,9 +809,9 @@ def send_message_outside_24_hours_window(template, pm = nil) test 'should process menu option using NLU' do # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns(true) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns(true) # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords for the newsletter menu option nlu = SmoochNlu.new(@team.slug) @@ -824,7 +824,7 @@ def send_message_outside_24_hours_window(template, pm = nil) subscription_option_id = @installation.get_smooch_workflows[0]['smooch_state_main']['smooch_menu_options'][2]['smooch_menu_option_id'] # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /want/ }.returns({ 'result' => [ + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns({ 'result' => [ { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, { '_score' => 0.2, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } ]}) @@ -838,7 +838,7 @@ def send_message_outside_24_hours_window(template, pm = nil) assert_state 'main' # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /want/ }.returns({ 'result' => [ + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns({ 'result' => [ { '_score' => 0.96, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, { '_score' => 0.91, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } ]}) @@ -877,9 +877,9 @@ def send_message_outside_24_hours_window(template, pm = nil) Sidekiq::Testing.fake! do WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /who are you/ }.returns(true) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /who are you/ }.returns(true) # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && (z[:text] =~ /who are you/).nil? }.returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && (z[:text] =~ /who are you/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords to a new "About Us" resource nlu = SmoochNlu.new(@team.slug) @@ -889,7 +889,7 @@ def send_message_outside_24_hours_window(template, pm = nil) r.add_keyword('who are you') # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /who are you/ }.returns({ 'result' => [ + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /who are you/ }.returns({ 'result' => [ { '_score' => 0.9, '_source' => { 'context' => { 'resource_id' => 0 } } }, { '_score' => 0.8, '_source' => { 'context' => { 'resource_id' => r.id } } } ]}) From 72c0ce065a5fdaad33500e4c80dee804a703929d Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 4 Sep 2024 07:28:42 -0700 Subject: [PATCH 04/12] fix typo brought over from previous PR --- app/lib/smooch_nlu.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index f264dc96ae..e1b8b0d337 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -87,7 +87,7 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k language: language, }.merge(context) } - response = Bot::Alegre.get_sync_raw_params(params, "text") + response = Bot::Alegre.get_sync_with_params(params, "text") # One approach would be to take the option that has the most matches # Unfortunately this approach is influenced by the number of keywords per option From 5a57f5cd7a8ee98556073877c1c17bb55c4feaa8 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Thu, 5 Sep 2024 07:43:54 -0700 Subject: [PATCH 05/12] alias and rename per caio review --- app/lib/smooch_nlu.rb | 4 ++-- app/models/concerns/alegre_v2.rb | 5 +++-- app/models/explainer.rb | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index e1b8b0d337..71ca744fea 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -44,7 +44,7 @@ def update_keywords(language, keywords, keyword, operation, doc_id, context) } if operation == 'add' && !keywords.include?(keyword) keywords << keyword - Bot::Alegre.get_sync_with_params(common_alegre_params.merge({ text: keyword, models: ALEGRE_MODELS_AND_THRESHOLDS.keys }), "text") + Bot::Alegre.index_sync_with_params(common_alegre_params.merge({ text: keyword, models: ALEGRE_MODELS_AND_THRESHOLDS.keys }), "text") elsif operation == 'remove' keywords -= [keyword] Bot::Alegre.request_delete_from_raw(common_alegre_params.merge({ quiet: true }), "text") @@ -87,7 +87,7 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k language: language, }.merge(context) } - response = Bot::Alegre.get_sync_with_params(params, "text") + response = Bot::Alegre.query_sync_with_params(params, "text") # One approach would be to take the option that has the most matches # Unfortunately this approach is influenced by the number of keywords per option diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index 46db6ea13c..5f63061908 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -290,11 +290,11 @@ def index_async_with_params(params, type, suppress_search_response=true) request("post", async_path_for_type(type), params.merge(suppress_search_response: suppress_search_response)) end - def get_sync_with_params(params, type) + def query_sync_with_params(params, type) request("post", sync_path_for_type(type), params) end - def get_async_with_params(params, type) + def query_async_with_params(params, type) request("post", async_path_for_type(type), params) end @@ -576,4 +576,5 @@ def restrict_contexts(project_media, project_media_id_scores) }] end end + alias :index_sync_with_params, :query_sync_with_params end diff --git a/app/models/explainer.rb b/app/models/explainer.rb index 35b1bf2887..50abca75be 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -116,7 +116,7 @@ def self.search_by_similarity(text, language, team_id) language: language } } - response = Bot::Alegre.get_async_with_params(params, "text") + response = Bot::Alegre.query_async_with_params(params, "text") results = response['result'].to_a.sort_by{ |result| result['_score'] } explainer_ids = results.collect{ |result| result.dig('_source', 'context', 'explainer_id').to_i }.uniq.first(3) explainer_ids.empty? ? Explainer.none : Explainer.where(team_id: team_id, id: explainer_ids) From 5a386aa0af9a035a3349331d0b9e00e49471d8af Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Thu, 5 Sep 2024 07:51:05 -0700 Subject: [PATCH 06/12] fix syntax --- app/models/concerns/alegre_v2.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index 5f63061908..3b6ebb8631 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -576,5 +576,5 @@ def restrict_contexts(project_media, project_media_id_scores) }] end end - alias :index_sync_with_params, :query_sync_with_params + alias_method :index_sync_with_params, :query_sync_with_params end From 1b6039af769dd617e2c54a375e48adbd05d278b2 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 9 Sep 2024 21:22:33 -0700 Subject: [PATCH 07/12] Mayyyyybe its the alias? --- app/models/concerns/alegre_v2.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index 3b6ebb8631..ee862aa47b 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -290,6 +290,14 @@ def index_async_with_params(params, type, suppress_search_response=true) request("post", async_path_for_type(type), params.merge(suppress_search_response: suppress_search_response)) end + def index_sync_with_params(params, type) + request("post", sync_path_for_type(type), params) + end + + def index_async_with_params(params, type) + request("post", async_path_for_type(type), params) + end + def query_sync_with_params(params, type) request("post", sync_path_for_type(type), params) end @@ -576,5 +584,4 @@ def restrict_contexts(project_media, project_media_id_scores) }] end end - alias_method :index_sync_with_params, :query_sync_with_params end From bf6ff469c2c038e39db53834315960cd47a305ff Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Tue, 10 Sep 2024 05:40:08 -0700 Subject: [PATCH 08/12] fix old reference --- app/models/request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/request.rb b/app/models/request.rb index d3e996bf35..74a9a9ac30 100644 --- a/app/models/request.rb +++ b/app/models/request.rb @@ -194,7 +194,7 @@ def self.send_to_alegre(id) models: request.text_similarity_settings.keys(), context: context } - ::Bot::Alegre.get_async_with_params(params, "text") + ::Bot::Alegre.index_async_with_params(params, "text") # elsif ['UploadedImage', 'UploadedAudio', 'UploadedVideo'].include?(media.type) # type = media.type.gsub(/^Uploaded/, '').downcase # url = media.file&.file&.public_url From 7646b40221c578606dbd27068a0e48ca477c2091 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Tue, 10 Sep 2024 07:20:06 -0700 Subject: [PATCH 09/12] move another stale function reference --- app/models/request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/request.rb b/app/models/request.rb index 74a9a9ac30..4f3242f3a1 100644 --- a/app/models/request.rb +++ b/app/models/request.rb @@ -53,7 +53,7 @@ def attach_to_similar_request!(alegre_limit = 20) models_thresholds = self.text_similarity_settings.reject{ |_k, v| v['min_words'] > words } if models_thresholds.count > 0 params = { text: media.quote, models: models_thresholds.keys, per_model_threshold: models_thresholds.transform_values{ |v| v['threshold'] }, limit: alegre_limit, context: context } - similar_request_id = ::Bot::Alegre.get_sync_with_params(params, "text")&.dig('result').to_a.collect{ |result| result&.dig('_source', 'context', 'request_id').to_i }.find{ |id| id != 0 && id < self.id } + similar_request_id = ::Bot::Alegre.query_sync_with_params(params, "text")&.dig('result').to_a.collect{ |result| result&.dig('_source', 'context', 'request_id').to_i }.find{ |id| id != 0 && id < self.id } end # elsif ['UploadedImage', 'UploadedAudio', 'UploadedVideo'].include?(media.type) # threshold = 0.85 #FIXME: Should be feed setting From bda48a9e88df7202c55e6a98d5ccd5f1e93e9f49 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Tue, 10 Sep 2024 09:06:06 -0700 Subject: [PATCH 10/12] Replace with alias --- app/models/concerns/alegre_v2.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index ee862aa47b..b3308213fb 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -290,10 +290,6 @@ def index_async_with_params(params, type, suppress_search_response=true) request("post", async_path_for_type(type), params.merge(suppress_search_response: suppress_search_response)) end - def index_sync_with_params(params, type) - request("post", sync_path_for_type(type), params) - end - def index_async_with_params(params, type) request("post", async_path_for_type(type), params) end @@ -584,4 +580,5 @@ def restrict_contexts(project_media, project_media_id_scores) }] end end + alias index_sync_with_params query_sync_with_params end From 81e12df4360e8cae2f7e837c110e1f11828646e2 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Tue, 10 Sep 2024 15:01:22 -0700 Subject: [PATCH 11/12] symbolize aliased method names --- app/models/concerns/alegre_v2.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index b3308213fb..91e408d528 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -580,5 +580,5 @@ def restrict_contexts(project_media, project_media_id_scores) }] end end - alias index_sync_with_params query_sync_with_params + alias :index_sync_with_params :query_sync_with_params end From 396f3b03970bf3d280e7165c27a284b863f5996d Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 11 Sep 2024 05:01:25 -0700 Subject: [PATCH 12/12] Revert to proper function --- app/models/concerns/alegre_v2.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index 91e408d528..61041304e6 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -294,6 +294,10 @@ def index_async_with_params(params, type) request("post", async_path_for_type(type), params) end + def index_sync_with_params(params, type) + query_sync_with_params(params, type) + end + def query_sync_with_params(params, type) request("post", sync_path_for_type(type), params) end @@ -580,5 +584,4 @@ def restrict_contexts(project_media, project_media_id_scores) }] end end - alias :index_sync_with_params :query_sync_with_params end