Tipline: NLU disambiguation

If NLU identifies that two options are very similar, present both to the user and let them choose.

Reference: CV2-3710.

app/lib/smooch_nlu.rb

@@ -5,9 +5,9 @@ class SmoochBotNotInstalledError < ::ArgumentError
   # FIXME: Make it more flexible
   # FIXME: Once we support paraphrase-multilingual-mpnet-base-v2 make it the only model used
   ALEGRE_MODELS_AND_THRESHOLDS = {
-    # Bot::Alegre::ELASTICSEARCH_MODEL => 0.8 # , Sometimes this is easier for local development
-    Bot::Alegre::OPENAI_ADA_MODEL => 0.8,
-    Bot::Alegre::MEAN_TOKENS_MODEL => 0.6
+    # Bot::Alegre::ELASTICSEARCH_MODEL => 0.8 # Sometimes this is easier for local development
+    # Bot::Alegre::OPENAI_ADA_MODEL => 0.8 # Not in use right now
+    Bot::Alegre::PARAPHRASE_MULTILINGUAL_MODEL => 0.6
   }
 
   include SmoochNluMenus
@@ -54,6 +54,11 @@ def update_keywords(language, keywords, keyword, operation, doc_id, context)
     keywords
   end
 
+  # If NLU matches two results that have at least this distance between them, they are both presented to the user for disambiguation
+  def self.disambiguation_threshold
+    CheckConfig.get('nlu_disambiguation_threshold', 0.11, :float).to_f
+  end
+
   def self.alegre_matches_from_message(message, language, context, alegre_result_key)
     # FIXME: Raise exception if not in a tipline context (so, if Bot::Smooch.config is nil)
     matches = []
@@ -86,7 +91,7 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k
 
       # Second approach is to sort the results from best to worst
       sorted_options = response['result'].to_a.sort_by{ |result| result['_score'] }.reverse
-      ranked_options = sorted_options.map{ |o| o.dig('_source', 'context', alegre_result_key) }
+      ranked_options = sorted_options.map{ |o| { 'key' => o.dig('_source', 'context', alegre_result_key), 'score' => o['_score'] } }
       matches = ranked_options
 
       # FIXME: Deal with ties (i.e., where two options have an equal _score or count)

app/lib/smooch_nlu_menus.rb

@@ -67,20 +67,38 @@ def update_menu_option_keywords(language, menu, menu_option_index, keyword, oper
   end
 
   module ClassMethods
-    def menu_option_from_message(message, language, options)
-      return nil if options.blank?
-      option = nil
+    def menu_options_from_message(message, language, options)
+      return [{ 'smooch_menu_option_value' => 'main_state' }] if message == 'cancel_nlu'
+      return [] if options.blank?
       context = {
         context: ALEGRE_CONTEXT_KEY_MENU
       }
       matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'menu_option_id')
-      # Select the top menu option that exists in `options`
+      # Select the top two menu options that exists in `options`
+      top_options = []
       matches.each do |r|
-        option = options.find{ |o| !o['smooch_menu_option_id'].blank? && o['smooch_menu_option_id'] == r }
-        break unless option.nil?
+        option = options.find { |o| !o['smooch_menu_option_id'].blank? && o['smooch_menu_option_id'] == r['key'] }
+        top_options << { 'option' => option, 'score' => r['score'] } if !option.nil? && (top_options.empty? || (top_options.first['score'] - r['score']) <= SmoochNlu.disambiguation_threshold)
+        break if top_options.size == 2
+      end
+      Rails.logger.info("[Smooch NLU] [Menu Option From Message] Menu options: #{top_options.inspect} | Message: #{message}")
+      top_options.collect{ |o| o['option'] }
+    end
+
+    def process_menu_options(uid, options, message, language, workflow, app_id)
+
+      if options.size == 1
+        Bot::Smooch.process_menu_option_value(options.first['smooch_menu_option_value'], options.first, message, language, workflow, app_id)
+      # Disambiguation
+      else
+        buttons = options.collect do |option|
+          {
+            value: { keyword: option['smooch_menu_option_keyword'] }.to_json,
+            label: option['smooch_menu_option_label']
+          }
+        end.concat([{ value: { keyword: 'cancel_nlu' }.to_json, label: Bot::Smooch.get_string('main_state_button_label', language, 20) }])
+        Bot::Smooch.send_message_to_user_with_buttons(uid, Bot::Smooch.get_string('nlu_disambiguation', language), buttons)
       end
-      Rails.logger.info("[Smooch NLU] [Menu Option From Message] Menu option: #{option} | Message: #{message}")
-      option
     end
   end
 end

app/models/bot/smooch.rb

@@ -573,9 +573,9 @@ def self.process_menu_option(message, state, app_id)
     end
     # ...if nothing is matched, try using the NLU feature
     if state != 'query'
-      option = SmoochNlu.menu_option_from_message(typed, language, options)
-      unless option.nil?
-        self.process_menu_option_value(option['smooch_menu_option_value'], option, message, language, workflow, app_id)
+      options = SmoochNlu.menu_options_from_message(typed, language, options)
+      unless options.blank?
+        SmoochNlu.process_menu_options(uid, options, message, language, workflow, app_id)
         return true
       end
       resource = TiplineResource.resource_from_message(typed, language)

app/models/concerns/tipline_resource_nlu.rb

@@ -36,7 +36,7 @@ def resource_from_message(message, language)
       context = {
         context: ALEGRE_CONTEXT_KEY_RESOURCE
       }
-      matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'resource_id')
+      matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'resource_id').collect{ |m| m['key'] }
       # Select the top resource that exists
       resource_id = matches.find { |id| TiplineResource.where(id: id).exists? }
       Rails.logger.info("[Smooch NLU] [Resource From Message] Resource ID: #{resource_id} | Message: #{message}")

config/config.yml.example

@@ -44,6 +44,7 @@ development: &default
   text_cluster_similarity_threshold: 0.9
   similarity_media_file_url_host: ''
   min_number_of_words_for_tipline_submit_shortcut: 10
+  nlu_disambiguation_threshold: 0.11
 
   # Localization
   #

config/tipline_strings.yml

@@ -261,6 +261,7 @@ en:
   subscribed: You are currently subscribed to our newsletter.
   unsubscribe_button_label: Unsubscribe
   unsubscribed: You are currently not subscribed to our newsletter.
+  nlu_disambiguation: "Choose one of the options below:"
 fil:
   add_more_details_state_button_label: Dagdagan pa
   ask_if_ready_state_button_label: Kanselahin

test/lib/smooch_nlu_test.rb

@@ -110,7 +110,7 @@ def create_team_with_smooch_bot_installed
     team = create_team_with_smooch_bot_installed
     SmoochNlu.new(team.slug).disable!
     Bot::Smooch.get_installation('smooch_id', 'test')
-    assert_nil SmoochNlu.menu_option_from_message('I want to subscribe to the newsletter', 'en', @menu_options)
+    assert_equal [], SmoochNlu.menu_options_from_message('I want to subscribe to the newsletter', 'en', @menu_options)
   end
 
   test 'should return a menu option if NLU is enabled' do
@@ -120,6 +120,6 @@ def create_team_with_smooch_bot_installed
     team = create_team_with_smooch_bot_installed
     SmoochNlu.new(team.slug).enable!
     Bot::Smooch.get_installation('smooch_id', 'test')
-    assert_not_nil SmoochNlu.menu_option_from_message('I want to subscribe to the newsletter', 'en', @menu_options)
+    assert_not_nil SmoochNlu.menu_options_from_message('I want to subscribe to the newsletter', 'en', @menu_options)
   end
 end

test/models/bot/smooch_6_test.rb

@@ -752,34 +752,45 @@ def send_message_outside_24_hours_window(template, pm = nil)
   end
 
   test 'should process menu option using NLU' do
-    # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "newsletter"
-    Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /newsletter/ }.returns(true)
-    # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "newsletter"
-    Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && (z[:text] =~ /newsletter/).nil? }.returns({ 'result' => [] })
+    # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "want"
+    Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns(true)
+    # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "want"
+    Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] })
 
     # Enable NLU and add a couple of keywords for the newsletter menu option
     nlu = SmoochNlu.new(@team.slug)
     nlu.enable!
+    nlu.add_keyword_to_menu_option('en', 'main', 1, 'I want to query')
     nlu.add_keyword_to_menu_option('en', 'main', 2, 'I want to subscribe to the newsletter')
     nlu.add_keyword_to_menu_option('en', 'main', 2, 'I want to unsubscribe from the newsletter')
     reload_tipline_settings
     query_option_id = @installation.get_smooch_workflows[0]['smooch_state_main']['smooch_menu_options'][1]['smooch_menu_option_id']
     subscription_option_id = @installation.get_smooch_workflows[0]['smooch_state_main']['smooch_menu_options'][2]['smooch_menu_option_id']
 
-    # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "newsletter"
-    Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /newsletter/ }.returns({ 'result' => [
+    # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want"
+    Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns({ 'result' => [
       { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } },
       { '_score' => 0.2, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } }
     ]})
 
     # Sending a message about the newsletter should take to the newsletter state, as per configurations done above
     send_message 'hello', '1' # Sends a first message and confirms language as English
     assert_state 'main'
-    send_message 'Can I subscribe to the newsletter?'
+    send_message 'I want to subscribe to the newsletter?'
     assert_state 'subscription'
     send_message '2' # Keep subscription
     assert_state 'main'
 
+    # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want"
+    Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns({ 'result' => [
+      { '_score' => 0.96, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } },
+      { '_score' => 0.91, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } }
+    ]})
+
+    # Sending a message that returns more than one option (disambiguation)
+    send_message 'I want to subscribe to the newsletter?'
+    assert_state 'main'
+
     # After disabling NLU
     nlu.disable!
     reload_tipline_settings