Merge branch 'develop'

.gitlab-ci.yml

@@ -17,7 +17,7 @@ build_qa:
 
   script:
     - apk add --no-cache curl python3 py3-pip
-    - pip install awscli==1.18.194
+    - pip install awscli==1.29.59
     - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
     - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/qa/check/api:$CI_COMMIT_SHA" .
     - docker push "$ECR_API_BASE_URL/qa/check/api:$CI_COMMIT_SHA"
@@ -39,10 +39,10 @@ deploy_qa:
     GITHUB_TOKEN: $GITHUB_TOKEN
   script:
     - apk add --no-cache curl python3 py3-pip git
-    - pip install awscli==1.18.194
-    - pip install botocore==1.17.47
-    - pip install boto3==1.14.47
-    - pip install ecs-deploy==1.11.0
+    - pip install botocore==1.31.58
+    - pip install boto3==1.28.58
+    - pip install ecs-deploy==1.14.0
+    - pip install awscli==1.29.59
     - alias aws='docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION --rm amazon/aws-cli'
     - aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /qa/check-api/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/qa/check-api/##' > env.qa.names
     - for NAME in `cat env.qa.names`; do echo -n "-s qa-check-api-migration $NAME /qa/check-api/$NAME " >> qa-check-api-migration.env.args; done
@@ -71,7 +71,7 @@ build_batch:
     AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
   script:
     - apk add --no-cache curl python3 py3-pip git
-    - pip install awscli==1.18.194
+    - pip install awscli==1.29.59
     - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
     - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/batch/check/api:$CI_COMMIT_SHA" .
     - docker push "$ECR_API_BASE_URL/batch/check/api:$CI_COMMIT_SHA"
@@ -91,7 +91,7 @@ build_live:
     AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
   script:
     - apk add --no-cache curl python3 py3-pip git
-    - pip install awscli==1.18.194
+    - pip install awscli==1.29.59
     - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
     - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/live/check/api:$CI_COMMIT_SHA" .
     - docker push "$ECR_API_BASE_URL/live/check/api:$CI_COMMIT_SHA"
@@ -114,10 +114,10 @@ deploy_live:
     GITHUB_TOKEN: $GITHUB_TOKEN
   script:
     - apk add --no-cache curl python3 py3-pip git
-    - pip install awscli==1.18.194
-    - pip install botocore==1.17.47
-    - pip install boto3==1.14.47
-    - pip install ecs-deploy==1.11.0
+    - pip install botocore==1.31.58
+    - pip install boto3==1.28.58
+    - pip install ecs-deploy==1.14.0
+    - pip install awscli==1.29.59
     - alias aws='docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION --rm amazon/aws-cli'
     - aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /live/check-api/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/live/check-api/##' > env.live.names
     - for NAME in `cat env.live.names`; do echo -n "-s live-check-api-migration $NAME /live/check-api/$NAME " >> live-check-api-migration.env.args; done

app/controllers/api/v1/webhooks_controller.rb

@@ -7,7 +7,8 @@ def index
         bot_name_to_class = {
           smooch: Bot::Smooch,
           keep: Bot::Keep,
-          fetch: Bot::Fetch
+          fetch: Bot::Fetch,
+          alegre: Bot::Alegre,
         }
         unless bot_name_to_class.has_key?(params[:name].to_sym)
           render_error('Bot not found', 'ID_NOT_FOUND', 404) and return

app/graph/mutations/graphql_crud_operations.rb

@@ -1,5 +1,6 @@
 class GraphqlCrudOperations
   def self.safe_save(obj, attrs, parent_names = [])
+    raise "Can't save a null object." if obj.nil?
     raise 'This operation must be done by a signed-in user' if User.current.nil? && ApiKey.current.nil?
     attrs.each do |key, value|
       method = key == "clientMutationId" ? "client_mutation_id=" : "#{key}="
@@ -192,7 +193,7 @@ def self.apply_bulk_update_or_destroy(inputs, ctx, update_or_destroy, klass)
       method_mapping = { update: :bulk_update, destroy: :bulk_destroy, mark_read: :bulk_mark_read }
       method = method_mapping[update_or_destroy.to_sym]
       result = klass.send(method, sql_ids, filtered_inputs, Team.current)
-      if update_or_destroy.to_s == "update"
+      if update_or_destroy.to_s != "destroy"
         result.merge!({ updated_objects: klass.where(id: sql_ids) })
       end
       { ids: processed_ids }.merge(result)

app/graph/types/project_media_type.rb

@@ -41,6 +41,8 @@ class ProjectMediaType < DefaultObject
   field :cluster, ClusterType, null: true
   field :is_suggested, GraphQL::Types::Boolean, null: true
   field :is_confirmed, GraphQL::Types::Boolean, null: true
+  field :positive_tipline_search_results_count, GraphQL::Types::Int, null: true
+  field :tipline_search_results_count, GraphQL::Types::Int, null: true
 
   field :claim_description, ClaimDescriptionType, null: true
 

app/graph/types/team_type.rb

@@ -301,6 +301,6 @@ def shared_teams
   end
 
   def tipline_messages(uid:)
-    object.tipline_messages.where(uid: uid).last(100)
+    object.tipline_messages.where(uid: uid).order("id DESC")
   end
 end

app/models/blocked_tipline_user.rb

@@ -0,0 +1,2 @@
+class BlockedTiplineUser < ApplicationRecord
+end

app/models/bot/alegre.rb

@@ -6,12 +6,14 @@ class Error < ::StandardError
   end
 
   include AlegreSimilarity
+  include AlegreWebhooks
 
   # Text similarity models
   MEAN_TOKENS_MODEL = 'xlm-r-bert-base-nli-stsb-mean-tokens'
   INDIAN_MODEL = 'indian-sbert'
   FILIPINO_MODEL = 'paraphrase-filipino-mpnet-base-v2'
   OPENAI_ADA_MODEL = 'openai-text-embedding-ada-002'
+  PARAPHRASE_MULTILINGUAL_MODEL = 'paraphrase-multilingual-mpnet-base-v2'
   ELASTICSEARCH_MODEL = 'elasticsearch'
   DEFAULT_ES_SCORE = 10
 
@@ -490,7 +492,13 @@ def self.request_api(method, path, params = {}, query_or_body = 'body', retries
       response_body = response.body
       Rails.logger.info("[Alegre Bot] Alegre response: #{response_body.inspect}")
       ActiveRecord::Base.connection.reconnect! if RequestStore.store[:pause_database_connection]
-      JSON.parse(response_body)
+      parsed_response = JSON.parse(response_body)
+      if parsed_response.dig("queue") == 'audio__Model' && parsed_response.dig("body", "callback_url") != nil
+        redis = Redis.new(REDIS_CONFIG)
+        redis_response = redis.blpop("alegre:webhook:#{parsed_response.dig("body", "id")}", 120)
+        return JSON.parse(redis_response[1])
+      end
+      parsed_response
     rescue StandardError => e
       if retries > 0
         sleep 1
@@ -724,7 +732,7 @@ def self.send_post_create_message(source, target, relationship)
   end
 
   def self.relationship_model_not_allowed(relationship_model)
-    allowed_models = [MEAN_TOKENS_MODEL, INDIAN_MODEL, FILIPINO_MODEL, OPENAI_ADA_MODEL, ELASTICSEARCH_MODEL, 'audio', 'image', 'video']
+    allowed_models = [MEAN_TOKENS_MODEL, INDIAN_MODEL, FILIPINO_MODEL, OPENAI_ADA_MODEL, PARAPHRASE_MULTILINGUAL_MODEL, ELASTICSEARCH_MODEL, 'audio', 'image', 'video']
     models = relationship_model.split("|").collect{ |m| m.split('/').first }
     models.length != (allowed_models&models).length
   end

app/models/bot/smooch.rb

@@ -31,6 +31,7 @@ class CapiUnhandledMessageWarning < MessageDeliveryError; end
   include SmoochMenus
   include SmoochFields
   include SmoochLanguage
+  include SmoochBlocking
 
   ::ProjectMedia.class_eval do
     attr_accessor :smooch_message
@@ -380,6 +381,11 @@ def self.parse_message_based_on_state(message, app_id)
       return true
     end
 
+    if self.clicked_on_search_result_button?(message)
+      self.search_result_button_click_callback(message, uid, app_id, workflow, language)
+      return true
+    end
+
     case state
     when 'waiting_for_message'
       self.bundle_message(message)
@@ -538,6 +544,10 @@ def self.process_menu_option_value(value, option, message, language, workflow, a
     end
   end
 
+  def self.is_a_shortcut_for_submission?(state, message)
+    self.is_v2? && (state == 'main' || state == 'waiting_for_message') && (!message['mediaUrl'].blank? || ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer))
+  end
+
   def self.process_menu_option(message, state, app_id)
     uid = message['authorId']
     sm = CheckStateMachine.new(uid)
@@ -577,8 +587,15 @@ def self.process_menu_option(message, state, app_id)
         return true
       end
     end
+    # Lastly, check if it's a submission shortcut
+    if self.is_a_shortcut_for_submission?(sm.state, message)
+      self.bundle_message(message)
+      sm.go_to_ask_if_ready
+      self.send_message_for_state(uid, workflow, 'ask_if_ready', language)
+      return true
+    end
     self.bundle_message(message)
-    return false
+    false
   end
 
   def self.user_received_report(message)
@@ -789,19 +806,6 @@ def self.add_hashtags(text, pm)
     end
   end
 
-  def self.ban_user(message)
-    unless message.nil?
-      uid = message['authorId']
-      Rails.logger.info("[Smooch Bot] Banned user #{uid}")
-      Rails.cache.write("smooch:banned:#{uid}", message.to_json)
-    end
-  end
-
-  def self.user_banned?(payload)
-    uid = payload.dig('appUser', '_id')
-    !uid.blank? && !Rails.cache.read("smooch:banned:#{uid}").nil?
-  end
-
   # Don't save as a ProjectMedia if it contains only menu options
   def self.is_a_valid_text_message?(text)
     !text.split(/#{MESSAGE_BOUNDARY}|\s+/).reject{ |m| m =~ /^[0-9]*$/ }.empty?
@@ -821,6 +825,10 @@ def self.save_text_message(message)
         claim = self.extract_claim(text).gsub(/\s+/, ' ').strip
         extra = { quote: claim }
         pm = ProjectMedia.joins(:media).where('trim(lower(quote)) = ?', claim.downcase).where('project_medias.team_id' => team.id).last
+        # Don't create a new text media if it's an unconfirmed request with just a few words
+        if pm.nil? && message['archived'] == CheckArchivedFlags::FlagCodes::UNCONFIRMED && ::Bot::Alegre.get_number_of_words(claim) < CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer)
+          return team
+        end
       else
         extra = { url: link.url }
         pm = ProjectMedia.joins(:media).where('medias.url' => link.url, 'project_medias.team_id' => team.id).last

app/models/concerns/alegre_similarity.rb

@@ -19,7 +19,7 @@ def get_similar_items(pm)
       type = Bot::Alegre.get_pm_type(pm)
       Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 2/5] Type is #{type.blank? ? "blank" : type}"
       unless type.blank?
-        if !self.should_get_similar_items_of_type?('master', pm.team_id) || !self.should_get_similar_items_of_type?(type, pm.team_id)
+        if !Bot::Alegre.should_get_similar_items_of_type?('master', pm.team_id) || !Bot::Alegre.should_get_similar_items_of_type?(type, pm.team_id)
           Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 3/5] ProjectMedia cannot be checked for similar items"
           return {}
         else

app/models/concerns/alegre_webhooks.rb

@@ -0,0 +1,26 @@
+class AlegreCallbackError < StandardError
+end
+
+module AlegreWebhooks
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def valid_request?(request)
+      token = request.params['token'] || request.query_parameters['token']
+      !token.blank? && token == CheckConfig.get('alegre_token')
+    end
+
+    def webhook(request)
+      begin
+        doc_id = request.params.dig('data', 'requested', 'body', 'id')
+        raise 'Unexpected params format' if doc_id.blank?
+        redis = Redis.new(REDIS_CONFIG)
+        key = "alegre:webhook:#{doc_id}"
+        redis.lpush(key, request.params.to_json)
+        redis.expire(key, 1.day.to_i)
+      rescue StandardError => e
+        CheckSentry.notify(AlegreCallbackError.new(e.message), { alegre_response: request.params })
+      end
+    end
+  end
+end

app/models/concerns/project_media_cached_fields.rb

@@ -446,6 +446,36 @@ def title_or_description_update
         },
       ]
 
+    cached_field :positive_tipline_search_results_count,
+      update_es: true,
+      recalculate: :recalculate_positive_tipline_search_results_count,
+      update_on: [
+        {
+          model: DynamicAnnotation::Field,
+          if: proc { |f| f.field_name == 'smooch_request_type' && f.value == 'relevant_search_result_requests' },
+          affected_ids: proc { |f| [f.annotation&.annotated_id.to_i] },
+          events: {
+            save: :recalculate,
+            destroy: :recalculate,
+          }
+        }
+      ]
+
+    cached_field :tipline_search_results_count,
+      update_es: true,
+      recalculate: :recalculate_tipline_search_results_count,
+      update_on: [
+        {
+          model: DynamicAnnotation::Field,
+          if: proc { |f| f.field_name == 'smooch_request_type' && ['relevant_search_result_requests', 'irrelevant_search_result_requests', 'timeout_search_requests'].include?(f.value) },
+          affected_ids: proc { |f| [f.annotation&.annotated_id.to_i] },
+          events: {
+            save: :recalculate,
+            destroy: :recalculate,
+          }
+        }
+      ]
+
     def recalculate_linked_items_count
       count = Relationship.send('confirmed').where(source_id: self.id).count
       count += 1 unless self.media.type == 'Blank'
@@ -605,6 +635,19 @@ def cached_field_tags_as_sentence_es(value)
     def cached_field_published_by_es(value)
       value.keys.first || 0
     end
+
+    def recalculate_positive_tipline_search_results_count
+      DynamicAnnotation::Field.where(annotation_type: 'smooch',field_name: 'smooch_request_type', value: 'relevant_search_result_requests')
+      .joins('INNER JOIN annotations a ON a.id = dynamic_annotation_fields.annotation_id')
+      .where('a.annotated_type = ? AND a.annotated_id = ?', 'ProjectMedia', self.id).count
+    end
+
+    def recalculate_tipline_search_results_count
+      DynamicAnnotation::Field.where(annotation_type: 'smooch',field_name: 'smooch_request_type')
+      .where('value IN (?)', ['"relevant_search_result_requests"', '"irrelevant_search_result_requests"', '"timeout_search_requests"'])
+      .joins('INNER JOIN annotations a ON a.id = dynamic_annotation_fields.annotation_id')
+      .where('a.annotated_type = ? AND a.annotated_id = ?', 'ProjectMedia', self.id).count
+    end
   end
 
   DynamicAnnotation::Field.class_eval do

app/models/concerns/smooch_blocking.rb

@@ -0,0 +1,46 @@
+require 'active_support/concern'
+
+module SmoochBlocking
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def ban_user(message)
+      unless message.nil?
+        uid = message['authorId']
+        self.block_user(uid)
+      end
+    end
+
+    def block_user_from_error_code(uid, error_code)
+      self.block_user(uid) if error_code == 131056 # Error of type "pair rate limit hit"
+    end
+
+    def block_user(uid)
+      begin
+        block = BlockedTiplineUser.new(uid: uid)
+        block.skip_check_ability = true
+        block.save!
+        Rails.logger.info("[Smooch Bot] Blocked user #{uid}")
+        Rails.cache.write("smooch:banned:#{uid}", Time.now.to_i)
+      rescue ActiveRecord::RecordNotUnique
+        # User already blocked
+        Rails.logger.info("[Smooch Bot] User #{uid} already blocked")
+      end
+    end
+
+    def unblock_user(uid)
+      BlockedTiplineUser.where(uid: uid).last.destroy!
+      Rails.logger.info("[Smooch Bot] Unblocked user #{uid}")
+      Rails.cache.delete("smooch:banned:#{uid}")
+    end
+
+    def user_blocked?(uid)
+      !uid.blank? && (!Rails.cache.read("smooch:banned:#{uid}").nil? || BlockedTiplineUser.where(uid: uid).exists?)
+    end
+
+    def user_banned?(payload)
+      uid = payload.dig('appUser', '_id')
+      self.user_blocked?(uid)
+    end
+  end
+end

app/models/concerns/smooch_capi.rb

@@ -281,13 +281,17 @@ def capi_send_message_to_user(uid, text, extra = {}, _force = false, preview_url
       response = http.request(req)
       if response.code.to_i >= 400
         error_message = begin JSON.parse(response.body)['error']['message'] rescue response.body end
+        error_code = begin JSON.parse(response.body)['error']['code'] rescue nil end
         e = Bot::Smooch::CapiMessageDeliveryError.new(error_message)
+        self.block_user_from_error_code(uid, error_code)
         CheckSentry.notify(e,
           uid: uid,
           type: payload.dig(:type),
           template_name: payload.dig(:template, :name),
           template_language: payload.dig(:template, :language, :code),
-          error: response.body
+          error: response.body,
+          error_message: error_message,
+          error_code: error_code
         )
       end
       response