From 721e5c49230af1a21a1e36a8685002ec9b894c2e Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Sun, 1 Oct 2023 17:16:39 -0300
Subject: [PATCH 01/18] Tipline submission shortcuts

When a tipline user sends a long text message or a file, jump right away into the "are you ready to submit" state.

Reference: CV2-2326.
---
 app/models/bot/smooch.rb         | 13 ++++++++++++-
 config/config.yml.example        |  1 +
 test/models/bot/smooch_6_test.rb | 12 +++++++++++-
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb
index 7f98916df1..803e631e91 100644
--- a/app/models/bot/smooch.rb
+++ b/app/models/bot/smooch.rb
@@ -538,6 +538,10 @@ def self.process_menu_option_value(value, option, message, language, workflow, a
     end
   end
 
+  def self.is_a_shortcut_for_submission?(state, message)
+    self.is_v2? && (state == 'main' || state == 'waiting_for_message') && (!message['mediaUrl'].blank? || ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_shortcut', 3, :integer))
+  end
+
   def self.process_menu_option(message, state, app_id)
     uid = message['authorId']
     sm = CheckStateMachine.new(uid)
@@ -577,8 +581,15 @@ def self.process_menu_option(message, state, app_id)
         return true
       end
     end
+    # Lastly, check if it's a submission shortcut
+    if self.is_a_shortcut_for_submission?(sm.state, message)
+      self.bundle_message(message)
+      sm.go_to_ask_if_ready
+      self.send_message_for_state(uid, workflow, 'ask_if_ready', language)
+      return true
+    end
     self.bundle_message(message)
-    return false
+    false
   end
 
   def self.user_received_report(message)
diff --git a/config/config.yml.example b/config/config.yml.example
index 73aabf012e..2c6ff6cb39 100644
--- a/config/config.yml.example
+++ b/config/config.yml.example
@@ -44,6 +44,7 @@ development: &default
   image_cluster_similarity_threshold: 0.9
   text_cluster_similarity_threshold: 0.9
   similarity_media_file_url_host: ''
+  min_number_of_words_for_tipline_shortcut: 3
 
   # Localization
   #
diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb
index d9be1df6cb..ff608d617c 100644
--- a/test/models/bot/smooch_6_test.rb
+++ b/test/models/bot/smooch_6_test.rb
@@ -784,7 +784,7 @@ def send_message_outside_24_hours_window(template, pm = nil)
     nlu.disable!
     reload_tipline_settings
     send_message 'Can I subscribe to the newsletter?'
-    assert_state 'main'
+    assert_state 'ask_if_ready'
 
     # Delete two keywords, so expect two calls to Alegre
     Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'delete' && y == '/text/similarity/' }.twice
@@ -844,4 +844,14 @@ def send_message_outside_24_hours_window(template, pm = nil)
       r.remove_keyword('who are you')
     end
   end
+
+  test 'should have shortcuts for submission' do
+    send_message 'This is message is so long that it is considered a media'
+    assert_state 'ask_if_ready'
+  end
+
+  test 'should not have shortcuts for submission' do
+    send_message 'Hello'
+    assert_state 'main'
+  end
 end

From 2fe41c109dc815c6fcc690069918068cdaea7449 Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Mon, 2 Oct 2023 10:55:14 -0300
Subject: [PATCH 02/18] Changing configuration key name and default value for
 tipline submission shortcut

Reference: CV2-2326.
---
 app/models/bot/smooch.rb         | 2 +-
 config/config.yml.example        | 2 +-
 test/models/bot/smooch_6_test.rb | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb
index 803e631e91..a2e50b2206 100644
--- a/app/models/bot/smooch.rb
+++ b/app/models/bot/smooch.rb
@@ -539,7 +539,7 @@ def self.process_menu_option_value(value, option, message, language, workflow, a
   end
 
   def self.is_a_shortcut_for_submission?(state, message)
-    self.is_v2? && (state == 'main' || state == 'waiting_for_message') && (!message['mediaUrl'].blank? || ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_shortcut', 3, :integer))
+    self.is_v2? && (state == 'main' || state == 'waiting_for_message') && (!message['mediaUrl'].blank? || ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer))
   end
 
   def self.process_menu_option(message, state, app_id)
diff --git a/config/config.yml.example b/config/config.yml.example
index 2c6ff6cb39..ebbd6d49c9 100644
--- a/config/config.yml.example
+++ b/config/config.yml.example
@@ -44,7 +44,7 @@ development: &default
   image_cluster_similarity_threshold: 0.9
   text_cluster_similarity_threshold: 0.9
   similarity_media_file_url_host: ''
-  min_number_of_words_for_tipline_shortcut: 3
+  min_number_of_words_for_tipline_submit_shortcut: 10
 
   # Localization
   #
diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb
index ff608d617c..77d18a14bb 100644
--- a/test/models/bot/smooch_6_test.rb
+++ b/test/models/bot/smooch_6_test.rb
@@ -784,7 +784,7 @@ def send_message_outside_24_hours_window(template, pm = nil)
     nlu.disable!
     reload_tipline_settings
     send_message 'Can I subscribe to the newsletter?'
-    assert_state 'ask_if_ready'
+    assert_state 'main'
 
     # Delete two keywords, so expect two calls to Alegre
     Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'delete' && y == '/text/similarity/' }.twice

From 69641f14774897c075dd52171b1ece4ad7c4d1e1 Mon Sep 17 00:00:00 2001
From: ahmednasserswe <ahmed.nasser.swe@gmail.com>
Date: Tue, 3 Oct 2023 04:41:31 +0200
Subject: [PATCH 03/18] CV2-3482: Add paraphrase-multilingual-mpnet-base-v2
 vector model

---
 app/models/bot/alegre.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb
index b0e56c501d..d8997e3a3f 100644
--- a/app/models/bot/alegre.rb
+++ b/app/models/bot/alegre.rb
@@ -12,6 +12,7 @@ class Error < ::StandardError
   INDIAN_MODEL = 'indian-sbert'
   FILIPINO_MODEL = 'paraphrase-filipino-mpnet-base-v2'
   OPENAI_ADA_MODEL = 'openai-text-embedding-ada-002'
+  PARAPHRASE_MULTILINGUAL_MODEL = 'paraphrase-multilingual-mpnet-base-v2'
   ELASTICSEARCH_MODEL = 'elasticsearch'
   DEFAULT_ES_SCORE = 10
 
@@ -724,7 +725,7 @@ def self.send_post_create_message(source, target, relationship)
   end
 
   def self.relationship_model_not_allowed(relationship_model)
-    allowed_models = [MEAN_TOKENS_MODEL, INDIAN_MODEL, FILIPINO_MODEL, OPENAI_ADA_MODEL, ELASTICSEARCH_MODEL, 'audio', 'image', 'video']
+    allowed_models = [MEAN_TOKENS_MODEL, INDIAN_MODEL, FILIPINO_MODEL, OPENAI_ADA_MODEL, PARAPHRASE_MULTILINGUAL_MODEL, ELASTICSEARCH_MODEL, 'audio', 'image', 'video']
     models = relationship_model.split("|").collect{ |m| m.split('/').first }
     models.length != (allowed_models&models).length
   end

From 99a6aa99e31f3e832c532b7e86fb96ba298cb2f9 Mon Sep 17 00:00:00 2001
From: Mohamed El-Sawy <melsawy.be7eri@gmail.com>
Date: Wed, 4 Oct 2023 03:04:50 +0300
Subject: [PATCH 04/18] CV2-3776: paginate tipline messages (#1677)

* CV2-3776: paginate tipline messages

* CV2-3776: add a unit test for pagination using specific message
---
 app/graph/types/team_type.rb                  |  2 +-
 test/controllers/graphql_controller_9_test.rb | 76 +++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb
index 073f9293ab..693b9d2e34 100644
--- a/app/graph/types/team_type.rb
+++ b/app/graph/types/team_type.rb
@@ -301,6 +301,6 @@ def shared_teams
   end
 
   def tipline_messages(uid:)
-    object.tipline_messages.where(uid: uid).last(100)
+    object.tipline_messages.where(uid: uid).order("id DESC")
   end
 end
diff --git a/test/controllers/graphql_controller_9_test.rb b/test/controllers/graphql_controller_9_test.rb
index c3edeaa351..3ae958b198 100644
--- a/test/controllers/graphql_controller_9_test.rb
+++ b/test/controllers/graphql_controller_9_test.rb
@@ -409,6 +409,82 @@ def setup
     assert_equal "Not Found", JSON.parse(@response.body)['errors'][0]['message']
   end
 
+  test "should paginate tipline messages" do
+    t = create_team slug: 'test', private: true
+    u = create_user
+    create_team_user user: u, team: t, role: 'admin'
+    uid = random_string
+
+    tp1_uid = create_tipline_message team_id: t.id, uid: uid
+    tp2_uid = create_tipline_message team_id: t.id, uid: uid
+    tp3_uid = create_tipline_message team_id: t.id, uid: uid
+
+    authenticate_with_user(u)
+
+    # Paginating one item per page
+
+    # Page 1
+    query = 'query read { team(slug: "test") { tipline_messages(first: 1, uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }'
+    post :create, params: { query: query }
+    assert_response :success
+    data = JSON.parse(@response.body)['data']['team']['tipline_messages']
+    results = data['edges'].to_a.collect{ |e| e['node']['dbid'] }
+    assert_equal 1, results.size
+    assert_equal tp3_uid.id, results[0]
+    page_info = data['pageInfo']
+    assert page_info['hasNextPage']
+
+    # Page 2
+    query = 'query read { team(slug: "test") { tipline_messages(first: 1, after: "' + page_info['endCursor'] + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }'
+    post :create, params: { query: query }
+    assert_response :success
+    data = JSON.parse(@response.body)['data']['team']['tipline_messages']
+    results = data['edges'].to_a.collect{ |e| e['node']['dbid'] }
+    assert_equal 1, results.size
+    assert_equal tp2_uid.id, results[0]
+    page_info = data['pageInfo']
+    assert page_info['hasNextPage']
+
+    # Page 3
+    query = 'query read { team(slug: "test") { tipline_messages(first: 1, after: "' + page_info['endCursor'] + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }'
+    post :create, params: { query: query }
+    assert_response :success
+    data = JSON.parse(@response.body)['data']['team']['tipline_messages']
+    results = data['edges'].to_a.collect{ |e| e['node']['dbid'] }
+    assert_equal 1, results.size
+    assert_equal tp1_uid.id, results[0]
+    page_info = data['pageInfo']
+    assert !page_info['hasNextPage']
+    # paginate using specific message id
+    tp4_uid = create_tipline_message team_id: t.id, uid: uid
+    query = 'query read { team(slug: "test") { tipline_messages(uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid }, cursor } } } }'
+    post :create, params: { query: query }
+    assert_response :success
+    data = JSON.parse(@response.body)['data']['team']['tipline_messages']
+    id_cursor = {}
+    data['edges'].to_a.each{ |e| id_cursor[e['node']['dbid']] = e['cursor'] }
+    # Start with tp4_uid message
+    query = 'query read { team(slug: "test") { tipline_messages(first: 1, after: "' + id_cursor[tp4_uid.id] + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }'
+    post :create, params: { query: query }
+    assert_response :success
+    data = JSON.parse(@response.body)['data']['team']['tipline_messages']
+    results = data['edges'].to_a.collect{ |e| e['node']['dbid'] }
+    assert_equal 1, results.size
+    assert_equal tp3_uid.id, results[0]
+    page_info = data['pageInfo']
+    assert page_info['hasNextPage']
+    # Next page
+    query = 'query read { team(slug: "test") { tipline_messages(first: 1, after: "' + page_info['endCursor'] + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }'
+    post :create, params: { query: query }
+    assert_response :success
+    data = JSON.parse(@response.body)['data']['team']['tipline_messages']
+    results = data['edges'].to_a.collect{ |e| e['node']['dbid'] }
+    assert_equal 1, results.size
+    assert_equal tp2_uid.id, results[0]
+    page_info = data['pageInfo']
+    assert page_info['hasNextPage']
+  end
+
   protected
 
   def assert_error_message(expected)

From 68fdd4280389ee0ecca98e6363da116b132b7c5d Mon Sep 17 00:00:00 2001
From: Martin Peck <51542678+sonoransun@users.noreply.github.com>
Date: Tue, 3 Oct 2023 17:22:39 -0700
Subject: [PATCH 05/18] Update version pins for Python dependencies to fix
 deployments. (#1680)

* Update version pins for awscli, boto3, botocore, and ecs-deploy.
---
 .gitlab-ci.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f741755a65..97db4df4d9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,7 +16,7 @@ build_qa:
 
   script:
     - apk add --no-cache curl python3 py3-pip
-    - pip install awscli==1.18.194
+    - pip install awscli==1.29.59
     - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
     - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/qa/check/api:$CI_COMMIT_SHA" .
     - docker push "$ECR_API_BASE_URL/qa/check/api:$CI_COMMIT_SHA"
@@ -38,10 +38,10 @@ deploy_qa:
     GITHUB_TOKEN: $GITHUB_TOKEN
   script:
     - apk add --no-cache curl python3 py3-pip git
-    - pip install awscli==1.18.194
-    - pip install botocore==1.17.47
-    - pip install boto3==1.14.47
-    - pip install ecs-deploy==1.11.0
+    - pip install botocore==1.31.58
+    - pip install boto3==1.28.58
+    - pip install ecs-deploy==1.14.0
+    - pip install awscli==1.29.59
     - alias aws='docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION --rm amazon/aws-cli'
     - aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /qa/check-api/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/qa/check-api/##' > env.qa.names
     - for NAME in `cat env.qa.names`; do echo -n "-s qa-check-api-migration $NAME /qa/check-api/$NAME " >> qa-check-api-migration.env.args; done
@@ -70,7 +70,7 @@ build_batch:
     AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
   script:
     - apk add --no-cache curl python3 py3-pip git
-    - pip install awscli==1.18.194
+    - pip install awscli==1.29.59
     - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
     - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/batch/check/api:$CI_COMMIT_SHA" .
     - docker push "$ECR_API_BASE_URL/batch/check/api:$CI_COMMIT_SHA"
@@ -90,7 +90,7 @@ build_live:
     AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
   script:
     - apk add --no-cache curl python3 py3-pip git
-    - pip install awscli==1.18.194
+    - pip install awscli==1.29.59
     - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
     - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/live/check/api:$CI_COMMIT_SHA" .
     - docker push "$ECR_API_BASE_URL/live/check/api:$CI_COMMIT_SHA"
@@ -113,10 +113,10 @@ deploy_live:
     GITHUB_TOKEN: $GITHUB_TOKEN
   script:
     - apk add --no-cache curl python3 py3-pip git
-    - pip install awscli==1.18.194
-    - pip install botocore==1.17.47
-    - pip install boto3==1.14.47
-    - pip install ecs-deploy==1.11.0
+    - pip install botocore==1.31.58
+    - pip install boto3==1.28.58
+    - pip install ecs-deploy==1.14.0
+    - pip install awscli==1.29.59
     - alias aws='docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION --rm amazon/aws-cli'
     - aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /live/check-api/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/live/check-api/##' > env.live.names
     - for NAME in `cat env.live.names`; do echo -n "-s live-check-api-migration $NAME /live/check-api/$NAME " >> live-check-api-migration.env.args; done

From 54fb3776a1234d5cf13846fde0c92d801cf0489a Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Tue, 3 Oct 2023 21:54:11 -0300
Subject: [PATCH 06/18] Block WhatsApp user if WhatsApp API reports a "pair
 rate limit hit" error

If WhatsApp Cloud API reports an error of type "pair rate limit hit" when we try to send a message to a WhatsApp user, block that user automatically, so we don't need to answer them again.

It helps with SPAM and abuse.

Reference: CV2-3489.
---
 app/models/blocked_tipline_user.rb            |  2 +
 app/models/bot/smooch.rb                      | 14 +-----
 app/models/concerns/smooch_blocking.rb        | 46 +++++++++++++++++++
 app/models/concerns/smooch_capi.rb            |  6 ++-
 ...1002202443_create_blocked_tipline_users.rb |  9 ++++
 db/schema.rb                                  |  9 +++-
 lib/sample_data.rb                            |  4 ++
 test/models/blocked_tipline_user_test.rb      | 33 +++++++++++++
 test/models/concerns/smooch_capi_test.rb      | 10 ++++
 9 files changed, 118 insertions(+), 15 deletions(-)
 create mode 100644 app/models/blocked_tipline_user.rb
 create mode 100644 app/models/concerns/smooch_blocking.rb
 create mode 100644 db/migrate/20231002202443_create_blocked_tipline_users.rb
 create mode 100644 test/models/blocked_tipline_user_test.rb

diff --git a/app/models/blocked_tipline_user.rb b/app/models/blocked_tipline_user.rb
new file mode 100644
index 0000000000..5eb56c7612
--- /dev/null
+++ b/app/models/blocked_tipline_user.rb
@@ -0,0 +1,2 @@
+class BlockedTiplineUser < ApplicationRecord
+end
diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb
index a2e50b2206..31fd749316 100644
--- a/app/models/bot/smooch.rb
+++ b/app/models/bot/smooch.rb
@@ -31,6 +31,7 @@ class CapiUnhandledMessageWarning < MessageDeliveryError; end
   include SmoochMenus
   include SmoochFields
   include SmoochLanguage
+  include SmoochBlocking
 
   ::ProjectMedia.class_eval do
     attr_accessor :smooch_message
@@ -800,19 +801,6 @@ def self.add_hashtags(text, pm)
     end
   end
 
-  def self.ban_user(message)
-    unless message.nil?
-      uid = message['authorId']
-      Rails.logger.info("[Smooch Bot] Banned user #{uid}")
-      Rails.cache.write("smooch:banned:#{uid}", message.to_json)
-    end
-  end
-
-  def self.user_banned?(payload)
-    uid = payload.dig('appUser', '_id')
-    !uid.blank? && !Rails.cache.read("smooch:banned:#{uid}").nil?
-  end
-
   # Don't save as a ProjectMedia if it contains only menu options
   def self.is_a_valid_text_message?(text)
     !text.split(/#{MESSAGE_BOUNDARY}|\s+/).reject{ |m| m =~ /^[0-9]*$/ }.empty?
diff --git a/app/models/concerns/smooch_blocking.rb b/app/models/concerns/smooch_blocking.rb
new file mode 100644
index 0000000000..cdf94b9e48
--- /dev/null
+++ b/app/models/concerns/smooch_blocking.rb
@@ -0,0 +1,46 @@
+require 'active_support/concern'
+
+module SmoochBlocking
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def ban_user(message)
+      unless message.nil?
+        uid = message['authorId']
+        self.block_user(uid)
+      end
+    end
+
+    def block_user_from_error_code(uid, error_code)
+      self.block_user(uid) if error_code == 131056 # Error of type "pair rate limit hit"
+    end
+
+    def block_user(uid)
+      begin
+        block = BlockedTiplineUser.new(uid: uid)
+        block.skip_check_ability = true
+        block.save!
+        Rails.logger.info("[Smooch Bot] Blocked user #{uid}")
+        Rails.cache.write("smooch:banned:#{uid}", Time.now.to_i)
+      rescue ActiveRecord::RecordNotUnique
+        # User already blocked
+        Rails.logger.info("[Smooch Bot] User #{uid} already blocked")
+      end
+    end
+
+    def unblock_user(uid)
+      BlockedTiplineUser.where(uid: uid).last.destroy!
+      Rails.logger.info("[Smooch Bot] Unblocked user #{uid}")
+      Rails.cache.delete("smooch:banned:#{uid}")
+    end
+
+    def user_blocked?(uid)
+      !uid.blank? && (!Rails.cache.read("smooch:banned:#{uid}").nil? || BlockedTiplineUser.where(uid: uid).exists?)
+    end
+
+    def user_banned?(payload)
+      uid = payload.dig('appUser', '_id')
+      self.user_blocked?(uid)
+    end
+  end
+end
diff --git a/app/models/concerns/smooch_capi.rb b/app/models/concerns/smooch_capi.rb
index 4ad790ea7c..469ce56d51 100644
--- a/app/models/concerns/smooch_capi.rb
+++ b/app/models/concerns/smooch_capi.rb
@@ -281,13 +281,17 @@ def capi_send_message_to_user(uid, text, extra = {}, _force = false, preview_url
       response = http.request(req)
       if response.code.to_i >= 400
         error_message = begin JSON.parse(response.body)['error']['message'] rescue response.body end
+        error_code = begin JSON.parse(response.body)['error']['code'] rescue nil end
         e = Bot::Smooch::CapiMessageDeliveryError.new(error_message)
+        self.block_user_from_error_code(uid, error_code)
         CheckSentry.notify(e,
           uid: uid,
           type: payload.dig(:type),
           template_name: payload.dig(:template, :name),
           template_language: payload.dig(:template, :language, :code),
-          error: response.body
+          error: response.body,
+          error_message: error_message,
+          error_code: error_code
         )
       end
       response
diff --git a/db/migrate/20231002202443_create_blocked_tipline_users.rb b/db/migrate/20231002202443_create_blocked_tipline_users.rb
new file mode 100644
index 0000000000..3cc9c5bdaa
--- /dev/null
+++ b/db/migrate/20231002202443_create_blocked_tipline_users.rb
@@ -0,0 +1,9 @@
+class CreateBlockedTiplineUsers < ActiveRecord::Migration[6.1]
+  def change
+    create_table :blocked_tipline_users do |t|
+      t.string :uid, null: false
+      t.timestamps
+    end
+    add_index :blocked_tipline_users, :uid, unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index c8beea6b6d..b8982c24e3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2023_09_22_174044) do
+ActiveRecord::Schema.define(version: 2023_10_02_202443) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -195,6 +195,13 @@
     t.index ["user_id"], name: "index_assignments_on_user_id"
   end
 
+  create_table "blocked_tipline_users", force: :cascade do |t|
+    t.string "uid", null: false
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["uid"], name: "index_blocked_tipline_users_on_uid", unique: true
+  end
+
   create_table "bounces", id: :serial, force: :cascade do |t|
     t.string "email", null: false
     t.datetime "created_at", null: false
diff --git a/lib/sample_data.rb b/lib/sample_data.rb
index bd851ff9bf..7bacb897f1 100644
--- a/lib/sample_data.rb
+++ b/lib/sample_data.rb
@@ -1096,4 +1096,8 @@ def create_task_stuff(delete_existing = true)
     at = create_annotation_type annotation_type: 'task_response_free_text', label: 'Task Response Free Text'
     create_field_instance annotation_type_object: at, name: 'response_free_text', label: 'Response', field_type_object: text, optional: false
   end
+
+  def create_blocked_tipline_user(options = {})
+    BlockedTiplineUser.create!({ uid: random_string }.merge(options))
+  end
 end
diff --git a/test/models/blocked_tipline_user_test.rb b/test/models/blocked_tipline_user_test.rb
new file mode 100644
index 0000000000..b6374f5555
--- /dev/null
+++ b/test/models/blocked_tipline_user_test.rb
@@ -0,0 +1,33 @@
+require_relative '../test_helper'
+
+class BlockedTiplineUserTest < ActiveSupport::TestCase
+  def setup
+  end
+
+  def teardown
+  end
+
+  test 'should create blocked tipline user' do
+    assert_difference 'BlockedTiplineUser.count' do
+      create_blocked_tipline_user
+    end
+  end
+
+  test 'should not create blocked tipline user if UID is blank' do
+    assert_no_difference 'BlockedTiplineUser.count' do
+      assert_raises ActiveRecord::NotNullViolation do
+        create_blocked_tipline_user uid: nil
+      end
+    end
+  end
+
+  test 'should not block the same user more than once' do
+    uid = random_string
+    create_blocked_tipline_user uid: uid
+    assert_no_difference 'BlockedTiplineUser.count' do
+      assert_raises ActiveRecord::RecordNotUnique do
+        create_blocked_tipline_user uid: uid
+      end
+    end
+  end
+end
diff --git a/test/models/concerns/smooch_capi_test.rb b/test/models/concerns/smooch_capi_test.rb
index c73665c03f..791e18a7e0 100644
--- a/test/models/concerns/smooch_capi_test.rb
+++ b/test/models/concerns/smooch_capi_test.rb
@@ -286,4 +286,14 @@ def teardown
     WebMock.stub_request(:post, 'https://graph.facebook.com/v15.0/123456/messages').to_return(status: 200, body: { id: '123456' }.to_json)
     assert_equal 200, Bot::Smooch.send_message_to_user(@uid, 'Test', { 'type' => 'video', 'mediaUrl' => 'https://test.test/video.mp4' }).code.to_i
   end
+
+  test 'should block user if WhatsApp reports pair rate limit hit' do
+    assert !Bot::Smooch.user_blocked?(@uid)
+    WebMock.stub_request(:post, 'https://graph.facebook.com/v15.0/123456/messages').to_return(status: 400, body: { error: { message: '(#131056) (Business Account, Consumer Account) pair rate limit hit', code: 131056 } }.to_json)
+    Bot::Smooch.send_message_to_user(@uid, 'Test', { 'type' => 'text', 'text' => 'Test' })
+    Bot::Smooch.send_message_to_user(@uid, 'Test', { 'type' => 'text', 'text' => 'Test' }) # Race condition
+    assert Bot::Smooch.user_blocked?(@uid)
+    Bot::Smooch.unblock_user(@uid)
+    assert !Bot::Smooch.user_blocked?(@uid)
+  end
 end

From dfbb1bc355bd003a899fdda789ab6022bbd28910 Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Thu, 5 Oct 2023 05:40:53 -0700
Subject: [PATCH 07/18] Cv2 3435 alegre with presto for audio (#1647)

* CV2-3435 add alegre webhook

* update fixtures to match actual alegre resposne

* remove unnecessary copypasting

* fix test call

* small fix

* try different setup

* readd request

* downgrade to type checking for the moment

* replace test with known direct relationship count changing test

* update stub

* finally fix test

* add newline per codeclimate

* CV2-3435 updates to alegre to work locally

* update test

* remove token passing

* resolve error in returned value of request_api

* add prefix, add blank safety check

* CV2-3435 update tests as per @caiosba commentary

* Clean-up and test simplification

---------

Co-authored-by: Caio <117518+caiosba@users.noreply.github.com>
---
 app/controllers/api/v1/webhooks_controller.rb |  3 ++-
 app/models/bot/alegre.rb                      |  9 ++++++-
 app/models/concerns/alegre_similarity.rb      |  2 +-
 app/models/concerns/alegre_webhooks.rb        | 26 +++++++++++++++++++
 config/config.yml.example                     |  1 -
 test/controllers/webhooks_controller_test.rb  | 20 ++++++++++++++
 test/models/bot/alegre_2_test.rb              | 10 +++++++
 7 files changed, 67 insertions(+), 4 deletions(-)
 create mode 100644 app/models/concerns/alegre_webhooks.rb

diff --git a/app/controllers/api/v1/webhooks_controller.rb b/app/controllers/api/v1/webhooks_controller.rb
index e08c1cc090..d489014cf6 100644
--- a/app/controllers/api/v1/webhooks_controller.rb
+++ b/app/controllers/api/v1/webhooks_controller.rb
@@ -7,7 +7,8 @@ def index
         bot_name_to_class = {
           smooch: Bot::Smooch,
           keep: Bot::Keep,
-          fetch: Bot::Fetch
+          fetch: Bot::Fetch,
+          alegre: Bot::Alegre,
         }
         unless bot_name_to_class.has_key?(params[:name].to_sym)
           render_error('Bot not found', 'ID_NOT_FOUND', 404) and return
diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb
index d8997e3a3f..d0b1e76aa3 100644
--- a/app/models/bot/alegre.rb
+++ b/app/models/bot/alegre.rb
@@ -6,6 +6,7 @@ class Error < ::StandardError
   end
 
   include AlegreSimilarity
+  include AlegreWebhooks
 
   # Text similarity models
   MEAN_TOKENS_MODEL = 'xlm-r-bert-base-nli-stsb-mean-tokens'
@@ -491,7 +492,13 @@ def self.request_api(method, path, params = {}, query_or_body = 'body', retries
       response_body = response.body
       Rails.logger.info("[Alegre Bot] Alegre response: #{response_body.inspect}")
       ActiveRecord::Base.connection.reconnect! if RequestStore.store[:pause_database_connection]
-      JSON.parse(response_body)
+      parsed_response = JSON.parse(response_body)
+      if parsed_response.dig("queue") == 'audio__Model' && parsed_response.dig("body", "callback_url") != nil
+        redis = Redis.new(REDIS_CONFIG)
+        redis_response = redis.blpop("alegre:webhook:#{parsed_response.dig("body", "id")}", 120)
+        return JSON.parse(redis_response[1])
+      end
+      parsed_response
     rescue StandardError => e
       if retries > 0
         sleep 1
diff --git a/app/models/concerns/alegre_similarity.rb b/app/models/concerns/alegre_similarity.rb
index d331405478..bc5d6faf61 100644
--- a/app/models/concerns/alegre_similarity.rb
+++ b/app/models/concerns/alegre_similarity.rb
@@ -19,7 +19,7 @@ def get_similar_items(pm)
       type = Bot::Alegre.get_pm_type(pm)
       Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 2/5] Type is #{type.blank? ? "blank" : type}"
       unless type.blank?
-        if !self.should_get_similar_items_of_type?('master', pm.team_id) || !self.should_get_similar_items_of_type?(type, pm.team_id)
+        if !Bot::Alegre.should_get_similar_items_of_type?('master', pm.team_id) || !Bot::Alegre.should_get_similar_items_of_type?(type, pm.team_id)
           Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 3/5] ProjectMedia cannot be checked for similar items"
           return {}
         else
diff --git a/app/models/concerns/alegre_webhooks.rb b/app/models/concerns/alegre_webhooks.rb
new file mode 100644
index 0000000000..4b71b2d582
--- /dev/null
+++ b/app/models/concerns/alegre_webhooks.rb
@@ -0,0 +1,26 @@
+class AlegreCallbackError < StandardError
+end
+
+module AlegreWebhooks
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def valid_request?(request)
+      token = request.params['token'] || request.query_parameters['token']
+      !token.blank? && token == CheckConfig.get('alegre_token')
+    end
+
+    def webhook(request)
+      begin
+        doc_id = request.params.dig('data', 'requested', 'body', 'id')
+        raise 'Unexpected params format' if doc_id.blank?
+        redis = Redis.new(REDIS_CONFIG)
+        key = "alegre:webhook:#{doc_id}"
+        redis.lpush(key, request.params.to_json)
+        redis.expire(key, 1.day.to_i)
+      rescue StandardError => e
+        CheckSentry.notify(AlegreCallbackError.new(e.message), { alegre_response: request.params })
+      end
+    end
+  end
+end
diff --git a/config/config.yml.example b/config/config.yml.example
index ebbd6d49c9..3943600acd 100644
--- a/config/config.yml.example
+++ b/config/config.yml.example
@@ -11,7 +11,6 @@ development: &default
   elasticsearch_index:
   elasticsearch_log: true
   elasticsearch_sync: false
-
   # WARNING For production, don't use a wildcard: set the allowed domains explicitly as a regular expression, e.g.
   # '(https?://.*\.?(meedan.com|meedan.org))'
   allowed_origins: '.*'
diff --git a/test/controllers/webhooks_controller_test.rb b/test/controllers/webhooks_controller_test.rb
index f91de407b7..e809d0001d 100644
--- a/test/controllers/webhooks_controller_test.rb
+++ b/test/controllers/webhooks_controller_test.rb
@@ -232,4 +232,24 @@ def setup
     assert_equal '200', response.code
     assert_match /ignored/, response.body
   end
+
+  test "should process Alegre webhook" do
+    CheckSentry.expects(:notify).never
+    redis = Redis.new(REDIS_CONFIG)
+    redis.del('foo')
+    payload = { 'action' => 'audio', 'data' => { 'requested' => { 'body' => { 'id' => 'foo', 'context' => { 'project_media_id' => random_number } } } } }
+    assert_nil redis.lpop('alegre:webhook:foo')
+
+    post :index, params: { name: :alegre, token: CheckConfig.get('alegre_token') }.merge(payload)
+    response = JSON.parse(redis.lpop('alegre:webhook:foo'))
+    assert_equal 'foo', response.dig('data', 'requested', 'body', 'id')
+
+    travel_to Time.now.since(2.days)
+    assert_nil redis.lpop('alegre:webhook:foo')
+  end
+
+  test "should report error if can't process Alegre webhook" do
+    CheckSentry.expects(:notify).once
+    post :index, params: { name: :alegre, token: CheckConfig.get('alegre_token') }.merge({ foo: 'bar' })
+  end
 end
diff --git a/test/models/bot/alegre_2_test.rb b/test/models/bot/alegre_2_test.rb
index 804ce5e991..120b3c25fc 100644
--- a/test/models/bot/alegre_2_test.rb
+++ b/test/models/bot/alegre_2_test.rb
@@ -315,6 +315,16 @@ def teardown
     RequestStore.store[:pause_database_connection] = false
   end
 
+  test "should block calls on redis blpop for audio request" do
+    stubbed_response = Net::HTTPSuccess.new(1.0, '200', 'OK')
+    stubbed_response.stubs(:body).returns({"queue" => "audio__Model", "body" => {"id" => "123", "callback_url" => "http://example.com/callback"}}.to_json)
+    Net::HTTP.any_instance.stubs(:request).returns(stubbed_response)
+    Redis.any_instance.stubs(:blpop).with("alegre:webhook:123", 120).returns(["alegre:webhook:123", {"tested" => true}.to_json])
+    assert_equal Bot::Alegre.request_api('get', '/audio/similarity/', @params, 'body'), {"tested" => true}
+    Net::HTTP.any_instance.unstub(:request)
+    Redis.any_instance.unstub(:blpop)
+  end
+
   test "should get items with similar title" do
     create_verification_status_stuff
     RequestStore.store[:skip_cached_field_update] = false

From 5da8afa867fd12093a1dc154f96ee067e31124e7 Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Thu, 5 Oct 2023 16:47:34 -0300
Subject: [PATCH 08/18] Fixing URL shortening for Arabic URLs.

We have to check if the URL itself contains Arabic characters, not only the full text where the URL is in.

Fixes: CV2-3814.
---
 lib/url_rewriter.rb           |  2 +-
 test/lib/url_rewriter_test.rb | 14 ++++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/lib/url_rewriter.rb b/lib/url_rewriter.rb
index e97fff262b..8955a21be0 100644
--- a/lib/url_rewriter.rb
+++ b/lib/url_rewriter.rb
@@ -23,7 +23,7 @@ def self.utmize(url, source)
   def self.shorten_and_utmize_urls(input_text, source = nil, owner = nil)
     text = input_text
     # Encode URLs in Arabic which are not detected by the URL extraction methods
-    text = text.gsub(/https?:\/\/[\S]+/) { |url| Addressable::URI.escape(url) } if input_text =~ /\p{Arabic}/
+    text = text.gsub(/https?:\/\/[\S]+/) { |url| url =~ /\p{Arabic}/ ? Addressable::URI.escape(url) : url } if input_text =~ /\p{Arabic}/
     entities = Twitter::TwitterText::Extractor.extract_urls_with_indices(text, extract_url_without_protocol: true)
     # Ruby 2.7 freezes the empty string from nil.to_s, which causes an error within the rewriter
     Twitter::TwitterText::Rewriter.rewrite_entities(text || '', entities) do |entity, _codepoints|
diff --git a/test/lib/url_rewriter_test.rb b/test/lib/url_rewriter_test.rb
index 4485cedf2c..4c83641f06 100644
--- a/test/lib/url_rewriter_test.rb
+++ b/test/lib/url_rewriter_test.rb
@@ -54,9 +54,19 @@ def teardown
   test 'should shorten Arabic URL' do
     shortened = nil
     stub_configs({ 'short_url_host_display' => 'https://chck.media' }) do
-      shortened = UrlRewriter.shorten_and_utmize_urls('Visit https://fatabyyano.net/هذا-المقطع-ليس-لاشتباكات-حديثة-بين-الج/ for more information.', nil)
+      shortened = UrlRewriter.shorten_and_utmize_urls('Visit https://fatabyyano.net/هذا-المقطع-قديم،-ولا-يبين-لحظة-إنقاذ-شا/ for more information.', nil)
     end
-    assert_equal 'https://fatabyyano.net/%D9%87%D8%B0%D8%A7-%D8%A7%D9%84%D9%85%D9%82%D8%B7%D8%B9-%D9%84%D9%8A%D8%B3-%D9%84%D8%A7%D8%B4%D8%AA%D8%A8%D8%A7%D9%83%D8%A7%D8%AA-%D8%AD%D8%AF%D9%8A%D8%AB%D8%A9-%D8%A8%D9%8A%D9%86-%D8%A7%D9%84%D8%AC/', Shortener::ShortenedUrl.last.url
+    assert_equal 'https://fatabyyano.net/%D9%87%D8%B0%D8%A7-%D8%A7%D9%84%D9%85%D9%82%D8%B7%D8%B9-%D9%82%D8%AF%D9%8A%D9%85%D8%8C-%D9%88%D9%84%D8%A7-%D9%8A%D8%A8%D9%8A%D9%86-%D9%84%D8%AD%D8%B8%D8%A9-%D8%A5%D9%86%D9%82%D8%A7%D8%B0-%D8%B4%D8%A7/', Shortener::ShortenedUrl.last.url
     assert_match /^Visit https:\/\/chck\.media\/[a-zA-Z0-9]+ for more information\.$/, shortened
   end
+
+  test 'should not shorten decoded Arabic URL' do
+    url = 'https://fatabyyano.net/%da%af%d8%b1%d8%aa%db%95-%da%a4%db%8c%d8%af%db%8c%db%86%db%8c%db%8c%db%95%da%a9%db%95-%d8%b3%d8%a7%d8%ae%d8%aa%db%95%db%8c%db%95-%d9%88-%d9%84%d8%a7%d9%81%d8%a7%d9%88-%d9%88-%d8%b2%d8%b1%db%8c%d8%a7/'
+    shortened = nil
+    stub_configs({ 'short_url_host_display' => 'https://chck.media' }) do
+      shortened = UrlRewriter.shorten_and_utmize_urls("فتبينوا  | Visit #{url} for more information.", nil)
+    end
+    assert_equal url, Shortener::ShortenedUrl.last.url
+    assert_match /^فتبينوا  \| Visit https:\/\/chck\.media\/[a-zA-Z0-9]+ for more information\.$/, shortened
+  end
 end

From a7de7219d9157e12445e3f9892e8de5f32005261 Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Sat, 7 Oct 2023 19:42:47 -0300
Subject: [PATCH 09/18] Do not submit as unconfirmed media a text with just a
 few words

In that case, the request should still be saved, but associated with the workspace, not with a text media.

Fixes CV2-3784.
---
 app/models/bot/smooch.rb                      |  4 ++
 app/models/concerns/smooch_messages.rb        |  4 +-
 .../controllers/graphql_controller_10_test.rb | 46 ++++++++++++++
 test/controllers/graphql_controller_3_test.rb | 43 -------------
 test/models/bot/smooch_4_test.rb              | 61 ++++++++++++++-----
 test/models/bot/smooch_6_test.rb              | 26 ++++++++
 6 files changed, 124 insertions(+), 60 deletions(-)

diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb
index 31fd749316..5c5220305a 100644
--- a/app/models/bot/smooch.rb
+++ b/app/models/bot/smooch.rb
@@ -820,6 +820,10 @@ def self.save_text_message(message)
         claim = self.extract_claim(text).gsub(/\s+/, ' ').strip
         extra = { quote: claim }
         pm = ProjectMedia.joins(:media).where('trim(lower(quote)) = ?', claim.downcase).where('project_medias.team_id' => team.id).last
+        # Don't create a new text media if it's an unconfirmed request with just a few words
+        if pm.nil? && message['archived'] == CheckArchivedFlags::FlagCodes::UNCONFIRMED && ::Bot::Alegre.get_number_of_words(claim) < CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer)
+          return team
+        end
       else
         extra = { url: link.url }
         pm = ProjectMedia.joins(:media).where('medias.url' => link.url, 'project_medias.team_id' => team.id).last
diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb
index c8f69a9461..fb433182f9 100644
--- a/app/models/concerns/smooch_messages.rb
+++ b/app/models/concerns/smooch_messages.rb
@@ -335,10 +335,10 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req
       self.get_installation(self.installation_setting_id_keys, app_id)
       Team.current = Team.where(id: self.config['team_id']).last
       annotated = nil
-      if ['default_requests', 'timeout_requests', 'resource_requests', 'irrelevant_search_result_requests'].include?(request_type)
+      if ['default_requests', 'timeout_requests', 'irrelevant_search_result_requests'].include?(request_type)
         message['archived'] = ['default_requests', 'irrelevant_search_result_requests'].include?(request_type) ? self.default_archived_flag : CheckArchivedFlags::FlagCodes::UNCONFIRMED
         annotated = self.create_project_media_from_message(message)
-      elsif ['menu_options_requests', 'relevant_search_result_requests', 'timeout_search_requests'].include?(request_type)
+      elsif ['menu_options_requests', 'relevant_search_result_requests', 'timeout_search_requests', 'resource_requests'].include?(request_type)
         annotated = annotated_obj
       end
 
diff --git a/test/controllers/graphql_controller_10_test.rb b/test/controllers/graphql_controller_10_test.rb
index 0193d1efb8..90479327bf 100644
--- a/test/controllers/graphql_controller_10_test.rb
+++ b/test/controllers/graphql_controller_10_test.rb
@@ -809,4 +809,50 @@ def setup
     assert_response :success
     assert !JSON.parse(@response.body)['data']['sendTiplineMessage']['success']
   end
+
+  test "should set smooch user Slack channel URL in background" do
+    u = create_user
+    t = create_team
+    create_team_user team: t, user: u, role: 'admin'
+    p = create_project team: t
+    author_id = random_string
+    set_fields = { smooch_user_data: { id: author_id }.to_json, smooch_user_app_id: 'fake', smooch_user_id: 'fake' }.to_json
+    d = create_dynamic_annotation annotated: p, annotation_type: 'smooch_user', set_fields: set_fields
+    authenticate_with_token
+    url = random_url
+    query = 'mutation { updateDynamicAnnotationSmoochUser(input: { clientMutationId: "1", id: "' + d.graphql_id + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url + '\"}" }) { project { dbid } } }'
+
+    Sidekiq::Testing.fake! do
+      post :create, params: { query: query }
+      assert_response :success
+    end
+    Sidekiq::Worker.drain_all
+    assert_equal url, Dynamic.find(d.id).get_field_value('smooch_user_slack_channel_url')
+
+    # Check that cache key exists
+    key = "SmoochUserSlackChannelUrl:Team:#{d.team_id}:#{author_id}"
+    assert_equal url, Rails.cache.read(key)
+
+    # Test using a new mutation `smoochBotAddSlackChannelUrl`
+    url2 = random_url
+    query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "' + d.id.to_s + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }'
+    Sidekiq::Testing.fake! do
+      post :create, params: { query: query }
+      assert_response :success
+    end
+    assert Sidekiq::Worker.jobs.size > 0
+    assert_equal url, d.reload.get_field_value('smooch_user_slack_channel_url')
+
+    # Execute job and check that URL was set
+    Sidekiq::Worker.drain_all
+    assert_equal url2, d.get_field_value('smooch_user_slack_channel_url')
+
+    # Check that cache key exists
+    assert_equal url2, Rails.cache.read(key)
+
+    # Call mutation with non existing ID
+    query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "99999", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }'
+    post :create, params: { query: query }
+    assert_response :success
+  end
 end
diff --git a/test/controllers/graphql_controller_3_test.rb b/test/controllers/graphql_controller_3_test.rb
index 57f9408831..e2892e565c 100644
--- a/test/controllers/graphql_controller_3_test.rb
+++ b/test/controllers/graphql_controller_3_test.rb
@@ -341,49 +341,6 @@ def setup
     assert_equal 1, response['item_navigation_offset']
   end
 
-  test "should set smooch user slack channel url in background" do
-    Sidekiq::Testing.fake! do
-      u = create_user
-      t = create_team
-      create_team_user team: t, user: u, role: 'admin'
-      p = create_project team: t
-      author_id = random_string
-      set_fields = { smooch_user_data: { id: author_id }.to_json, smooch_user_app_id: 'fake', smooch_user_id: 'fake' }.to_json
-      d = create_dynamic_annotation annotated: p, annotation_type: 'smooch_user', set_fields: set_fields
-      Sidekiq::Worker.drain_all
-      assert_equal 0, Sidekiq::Worker.jobs.size
-      authenticate_with_token
-      url = random_url
-      query = 'mutation { updateDynamicAnnotationSmoochUser(input: { clientMutationId: "1", id: "' + d.graphql_id + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url + '\"}" }) { project { dbid } } }'
-      post :create, params: { query: query }
-      assert_response :success
-      Sidekiq::Worker.drain_all
-      sleep 1
-      assert_equal url, Dynamic.find(d.id).get_field_value('smooch_user_slack_channel_url')
-      # check that cache key exists
-      key = "SmoochUserSlackChannelUrl:Team:#{d.team_id}:#{author_id}"
-      assert_equal url, Rails.cache.read(key)
-      # test using a new mutation `smoochBotAddSlackChannelUrl`
-      Sidekiq::Worker.drain_all
-      assert_equal 0, Sidekiq::Worker.jobs.size
-      url2 = random_url
-      query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "' + d.id.to_s + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }'
-      post :create, params: { query: query }
-      assert_response :success
-      assert Sidekiq::Worker.jobs.size > 0
-      assert_equal url, d.reload.get_field_value('smooch_user_slack_channel_url')
-      # execute job and check that url was set
-      Sidekiq::Worker.drain_all
-      assert_equal url2, d.get_field_value('smooch_user_slack_channel_url')
-      # check that cache key exists
-      assert_equal url2, Rails.cache.read(key)
-      # call mutation with non existing id
-      query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "99999", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }'
-      post :create, params: { query: query }
-      assert_response :success
-    end
-  end
-
   test "should get requests from media" do
     u = create_user is_admin: true
     t = create_team
diff --git a/test/models/bot/smooch_4_test.rb b/test/models/bot/smooch_4_test.rb
index 80db102e3a..1d619bc543 100644
--- a/test/models/bot/smooch_4_test.rb
+++ b/test/models/bot/smooch_4_test.rb
@@ -476,50 +476,81 @@ def teardown
 
   test "should request resource" do
     setup_smooch_bot(true)
-    RequestStore.store[:skip_cached_field_update] = false
     uid = random_string
     rss = '<rss version="1"><channel><title>x</title><link>x</link><description>x</description><item><title>x</title><link>x</link></item></channel></rss>'
     WebMock.stub_request(:get, 'http://test.com/feed.rss').to_return(status: 200, body: rss)
     Sidekiq::Testing.fake! do
       send_message_to_smooch_bot('Hello', uid)
       send_message_to_smooch_bot('1', uid)
+      send_message_to_smooch_bot('4', uid)
+    end
+    Sidekiq::Worker.drain_all
+    a = Dynamic.where(annotation_type: 'smooch').last
+    assert_equal 'TiplineResource', a.annotated_type
+    assert_not_nil a.get_field('smooch_resource_id')
+  end
+
+  test "should submit short unconfirmed request" do
+    setup_smooch_bot(true)
+    RequestStore.store[:skip_cached_field_update] = false
+    uid = random_string
+    message = "Hey ho, let's go"
+    Sidekiq::Testing.fake! do
+      send_message_to_smooch_bot('Hello', uid)
+      send_message_to_smooch_bot(message, uid)
+    end
+    assert_no_difference 'ProjectMedia.count' do
+      Sidekiq::Worker.drain_all
+    end
+    a = Dynamic.where(annotation_type: 'smooch').last
+    annotated = a.annotated
+    assert_equal 'Team', a.annotated_type
+  end
+
+  test "should submit long unconfirmed request" do
+    setup_smooch_bot(true)
+    RequestStore.store[:skip_cached_field_update] = false
+    uid = random_string
+    message = 'This is a message that has enough words to be considered a media'
+    Sidekiq::Testing.fake! do
+      send_message_to_smooch_bot('Hello', uid)
+      send_message_to_smooch_bot(message, uid)
+    end
+    assert_difference 'ProjectMedia.count' do
+      Sidekiq::Worker.drain_all
     end
-    Rails.cache.stubs(:read).returns(nil)
-    Rails.cache.stubs(:read).with("smooch:last_message_from_user:#{uid}").returns(Time.now + 10.seconds)
-    send_message_to_smooch_bot('4', uid)
     a = Dynamic.where(annotation_type: 'smooch').last
     annotated = a.annotated
     assert_equal 'ProjectMedia', a.annotated_type
     assert_equal CheckArchivedFlags::FlagCodes::UNCONFIRMED, annotated.archived
-    # verify requests_count & demand count
+    # Verify requests count & demand
     assert_equal 1, annotated.requests_count
     assert_equal 1, annotated.demand
-    assert_not_nil a.get_field('smooch_resource_id')
-    # Test auto confirm the media if resend same media as a default request
+    # Auto confirm the media if the same media is sent as a default request
     Sidekiq::Testing.fake! do
-      send_message_to_smooch_bot('Hello', uid)
+      send_message_to_smooch_bot(message, uid)
       send_message_to_smooch_bot('1', uid)
+      send_message_to_smooch_bot('2', uid)
     end
     Rails.cache.stubs(:read).returns(nil)
     Rails.cache.stubs(:read).with("smooch:last_message_from_user:#{uid}").returns(Time.now + 10.seconds)
     assert_no_difference 'ProjectMedia.count' do
-      send_message_to_smooch_bot('2', uid)
+      send_message_to_smooch_bot('Query', uid)
     end
+    Rails.cache.unstub(:read)
+    Sidekiq::Worker.drain_all
     assert_equal CheckArchivedFlags::FlagCodes::NONE, annotated.reload.archived
     assert_equal 2, annotated.reload.requests_count
-    # Test resend same media (should not update archived cloumn)
+    # Test resend the same media (should not update archived column)
     Sidekiq::Testing.fake! do
       send_message_to_smooch_bot('Hello', uid)
-      send_message_to_smooch_bot('1', uid)
+      send_message_to_smooch_bot(message, uid)
     end
-    Rails.cache.stubs(:read).returns(nil)
-    Rails.cache.stubs(:read).with("smooch:last_message_from_user:#{uid}").returns(Time.now + 10.seconds)
     assert_no_difference 'ProjectMedia.count' do
-      send_message_to_smooch_bot('2', uid)
+      Sidekiq::Worker.drain_all
     end
     assert_equal CheckArchivedFlags::FlagCodes::NONE, annotated.reload.archived
     assert_equal 3, annotated.reload.requests_count
-    Rails.cache.unstub(:read)
   end
 
   test "should get default TOS message" do
diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb
index 77d18a14bb..5f66ead55c 100644
--- a/test/models/bot/smooch_6_test.rb
+++ b/test/models/bot/smooch_6_test.rb
@@ -854,4 +854,30 @@ def send_message_outside_24_hours_window(template, pm = nil)
     send_message 'Hello'
     assert_state 'main'
   end
+
+  test 'should save unconfirmed media with enough words' do
+    @installation.set_smooch_disable_timeout = false
+    @installation.save!
+    reload_tipline_settings
+    send_message 'hello', '1' # Sends a first message and confirms language as English
+    send_message 'This is message is so long that it is considered a media'
+    assert_difference 'ProjectMedia.count' do
+      assert_difference "Dynamic.where(annotation_type: 'smooch').count" do
+        Sidekiq::Worker.drain_all
+      end
+    end
+  end
+
+  test 'should not save unconfirmed media with just a few words' do
+    @installation.set_smooch_disable_timeout = false
+    @installation.save!
+    reload_tipline_settings
+    send_message 'hello', '1' # Sends a first message and confirms language as English
+    send_message 'Hi, there!'
+    assert_no_difference 'ProjectMedia.count' do
+      assert_difference "Dynamic.where(annotation_type: 'smooch').count" do
+        Sidekiq::Worker.drain_all
+      end
+    end
+  end
 end

From a52dacc35fcfc746f3385860499b838ca0774eae Mon Sep 17 00:00:00 2001
From: Mohamed El-Sawy <melsawy.be7eri@gmail.com>
Date: Sun, 8 Oct 2023 07:28:49 +0300
Subject: [PATCH 10/18] CV2-2897: fix return updated_objects for bulk read
 action (#1685)

* CV2-2897: fix return updated_objects for bulk read action

* CV2-2897: fix tests
---
 app/graph/mutations/graphql_crud_operations.rb | 2 +-
 test/controllers/graphql_controller_4_test.rb  | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/graph/mutations/graphql_crud_operations.rb b/app/graph/mutations/graphql_crud_operations.rb
index d4f5ef209e..8b7a493136 100644
--- a/app/graph/mutations/graphql_crud_operations.rb
+++ b/app/graph/mutations/graphql_crud_operations.rb
@@ -192,7 +192,7 @@ def self.apply_bulk_update_or_destroy(inputs, ctx, update_or_destroy, klass)
       method_mapping = { update: :bulk_update, destroy: :bulk_destroy, mark_read: :bulk_mark_read }
       method = method_mapping[update_or_destroy.to_sym]
       result = klass.send(method, sql_ids, filtered_inputs, Team.current)
-      if update_or_destroy.to_s == "update"
+      if update_or_destroy.to_s != "destroy"
         result.merge!({ updated_objects: klass.where(id: sql_ids) })
       end
       { ids: processed_ids }.merge(result)
diff --git a/test/controllers/graphql_controller_4_test.rb b/test/controllers/graphql_controller_4_test.rb
index b1b32efe91..2bdd4ef54b 100644
--- a/test/controllers/graphql_controller_4_test.rb
+++ b/test/controllers/graphql_controller_4_test.rb
@@ -322,9 +322,11 @@ def teardown
     @pms.each { |pm| assert_not pm.read }
     assert_search_finds_all({ read: 0 })
     assert_search_finds_none({ read: 1 })
-    query = 'mutation { bulkProjectMediaMarkRead(input: { clientMutationId: "1", ids: ' + @ids + ', read: true }) { ids, team { dbid } } }'
+    query = 'mutation { bulkProjectMediaMarkRead(input: { clientMutationId: "1", ids: ' + @ids + ', read: true }) { updated_objects { id, is_read, dbid }, ids, team { dbid } } }'
     post :create, params: { query: query, team: @t.slug }
     assert_response :success
+    updated_objects = JSON.parse(@response.body)['data']['bulkProjectMediaMarkRead']['updated_objects']
+    assert_equal @pms.map(&:id).sort, updated_objects.collect{|obj| obj['dbid']}.sort
     @pms.each { |pm| assert pm.reload.read }
     assert_search_finds_all({ read: 1 })
     assert_search_finds_none({ read: 0 })

From d961a349e0e428fb4e8f722075098a32d11aa6bc Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Sun, 8 Oct 2023 14:34:47 -0300
Subject: [PATCH 11/18] Don't crop header images for tipline resources and
 newsletters

Previously, some tipline resources and newsletters header images would be cropped unless they had some specific aspect ratio. This PR fixes this, by always resizing the images while keeping aspect ratio and never cropping them.

Fixes: CV2-3356.
---
 app/models/concerns/tipline_content_image.rb |  9 ++++-
 config/config.yml.example                    |  1 +
 public/tipline-content-template.html         | 40 ++++++++------------
 3 files changed, 23 insertions(+), 27 deletions(-)

diff --git a/app/models/concerns/tipline_content_image.rb b/app/models/concerns/tipline_content_image.rb
index faa902dd83..a85c8af8c1 100644
--- a/app/models/concerns/tipline_content_image.rb
+++ b/app/models/concerns/tipline_content_image.rb
@@ -30,7 +30,12 @@ def convert_header_file_image
       temp_name = 'temp-' + self.id.to_s + '-' + self.language + '.html'
       temp = File.join(Rails.root, 'public', content_name, temp_name)
       output = File.open(temp, 'w+')
-      output.puts doc.to_s.gsub('%IMAGE_URL%', CheckS3.rewrite_url(self.header_file_url.to_s))
+
+      # Replace the image in the template
+      image_url = CheckS3.rewrite_url(self.header_file_url.to_s)
+      w, h = ::MiniMagick::Image.open(image_url)[:dimensions]
+      image_class = w > h ? 'wider' : 'taller'
+      output.puts doc.to_s.gsub('%IMAGE_URL%', image_url).gsub('%IMAGE_CLASS%', image_class)
       output.close
 
       # Upload the HTML to S3
@@ -39,7 +44,7 @@ def convert_header_file_image
       temp_url = CheckS3.public_url(path)
 
       # Convert the HTML to PNG
-      uri = URI("#{CheckConfig.get('narcissus_url')}/?url=#{CheckS3.rewrite_url(temp_url)}&selector=%23frame")
+      uri = URI("#{CheckConfig.get('narcissus_url')}/?selector=img&url=#{CheckS3.rewrite_url(temp_url)}")
       request = Net::HTTP::Get.new(uri)
       request['x-api-key'] = CheckConfig.get('narcissus_token')
       response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') { |http| http.request(request) }
diff --git a/config/config.yml.example b/config/config.yml.example
index 3943600acd..63a205f99e 100644
--- a/config/config.yml.example
+++ b/config/config.yml.example
@@ -286,6 +286,7 @@ test:
   otel_log_level: error
   otel_traces_sampler:
   sentry_dsn:
+  storage_rewrite_host: 'http://minio:9000'
 
   # Facebook social login
   #
diff --git a/public/tipline-content-template.html b/public/tipline-content-template.html
index 6a6bd4c22b..143597e2f8 100644
--- a/public/tipline-content-template.html
+++ b/public/tipline-content-template.html
@@ -1,7 +1,7 @@
 <!doctype html>
 <html>
   <head>
-    <title>Tipline Newsletter Template</title>
+    <title>Tipline Content Template</title>
     <style>
       @import url(https://cdnjs.cloudflare.com/ajax/libs/meyer-reset/2.0/reset.min.css);
       @import url('https://fonts.googleapis.com/css2?family=Roboto&display=swap');
@@ -12,9 +12,6 @@
         font-family: 'Roboto', sans-serif;
         font-weight: 400;
         line-height: 1.5em;
-        width: 282px;
-        height: 214px;
-        position: relative;
       }
 
       body.hi {
@@ -37,25 +34,20 @@
       }
 
       #frame {
-        background: white;
-        width: 282px;
-        height: 214px;
-        overflow: hidden;
-        position: relative;
-        box-sizing: border-box;
-      }
-
-      #image {
-        width: 282px;
-        height: 214px;
-        background-color: white;
-        background-position: center center;
-        background-repeat: no-repeat;
-        background-size: cover;
-        background-image: url('%IMAGE_URL%');
         display: flex;
         align-items: center;
         justify-content: center;
+        position: relative;
+      }
+
+      #frame img.taller {
+        width: 230px;
+        height: auto;
+      }
+
+      #frame img.wider {
+        height: 230px;
+        width: auto;
       }
 
       #text {
@@ -67,17 +59,15 @@
         background: rgba(34, 34, 34, 0.7);
         border-radius: 8px;
         padding: 6px 12px;
+        position: absolute;
       }
     </style>
   </head>
   <body class="content">
     <div id="frame">
-      <div id="image">
-        <div id="text">
-        </div>
+      <img src="%IMAGE_URL%" class="%IMAGE_CLASS%" alt="Background image" />
+      <div id="text">
       </div>
     </div>
-    <!-- Just to be sure the image will load -->
-    <img style="display: none;" src="%IMAGE_URL%" alt="Background image" />
   </body>
 </html>

From c92ecc7dea7fad25e58c6b81616dbf660e099b48 Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Mon, 9 Oct 2023 09:07:50 -0300
Subject: [PATCH 12/18] Raise error when trying to save a null object

Raise error when trying to save a null object through GraphQL update mutation.

Reference: CV2-3826.
---
 app/graph/mutations/graphql_crud_operations.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/graph/mutations/graphql_crud_operations.rb b/app/graph/mutations/graphql_crud_operations.rb
index 8b7a493136..15fe74f699 100644
--- a/app/graph/mutations/graphql_crud_operations.rb
+++ b/app/graph/mutations/graphql_crud_operations.rb
@@ -1,5 +1,6 @@
 class GraphqlCrudOperations
   def self.safe_save(obj, attrs, parent_names = [])
+    raise "Can't save a null object." if obj.nil?
     raise 'This operation must be done by a signed-in user' if User.current.nil? && ApiKey.current.nil?
     attrs.each do |key, value|
       method = key == "clientMutationId" ? "client_mutation_id=" : "#{key}="

From 2ea08824bda41a633dd09cc2a358c35f2b9311be Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Mon, 9 Oct 2023 12:18:25 -0300
Subject: [PATCH 13/18] Individual feedback for tipline search results on
 WhatsApp

For WhatsApp tiplines, instead of asking for feedback for all the results, ask for feedback individually by appending a thumbs-up button to each result. The timeout case is also individual as part of this change.

Reference: CV2-3414.
---
 app/models/bot/smooch.rb             |  5 ++
 app/models/concerns/smooch_menus.rb  |  8 ++-
 app/models/concerns/smooch_search.rb | 84 ++++++++++++++++++++++++----
 test/models/bot/smooch_6_test.rb     | 49 ++++++++++++++++
 4 files changed, 135 insertions(+), 11 deletions(-)

diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb
index 5c5220305a..3403e33020 100644
--- a/app/models/bot/smooch.rb
+++ b/app/models/bot/smooch.rb
@@ -381,6 +381,11 @@ def self.parse_message_based_on_state(message, app_id)
       return true
     end
 
+    if self.clicked_on_search_result_button?(message)
+      self.search_result_button_click_callback(message, uid, app_id, workflow, language)
+      return true
+    end
+
     case state
     when 'waiting_for_message'
       self.bundle_message(message)
diff --git a/app/models/concerns/smooch_menus.rb b/app/models/concerns/smooch_menus.rb
index 755e40d155..7f0469508b 100644
--- a/app/models/concerns/smooch_menus.rb
+++ b/app/models/concerns/smooch_menus.rb
@@ -166,7 +166,7 @@ def get_custom_string(key, language, truncate_at = 1024)
       label.to_s.truncate(truncate_at)
     end
 
-    def send_message_to_user_with_buttons(uid, text, options)
+    def send_message_to_user_with_buttons(uid, text, options, image_url = nil)
       buttons = []
       options.each_with_index do |option, i|
         buttons << {
@@ -196,6 +196,12 @@ def send_message_to_user_with_buttons(uid, text, options)
           }
         }
       }
+      extra[:override][:whatsapp][:payload][:interactive][:header] = {
+        type: 'image',
+        image: {
+          link: CheckS3.rewrite_url(image_url)
+        }
+      } unless image_url.blank?
       extra, fallback = self.format_fallback_text_menu_from_options(text, options, extra)
       self.send_message_to_user(uid, fallback.join("\n"), extra)
     end
diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb
index cbd339bd3a..d442c925df 100644
--- a/app/models/concerns/smooch_search.rb
+++ b/app/models/concerns/smooch_search.rb
@@ -19,10 +19,15 @@ def search(app_id, uid, language, message, team_id, workflow)
           self.bundle_messages(uid, '', app_id, 'default_requests', nil, true)
           self.send_final_message_to_user(uid, self.get_custom_string('search_no_results', language), workflow, language)
         else
-          self.send_search_results_to_user(uid, results, team_id)
-          sm.go_to_search_result
-          self.save_search_results_for_user(uid, results.map(&:id))
-          self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, 1)
+          self.send_search_results_to_user(uid, results, team_id, platform, app_id)
+          # For WhatsApp, each search result goes with a button where the user can give feedback individually, so, reset the conversation right away
+          if platform == 'WhatsApp'
+            sm.reset
+          else
+            sm.go_to_search_result
+            self.save_search_results_for_user(uid, results.map(&:id))
+            self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, 1)
+          end
         end
       rescue StandardError => e
         self.handle_search_error(uid, e, language)
@@ -214,21 +219,51 @@ def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids,
       results
     end
 
-    def send_search_results_to_user(uid, results, team_id)
+    def send_search_results_to_user(uid, results, team_id, platform, app_id)
       team = Team.find(team_id)
-      redis = Redis.new(REDIS_CONFIG)
       language = self.get_user_language(uid)
-      reports = results.collect{ |r| r.get_dynamic_annotation('report_design') }
+      reports = results.collect{ |r| r.get_dynamic_annotation('report_design') }.reject{ |r| r.blank? }
       # Get reports languages
-      reports_language = reports.map{|r| r&.report_design_field_value('language')}.uniq
+      reports_language = reports.map{ |r| r.report_design_field_value('language') }.uniq
       if team.get_languages.to_a.size > 1 && !reports_language.include?(language)
         self.send_message_to_user(uid, self.get_string(:no_results_in_language, language).gsub('%{language}', CheckCldr.language_code_to_name(language, language)))
         sleep 1
       end
+      if platform == 'WhatsApp'
+        self.send_search_results_to_whatsapp_user(uid, reports, app_id)
+      else
+        self.send_search_results_to_non_whatsapp_user(uid, reports)
+      end
+    end
+
+    def generate_search_id
+      SecureRandom.hex
+    end
+
+    def send_search_results_to_whatsapp_user(uid, reports, app_id)
+      search_id = self.generate_search_id
+      # Cache the current bundle of messages from this user related to this search, so a request can be created correctly
+      # Expires after the time to give feedback is expired
+      Rails.cache.write("smooch:user_search_bundle:#{uid}:#{search_id}", self.list_of_bundled_messages_from_user(uid), expires_in: 20.minutes)
+      self.clear_user_bundled_messages(uid)
+      reports.each do |report|
+        text = report.report_design_text if report.report_design_field_value('use_text_message')
+        image_url = report.report_design_image_url if report.report_design_field_value('use_visual_card')
+        options = [{
+          value: { project_media_id: report.annotated_id, keyword: 'search_result_is_relevant', search_id: search_id }.to_json,
+          label: '👍'
+        }]
+        self.send_message_to_user_with_buttons(uid, text || '-', options, image_url) # "text" is mandatory for WhatsApp interactive messages
+        self.delay_for(15.minutes, { queue: 'smooch_priority' }).timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, report.annotated_id)
+      end
+    end
+
+    def send_search_results_to_non_whatsapp_user(uid, reports)
+      redis = Redis.new(REDIS_CONFIG)
       reports.each do |report|
         response = nil
-        response = self.send_message_to_user(uid, report.report_design_text) if report&.report_design_field_value('use_text_message')
-        response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }) if !report&.report_design_field_value('use_text_message') && report&.report_design_field_value('use_visual_card')
+        response = self.send_message_to_user(uid, report.report_design_text) if report.report_design_field_value('use_text_message')
+        response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }) if !report.report_design_field_value('use_text_message') && report.report_design_field_value('use_visual_card')
         id = self.get_id_from_send_response(response)
         redis.rpush("smooch:search:#{uid}", id) unless id.blank?
       end
@@ -253,5 +288,34 @@ def ask_for_feedback_when_all_search_results_are_received(app_id, language, work
         self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, attempts + 1) if attempts < max # Try for 20 seconds
       end
     end
+
+    def timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, pmid)
+      key = "smooch:user_search_bundle:#{uid}:#{search_id}:#{pmid}"
+      if Rails.cache.read(key).nil? # User gave no feedback for the search result
+        bundle = Rails.cache.read("smooch:user_search_bundle:#{uid}:#{search_id}").to_a
+        self.delay_for(1.seconds, { queue: 'smooch', retry: false }).bundle_messages(uid, nil, app_id, 'timeout_search_requests', [ProjectMedia.find(pmid)], true, bundle)
+      else
+        Rails.cache.delete(key) # User gave feedback to search result
+      end
+    end
+
+    def clicked_on_search_result_button?(message)
+      begin
+        JSON.parse(message['payload'])['keyword'] == 'search_result_is_relevant'
+      rescue
+        false
+      end
+    end
+
+    def search_result_button_click_callback(message, uid, app_id, workflow, language)
+      payload = JSON.parse(message['payload'])
+      result = ProjectMedia.find(payload['project_media_id'])
+      bundle = Rails.cache.read("smooch:user_search_bundle:#{uid}:#{payload['search_id']}").to_a
+      unless bundle.empty?
+        Rails.cache.write("smooch:user_search_bundle:#{uid}:#{payload['search_id']}:#{result.id}", Time.now.to_i) # Store that the user has given feedback to this search result
+        self.delay_for(1.seconds, { queue: 'smooch', retry: false }).bundle_messages(uid, message['_id'], app_id, 'relevant_search_result_requests', [result], true, bundle)
+        self.send_final_message_to_user(uid, self.get_custom_string('search_result_is_relevant', language), workflow, language)
+      end
+    end
   end
 end
diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb
index 5f66ead55c..4d98d7ba06 100644
--- a/test/models/bot/smooch_6_test.rb
+++ b/test/models/bot/smooch_6_test.rb
@@ -880,4 +880,53 @@ def send_message_outside_24_hours_window(template, pm = nil)
       end
     end
   end
+
+  test 'should generate a unique ID for search results' do
+    assert_not_equal Bot::Smooch.generate_search_id, Bot::Smooch.generate_search_id
+  end
+
+  test 'should give individual feedback to search result on WhatsApp' do
+    search_id = random_string
+    Bot::Smooch.stubs(:generate_search_id).returns(search_id)
+    pm1 = create_project_media(team: @team)
+    pm2 = create_project_media(team: @team)
+    publish_report(pm1, {}, nil, { language: 'en', use_visual_card: true })
+    publish_report(pm2, {}, nil, { language: 'en', use_visual_card: false })
+    CheckSearch.any_instance.stubs(:medias).returns([pm1, pm2])
+    Sidekiq::Testing.inline! do
+      send_message 'hello', '1', '1', random_string
+    end
+    assert_state 'ask_if_ready'
+    Rails.cache.write("smooch:user_search_bundle:#{@uid}:#{search_id}:#{pm1.id}", Time.now.to_i) # User gave feedback to one result
+    Sidekiq::Testing.fake! do
+      send_message_to_smooch_bot('1', @uid, { 'source' => { 'type' => 'whatsapp' } })
+    end
+    assert_difference 'DynamicAnnotation::Field.where(field_name: "smooch_request_type", value: "timeout_search_requests").count' do
+      Sidekiq::Worker.drain_all
+    end
+  end
+
+  test 'should click on button to evaluate search result on WhatsApp' do
+    search_id = random_string
+    pm = create_project_media(team: @team)
+    bundle = [
+      {
+        '_id': random_string,
+        authorId: @uid,
+        type: 'text',
+        text: random_string
+      }.to_json
+    ]
+    Rails.cache.write("smooch:user_search_bundle:#{@uid}:#{search_id}", bundle)
+    Sidekiq::Testing.inline! do
+      assert_difference 'DynamicAnnotation::Field.where(field_name: "smooch_request_type", value: "relevant_search_result_requests").count' do
+        payload = {
+          'keyword' => 'search_result_is_relevant',
+          'project_media_id' => pm.id,
+          'search_id' => search_id
+        }
+        send_message_to_smooch_bot('Thumbs up', @uid, { 'source' => { 'type' => 'whatsapp' }, 'payload' => payload.to_json })
+      end
+    end
+  end
 end

From 73ca11fca7f44cce089fe21b8eb189c6bde6e928 Mon Sep 17 00:00:00 2001
From: Mohamed El-Sawy <melsawy.be7eri@gmail.com>
Date: Mon, 9 Oct 2023 19:24:08 +0300
Subject: [PATCH 14/18] CV2-3779: add tipline requests cached fields (#1689)

* CV2-3779: add tipline requests cached fields

* CV2-3779: add unit tests and expose graphql fields

* CV2-3779: update graphql schema
---
 app/graph/types/project_media_type.rb         |  2 +
 .../concerns/project_media_cached_fields.rb   | 43 ++++++++++++
 app/repositories/media_search.rb              |  4 ++
 ...pping_for_tipline_search_results_fields.rb | 14 ++++
 db/schema.rb                                  |  4 +-
 lib/relay.idl                                 |  2 +
 public/relay.json                             | 28 ++++++++
 test/models/bot/smooch_7_test.rb              | 66 +++++++++++++++++++
 8 files changed, 161 insertions(+), 2 deletions(-)
 create mode 100644 db/migrate/20231008074526_add_mapping_for_tipline_search_results_fields.rb

diff --git a/app/graph/types/project_media_type.rb b/app/graph/types/project_media_type.rb
index aaed722128..f92dd09bd2 100644
--- a/app/graph/types/project_media_type.rb
+++ b/app/graph/types/project_media_type.rb
@@ -41,6 +41,8 @@ class ProjectMediaType < DefaultObject
   field :cluster, ClusterType, null: true
   field :is_suggested, GraphQL::Types::Boolean, null: true
   field :is_confirmed, GraphQL::Types::Boolean, null: true
+  field :positive_tipline_search_results_count, GraphQL::Types::Int, null: true
+  field :tipline_search_results_count, GraphQL::Types::Int, null: true
 
   field :claim_description, ClaimDescriptionType, null: true
 
diff --git a/app/models/concerns/project_media_cached_fields.rb b/app/models/concerns/project_media_cached_fields.rb
index 1476300167..8d31662057 100644
--- a/app/models/concerns/project_media_cached_fields.rb
+++ b/app/models/concerns/project_media_cached_fields.rb
@@ -446,6 +446,36 @@ def title_or_description_update
         },
       ]
 
+    cached_field :positive_tipline_search_results_count,
+      update_es: true,
+      recalculate: :recalculate_positive_tipline_search_results_count,
+      update_on: [
+        {
+          model: DynamicAnnotation::Field,
+          if: proc { |f| f.field_name == 'smooch_request_type' && f.value == 'relevant_search_result_requests' },
+          affected_ids: proc { |f| [f.annotation&.annotated_id.to_i] },
+          events: {
+            save: :recalculate,
+            destroy: :recalculate,
+          }
+        }
+      ]
+
+    cached_field :tipline_search_results_count,
+      update_es: true,
+      recalculate: :recalculate_tipline_search_results_count,
+      update_on: [
+        {
+          model: DynamicAnnotation::Field,
+          if: proc { |f| f.field_name == 'smooch_request_type' && ['relevant_search_result_requests', 'irrelevant_search_result_requests', 'timeout_search_requests'].include?(f.value) },
+          affected_ids: proc { |f| [f.annotation&.annotated_id.to_i] },
+          events: {
+            save: :recalculate,
+            destroy: :recalculate,
+          }
+        }
+      ]
+
     def recalculate_linked_items_count
       count = Relationship.send('confirmed').where(source_id: self.id).count
       count += 1 unless self.media.type == 'Blank'
@@ -605,6 +635,19 @@ def cached_field_tags_as_sentence_es(value)
     def cached_field_published_by_es(value)
       value.keys.first || 0
     end
+
+    def recalculate_positive_tipline_search_results_count
+      DynamicAnnotation::Field.where(annotation_type: 'smooch',field_name: 'smooch_request_type', value: 'relevant_search_result_requests')
+      .joins('INNER JOIN annotations a ON a.id = dynamic_annotation_fields.annotation_id')
+      .where('a.annotated_type = ? AND a.annotated_id = ?', 'ProjectMedia', self.id).count
+    end
+
+    def recalculate_tipline_search_results_count
+      DynamicAnnotation::Field.where(annotation_type: 'smooch',field_name: 'smooch_request_type')
+      .where('value IN (?)', ['"relevant_search_result_requests"', '"irrelevant_search_result_requests"', '"timeout_search_requests"'])
+      .joins('INNER JOIN annotations a ON a.id = dynamic_annotation_fields.annotation_id')
+      .where('a.annotated_type = ? AND a.annotated_id = ?', 'ProjectMedia', self.id).count
+    end
   end
 
   DynamicAnnotation::Field.class_eval do
diff --git a/app/repositories/media_search.rb b/app/repositories/media_search.rb
index 33e969062a..3fcf8739b5 100644
--- a/app/repositories/media_search.rb
+++ b/app/repositories/media_search.rb
@@ -136,5 +136,9 @@ class MediaSearch
     indexes :report_language, { type: 'keyword', normalizer: 'check' }
 
     indexes :fact_check_published_on, { type: 'long' }
+
+    indexes :positive_tipline_search_results_count, { type: 'long' }
+
+    indexes :tipline_search_results_count, { type: 'long' }
   end
 end
diff --git a/db/migrate/20231008074526_add_mapping_for_tipline_search_results_fields.rb b/db/migrate/20231008074526_add_mapping_for_tipline_search_results_fields.rb
new file mode 100644
index 0000000000..9c038d4cf8
--- /dev/null
+++ b/db/migrate/20231008074526_add_mapping_for_tipline_search_results_fields.rb
@@ -0,0 +1,14 @@
+class AddMappingForTiplineSearchResultsFields < ActiveRecord::Migration[6.1]
+  def change
+    options = {
+      index: CheckElasticSearchModel.get_index_alias,
+      body: {
+        properties: {
+          positive_tipline_search_results_count: { type: 'long' },
+          tipline_search_results_count: { type: 'long' },
+        }
+      }
+    }
+    $repository.client.indices.put_mapping options
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index b8982c24e3..e75733f01b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2023_10_02_202443) do
+ActiveRecord::Schema.define(version: 2023_10_08_074526) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -268,7 +268,7 @@
     t.jsonb "value_json", default: "{}"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY ((ARRAY['external_id'::character varying, 'smooch_user_id'::character varying, 'verification_status_status'::character varying])::text[]))"
+    t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY (ARRAY[('external_id'::character varying)::text, ('smooch_user_id'::character varying)::text, ('verification_status_status'::character varying)::text]))"
     t.index ["annotation_id", "field_name"], name: "index_dynamic_annotation_fields_on_annotation_id_and_field_name"
     t.index ["annotation_id"], name: "index_dynamic_annotation_fields_on_annotation_id"
     t.index ["annotation_type"], name: "index_dynamic_annotation_fields_on_annotation_type"
diff --git a/lib/relay.idl b/lib/relay.idl
index 2303bb732a..9a958332b7 100644
--- a/lib/relay.idl
+++ b/lib/relay.idl
@@ -11211,6 +11211,7 @@ type ProjectMedia implements Node {
   oembed_metadata: String
   permissions: String
   picture: String
+  positive_tipline_search_results_count: Int
   project: Project
   project_group: ProjectGroup
   project_id: Int
@@ -11336,6 +11337,7 @@ type ProjectMedia implements Node {
   tasks_count: JsonStringType
   team: Team
   team_name: String
+  tipline_search_results_count: Int
   title: String
   type: String
   updated_at: String
diff --git a/public/relay.json b/public/relay.json
index df8894a54b..9c6da4dd74 100644
--- a/public/relay.json
+++ b/public/relay.json
@@ -58978,6 +58978,20 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
+            {
+              "name": "positive_tipline_search_results_count",
+              "description": null,
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "Int",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
             {
               "name": "project",
               "description": null,
@@ -59561,6 +59575,20 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
+            {
+              "name": "tipline_search_results_count",
+              "description": null,
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "Int",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
             {
               "name": "title",
               "description": null,
diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb
index 824478f3a8..234e523472 100644
--- a/test/models/bot/smooch_7_test.rb
+++ b/test/models/bot/smooch_7_test.rb
@@ -463,4 +463,70 @@ def teardown
       end
     end
   end
+
+  test "should store number of tipline requests by type" do
+    RequestStore.store[:skip_cached_field_update] = false
+    Sidekiq::Testing.inline! do
+      text = random_string
+      pm = create_project_media team: @team, quote: text, disable_es_callbacks: false
+      text2 = random_string
+      pm2 = create_project_media team: @team, quote: text2, disable_es_callbacks: false
+      message = {
+        type: 'text',
+        text: text,
+        role: 'appUser',
+        received: 1573082583.219,
+        name: random_string,
+        authorId: random_string,
+        '_id': random_string,
+        source: {
+          originalMessageId: random_string,
+          originalMessageTimestamp: 1573082582,
+          type: 'whatsapp',
+          integrationId: random_string
+        },
+      }
+      Bot::Smooch.save_message(message.to_json, @app_id, nil, 'relevant_search_result_requests', pm)
+      Bot::Smooch.save_message(message.to_json, @app_id, nil, 'relevant_search_result_requests', pm)
+      Bot::Smooch.save_message(message.to_json, @app_id, nil, 'timeout_search_requests', pm)
+      Bot::Smooch.save_message(message.to_json, @app_id, nil, 'irrelevant_search_result_requests')
+      Bot::Smooch.save_message(message.to_json, @app_id, nil, 'irrelevant_search_result_requests')
+       message = {
+        type: 'text',
+        text: text2,
+        role: 'appUser',
+        received: 1573082583.219,
+        name: random_string,
+        authorId: random_string,
+        '_id': random_string,
+        source: {
+          originalMessageId: random_string,
+          originalMessageTimestamp: 1573082582,
+          type: 'whatsapp',
+          integrationId: random_string
+        },
+      }
+      Bot::Smooch.save_message(message.to_json, @app_id, nil, 'relevant_search_result_requests', pm2)
+      Bot::Smooch.save_message(message.to_json, @app_id, nil, 'irrelevant_search_result_requests')
+      # Verify cached field
+      assert_equal 5, pm.tipline_search_results_count
+      assert_equal 2, pm.positive_tipline_search_results_count
+      assert_equal 2, pm2.tipline_search_results_count
+      assert_equal 1, pm2.positive_tipline_search_results_count
+      # Verify ES values
+      es = $repository.find(pm.get_es_doc_id)
+      assert_equal 5, es['tipline_search_results_count']
+      assert_equal 2, es['positive_tipline_search_results_count']
+      es2 = $repository.find(pm2.get_es_doc_id)
+      assert_equal 2, es2['tipline_search_results_count']
+      assert_equal 1, es2['positive_tipline_search_results_count']
+      # Verify destroy
+      DynamicAnnotation::Field.where(annotation_type: 'smooch',field_name: 'smooch_request_type')
+      .where('value IN (?)', ['"irrelevant_search_result_requests"', '"timeout_search_requests"'])
+      .joins('INNER JOIN annotations a ON a.id = dynamic_annotation_fields.annotation_id')
+      .where('a.annotated_type = ? AND a.annotated_id = ?', 'ProjectMedia', pm.id).destroy_all
+      assert_equal 2, pm.tipline_search_results_count
+      assert_equal 2, pm.positive_tipline_search_results_count
+    end
+  end
 end

From 5069d81df99724e6fd74dea04f6bcad083636e19 Mon Sep 17 00:00:00 2001
From: Mohamed El-Sawy <melsawy.be7eri@gmail.com>
Date: Mon, 9 Oct 2023 21:54:51 +0300
Subject: [PATCH 15/18] CV2-3785: limit tipline search fields and sort by
 recent activity (#1690)

* CV2-3785: limit tipline search fields and sort by recent activity
---
 app/models/concerns/smooch_search.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb
index d442c925df..343513761e 100644
--- a/app/models/concerns/smooch_search.rb
+++ b/app/models/concerns/smooch_search.rb
@@ -205,7 +205,8 @@ def should_restrict_by_language?(team_ids)
     end
 
     def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, feed_id = nil, language = nil)
-      filters = { keyword: words.join('+'), eslimit: 3 }
+      search_fields = %w(title description fact_check_title fact_check_summary extracted_text url claim_description_content')
+      filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: 3 }
       filters.merge!({ fc_language: [language] }) if should_restrict_by_language?(team_ids)
       filters.merge!({ sort: 'score' }) if words.size > 1 # We still want to be able to return the latest fact-checks if a meaninful query is not passed
       feed_id.blank? ? filters.merge!({ report_status: ['published'] }) : filters.merge!({ feed_id: feed_id })

From f151250dc56d6caf8d36c3d88a708f33177d1a65 Mon Sep 17 00:00:00 2001
From: Caio Almeida <117518+caiosba@users.noreply.github.com>
Date: Tue, 10 Oct 2023 09:14:18 -0300
Subject: [PATCH 16/18] Trying to fix flaky test

---
 .../controllers/graphql_controller_10_test.rb | 47 ---------------
 .../controllers/graphql_controller_11_test.rb | 60 +++++++++++++++++++
 2 files changed, 60 insertions(+), 47 deletions(-)
 create mode 100644 test/controllers/graphql_controller_11_test.rb

diff --git a/test/controllers/graphql_controller_10_test.rb b/test/controllers/graphql_controller_10_test.rb
index 90479327bf..4d256a7024 100644
--- a/test/controllers/graphql_controller_10_test.rb
+++ b/test/controllers/graphql_controller_10_test.rb
@@ -15,7 +15,6 @@ def setup
     Team.current = nil
   end
 
-
   test "should uninstall bot using mutation" do
     t = create_team slug: 'test'
     u = create_user
@@ -809,50 +808,4 @@ def setup
     assert_response :success
     assert !JSON.parse(@response.body)['data']['sendTiplineMessage']['success']
   end
-
-  test "should set smooch user Slack channel URL in background" do
-    u = create_user
-    t = create_team
-    create_team_user team: t, user: u, role: 'admin'
-    p = create_project team: t
-    author_id = random_string
-    set_fields = { smooch_user_data: { id: author_id }.to_json, smooch_user_app_id: 'fake', smooch_user_id: 'fake' }.to_json
-    d = create_dynamic_annotation annotated: p, annotation_type: 'smooch_user', set_fields: set_fields
-    authenticate_with_token
-    url = random_url
-    query = 'mutation { updateDynamicAnnotationSmoochUser(input: { clientMutationId: "1", id: "' + d.graphql_id + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url + '\"}" }) { project { dbid } } }'
-
-    Sidekiq::Testing.fake! do
-      post :create, params: { query: query }
-      assert_response :success
-    end
-    Sidekiq::Worker.drain_all
-    assert_equal url, Dynamic.find(d.id).get_field_value('smooch_user_slack_channel_url')
-
-    # Check that cache key exists
-    key = "SmoochUserSlackChannelUrl:Team:#{d.team_id}:#{author_id}"
-    assert_equal url, Rails.cache.read(key)
-
-    # Test using a new mutation `smoochBotAddSlackChannelUrl`
-    url2 = random_url
-    query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "' + d.id.to_s + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }'
-    Sidekiq::Testing.fake! do
-      post :create, params: { query: query }
-      assert_response :success
-    end
-    assert Sidekiq::Worker.jobs.size > 0
-    assert_equal url, d.reload.get_field_value('smooch_user_slack_channel_url')
-
-    # Execute job and check that URL was set
-    Sidekiq::Worker.drain_all
-    assert_equal url2, d.get_field_value('smooch_user_slack_channel_url')
-
-    # Check that cache key exists
-    assert_equal url2, Rails.cache.read(key)
-
-    # Call mutation with non existing ID
-    query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "99999", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }'
-    post :create, params: { query: query }
-    assert_response :success
-  end
 end
diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb
new file mode 100644
index 0000000000..890f7e3d2f
--- /dev/null
+++ b/test/controllers/graphql_controller_11_test.rb
@@ -0,0 +1,60 @@
+require_relative '../test_helper'
+require 'error_codes'
+require 'sidekiq/testing'
+
+class GraphqlController11Test < ActionController::TestCase
+  def setup
+    @controller = Api::V1::GraphqlController.new
+    TestDynamicAnnotationTables.load!
+  end
+
+  def teardown
+  end
+
+  test "should set Smooch user Slack channel URL in background" do
+    Sidekiq::Worker.clear_all
+    u = create_user
+    t = create_team
+    create_team_user team: t, user: u, role: 'admin'
+    p = create_project team: t
+    author_id = random_string
+    set_fields = { smooch_user_data: { id: author_id }.to_json, smooch_user_app_id: 'fake', smooch_user_id: 'fake' }.to_json
+    d = create_dynamic_annotation annotated: p, annotation_type: 'smooch_user', set_fields: set_fields
+    authenticate_with_token
+    url = random_url
+    query = 'mutation { updateDynamicAnnotationSmoochUser(input: { clientMutationId: "1", id: "' + d.graphql_id + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url + '\"}" }) { project { dbid } } }'
+
+    Sidekiq::Testing.fake! do
+      post :create, params: { query: query }
+      assert_response :success
+    end
+    Sidekiq::Worker.drain_all
+    assert_equal url, Dynamic.find(d.id).get_field_value('smooch_user_slack_channel_url')
+
+    # Check that cache key exists
+    key = "SmoochUserSlackChannelUrl:Team:#{d.team_id}:#{author_id}"
+    assert_equal url, Rails.cache.read(key)
+
+    # Test using a new mutation `smoochBotAddSlackChannelUrl`
+    url2 = random_url
+    query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "' + d.id.to_s + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }'
+    Sidekiq::Testing.fake! do
+      post :create, params: { query: query }
+      assert_response :success
+    end
+    assert Sidekiq::Worker.jobs.size > 0
+    assert_equal url, d.reload.get_field_value('smooch_user_slack_channel_url')
+
+    # Execute job and check that URL was set
+    Sidekiq::Worker.drain_all
+    assert_equal url2, d.get_field_value('smooch_user_slack_channel_url')
+
+    # Check that cache key exists
+    assert_equal url2, Rails.cache.read(key)
+
+    # Call mutation with non existing ID
+    query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "99999", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }'
+    post :create, params: { query: query }
+    assert_response :success
+  end
+end

From ebe860d7badbb537a8dbfa119a6120d919620ea5 Mon Sep 17 00:00:00 2001
From: Mohamed El-Sawy <melsawy.be7eri@gmail.com>
Date: Tue, 10 Oct 2023 20:56:08 +0300
Subject: [PATCH 17/18] CV2-3771: export published items without fact check to
 CSV file and copy reports to fact check (#1681)

* CV2-3771: add rake task to unpulish items with no fact check

* CV2-3771: export items into CSV and add another rake task to create fact-check from reports
---
 ...27_unpublish_items_without_fact_check.rake | 109 ++++++++++++++++++
 1 file changed, 109 insertions(+)
 create mode 100644 lib/tasks/migrate/20231003085827_unpublish_items_without_fact_check.rake

diff --git a/lib/tasks/migrate/20231003085827_unpublish_items_without_fact_check.rake b/lib/tasks/migrate/20231003085827_unpublish_items_without_fact_check.rake
new file mode 100644
index 0000000000..5ce9863a69
--- /dev/null
+++ b/lib/tasks/migrate/20231003085827_unpublish_items_without_fact_check.rake
@@ -0,0 +1,109 @@
+namespace :check do
+  namespace :migrate do
+    task export_published_items_without_fact_check: :environment do
+      started = Time.now.to_i
+      log_items = []
+      data_csv = []
+      Team.find_each do |team|
+        print '.'
+        total = 0
+        team.project_medias.find_in_batches(:batch_size => 1000) do |pms|
+          pm_ids = pms.map(&:id)
+          # Get published items
+          published_ids = Annotation.where(
+            annotation_type: "report_design",
+            annotated_type: "ProjectMedia",
+            annotated_id: pm_ids
+          ).select{ |a| a.data['state'] == 'published'}.map(&:annotated_id)
+          # Get items with fact checks
+          fact_checks_ids = ProjectMedia.where(id: published_ids)
+          .joins('INNER JOIN claim_descriptions cd ON project_medias.id = cd.project_media_id')
+          .joins('INNER JOIN fact_checks fc ON cd.id = fc.claim_description_id').map(&:id)
+          # Get published items without fact checks
+          diff = published_ids - fact_checks_ids
+          unless diff.empty?
+            total += diff.length
+            ProjectMedia.where(id: diff).find_each do |pm|
+              data_csv << [team.name, pm.full_url, pm.media_published_at]
+            end
+          end
+        end
+        log_items << { team_slug: team.slug, total: total } unless total == 0
+      end
+      unless data_csv.empty?
+        # Export items to CSV
+        require 'csv'
+        file = "#{Rails.root}/public/list_published_reports_without_fact_check_#{Time.now.to_i}.csv"
+        headers = ["Workspace", "URL", "Published at"]
+        CSV.open(file, 'w', write_headers: true, headers: headers) do |writer|
+          data_csv.each do |d|
+            writer << d
+          end
+        end
+        puts "\nExported items to file:: #{file}"
+      end
+      puts "Logs data:: #{log_items.inspect}" if log_items.length > 0
+      minutes = ((Time.now.to_i - started) / 60).to_i
+      puts "[#{Time.now}] Done in #{minutes} minutes."
+    end
+
+    # Copy reports to fact check and used `check:migrate:reports_to_fact_checks` rake task as a reference
+    task set_fact_check_for_published_items: :environment do
+      started = Time.now.to_i
+      n = 0
+      last_team_id = Rails.cache.read('check:migrate:set_fact_check_for_published_items:team_id') || 0
+      Team.where('id > ?', last_team_id).find_each do |team|
+        print '.'
+        languages = team.get_languages || ['en']
+        team.project_medias.find_in_batches(:batch_size => 1000) do |pms|
+          pm_ids = pms.map(&:id)
+          # Get published items
+          published_ids = Annotation.where(
+            annotation_type: "report_design",
+            annotated_type: "ProjectMedia",
+            annotated_id: pm_ids
+          ).select{ |a| a.data['state'] == 'published'}.map(&:annotated_id)
+          # Get items with fact checks
+          fact_checks_ids = ProjectMedia.where(id: published_ids)
+          .joins('INNER JOIN claim_descriptions cd ON project_medias.id = cd.project_media_id')
+          .joins('INNER JOIN fact_checks fc ON cd.id = fc.claim_description_id').map(&:id)
+          # Get published items without fact checks
+          diff = published_ids - fact_checks_ids
+          unless diff.empty?
+            Dynamic.where(annotation_type: "report_design", annotated_type: "ProjectMedia", annotated_id: diff).find_each do |report|
+              pm = report.annotated
+              begin
+                user_id = report.annotator_id
+                cd = pm.claim_description || ClaimDescription.create!(project_media: pm, description: '​', user_id: user_id)
+                language = report.report_design_field_value('language')
+                fc_language = languages.include?(language) ? language : 'und'
+                fields = { user_id: user_id, skip_report_update: true, language: fc_language }
+                if report.report_design_field_value('use_text_message')
+                  fields.merge!({
+                    title: report.report_design_field_value('title'),
+                    summary: report.report_design_field_value('text'),
+                    url: report.report_design_field_value('published_article_url')
+                  })
+                elsif report.report_design_field_value('use_visual_card')
+                  fields.merge!({
+                    title: report.report_design_field_value('headline'),
+                    summary: report.report_design_field_value('description'),
+                    url: report.report_design_field_value('published_article_url')
+                  })
+                end
+                fc = FactCheck.create!({ claim_description: cd }.merge(fields))
+                n += 1
+                puts "[#{Time.now}] #{n}. Created fact-check #{fc.id}"
+              rescue Exception => e
+                puts "[#{Time.now}] Could not create fact-check for report #{report.id}: #{e.message}"
+              end
+            end
+          end
+        end
+        Rails.cache.write('check:migrate:set_fact_check_for_published_items:team_id', team.id)
+      end
+      minutes = ((Time.now.to_i - started) / 60).to_i
+      puts "[#{Time.now}] Done in #{minutes} minutes."
+    end
+  end
+end
\ No newline at end of file

From 9661f3b8f3af07afb8dc270270e9cd50b72f9c59 Mon Sep 17 00:00:00 2001
From: Alexandre Amoedo Amorim <amoedoamorim@gmail.com>
Date: Wed, 11 Oct 2023 09:12:20 -0300
Subject: [PATCH 18/18] Add japanese tipline strings (#1693)

---
 config/tipline_strings.yml | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/config/tipline_strings.yml b/config/tipline_strings.yml
index 61379b6d86..9599a683dc 100644
--- a/config/tipline_strings.yml
+++ b/config/tipline_strings.yml
@@ -485,6 +485,43 @@ id:
   subscribed: Anda saat ini sedang berlangganan ke buletin kami.
   unsubscribe_button_label: Brhenti brlangganan
   unsubscribed: Saat ini Anda tidak berlangganan buletin kami.
+ja:
+  add_more_details_state_button_label: 追加
+  ask_if_ready_state_button_label: キャンセル
+  cancelled: オーケー
+  confirm_preferred_language: 選択した言語を確認してください。
+  invalid_format: "提出して頂いたファイルのフォーマットはサポートされていません。"
+  keep_subscription_button_label: 定期購読
+  languages: 言語
+  languages_and_privacy_title: 言語とプライバシー
+  main_menu: メニュー
+  main_state_button_label: キャンセル
+  navigation_button: ボタンを押して進んでください。
+  no_results_in_language: "%{language}で結果が見つかりませんでした。 関連する可能性のある他の言語での結果をいくつか示します。"
+  privacy_and_purpose: |-
+    プライバシーと目的について
+
+    %{team}ホットラインへようこそ！
+
+    あなたが検証したい情報をこのホットラインに提供してください。
+
+    あなたのデータは安全です。わたしたちはあなたの個人情報を保護することに真剣に取り組み非公開かつ安全に保ちます。シェアしたり、販売したりせず、個人識別情報（PII）をこのサービスの向上のためにのみ使用します。
+
+    拡散する誤情報を将来的に可能な限り早期発見していくために、私達はこのホットラインから得られた非PIIコンテンツを、吟味の上でリサーチャーやファクトチェック・パートナーと共有することがあります。
+
+    私達がリンクするウェブサイトには、独自のプライバシー・ポリシーがあることにご注意ください。
+
+    あなたの投稿がこの作業に使用されることを望まない場合は、私たちのシステムへの投稿をご遠慮ください。
+  privacy_statement: プライバシーに関する報告
+  privacy_title: プライバシー
+  report_updated: "次のファクトチェエックは新しい情報と共にアップデートされました"
+  search_result_is_not_relevant_button_label: "いいえ"
+  search_result_is_relevant_button_label: "はい"
+  search_state_button_label: 提出
+  subscribe_button_label: 購読
+  subscribed: あなたは現在ニュースレターを購読中です。
+  unsubscribe_button_label: 購読停止
+  unsubscribed: あなたは現在、ニュースレターの購読を停止中です。
 kn:
   add_more_details_state_button_label: ಹೆಚ್ಚು ಸೇರಿಸಿ
   ask_if_ready_state_button_label: ರದ್ದು