diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3bdba22b1f..c04b77fece 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,7 @@ build_qa: script: - apk add --no-cache curl python3 py3-pip - - pip install awscli==1.18.194 + - pip install awscli==1.29.59 - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/qa/check/api:$CI_COMMIT_SHA" . - docker push "$ECR_API_BASE_URL/qa/check/api:$CI_COMMIT_SHA" @@ -39,10 +39,10 @@ deploy_qa: GITHUB_TOKEN: $GITHUB_TOKEN script: - apk add --no-cache curl python3 py3-pip git - - pip install awscli==1.18.194 - - pip install botocore==1.17.47 - - pip install boto3==1.14.47 - - pip install ecs-deploy==1.11.0 + - pip install botocore==1.31.58 + - pip install boto3==1.28.58 + - pip install ecs-deploy==1.14.0 + - pip install awscli==1.29.59 - alias aws='docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION --rm amazon/aws-cli' - aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /qa/check-api/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/qa/check-api/##' > env.qa.names - for NAME in `cat env.qa.names`; do echo -n "-s qa-check-api-migration $NAME /qa/check-api/$NAME " >> qa-check-api-migration.env.args; done @@ -71,7 +71,7 @@ build_batch: AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY script: - apk add --no-cache curl python3 py3-pip git - - pip install awscli==1.18.194 + - pip install awscli==1.29.59 - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/batch/check/api:$CI_COMMIT_SHA" . - docker push "$ECR_API_BASE_URL/batch/check/api:$CI_COMMIT_SHA" @@ -91,7 +91,7 @@ build_live: AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY script: - apk add --no-cache curl python3 py3-pip git - - pip install awscli==1.18.194 + - pip install awscli==1.29.59 - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) - docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/live/check/api:$CI_COMMIT_SHA" . - docker push "$ECR_API_BASE_URL/live/check/api:$CI_COMMIT_SHA" @@ -114,10 +114,10 @@ deploy_live: GITHUB_TOKEN: $GITHUB_TOKEN script: - apk add --no-cache curl python3 py3-pip git - - pip install awscli==1.18.194 - - pip install botocore==1.17.47 - - pip install boto3==1.14.47 - - pip install ecs-deploy==1.11.0 + - pip install botocore==1.31.58 + - pip install boto3==1.28.58 + - pip install ecs-deploy==1.14.0 + - pip install awscli==1.29.59 - alias aws='docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION --rm amazon/aws-cli' - aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /live/check-api/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/live/check-api/##' > env.live.names - for NAME in `cat env.live.names`; do echo -n "-s live-check-api-migration $NAME /live/check-api/$NAME " >> live-check-api-migration.env.args; done diff --git a/app/controllers/api/v1/webhooks_controller.rb b/app/controllers/api/v1/webhooks_controller.rb index e08c1cc090..d489014cf6 100644 --- a/app/controllers/api/v1/webhooks_controller.rb +++ b/app/controllers/api/v1/webhooks_controller.rb @@ -7,7 +7,8 @@ def index bot_name_to_class = { smooch: Bot::Smooch, keep: Bot::Keep, - fetch: Bot::Fetch + fetch: Bot::Fetch, + alegre: Bot::Alegre, } unless bot_name_to_class.has_key?(params[:name].to_sym) render_error('Bot not found', 'ID_NOT_FOUND', 404) and return diff --git a/app/graph/mutations/graphql_crud_operations.rb b/app/graph/mutations/graphql_crud_operations.rb index d4f5ef209e..15fe74f699 100644 --- a/app/graph/mutations/graphql_crud_operations.rb +++ b/app/graph/mutations/graphql_crud_operations.rb @@ -1,5 +1,6 @@ class GraphqlCrudOperations def self.safe_save(obj, attrs, parent_names = []) + raise "Can't save a null object." if obj.nil? raise 'This operation must be done by a signed-in user' if User.current.nil? && ApiKey.current.nil? attrs.each do |key, value| method = key == "clientMutationId" ? "client_mutation_id=" : "#{key}=" @@ -192,7 +193,7 @@ def self.apply_bulk_update_or_destroy(inputs, ctx, update_or_destroy, klass) method_mapping = { update: :bulk_update, destroy: :bulk_destroy, mark_read: :bulk_mark_read } method = method_mapping[update_or_destroy.to_sym] result = klass.send(method, sql_ids, filtered_inputs, Team.current) - if update_or_destroy.to_s == "update" + if update_or_destroy.to_s != "destroy" result.merge!({ updated_objects: klass.where(id: sql_ids) }) end { ids: processed_ids }.merge(result) diff --git a/app/graph/types/project_media_type.rb b/app/graph/types/project_media_type.rb index aaed722128..f92dd09bd2 100644 --- a/app/graph/types/project_media_type.rb +++ b/app/graph/types/project_media_type.rb @@ -41,6 +41,8 @@ class ProjectMediaType < DefaultObject field :cluster, ClusterType, null: true field :is_suggested, GraphQL::Types::Boolean, null: true field :is_confirmed, GraphQL::Types::Boolean, null: true + field :positive_tipline_search_results_count, GraphQL::Types::Int, null: true + field :tipline_search_results_count, GraphQL::Types::Int, null: true field :claim_description, ClaimDescriptionType, null: true diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index 073f9293ab..693b9d2e34 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -301,6 +301,6 @@ def shared_teams end def tipline_messages(uid:) - object.tipline_messages.where(uid: uid).last(100) + object.tipline_messages.where(uid: uid).order("id DESC") end end diff --git a/app/models/blocked_tipline_user.rb b/app/models/blocked_tipline_user.rb new file mode 100644 index 0000000000..5eb56c7612 --- /dev/null +++ b/app/models/blocked_tipline_user.rb @@ -0,0 +1,2 @@ +class BlockedTiplineUser < ApplicationRecord +end diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb index b0e56c501d..d0b1e76aa3 100644 --- a/app/models/bot/alegre.rb +++ b/app/models/bot/alegre.rb @@ -6,12 +6,14 @@ class Error < ::StandardError end include AlegreSimilarity + include AlegreWebhooks # Text similarity models MEAN_TOKENS_MODEL = 'xlm-r-bert-base-nli-stsb-mean-tokens' INDIAN_MODEL = 'indian-sbert' FILIPINO_MODEL = 'paraphrase-filipino-mpnet-base-v2' OPENAI_ADA_MODEL = 'openai-text-embedding-ada-002' + PARAPHRASE_MULTILINGUAL_MODEL = 'paraphrase-multilingual-mpnet-base-v2' ELASTICSEARCH_MODEL = 'elasticsearch' DEFAULT_ES_SCORE = 10 @@ -490,7 +492,13 @@ def self.request_api(method, path, params = {}, query_or_body = 'body', retries response_body = response.body Rails.logger.info("[Alegre Bot] Alegre response: #{response_body.inspect}") ActiveRecord::Base.connection.reconnect! if RequestStore.store[:pause_database_connection] - JSON.parse(response_body) + parsed_response = JSON.parse(response_body) + if parsed_response.dig("queue") == 'audio__Model' && parsed_response.dig("body", "callback_url") != nil + redis = Redis.new(REDIS_CONFIG) + redis_response = redis.blpop("alegre:webhook:#{parsed_response.dig("body", "id")}", 120) + return JSON.parse(redis_response[1]) + end + parsed_response rescue StandardError => e if retries > 0 sleep 1 @@ -724,7 +732,7 @@ def self.send_post_create_message(source, target, relationship) end def self.relationship_model_not_allowed(relationship_model) - allowed_models = [MEAN_TOKENS_MODEL, INDIAN_MODEL, FILIPINO_MODEL, OPENAI_ADA_MODEL, ELASTICSEARCH_MODEL, 'audio', 'image', 'video'] + allowed_models = [MEAN_TOKENS_MODEL, INDIAN_MODEL, FILIPINO_MODEL, OPENAI_ADA_MODEL, PARAPHRASE_MULTILINGUAL_MODEL, ELASTICSEARCH_MODEL, 'audio', 'image', 'video'] models = relationship_model.split("|").collect{ |m| m.split('/').first } models.length != (allowed_models&models).length end diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb index 7f98916df1..3403e33020 100644 --- a/app/models/bot/smooch.rb +++ b/app/models/bot/smooch.rb @@ -31,6 +31,7 @@ class CapiUnhandledMessageWarning < MessageDeliveryError; end include SmoochMenus include SmoochFields include SmoochLanguage + include SmoochBlocking ::ProjectMedia.class_eval do attr_accessor :smooch_message @@ -380,6 +381,11 @@ def self.parse_message_based_on_state(message, app_id) return true end + if self.clicked_on_search_result_button?(message) + self.search_result_button_click_callback(message, uid, app_id, workflow, language) + return true + end + case state when 'waiting_for_message' self.bundle_message(message) @@ -538,6 +544,10 @@ def self.process_menu_option_value(value, option, message, language, workflow, a end end + def self.is_a_shortcut_for_submission?(state, message) + self.is_v2? && (state == 'main' || state == 'waiting_for_message') && (!message['mediaUrl'].blank? || ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer)) + end + def self.process_menu_option(message, state, app_id) uid = message['authorId'] sm = CheckStateMachine.new(uid) @@ -577,8 +587,15 @@ def self.process_menu_option(message, state, app_id) return true end end + # Lastly, check if it's a submission shortcut + if self.is_a_shortcut_for_submission?(sm.state, message) + self.bundle_message(message) + sm.go_to_ask_if_ready + self.send_message_for_state(uid, workflow, 'ask_if_ready', language) + return true + end self.bundle_message(message) - return false + false end def self.user_received_report(message) @@ -789,19 +806,6 @@ def self.add_hashtags(text, pm) end end - def self.ban_user(message) - unless message.nil? - uid = message['authorId'] - Rails.logger.info("[Smooch Bot] Banned user #{uid}") - Rails.cache.write("smooch:banned:#{uid}", message.to_json) - end - end - - def self.user_banned?(payload) - uid = payload.dig('appUser', '_id') - !uid.blank? && !Rails.cache.read("smooch:banned:#{uid}").nil? - end - # Don't save as a ProjectMedia if it contains only menu options def self.is_a_valid_text_message?(text) !text.split(/#{MESSAGE_BOUNDARY}|\s+/).reject{ |m| m =~ /^[0-9]*$/ }.empty? @@ -821,6 +825,10 @@ def self.save_text_message(message) claim = self.extract_claim(text).gsub(/\s+/, ' ').strip extra = { quote: claim } pm = ProjectMedia.joins(:media).where('trim(lower(quote)) = ?', claim.downcase).where('project_medias.team_id' => team.id).last + # Don't create a new text media if it's an unconfirmed request with just a few words + if pm.nil? && message['archived'] == CheckArchivedFlags::FlagCodes::UNCONFIRMED && ::Bot::Alegre.get_number_of_words(claim) < CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer) + return team + end else extra = { url: link.url } pm = ProjectMedia.joins(:media).where('medias.url' => link.url, 'project_medias.team_id' => team.id).last diff --git a/app/models/concerns/alegre_similarity.rb b/app/models/concerns/alegre_similarity.rb index d331405478..bc5d6faf61 100644 --- a/app/models/concerns/alegre_similarity.rb +++ b/app/models/concerns/alegre_similarity.rb @@ -19,7 +19,7 @@ def get_similar_items(pm) type = Bot::Alegre.get_pm_type(pm) Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 2/5] Type is #{type.blank? ? "blank" : type}" unless type.blank? - if !self.should_get_similar_items_of_type?('master', pm.team_id) || !self.should_get_similar_items_of_type?(type, pm.team_id) + if !Bot::Alegre.should_get_similar_items_of_type?('master', pm.team_id) || !Bot::Alegre.should_get_similar_items_of_type?(type, pm.team_id) Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 3/5] ProjectMedia cannot be checked for similar items" return {} else diff --git a/app/models/concerns/alegre_webhooks.rb b/app/models/concerns/alegre_webhooks.rb new file mode 100644 index 0000000000..4b71b2d582 --- /dev/null +++ b/app/models/concerns/alegre_webhooks.rb @@ -0,0 +1,26 @@ +class AlegreCallbackError < StandardError +end + +module AlegreWebhooks + extend ActiveSupport::Concern + + module ClassMethods + def valid_request?(request) + token = request.params['token'] || request.query_parameters['token'] + !token.blank? && token == CheckConfig.get('alegre_token') + end + + def webhook(request) + begin + doc_id = request.params.dig('data', 'requested', 'body', 'id') + raise 'Unexpected params format' if doc_id.blank? + redis = Redis.new(REDIS_CONFIG) + key = "alegre:webhook:#{doc_id}" + redis.lpush(key, request.params.to_json) + redis.expire(key, 1.day.to_i) + rescue StandardError => e + CheckSentry.notify(AlegreCallbackError.new(e.message), { alegre_response: request.params }) + end + end + end +end diff --git a/app/models/concerns/project_media_cached_fields.rb b/app/models/concerns/project_media_cached_fields.rb index 1476300167..8d31662057 100644 --- a/app/models/concerns/project_media_cached_fields.rb +++ b/app/models/concerns/project_media_cached_fields.rb @@ -446,6 +446,36 @@ def title_or_description_update }, ] + cached_field :positive_tipline_search_results_count, + update_es: true, + recalculate: :recalculate_positive_tipline_search_results_count, + update_on: [ + { + model: DynamicAnnotation::Field, + if: proc { |f| f.field_name == 'smooch_request_type' && f.value == 'relevant_search_result_requests' }, + affected_ids: proc { |f| [f.annotation&.annotated_id.to_i] }, + events: { + save: :recalculate, + destroy: :recalculate, + } + } + ] + + cached_field :tipline_search_results_count, + update_es: true, + recalculate: :recalculate_tipline_search_results_count, + update_on: [ + { + model: DynamicAnnotation::Field, + if: proc { |f| f.field_name == 'smooch_request_type' && ['relevant_search_result_requests', 'irrelevant_search_result_requests', 'timeout_search_requests'].include?(f.value) }, + affected_ids: proc { |f| [f.annotation&.annotated_id.to_i] }, + events: { + save: :recalculate, + destroy: :recalculate, + } + } + ] + def recalculate_linked_items_count count = Relationship.send('confirmed').where(source_id: self.id).count count += 1 unless self.media.type == 'Blank' @@ -605,6 +635,19 @@ def cached_field_tags_as_sentence_es(value) def cached_field_published_by_es(value) value.keys.first || 0 end + + def recalculate_positive_tipline_search_results_count + DynamicAnnotation::Field.where(annotation_type: 'smooch',field_name: 'smooch_request_type', value: 'relevant_search_result_requests') + .joins('INNER JOIN annotations a ON a.id = dynamic_annotation_fields.annotation_id') + .where('a.annotated_type = ? AND a.annotated_id = ?', 'ProjectMedia', self.id).count + end + + def recalculate_tipline_search_results_count + DynamicAnnotation::Field.where(annotation_type: 'smooch',field_name: 'smooch_request_type') + .where('value IN (?)', ['"relevant_search_result_requests"', '"irrelevant_search_result_requests"', '"timeout_search_requests"']) + .joins('INNER JOIN annotations a ON a.id = dynamic_annotation_fields.annotation_id') + .where('a.annotated_type = ? AND a.annotated_id = ?', 'ProjectMedia', self.id).count + end end DynamicAnnotation::Field.class_eval do diff --git a/app/models/concerns/smooch_blocking.rb b/app/models/concerns/smooch_blocking.rb new file mode 100644 index 0000000000..cdf94b9e48 --- /dev/null +++ b/app/models/concerns/smooch_blocking.rb @@ -0,0 +1,46 @@ +require 'active_support/concern' + +module SmoochBlocking + extend ActiveSupport::Concern + + module ClassMethods + def ban_user(message) + unless message.nil? + uid = message['authorId'] + self.block_user(uid) + end + end + + def block_user_from_error_code(uid, error_code) + self.block_user(uid) if error_code == 131056 # Error of type "pair rate limit hit" + end + + def block_user(uid) + begin + block = BlockedTiplineUser.new(uid: uid) + block.skip_check_ability = true + block.save! + Rails.logger.info("[Smooch Bot] Blocked user #{uid}") + Rails.cache.write("smooch:banned:#{uid}", Time.now.to_i) + rescue ActiveRecord::RecordNotUnique + # User already blocked + Rails.logger.info("[Smooch Bot] User #{uid} already blocked") + end + end + + def unblock_user(uid) + BlockedTiplineUser.where(uid: uid).last.destroy! + Rails.logger.info("[Smooch Bot] Unblocked user #{uid}") + Rails.cache.delete("smooch:banned:#{uid}") + end + + def user_blocked?(uid) + !uid.blank? && (!Rails.cache.read("smooch:banned:#{uid}").nil? || BlockedTiplineUser.where(uid: uid).exists?) + end + + def user_banned?(payload) + uid = payload.dig('appUser', '_id') + self.user_blocked?(uid) + end + end +end diff --git a/app/models/concerns/smooch_capi.rb b/app/models/concerns/smooch_capi.rb index 4ad790ea7c..469ce56d51 100644 --- a/app/models/concerns/smooch_capi.rb +++ b/app/models/concerns/smooch_capi.rb @@ -281,13 +281,17 @@ def capi_send_message_to_user(uid, text, extra = {}, _force = false, preview_url response = http.request(req) if response.code.to_i >= 400 error_message = begin JSON.parse(response.body)['error']['message'] rescue response.body end + error_code = begin JSON.parse(response.body)['error']['code'] rescue nil end e = Bot::Smooch::CapiMessageDeliveryError.new(error_message) + self.block_user_from_error_code(uid, error_code) CheckSentry.notify(e, uid: uid, type: payload.dig(:type), template_name: payload.dig(:template, :name), template_language: payload.dig(:template, :language, :code), - error: response.body + error: response.body, + error_message: error_message, + error_code: error_code ) end response diff --git a/app/models/concerns/smooch_menus.rb b/app/models/concerns/smooch_menus.rb index 755e40d155..7f0469508b 100644 --- a/app/models/concerns/smooch_menus.rb +++ b/app/models/concerns/smooch_menus.rb @@ -166,7 +166,7 @@ def get_custom_string(key, language, truncate_at = 1024) label.to_s.truncate(truncate_at) end - def send_message_to_user_with_buttons(uid, text, options) + def send_message_to_user_with_buttons(uid, text, options, image_url = nil) buttons = [] options.each_with_index do |option, i| buttons << { @@ -196,6 +196,12 @@ def send_message_to_user_with_buttons(uid, text, options) } } } + extra[:override][:whatsapp][:payload][:interactive][:header] = { + type: 'image', + image: { + link: CheckS3.rewrite_url(image_url) + } + } unless image_url.blank? extra, fallback = self.format_fallback_text_menu_from_options(text, options, extra) self.send_message_to_user(uid, fallback.join("\n"), extra) end diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index c8f69a9461..fb433182f9 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -335,10 +335,10 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req self.get_installation(self.installation_setting_id_keys, app_id) Team.current = Team.where(id: self.config['team_id']).last annotated = nil - if ['default_requests', 'timeout_requests', 'resource_requests', 'irrelevant_search_result_requests'].include?(request_type) + if ['default_requests', 'timeout_requests', 'irrelevant_search_result_requests'].include?(request_type) message['archived'] = ['default_requests', 'irrelevant_search_result_requests'].include?(request_type) ? self.default_archived_flag : CheckArchivedFlags::FlagCodes::UNCONFIRMED annotated = self.create_project_media_from_message(message) - elsif ['menu_options_requests', 'relevant_search_result_requests', 'timeout_search_requests'].include?(request_type) + elsif ['menu_options_requests', 'relevant_search_result_requests', 'timeout_search_requests', 'resource_requests'].include?(request_type) annotated = annotated_obj end diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index cbd339bd3a..343513761e 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -19,10 +19,15 @@ def search(app_id, uid, language, message, team_id, workflow) self.bundle_messages(uid, '', app_id, 'default_requests', nil, true) self.send_final_message_to_user(uid, self.get_custom_string('search_no_results', language), workflow, language) else - self.send_search_results_to_user(uid, results, team_id) - sm.go_to_search_result - self.save_search_results_for_user(uid, results.map(&:id)) - self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, 1) + self.send_search_results_to_user(uid, results, team_id, platform, app_id) + # For WhatsApp, each search result goes with a button where the user can give feedback individually, so, reset the conversation right away + if platform == 'WhatsApp' + sm.reset + else + sm.go_to_search_result + self.save_search_results_for_user(uid, results.map(&:id)) + self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, 1) + end end rescue StandardError => e self.handle_search_error(uid, e, language) @@ -200,7 +205,8 @@ def should_restrict_by_language?(team_ids) end def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, feed_id = nil, language = nil) - filters = { keyword: words.join('+'), eslimit: 3 } + search_fields = %w(title description fact_check_title fact_check_summary extracted_text url claim_description_content') + filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: 3 } filters.merge!({ fc_language: [language] }) if should_restrict_by_language?(team_ids) filters.merge!({ sort: 'score' }) if words.size > 1 # We still want to be able to return the latest fact-checks if a meaninful query is not passed feed_id.blank? ? filters.merge!({ report_status: ['published'] }) : filters.merge!({ feed_id: feed_id }) @@ -214,21 +220,51 @@ def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, results end - def send_search_results_to_user(uid, results, team_id) + def send_search_results_to_user(uid, results, team_id, platform, app_id) team = Team.find(team_id) - redis = Redis.new(REDIS_CONFIG) language = self.get_user_language(uid) - reports = results.collect{ |r| r.get_dynamic_annotation('report_design') } + reports = results.collect{ |r| r.get_dynamic_annotation('report_design') }.reject{ |r| r.blank? } # Get reports languages - reports_language = reports.map{|r| r&.report_design_field_value('language')}.uniq + reports_language = reports.map{ |r| r.report_design_field_value('language') }.uniq if team.get_languages.to_a.size > 1 && !reports_language.include?(language) self.send_message_to_user(uid, self.get_string(:no_results_in_language, language).gsub('%{language}', CheckCldr.language_code_to_name(language, language))) sleep 1 end + if platform == 'WhatsApp' + self.send_search_results_to_whatsapp_user(uid, reports, app_id) + else + self.send_search_results_to_non_whatsapp_user(uid, reports) + end + end + + def generate_search_id + SecureRandom.hex + end + + def send_search_results_to_whatsapp_user(uid, reports, app_id) + search_id = self.generate_search_id + # Cache the current bundle of messages from this user related to this search, so a request can be created correctly + # Expires after the time to give feedback is expired + Rails.cache.write("smooch:user_search_bundle:#{uid}:#{search_id}", self.list_of_bundled_messages_from_user(uid), expires_in: 20.minutes) + self.clear_user_bundled_messages(uid) + reports.each do |report| + text = report.report_design_text if report.report_design_field_value('use_text_message') + image_url = report.report_design_image_url if report.report_design_field_value('use_visual_card') + options = [{ + value: { project_media_id: report.annotated_id, keyword: 'search_result_is_relevant', search_id: search_id }.to_json, + label: '👍' + }] + self.send_message_to_user_with_buttons(uid, text || '-', options, image_url) # "text" is mandatory for WhatsApp interactive messages + self.delay_for(15.minutes, { queue: 'smooch_priority' }).timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, report.annotated_id) + end + end + + def send_search_results_to_non_whatsapp_user(uid, reports) + redis = Redis.new(REDIS_CONFIG) reports.each do |report| response = nil - response = self.send_message_to_user(uid, report.report_design_text) if report&.report_design_field_value('use_text_message') - response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }) if !report&.report_design_field_value('use_text_message') && report&.report_design_field_value('use_visual_card') + response = self.send_message_to_user(uid, report.report_design_text) if report.report_design_field_value('use_text_message') + response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }) if !report.report_design_field_value('use_text_message') && report.report_design_field_value('use_visual_card') id = self.get_id_from_send_response(response) redis.rpush("smooch:search:#{uid}", id) unless id.blank? end @@ -253,5 +289,34 @@ def ask_for_feedback_when_all_search_results_are_received(app_id, language, work self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, attempts + 1) if attempts < max # Try for 20 seconds end end + + def timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, pmid) + key = "smooch:user_search_bundle:#{uid}:#{search_id}:#{pmid}" + if Rails.cache.read(key).nil? # User gave no feedback for the search result + bundle = Rails.cache.read("smooch:user_search_bundle:#{uid}:#{search_id}").to_a + self.delay_for(1.seconds, { queue: 'smooch', retry: false }).bundle_messages(uid, nil, app_id, 'timeout_search_requests', [ProjectMedia.find(pmid)], true, bundle) + else + Rails.cache.delete(key) # User gave feedback to search result + end + end + + def clicked_on_search_result_button?(message) + begin + JSON.parse(message['payload'])['keyword'] == 'search_result_is_relevant' + rescue + false + end + end + + def search_result_button_click_callback(message, uid, app_id, workflow, language) + payload = JSON.parse(message['payload']) + result = ProjectMedia.find(payload['project_media_id']) + bundle = Rails.cache.read("smooch:user_search_bundle:#{uid}:#{payload['search_id']}").to_a + unless bundle.empty? + Rails.cache.write("smooch:user_search_bundle:#{uid}:#{payload['search_id']}:#{result.id}", Time.now.to_i) # Store that the user has given feedback to this search result + self.delay_for(1.seconds, { queue: 'smooch', retry: false }).bundle_messages(uid, message['_id'], app_id, 'relevant_search_result_requests', [result], true, bundle) + self.send_final_message_to_user(uid, self.get_custom_string('search_result_is_relevant', language), workflow, language) + end + end end end diff --git a/app/models/concerns/tipline_content_image.rb b/app/models/concerns/tipline_content_image.rb index faa902dd83..a85c8af8c1 100644 --- a/app/models/concerns/tipline_content_image.rb +++ b/app/models/concerns/tipline_content_image.rb @@ -30,7 +30,12 @@ def convert_header_file_image temp_name = 'temp-' + self.id.to_s + '-' + self.language + '.html' temp = File.join(Rails.root, 'public', content_name, temp_name) output = File.open(temp, 'w+') - output.puts doc.to_s.gsub('%IMAGE_URL%', CheckS3.rewrite_url(self.header_file_url.to_s)) + + # Replace the image in the template + image_url = CheckS3.rewrite_url(self.header_file_url.to_s) + w, h = ::MiniMagick::Image.open(image_url)[:dimensions] + image_class = w > h ? 'wider' : 'taller' + output.puts doc.to_s.gsub('%IMAGE_URL%', image_url).gsub('%IMAGE_CLASS%', image_class) output.close # Upload the HTML to S3 @@ -39,7 +44,7 @@ def convert_header_file_image temp_url = CheckS3.public_url(path) # Convert the HTML to PNG - uri = URI("#{CheckConfig.get('narcissus_url')}/?url=#{CheckS3.rewrite_url(temp_url)}&selector=%23frame") + uri = URI("#{CheckConfig.get('narcissus_url')}/?selector=img&url=#{CheckS3.rewrite_url(temp_url)}") request = Net::HTTP::Get.new(uri) request['x-api-key'] = CheckConfig.get('narcissus_token') response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') { |http| http.request(request) } diff --git a/app/repositories/media_search.rb b/app/repositories/media_search.rb index 33e969062a..3fcf8739b5 100644 --- a/app/repositories/media_search.rb +++ b/app/repositories/media_search.rb @@ -136,5 +136,9 @@ class MediaSearch indexes :report_language, { type: 'keyword', normalizer: 'check' } indexes :fact_check_published_on, { type: 'long' } + + indexes :positive_tipline_search_results_count, { type: 'long' } + + indexes :tipline_search_results_count, { type: 'long' } end end diff --git a/config/config.yml.example b/config/config.yml.example index 73aabf012e..63a205f99e 100644 --- a/config/config.yml.example +++ b/config/config.yml.example @@ -11,7 +11,6 @@ development: &default elasticsearch_index: elasticsearch_log: true elasticsearch_sync: false - # WARNING For production, don't use a wildcard: set the allowed domains explicitly as a regular expression, e.g. # '(https?://.*\.?(meedan.com|meedan.org))' allowed_origins: '.*' @@ -44,6 +43,7 @@ development: &default image_cluster_similarity_threshold: 0.9 text_cluster_similarity_threshold: 0.9 similarity_media_file_url_host: '' + min_number_of_words_for_tipline_submit_shortcut: 10 # Localization # @@ -286,6 +286,7 @@ test: otel_log_level: error otel_traces_sampler: sentry_dsn: + storage_rewrite_host: 'http://minio:9000' # Facebook social login # diff --git a/config/tipline_strings.yml b/config/tipline_strings.yml index 61379b6d86..9599a683dc 100644 --- a/config/tipline_strings.yml +++ b/config/tipline_strings.yml @@ -485,6 +485,43 @@ id: subscribed: Anda saat ini sedang berlangganan ke buletin kami. unsubscribe_button_label: Brhenti brlangganan unsubscribed: Saat ini Anda tidak berlangganan buletin kami. +ja: + add_more_details_state_button_label: 追加 + ask_if_ready_state_button_label: キャンセル + cancelled: オーケー + confirm_preferred_language: 選択した言語を確認してください。 + invalid_format: "提出して頂いたファイルのフォーマットはサポートされていません。" + keep_subscription_button_label: 定期購読 + languages: 言語 + languages_and_privacy_title: 言語とプライバシー + main_menu: メニュー + main_state_button_label: キャンセル + navigation_button: ボタンを押して進んでください。 + no_results_in_language: "%{language}で結果が見つかりませんでした。 関連する可能性のある他の言語での結果をいくつか示します。" + privacy_and_purpose: |- + プライバシーと目的について + + %{team}ホットラインへようこそ! + + あなたが検証したい情報をこのホットラインに提供してください。 + + あなたのデータは安全です。わたしたちはあなたの個人情報を保護することに真剣に取り組み非公開かつ安全に保ちます。シェアしたり、販売したりせず、個人識別情報(PII)をこのサービスの向上のためにのみ使用します。 + + 拡散する誤情報を将来的に可能な限り早期発見していくために、私達はこのホットラインから得られた非PIIコンテンツを、吟味の上でリサーチャーやファクトチェック・パートナーと共有することがあります。 + + 私達がリンクするウェブサイトには、独自のプライバシー・ポリシーがあることにご注意ください。 + + あなたの投稿がこの作業に使用されることを望まない場合は、私たちのシステムへの投稿をご遠慮ください。 + privacy_statement: プライバシーに関する報告 + privacy_title: プライバシー + report_updated: "次のファクトチェエックは新しい情報と共にアップデートされました" + search_result_is_not_relevant_button_label: "いいえ" + search_result_is_relevant_button_label: "はい" + search_state_button_label: 提出 + subscribe_button_label: 購読 + subscribed: あなたは現在ニュースレターを購読中です。 + unsubscribe_button_label: 購読停止 + unsubscribed: あなたは現在、ニュースレターの購読を停止中です。 kn: add_more_details_state_button_label: ಹೆಚ್ಚು ಸೇರಿಸಿ ask_if_ready_state_button_label: ರದ್ದು diff --git a/db/migrate/20231002202443_create_blocked_tipline_users.rb b/db/migrate/20231002202443_create_blocked_tipline_users.rb new file mode 100644 index 0000000000..3cc9c5bdaa --- /dev/null +++ b/db/migrate/20231002202443_create_blocked_tipline_users.rb @@ -0,0 +1,9 @@ +class CreateBlockedTiplineUsers < ActiveRecord::Migration[6.1] + def change + create_table :blocked_tipline_users do |t| + t.string :uid, null: false + t.timestamps + end + add_index :blocked_tipline_users, :uid, unique: true + end +end diff --git a/db/migrate/20231008074526_add_mapping_for_tipline_search_results_fields.rb b/db/migrate/20231008074526_add_mapping_for_tipline_search_results_fields.rb new file mode 100644 index 0000000000..9c038d4cf8 --- /dev/null +++ b/db/migrate/20231008074526_add_mapping_for_tipline_search_results_fields.rb @@ -0,0 +1,14 @@ +class AddMappingForTiplineSearchResultsFields < ActiveRecord::Migration[6.1] + def change + options = { + index: CheckElasticSearchModel.get_index_alias, + body: { + properties: { + positive_tipline_search_results_count: { type: 'long' }, + tipline_search_results_count: { type: 'long' }, + } + } + } + $repository.client.indices.put_mapping options + end +end diff --git a/db/schema.rb b/db/schema.rb index c8beea6b6d..e75733f01b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_09_22_174044) do +ActiveRecord::Schema.define(version: 2023_10_08_074526) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -195,6 +195,13 @@ t.index ["user_id"], name: "index_assignments_on_user_id" end + create_table "blocked_tipline_users", force: :cascade do |t| + t.string "uid", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["uid"], name: "index_blocked_tipline_users_on_uid", unique: true + end + create_table "bounces", id: :serial, force: :cascade do |t| t.string "email", null: false t.datetime "created_at", null: false @@ -261,7 +268,7 @@ t.jsonb "value_json", default: "{}" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY ((ARRAY['external_id'::character varying, 'smooch_user_id'::character varying, 'verification_status_status'::character varying])::text[]))" + t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY (ARRAY[('external_id'::character varying)::text, ('smooch_user_id'::character varying)::text, ('verification_status_status'::character varying)::text]))" t.index ["annotation_id", "field_name"], name: "index_dynamic_annotation_fields_on_annotation_id_and_field_name" t.index ["annotation_id"], name: "index_dynamic_annotation_fields_on_annotation_id" t.index ["annotation_type"], name: "index_dynamic_annotation_fields_on_annotation_type" diff --git a/lib/relay.idl b/lib/relay.idl index 2303bb732a..9a958332b7 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -11211,6 +11211,7 @@ type ProjectMedia implements Node { oembed_metadata: String permissions: String picture: String + positive_tipline_search_results_count: Int project: Project project_group: ProjectGroup project_id: Int @@ -11336,6 +11337,7 @@ type ProjectMedia implements Node { tasks_count: JsonStringType team: Team team_name: String + tipline_search_results_count: Int title: String type: String updated_at: String diff --git a/lib/sample_data.rb b/lib/sample_data.rb index bd851ff9bf..7bacb897f1 100644 --- a/lib/sample_data.rb +++ b/lib/sample_data.rb @@ -1096,4 +1096,8 @@ def create_task_stuff(delete_existing = true) at = create_annotation_type annotation_type: 'task_response_free_text', label: 'Task Response Free Text' create_field_instance annotation_type_object: at, name: 'response_free_text', label: 'Response', field_type_object: text, optional: false end + + def create_blocked_tipline_user(options = {}) + BlockedTiplineUser.create!({ uid: random_string }.merge(options)) + end end diff --git a/lib/tasks/migrate/20231003085827_unpublish_items_without_fact_check.rake b/lib/tasks/migrate/20231003085827_unpublish_items_without_fact_check.rake new file mode 100644 index 0000000000..5ce9863a69 --- /dev/null +++ b/lib/tasks/migrate/20231003085827_unpublish_items_without_fact_check.rake @@ -0,0 +1,109 @@ +namespace :check do + namespace :migrate do + task export_published_items_without_fact_check: :environment do + started = Time.now.to_i + log_items = [] + data_csv = [] + Team.find_each do |team| + print '.' + total = 0 + team.project_medias.find_in_batches(:batch_size => 1000) do |pms| + pm_ids = pms.map(&:id) + # Get published items + published_ids = Annotation.where( + annotation_type: "report_design", + annotated_type: "ProjectMedia", + annotated_id: pm_ids + ).select{ |a| a.data['state'] == 'published'}.map(&:annotated_id) + # Get items with fact checks + fact_checks_ids = ProjectMedia.where(id: published_ids) + .joins('INNER JOIN claim_descriptions cd ON project_medias.id = cd.project_media_id') + .joins('INNER JOIN fact_checks fc ON cd.id = fc.claim_description_id').map(&:id) + # Get published items without fact checks + diff = published_ids - fact_checks_ids + unless diff.empty? + total += diff.length + ProjectMedia.where(id: diff).find_each do |pm| + data_csv << [team.name, pm.full_url, pm.media_published_at] + end + end + end + log_items << { team_slug: team.slug, total: total } unless total == 0 + end + unless data_csv.empty? + # Export items to CSV + require 'csv' + file = "#{Rails.root}/public/list_published_reports_without_fact_check_#{Time.now.to_i}.csv" + headers = ["Workspace", "URL", "Published at"] + CSV.open(file, 'w', write_headers: true, headers: headers) do |writer| + data_csv.each do |d| + writer << d + end + end + puts "\nExported items to file:: #{file}" + end + puts "Logs data:: #{log_items.inspect}" if log_items.length > 0 + minutes = ((Time.now.to_i - started) / 60).to_i + puts "[#{Time.now}] Done in #{minutes} minutes." + end + + # Copy reports to fact check and used `check:migrate:reports_to_fact_checks` rake task as a reference + task set_fact_check_for_published_items: :environment do + started = Time.now.to_i + n = 0 + last_team_id = Rails.cache.read('check:migrate:set_fact_check_for_published_items:team_id') || 0 + Team.where('id > ?', last_team_id).find_each do |team| + print '.' + languages = team.get_languages || ['en'] + team.project_medias.find_in_batches(:batch_size => 1000) do |pms| + pm_ids = pms.map(&:id) + # Get published items + published_ids = Annotation.where( + annotation_type: "report_design", + annotated_type: "ProjectMedia", + annotated_id: pm_ids + ).select{ |a| a.data['state'] == 'published'}.map(&:annotated_id) + # Get items with fact checks + fact_checks_ids = ProjectMedia.where(id: published_ids) + .joins('INNER JOIN claim_descriptions cd ON project_medias.id = cd.project_media_id') + .joins('INNER JOIN fact_checks fc ON cd.id = fc.claim_description_id').map(&:id) + # Get published items without fact checks + diff = published_ids - fact_checks_ids + unless diff.empty? + Dynamic.where(annotation_type: "report_design", annotated_type: "ProjectMedia", annotated_id: diff).find_each do |report| + pm = report.annotated + begin + user_id = report.annotator_id + cd = pm.claim_description || ClaimDescription.create!(project_media: pm, description: '', user_id: user_id) + language = report.report_design_field_value('language') + fc_language = languages.include?(language) ? language : 'und' + fields = { user_id: user_id, skip_report_update: true, language: fc_language } + if report.report_design_field_value('use_text_message') + fields.merge!({ + title: report.report_design_field_value('title'), + summary: report.report_design_field_value('text'), + url: report.report_design_field_value('published_article_url') + }) + elsif report.report_design_field_value('use_visual_card') + fields.merge!({ + title: report.report_design_field_value('headline'), + summary: report.report_design_field_value('description'), + url: report.report_design_field_value('published_article_url') + }) + end + fc = FactCheck.create!({ claim_description: cd }.merge(fields)) + n += 1 + puts "[#{Time.now}] #{n}. Created fact-check #{fc.id}" + rescue Exception => e + puts "[#{Time.now}] Could not create fact-check for report #{report.id}: #{e.message}" + end + end + end + end + Rails.cache.write('check:migrate:set_fact_check_for_published_items:team_id', team.id) + end + minutes = ((Time.now.to_i - started) / 60).to_i + puts "[#{Time.now}] Done in #{minutes} minutes." + end + end +end \ No newline at end of file diff --git a/public/relay.json b/public/relay.json index df8894a54b..9c6da4dd74 100644 --- a/public/relay.json +++ b/public/relay.json @@ -58978,6 +58978,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "positive_tipline_search_results_count", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "project", "description": null, @@ -59561,6 +59575,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "tipline_search_results_count", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "title", "description": null, diff --git a/public/tipline-content-template.html b/public/tipline-content-template.html index 6a6bd4c22b..143597e2f8 100644 --- a/public/tipline-content-template.html +++ b/public/tipline-content-template.html @@ -1,7 +1,7 @@
-