From 3e0449e257b9ea5d98a3595613aa6192a55527a5 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Mon, 16 Oct 2023 11:12:23 -0300 Subject: [PATCH 01/54] A few fixes/changes for `TiplineMessages` API * Set `event` for tipline messages that are sent * Return media URL for tipline messages on GraphQL API Reference: CV2-3853. --- app/graph/types/team_type.rb | 2 +- app/graph/types/tipline_message_type.rb | 1 + app/models/bot/smooch.rb | 15 +++++++------- app/models/concerns/smooch_menus.rb | 8 ++++---- app/models/concerns/smooch_messages.rb | 16 +++++++-------- app/models/concerns/smooch_resources.rb | 4 ++-- app/models/concerns/smooch_search.rb | 10 +++++----- app/models/tipline_message.rb | 12 ++++++++++++ app/models/tipline_newsletter.rb | 2 +- app/workers/tipline_newsletter_worker.rb | 2 +- lib/relay.idl | 1 + public/relay.json | 14 +++++++++++++ test/models/tipline_message_test.rb | 25 ++++++++++++++++++++++++ 13 files changed, 83 insertions(+), 29 deletions(-) diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index 693b9d2e34..dbd44828fb 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -301,6 +301,6 @@ def shared_teams end def tipline_messages(uid:) - object.tipline_messages.where(uid: uid).order("id DESC") + object.tipline_messages.where(uid: uid).order('sent_at DESC') end end diff --git a/app/graph/types/tipline_message_type.rb b/app/graph/types/tipline_message_type.rb index 4fcbd51939..7bc364551c 100644 --- a/app/graph/types/tipline_message_type.rb +++ b/app/graph/types/tipline_message_type.rb @@ -15,6 +15,7 @@ class TiplineMessageType < DefaultObject field :state, GraphQL::Types::String, null: true field :team, TeamType, null: true field :sent_at, GraphQL::Types::String, null: true, camelize: false + field :media_url, GraphQL::Types::String, null: true def sent_at object.sent_at.to_i.to_s diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb index 3403e33020..9f4057bd29 100644 --- a/app/models/bot/smooch.rb +++ b/app/models/bot/smooch.rb @@ -399,7 +399,7 @@ def self.parse_message_based_on_state(message, app_id) end when 'main', 'secondary', 'subscription', 'search_result' unless self.process_menu_option(message, state, app_id) - self.send_message_for_state(uid, workflow, state, language, self.get_custom_string(:option_not_available, language)) + self.send_message_for_state(uid, workflow, state, language, self.get_custom_string(:option_not_available, language), 'option_not_available') end when 'search' self.send_message_to_user(uid, self.get_message_for_state(workflow, state, language, uid)) @@ -710,7 +710,7 @@ def self.send_error_message(message, is_supported) self.send_message_to_user(message['authorId'], error_message) end - def self.send_message_to_user(uid, text, extra = {}, force = false, preview_url = true) + def self.send_message_to_user(uid, text, extra = {}, force = false, preview_url = true, event = nil) return if self.config['smooch_disabled'] && !force if RequestStore.store[:smooch_bot_provider] == 'TURN' response = self.turnio_send_message_to_user(uid, text, extra, force, preview_url) @@ -719,22 +719,23 @@ def self.send_message_to_user(uid, text, extra = {}, force = false, preview_url else response = self.zendesk_send_message_to_user(uid, text, extra, force, preview_url) end - # store a TiplineMessage + # Store a TiplineMessage external_id = self.get_id_from_send_response(response) sent_at = DateTime.now payload_json = { text: text }.merge(extra).to_json team_id = self.config['team_id'].to_i platform = RequestStore.store[:smooch_bot_platform] || 'Unknown' language = self.get_user_language(uid) - self.delay.store_sent_tipline_message(uid, external_id, sent_at, payload_json, team_id, platform, language) + self.delay.store_sent_tipline_message(uid, external_id, sent_at, payload_json, team_id, platform, language, event) response end - def self.store_sent_tipline_message(uid, external_id, sent_at, payload_json, team_id, platform, language) + def self.store_sent_tipline_message(uid, external_id, sent_at, payload_json, team_id, platform, language, event = nil) payload = JSON.parse(payload_json) tm = TiplineMessage.new tm.uid = uid tm.state = 'sent' + tm.event = event tm.direction = :outgoing tm.language = language tm.platform = platform @@ -976,10 +977,10 @@ def self.send_report_to_user(uid, data, pm, lang = 'en', fallback_template = nil end if report.report_design_field_value('use_text_message') workflow = self.get_workflow(lang) - last_smooch_response = self.send_final_messages_to_user(uid, report.report_design_text(lang), workflow, lang) + last_smooch_response = self.send_final_messages_to_user(uid, report.report_design_text(lang), workflow, lang, 1, true, 'report') Rails.logger.info "[Smooch Bot] Sent text report to user #{uid} for item with ID #{pm.id}, response was: #{last_smooch_response&.body}" elsif report.report_design_field_value('use_visual_card') - last_smooch_response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }) + last_smooch_response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }, false, true, 'report') Rails.logger.info "[Smooch Bot] Sent report visual card to user #{uid} for item with ID #{pm.id}, response was: #{last_smooch_response&.body}" end self.save_smooch_response(last_smooch_response, parent, data['received'], fallback_template, lang) diff --git a/app/models/concerns/smooch_menus.rb b/app/models/concerns/smooch_menus.rb index 7f0469508b..49d5b53efa 100644 --- a/app/models/concerns/smooch_menus.rb +++ b/app/models/concerns/smooch_menus.rb @@ -8,7 +8,7 @@ def is_v2? self.config['smooch_version'] == 'v2' end - def send_message_to_user_with_main_menu_appended(uid, text, workflow, language, tbi_id = nil) + def send_message_to_user_with_main_menu_appended(uid, text, workflow, language, tbi_id = nil, event = nil) self.get_installation('team_bot_installation_id', tbi_id) { |i| i.id == tbi_id } if self.config.blank? && !tbi_id.nil? main = [] counter = 1 @@ -114,7 +114,7 @@ def send_message_to_user_with_main_menu_appended(uid, text, workflow, language, fallback = [text] end - self.send_message_to_user(uid, fallback.join("\n"), extra) + self.send_message_to_user(uid, fallback.join("\n"), extra, false, true, event) end def adjust_language_options(rows, language, number_of_options) @@ -166,7 +166,7 @@ def get_custom_string(key, language, truncate_at = 1024) label.to_s.truncate(truncate_at) end - def send_message_to_user_with_buttons(uid, text, options, image_url = nil) + def send_message_to_user_with_buttons(uid, text, options, image_url = nil, event = nil) buttons = [] options.each_with_index do |option, i| buttons << { @@ -203,7 +203,7 @@ def send_message_to_user_with_buttons(uid, text, options, image_url = nil) } } unless image_url.blank? extra, fallback = self.format_fallback_text_menu_from_options(text, options, extra) - self.send_message_to_user(uid, fallback.join("\n"), extra) + self.send_message_to_user(uid, fallback.join("\n"), extra, false, true, event) end def send_message_to_user_with_single_section_menu(uid, text, options, menu_label) diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index fb433182f9..4dd71298fe 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -77,17 +77,17 @@ def bundle_messages(uid, id, app_id, type = 'default_requests', annotated = nil, end end - def send_final_message_to_user(uid, text, workflow, language) + def send_final_message_to_user(uid, text, workflow, language, event = nil) if self.is_v2? CheckStateMachine.new(uid).go_to_main - self.send_message_to_user_with_main_menu_appended(uid, text, workflow, language) + self.send_message_to_user_with_main_menu_appended(uid, text, workflow, language, nil, event) else self.send_message_to_user(uid, text) end end - def send_final_messages_to_user(uid, text, workflow, language, interval = 1, preview_url = true) - response = self.send_message_to_user(uid, text, {}, false, preview_url) + def send_final_messages_to_user(uid, text, workflow, language, interval = 1, preview_url = true, event = nil) + response = self.send_message_to_user(uid, text, {}, false, preview_url, event) if self.is_v2? label = self.get_string('navigation_button', language) CheckStateMachine.new(uid).go_to_main @@ -101,7 +101,7 @@ def send_final_messages_to_user(uid, text, workflow, language, interval = 1, pre response end - def send_message_for_state(uid, workflow, state, language, pretext = '') + def send_message_for_state(uid, workflow, state, language, pretext = '', event = nil) team = Team.find(self.config['team_id'].to_i) message = self.get_message_for_state(workflow, state, language, uid).to_s message = UrlRewriter.shorten_and_utmize_urls(message, team.get_outgoing_urls_utm_code) if team.get_shorten_outgoing_urls @@ -111,7 +111,7 @@ def send_message_for_state(uid, workflow, state, language, pretext = '') self.ask_for_language_confirmation(workflow, language, uid) else # On v2, when we go to the "main" state, we need to show the main menu - state == 'main' ? self.send_message_to_user_with_main_menu_appended(uid, text, workflow, language) : self.send_message_for_state_with_buttons(uid, text, workflow, state, language) + state == 'main' ? self.send_message_to_user_with_main_menu_appended(uid, text, workflow, language, nil, event) : self.send_message_for_state_with_buttons(uid, text, workflow, state, language) end else self.send_message_to_user(uid, text) @@ -433,7 +433,7 @@ def send_message_to_user_on_timeout(uid, language) redis = Redis.new(REDIS_CONFIG) user_messages_count = redis.llen("smooch:bundle:#{uid}") message = self.get_custom_string(:timeout, language) - self.send_message_to_user(uid, message) if user_messages_count > 0 && sm.state.value != 'main' + self.send_message_to_user(uid, message, {}, false, true, 'timeout') if user_messages_count > 0 && sm.state.value != 'main' sm.reset end @@ -449,7 +449,7 @@ def send_custom_message_to_user(team, uid, timestamp, message, language) Bot::Smooch.get_installation('team_bot_installation_id', tbi&.id) { |i| i.id == tbi&.id } date = I18n.l(Time.at(timestamp), locale: language, format: :short) message = self.format_template_message('custom_message', [date, message.to_s.gsub(/\s+/, ' ')], nil, message, language, nil, true) if platform == 'WhatsApp' - response = self.send_message_to_user(uid, message) + response = self.send_message_to_user(uid, message, {}, false, true, 'custom_message') success = (response.code.to_i < 400) success end diff --git a/app/models/concerns/smooch_resources.rb b/app/models/concerns/smooch_resources.rb index b3dc2b258e..33078e918c 100644 --- a/app/models/concerns/smooch_resources.rb +++ b/app/models/concerns/smooch_resources.rb @@ -13,12 +13,12 @@ def send_resource_to_user(uid, workflow, resource_uuid, language) type = resource.header_type type = 'video' if type == 'audio' # Audio gets converted to video with a cover type = 'file' if type == 'video' && RequestStore.store[:smooch_bot_provider] == 'ZENDESK' # Smooch doesn't support video - self.send_message_to_user(uid, message, { 'type' => type, 'mediaUrl' => CheckS3.rewrite_url(resource.header_media_url) }) + self.send_message_to_user(uid, message, { 'type' => type, 'mediaUrl' => CheckS3.rewrite_url(resource.header_media_url) }, false, true, 'resource') sleep 2 # Wait a couple of seconds before sending the main menu self.send_message_for_state(uid, workflow, 'main', language) else preview_url = (resource.header_type == 'link_preview') - self.send_final_messages_to_user(uid, message, workflow, language, 1, preview_url) unless message.blank? + self.send_final_messages_to_user(uid, message, workflow, language, 1, preview_url, 'resource') unless message.blank? end end resource diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index 343513761e..083f000ad0 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -17,7 +17,7 @@ def search(app_id, uid, language, message, team_id, workflow) end.uniq if results.empty? self.bundle_messages(uid, '', app_id, 'default_requests', nil, true) - self.send_final_message_to_user(uid, self.get_custom_string('search_no_results', language), workflow, language) + self.send_final_message_to_user(uid, self.get_custom_string('search_no_results', language), workflow, language, 'no_results') else self.send_search_results_to_user(uid, results, team_id, platform, app_id) # For WhatsApp, each search result goes with a button where the user can give feedback individually, so, reset the conversation right away @@ -227,7 +227,7 @@ def send_search_results_to_user(uid, results, team_id, platform, app_id) # Get reports languages reports_language = reports.map{ |r| r.report_design_field_value('language') }.uniq if team.get_languages.to_a.size > 1 && !reports_language.include?(language) - self.send_message_to_user(uid, self.get_string(:no_results_in_language, language).gsub('%{language}', CheckCldr.language_code_to_name(language, language))) + self.send_message_to_user(uid, self.get_string(:no_results_in_language, language).gsub('%{language}', CheckCldr.language_code_to_name(language, language)), {}, false, true, 'no_results') sleep 1 end if platform == 'WhatsApp' @@ -254,7 +254,7 @@ def send_search_results_to_whatsapp_user(uid, reports, app_id) value: { project_media_id: report.annotated_id, keyword: 'search_result_is_relevant', search_id: search_id }.to_json, label: '👍' }] - self.send_message_to_user_with_buttons(uid, text || '-', options, image_url) # "text" is mandatory for WhatsApp interactive messages + self.send_message_to_user_with_buttons(uid, text || '-', options, image_url, 'search_result') # "text" is mandatory for WhatsApp interactive messages self.delay_for(15.minutes, { queue: 'smooch_priority' }).timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, report.annotated_id) end end @@ -263,8 +263,8 @@ def send_search_results_to_non_whatsapp_user(uid, reports) redis = Redis.new(REDIS_CONFIG) reports.each do |report| response = nil - response = self.send_message_to_user(uid, report.report_design_text) if report.report_design_field_value('use_text_message') - response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }) if !report.report_design_field_value('use_text_message') && report.report_design_field_value('use_visual_card') + response = self.send_message_to_user(uid, report.report_design_text, {}, false, true, 'search_result') if report.report_design_field_value('use_text_message') + response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }, false, true, 'search_result') if !report.report_design_field_value('use_text_message') && report.report_design_field_value('use_visual_card') id = self.get_id_from_send_response(response) redis.rpush("smooch:search:#{uid}", id) unless id.blank? end diff --git a/app/models/tipline_message.rb b/app/models/tipline_message.rb index d648d8a3ca..46d968289f 100644 --- a/app/models/tipline_message.rb +++ b/app/models/tipline_message.rb @@ -14,6 +14,18 @@ def save_ignoring_duplicate! end end + def media_url + payload = begin JSON.parse(self.payload).to_h rescue self.payload.to_h end + media_url = nil + if self.direction == 'incoming' + media_url = payload.dig('messages', 0, 'mediaUrl') + elsif self.direction == 'outgoing' + header = payload.dig('override', 'whatsapp', 'payload', 'interactive', 'header') + media_url = header[header['type']]['link'] unless header.nil? + end + media_url || payload['mediaUrl'] + end + class << self def from_smooch_payload(msg, payload, event = nil, language = nil) msg = msg.with_indifferent_access diff --git a/app/models/tipline_newsletter.rb b/app/models/tipline_newsletter.rb index af64e6b4bd..036588f8aa 100644 --- a/app/models/tipline_newsletter.rb +++ b/app/models/tipline_newsletter.rb @@ -164,7 +164,7 @@ def format_as_tipline_message message = self.build_content message = (self.team.get_shorten_outgoing_urls ? UrlRewriter.shorten_and_utmize_urls(message, self.team.get_outgoing_urls_utm_code) : message) params = (['image', 'audio', 'video'].include?(self.header_type) ? { 'type' => NON_WHATSAPP_HEADER_TYPE_MAPPING[self.header_type], 'mediaUrl' => CheckS3.rewrite_url(self.header_media_url) } : {}) - [message, params] + [message, params, false, true, 'newsletter'] end def self.content_name diff --git a/app/workers/tipline_newsletter_worker.rb b/app/workers/tipline_newsletter_worker.rb index e3f8faa61b..45adefa1ae 100644 --- a/app/workers/tipline_newsletter_worker.rb +++ b/app/workers/tipline_newsletter_worker.rb @@ -40,7 +40,7 @@ def perform(team_id, language, job_created_at = 0) RequestStore.store[:smooch_bot_platform] = ts.platform Bot::Smooch.get_installation('team_bot_installation_id', tbi.id) { |i| i.id == tbi.id } - response = (ts.platform == 'WhatsApp' ? Bot::Smooch.send_message_to_user(ts.uid, newsletter.format_as_template_message) : Bot::Smooch.send_message_to_user(ts.uid, *newsletter.format_as_tipline_message)) + response = (ts.platform == 'WhatsApp' ? Bot::Smooch.send_message_to_user(ts.uid, newsletter.format_as_template_message, {}, false, true, 'newsletter') : Bot::Smooch.send_message_to_user(ts.uid, *newsletter.format_as_tipline_message)) if response.code.to_i < 400 log team_id, language, "Newsletter sent to subscriber ##{ts.id}, response: (#{response.code}) #{response.body.inspect}" diff --git a/lib/relay.idl b/lib/relay.idl index 9a958332b7..4aa2dd87df 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13177,6 +13177,7 @@ type TiplineMessage implements Node { external_id: String id: ID! language: String + media_url: String payload: JsonStringType permissions: String platform: String diff --git a/public/relay.json b/public/relay.json index 9c6da4dd74..df4df71757 100644 --- a/public/relay.json +++ b/public/relay.json @@ -69032,6 +69032,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "media_url", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "payload", "description": null, diff --git a/test/models/tipline_message_test.rb b/test/models/tipline_message_test.rb index 943f91e53e..979a1a8dce 100644 --- a/test/models/tipline_message_test.rb +++ b/test/models/tipline_message_test.rb @@ -198,4 +198,29 @@ def setup end end end + + test "should return media URL for tipline messages" do + url = random_url + payload = { 'mediaUrl' => url } + incoming_payload = { 'messages' => [{ 'mediaUrl' => url }] } + outgoing_payload = { + 'override' => { + 'whatsapp' => { + 'payload' => { + 'interactive' => { + 'header' => { + 'type' => 'image', + 'image' => { + 'link' => url + } + } + } + } + } + } + } + assert_equal url, create_tipline_message(payload: payload).media_url + assert_equal url, create_tipline_message(direction: 'incoming', payload: incoming_payload).media_url + assert_equal url, create_tipline_message(direction: 'outgoing', payload: outgoing_payload).media_url + end end From 09cf56111f5a44f2ea0322d2f2a6423a3875c1ee Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:02:06 -0300 Subject: [PATCH 02/54] Shared feed invitations - Added a new model `FeedInvitation` - Configured permissions for new model - Expose in GraphQL (type and mutations) Reference: CV2-3801. --- .../mutations/feed_invitation_mutations.rb | 44 + app/graph/mutations/feed_mutations.rb | 2 + app/graph/mutations/feed_team_mutations.rb | 2 + app/graph/types/feed_invitation_type.rb | 13 + app/graph/types/feed_type.rb | 3 + app/graph/types/mutation_type.rb | 7 + app/graph/types/query_type.rb | 1 + app/graph/types/user_type.rb | 7 + app/models/ability.rb | 10 +- app/models/feed.rb | 1 + app/models/feed_invitation.rb | 27 + app/models/user.rb | 1 + .../20231015203900_create_feed_invitations.rb | 13 + db/schema.rb | 22 +- lib/check_basic_abilities.rb | 4 + lib/relay.idl | 299 ++++ lib/sample_data.rb | 4 + public/relay.json | 1372 ++++++++++++++++- .../controllers/graphql_controller_12_test.rb | 178 +++ test/models/ability_test.rb | 18 + test/models/feed_invitation_test.rb | 66 + 21 files changed, 2068 insertions(+), 26 deletions(-) create mode 100644 app/graph/mutations/feed_invitation_mutations.rb create mode 100644 app/graph/types/feed_invitation_type.rb create mode 100644 app/models/feed_invitation.rb create mode 100644 db/migrate/20231015203900_create_feed_invitations.rb create mode 100644 test/controllers/graphql_controller_12_test.rb create mode 100644 test/models/feed_invitation_test.rb diff --git a/app/graph/mutations/feed_invitation_mutations.rb b/app/graph/mutations/feed_invitation_mutations.rb new file mode 100644 index 0000000000..89e872ed53 --- /dev/null +++ b/app/graph/mutations/feed_invitation_mutations.rb @@ -0,0 +1,44 @@ +module FeedInvitationMutations + MUTATION_TARGET = 'feed_invitation'.freeze + PARENTS = ['feed'].freeze + + class Create < Mutations::CreateMutation + argument :email, GraphQL::Types::String, required: true + argument :feed_id, GraphQL::Types::Int, required: true, camelize: false + end + + class Destroy < Mutations::DestroyMutation; end + + class Accept < Mutations::UpdateMutation + argument :id, GraphQL::Types::Int, required: true + argument :team_id, GraphQL::Types::Int, required: true, camelize: false + + field :success, GraphQL::Types::Boolean, null: true + + def resolve(id:, team_id:) + success = false + feed_invitation = FeedInvitation.find_if_can(id, context[:ability]) + if User.current && Team.current && User.current.team_ids.include?(team_id) && feed_invitation.email == User.current.email + feed_invitation.accept!(team_id) + success = true + end + { success: success } + end + end + + class Reject < Mutations::BaseMutation + argument :id, GraphQL::Types::Int, required: true + + field :success, GraphQL::Types::Boolean, null: true + + def resolve(id:) + success = false + feed_invitation = FeedInvitation.find_if_can(id, context[:ability]) + if User.current && Team.current && feed_invitation.email == User.current.email && feed_invitation.state == 'invited' + feed_invitation.reject! + success = true + end + { success: success } + end + end +end diff --git a/app/graph/mutations/feed_mutations.rb b/app/graph/mutations/feed_mutations.rb index 8d2e047716..d831636805 100644 --- a/app/graph/mutations/feed_mutations.rb +++ b/app/graph/mutations/feed_mutations.rb @@ -26,4 +26,6 @@ class Update < Mutations::UpdateMutation argument :name, GraphQL::Types::String, required: false end + + class Destroy < Mutations::DestroyMutation; end end diff --git a/app/graph/mutations/feed_team_mutations.rb b/app/graph/mutations/feed_team_mutations.rb index 1554322b76..e6f2642df8 100644 --- a/app/graph/mutations/feed_team_mutations.rb +++ b/app/graph/mutations/feed_team_mutations.rb @@ -7,4 +7,6 @@ class Update < Mutations::UpdateMutation argument :shared, GraphQL::Types::Boolean, required: false argument :requests_filters, JsonStringType, required: false, camelize: false end + + class Destroy < Mutations::DestroyMutation; end end diff --git a/app/graph/types/feed_invitation_type.rb b/app/graph/types/feed_invitation_type.rb new file mode 100644 index 0000000000..05106104c1 --- /dev/null +++ b/app/graph/types/feed_invitation_type.rb @@ -0,0 +1,13 @@ +class FeedInvitationType < DefaultObject + description "Feed invitation type" + + implements GraphQL::Types::Relay::Node + + field :dbid, GraphQL::Types::Int, null: false + field :feed_id, GraphQL::Types::Int, null: false + field :feed, FeedType, null: false + field :user_id, GraphQL::Types::Int, null: false + field :user, UserType, null: false + field :state, GraphQL::Types::String, null: false + field :email, GraphQL::Types::String, null: false +end diff --git a/app/graph/types/feed_type.rb b/app/graph/types/feed_type.rb index 00606a7752..7e5593c0e4 100644 --- a/app/graph/types/feed_type.rb +++ b/app/graph/types/feed_type.rb @@ -41,4 +41,7 @@ class FeedType < DefaultObject def requests(**args) object.search(args) end + + field :feed_invitations, FeedInvitationType.connection_type, null: false + field :teams, TeamType.connection_type, null: false end diff --git a/app/graph/types/mutation_type.rb b/app/graph/types/mutation_type.rb index 7632264e00..ffc94151fb 100644 --- a/app/graph/types/mutation_type.rb +++ b/app/graph/types/mutation_type.rb @@ -140,8 +140,15 @@ class MutationType < BaseObject field :createFeed, mutation: FeedMutations::Create field :updateFeed, mutation: FeedMutations::Update + field :destroyFeed, mutation: FeedMutations::Destroy field :updateFeedTeam, mutation: FeedTeamMutations::Update + field :destroyFeedTeam, mutation: FeedTeamMutations::Destroy + + field :createFeedInvitation, mutation: FeedInvitationMutations::Create + field :destroyFeedInvitation, mutation: FeedInvitationMutations::Destroy + field :acceptFeedInvitation, mutation: FeedInvitationMutations::Accept + field :rejectFeedInvitation, mutation: FeedInvitationMutations::Reject field :createTiplineNewsletter, mutation: TiplineNewsletterMutations::Create field :updateTiplineNewsletter, mutation: TiplineNewsletterMutations::Update diff --git a/app/graph/types/query_type.rb b/app/graph/types/query_type.rb index 0a22788984..efe650cca3 100644 --- a/app/graph/types/query_type.rb +++ b/app/graph/types/query_type.rb @@ -214,6 +214,7 @@ def dynamic_annotation_field(query:, only_cache: nil) cluster feed request + feed_invitation ].each do |type| field type, "#{type.to_s.camelize}Type", diff --git a/app/graph/types/user_type.rb b/app/graph/types/user_type.rb index 95cdbd1d31..31d4f31d9f 100644 --- a/app/graph/types/user_type.rb +++ b/app/graph/types/user_type.rb @@ -124,4 +124,11 @@ def assignments(team_id: nil) # pms.reject { |pm| pm.is_finished? } pms end + + field :feed_invitations, FeedInvitationType.connection_type, null: false + + def feed_invitations + return FeedInvitation.none if object.email.blank? + FeedInvitation.where(email: object.email) + end end diff --git a/app/models/ability.rb b/app/models/ability.rb index ab1b100ef5..84f21cd749 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -59,10 +59,6 @@ def admin_perms can [:update, :destroy], TeamUser, team_id: @context_team.id can :duplicate, Team, :id => @context_team.id can :set_privacy, Project, :team_id => @context_team.id - can :cud, Feed do |obj| - obj.get_team_ids.include?(@context_team.id) - end - can [:create, :destroy], FeedTeam, team_id: @context_team.id end def editor_perms @@ -87,7 +83,8 @@ def editor_perms can [:cud], DynamicAnnotation::Field do |obj| obj.annotation.team&.id == @context_team.id end - can [:create, :update, :read, :destroy], [Account, Source, TiplineNewsletter, TiplineResource], :team_id => @context_team.id + can [:create, :update, :read, :destroy], [Account, Source, TiplineNewsletter, TiplineResource, Feed, FeedTeam], :team_id => @context_team.id + can [:create, :update, :destroy], FeedInvitation, { feed: { team_id: @context_team.id } } can [:cud], AccountSource, source: { team: { team_users: { team_id: @context_team.id }}} %w(annotation comment dynamic task tag).each do |annotation_type| can [:cud], annotation_type.classify.constantize do |obj| @@ -102,9 +99,6 @@ def editor_perms teams << v_obj_parent.team&.id if v_obj_parent teams.include?(@context_team.id) end - can :update, FeedTeam do |obj| - obj.team_id == @context_team.id - end can :send, TiplineMessage do |obj| obj.team_id == @context_team.id end diff --git a/app/models/feed.rb b/app/models/feed.rb index 7d014b1470..eaed8392dd 100755 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -6,6 +6,7 @@ class Feed < ApplicationRecord has_many :requests has_many :feed_teams has_many :teams, through: :feed_teams + has_many :feed_invitations belongs_to :user, optional: true belongs_to :saved_search, optional: true belongs_to :team, optional: true diff --git a/app/models/feed_invitation.rb b/app/models/feed_invitation.rb new file mode 100644 index 0000000000..69b3b7b223 --- /dev/null +++ b/app/models/feed_invitation.rb @@ -0,0 +1,27 @@ +class FeedInvitation < ApplicationRecord + enum state: { invited: 0, accepted: 1, rejected: 2 } # default: invited + + belongs_to :feed + belongs_to :user + + before_validation :set_user, on: :create + validates_presence_of :email, :feed, :user + validates :email, format: { with: URI::MailTo::EMAIL_REGEXP } + + def accept!(team_id) + feed_team = FeedTeam.new(feed_id: self.feed_id, team_id: team_id, shared: true) + feed_team.skip_check_ability = true + feed_team.save! + self.update_column(:state, :accepted) + end + + def reject! + self.update_column(:state, :rejected) + end + + private + + def set_user + self.user ||= User.current + end +end diff --git a/app/models/user.rb b/app/models/user.rb index c59707adf1..c1430ec4d2 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -26,6 +26,7 @@ class ToSOrPrivacyPolicyReadError < StandardError; end has_many :claim_descriptions has_many :fact_checks has_many :feeds + has_many :feed_invitations devise :registerable, :recoverable, :rememberable, :trackable, :validatable, :confirmable, diff --git a/db/migrate/20231015203900_create_feed_invitations.rb b/db/migrate/20231015203900_create_feed_invitations.rb new file mode 100644 index 0000000000..e032d36ae0 --- /dev/null +++ b/db/migrate/20231015203900_create_feed_invitations.rb @@ -0,0 +1,13 @@ +class CreateFeedInvitations < ActiveRecord::Migration[6.1] + def change + create_table :feed_invitations do |t| + t.string :email, null: false + t.integer :state, null: false, default: 0 + t.references :feed, foreign_key: true, null: false + t.references :user, foreign_key: true, null: false # User who invited + + t.timestamps + end + add_index :feed_invitations, [:email, :feed_id], unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index e75733f01b..14b6c60131 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_10_08_074526) do +ActiveRecord::Schema.define(version: 2023_10_15_203900) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -268,7 +268,7 @@ t.jsonb "value_json", default: "{}" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY (ARRAY[('external_id'::character varying)::text, ('smooch_user_id'::character varying)::text, ('verification_status_status'::character varying)::text]))" + t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY ((ARRAY['external_id'::character varying, 'smooch_user_id'::character varying, 'verification_status_status'::character varying])::text[]))" t.index ["annotation_id", "field_name"], name: "index_dynamic_annotation_fields_on_annotation_id_and_field_name" t.index ["annotation_id"], name: "index_dynamic_annotation_fields_on_annotation_id" t.index ["annotation_type"], name: "index_dynamic_annotation_fields_on_annotation_type" @@ -297,6 +297,18 @@ t.index ["user_id"], name: "index_fact_checks_on_user_id" end + create_table "feed_invitations", force: :cascade do |t| + t.string "email", null: false + t.integer "state", default: 0, null: false + t.bigint "feed_id", null: false + t.bigint "user_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["email", "feed_id"], name: "index_feed_invitations_on_email_and_feed_id", unique: true + t.index ["feed_id"], name: "index_feed_invitations_on_feed_id" + t.index ["user_id"], name: "index_feed_invitations_on_user_id" + end + create_table "feed_teams", force: :cascade do |t| t.bigint "team_id", null: false t.bigint "feed_id", null: false @@ -648,6 +660,7 @@ t.datetime "updated_at", null: false t.string "state" t.index ["external_id", "state"], name: "index_tipline_messages_on_external_id_and_state", unique: true + t.index ["external_id"], name: "index_tipline_messages_on_external_id" t.index ["team_id"], name: "index_tipline_messages_on_team_id" t.index ["uid"], name: "index_tipline_messages_on_uid" end @@ -788,8 +801,7 @@ end create_table "versions", id: :serial, force: :cascade do |t| - t.string "item_type" - t.string "{:null=>false}" + t.string "item_type", null: false t.string "item_id", null: false t.string "event", null: false t.string "whodunnit" @@ -812,6 +824,8 @@ add_foreign_key "claim_descriptions", "users" add_foreign_key "fact_checks", "claim_descriptions" add_foreign_key "fact_checks", "users" + add_foreign_key "feed_invitations", "feeds" + add_foreign_key "feed_invitations", "users" add_foreign_key "feed_teams", "feeds" add_foreign_key "feed_teams", "teams" add_foreign_key "project_media_requests", "project_medias" diff --git a/lib/check_basic_abilities.rb b/lib/check_basic_abilities.rb index 1d9645591d..8b5691d720 100644 --- a/lib/check_basic_abilities.rb +++ b/lib/check_basic_abilities.rb @@ -124,6 +124,10 @@ def extra_perms_for_all_users can :read, Request do |obj| !(@user.cached_teams & obj.feed.team_ids).empty? end + + can :read, FeedInvitation do |obj| + @user.email == obj.email || @user.id == obj.user_id + end end def annotation_perms_for_all_users diff --git a/lib/relay.idl b/lib/relay.idl index 4aa2dd87df..737442f109 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -2350,6 +2350,31 @@ input CreateFeedInput { tags: [String] } +""" +Autogenerated input type of CreateFeedInvitation +""" +input CreateFeedInvitationInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + email: String! + feed_id: Int! +} + +""" +Autogenerated return type of CreateFeedInvitation +""" +type CreateFeedInvitationPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + feed: Feed + feed_invitation: FeedInvitation + feed_invitationEdge: FeedInvitationEdge +} + """ Autogenerated return type of CreateFeed """ @@ -4011,6 +4036,75 @@ type DestroyFactCheckPayload { deletedId: ID } +""" +Autogenerated input type of DestroyFeed +""" +input DestroyFeedInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + id: ID +} + +""" +Autogenerated input type of DestroyFeedInvitation +""" +input DestroyFeedInvitationInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + id: ID +} + +""" +Autogenerated return type of DestroyFeedInvitation +""" +type DestroyFeedInvitationPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + deletedId: ID + feed: Feed +} + +""" +Autogenerated return type of DestroyFeed +""" +type DestroyFeedPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + deletedId: ID + team: Team +} + +""" +Autogenerated input type of DestroyFeedTeam +""" +input DestroyFeedTeamInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + id: ID +} + +""" +Autogenerated return type of DestroyFeedTeam +""" +type DestroyFeedTeamPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + deletedId: ID + feed: Feed +} + """ Autogenerated input type of DestroyProjectGroup """ @@ -8326,6 +8420,27 @@ type Feed implements Node { dbid: Int description: String discoverable: Boolean + feed_invitations( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): FeedInvitationConnection! filters: JsonStringType id: ID! licenses: [Int] @@ -8371,6 +8486,27 @@ type Feed implements Node { tags: [String] team: Team team_id: Int + teams( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): TeamConnection! teams_count: Int updated_at: String user: User @@ -8413,6 +8549,59 @@ type FeedEdge { node: Feed } +""" +Feed invitation type +""" +type FeedInvitation implements Node { + created_at: String + dbid: Int! + email: String! + feed: Feed! + feed_id: Int! + id: ID! + permissions: String + state: String! + updated_at: String + user: User! + user_id: Int! +} + +""" +The connection type for FeedInvitation. +""" +type FeedInvitationConnection { + """ + A list of edges. + """ + edges: [FeedInvitationEdge] + + """ + A list of nodes. + """ + nodes: [FeedInvitation] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! + totalCount: Int +} + +""" +An edge in a connection. +""" +type FeedInvitationEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: FeedInvitation +} + """ Feed team type """ @@ -8746,6 +8935,12 @@ type MoveTeamTaskUpPayload { } type MutationType { + acceptFeedInvitation( + """ + Parameters for UpdateFeedInvitation + """ + input: UpdateFeedInvitationInput! + ): UpdateFeedInvitationPayload addFilesToTask( """ Parameters for AddFilesToTask @@ -9026,6 +9221,12 @@ type MutationType { """ input: CreateFeedInput! ): CreateFeedPayload + createFeedInvitation( + """ + Parameters for CreateFeedInvitation + """ + input: CreateFeedInvitationInput! + ): CreateFeedInvitationPayload createProject( """ Parameters for CreateProject @@ -9392,6 +9593,24 @@ type MutationType { """ input: DestroyFactCheckInput! ): DestroyFactCheckPayload + destroyFeed( + """ + Parameters for DestroyFeed + """ + input: DestroyFeedInput! + ): DestroyFeedPayload + destroyFeedInvitation( + """ + Parameters for DestroyFeedInvitation + """ + input: DestroyFeedInvitationInput! + ): DestroyFeedInvitationPayload + destroyFeedTeam( + """ + Parameters for DestroyFeedTeam + """ + input: DestroyFeedTeamInput! + ): DestroyFeedTeamPayload destroyProject( """ Parameters for DestroyProject @@ -9542,6 +9761,12 @@ type MutationType { """ input: MoveTeamTaskUpInput! ): MoveTeamTaskUpPayload + rejectFeedInvitation( + """ + Parameters for Reject + """ + input: RejectInput! + ): RejectPayload removeFilesFromTask( """ Parameters for RemoveFilesFromTask @@ -11459,6 +11684,11 @@ type Query { """ feed(id: ID!): Feed + """ + Information about the feed_invitation with given id + """ + feed_invitation(id: ID!): FeedInvitation + """ Find whether a team exists """ @@ -11564,6 +11794,28 @@ type Query { user(id: ID!): User } +""" +Autogenerated input type of Reject +""" +input RejectInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + id: Int! +} + +""" +Autogenerated return type of Reject +""" +type RejectPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + success: Boolean +} + """ A relationship between two items """ @@ -15018,6 +15270,32 @@ input UpdateFeedInput { tags: [String] } +""" +Autogenerated input type of UpdateFeedInvitation +""" +input UpdateFeedInvitationInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + id: Int! + team_id: Int! +} + +""" +Autogenerated return type of UpdateFeedInvitation +""" +type UpdateFeedInvitationPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + feed: Feed + feed_invitation: FeedInvitation + feed_invitationEdge: FeedInvitationEdge + success: Boolean +} + """ Autogenerated return type of UpdateFeed """ @@ -15754,6 +16032,27 @@ type User implements Node { current_team_id: Int dbid: Int email: String + feed_invitations( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): FeedInvitationConnection! get_send_email_notifications: Boolean get_send_failed_login_notifications: Boolean get_send_successful_login_notifications: Boolean diff --git a/lib/sample_data.rb b/lib/sample_data.rb index 7bacb897f1..ce4e68b4e9 100644 --- a/lib/sample_data.rb +++ b/lib/sample_data.rb @@ -1100,4 +1100,8 @@ def create_task_stuff(delete_existing = true) def create_blocked_tipline_user(options = {}) BlockedTiplineUser.create!({ uid: random_string }.merge(options)) end + + def create_feed_invitation(options = {}) + FeedInvitation.create!({ email: random_email, feed: create_feed, user: create_user, state: :invited }.merge(options)) + end end diff --git a/public/relay.json b/public/relay.json index df4df71757..52af15b27d 100644 --- a/public/relay.json +++ b/public/relay.json @@ -14465,6 +14465,130 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "INPUT_OBJECT", + "name": "CreateFeedInvitationInput", + "description": "Autogenerated input type of CreateFeedInvitation", + "fields": null, + "inputFields": [ + { + "name": "email", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateFeedInvitationPayload", + "description": "Autogenerated return type of CreateFeedInvitation", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Feed", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed_invitation", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "FeedInvitation", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed_invitationEdge", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "FeedInvitationEdge", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "CreateFeedPayload", @@ -23185,8 +23309,43 @@ }, { "kind": "INPUT_OBJECT", - "name": "DestroyProjectGroupInput", - "description": "Autogenerated input type of DestroyProjectGroup", + "name": "DestroyFeedInput", + "description": "Autogenerated input type of DestroyFeed", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": null, + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "DestroyFeedInvitationInput", + "description": "Autogenerated input type of DestroyFeedInvitation", "fields": null, "inputFields": [ { @@ -23220,8 +23379,63 @@ }, { "kind": "OBJECT", - "name": "DestroyProjectGroupPayload", - "description": "Autogenerated return type of DestroyProjectGroup", + "name": "DestroyFeedInvitationPayload", + "description": "Autogenerated return type of DestroyFeedInvitation", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "deletedId", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Feed", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DestroyFeedPayload", + "description": "Autogenerated return type of DestroyFeed", "fields": [ { "name": "clientMutationId", @@ -23275,8 +23489,8 @@ }, { "kind": "INPUT_OBJECT", - "name": "DestroyProjectInput", - "description": "Autogenerated input type of DestroyProject", + "name": "DestroyFeedTeamInput", + "description": "Autogenerated input type of DestroyFeedTeam", "fields": null, "inputFields": [ { @@ -23292,38 +23506,218 @@ "deprecationReason": null }, { - "name": "items_destination_project_id", - "description": null, + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", "type": { "kind": "SCALAR", - "name": "Int", + "name": "String", "ofType": null }, "defaultValue": null, "isDeprecated": false, "deprecationReason": null - }, + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DestroyFeedTeamPayload", + "description": "Autogenerated return type of DestroyFeedTeam", + "fields": [ { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "deletedId", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Feed", + "ofType": null + }, "isDeprecated": false, "deprecationReason": null } ], - "interfaces": null, + "inputFields": null, + "interfaces": [ + + ], "enumValues": null, "possibleTypes": null }, { "kind": "INPUT_OBJECT", - "name": "DestroyProjectMediaInput", - "description": "Autogenerated input type of DestroyProjectMedia", + "name": "DestroyProjectGroupInput", + "description": "Autogenerated input type of DestroyProjectGroup", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": null, + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "DestroyProjectGroupPayload", + "description": "Autogenerated return type of DestroyProjectGroup", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "deletedId", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "team", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Team", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "DestroyProjectInput", + "description": "Autogenerated input type of DestroyProject", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": null, + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "items_destination_project_id", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", + "name": "DestroyProjectMediaInput", + "description": "Autogenerated input type of DestroyProjectMedia", "fields": null, "inputFields": [ { @@ -45291,6 +45685,71 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "feed_invitations", + "description": null, + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "FeedInvitationConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "filters", "description": null, @@ -45678,6 +46137,71 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "teams", + "description": null, + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TeamConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "teams_count", "description": null, @@ -45872,6 +46396,335 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "FeedInvitation", + "description": "Feed invitation type", + "fields": [ + { + "name": "created_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dbid", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "email", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "Feed", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed_id", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "permissions", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "state", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updated_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "user", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "User", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "user_id", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "Node", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "FeedInvitationConnection", + "description": "The connection type for FeedInvitation.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "FeedInvitationEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "FeedInvitation", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "totalCount", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "FeedInvitationEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "FeedInvitation", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "FeedTeam", @@ -47622,6 +48475,35 @@ "name": "MutationType", "description": null, "fields": [ + { + "name": "acceptFeedInvitation", + "description": null, + "args": [ + { + "name": "input", + "description": "Parameters for UpdateFeedInvitation", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "UpdateFeedInvitationInput", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "UpdateFeedInvitationPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "addFilesToTask", "description": null, @@ -48956,6 +49838,35 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "createFeedInvitation", + "description": null, + "args": [ + { + "name": "input", + "description": "Parameters for CreateFeedInvitation", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateFeedInvitationInput", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateFeedInvitationPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "createProject", "description": null, @@ -50725,6 +51636,93 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "destroyFeed", + "description": null, + "args": [ + { + "name": "input", + "description": "Parameters for DestroyFeed", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DestroyFeedInput", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DestroyFeedPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "destroyFeedInvitation", + "description": null, + "args": [ + { + "name": "input", + "description": "Parameters for DestroyFeedInvitation", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DestroyFeedInvitationInput", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DestroyFeedInvitationPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "destroyFeedTeam", + "description": null, + "args": [ + { + "name": "input", + "description": "Parameters for DestroyFeedTeam", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "DestroyFeedTeamInput", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "DestroyFeedTeamPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "destroyProject", "description": null, @@ -51450,6 +52448,35 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "rejectFeedInvitation", + "description": null, + "args": [ + { + "name": "input", + "description": "Parameters for Reject", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "RejectInput", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "RejectPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "removeFilesFromTask", "description": null, @@ -53969,6 +54996,11 @@ "name": "Feed", "ofType": null }, + { + "kind": "OBJECT", + "name": "FeedInvitation", + "ofType": null + }, { "kind": "OBJECT", "name": "FeedTeam", @@ -60403,6 +61435,35 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "feed_invitation", + "description": "Information about the feed_invitation with given id", + "args": [ + { + "name": "id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "FeedInvitation", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "find_public_team", "description": "Find whether a team exists", @@ -60960,6 +62021,86 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "INPUT_OBJECT", + "name": "RejectInput", + "description": "Autogenerated input type of Reject", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RejectPayload", + "description": "Autogenerated return type of Reject", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "success", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "Relationship", @@ -82047,6 +83188,144 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "INPUT_OBJECT", + "name": "UpdateFeedInvitationInput", + "description": "Autogenerated input type of UpdateFeedInvitation", + "fields": null, + "inputFields": [ + { + "name": "id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "team_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "UpdateFeedInvitationPayload", + "description": "Autogenerated return type of UpdateFeedInvitation", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Feed", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed_invitation", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "FeedInvitation", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feed_invitationEdge", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "FeedInvitationEdge", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "success", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "UpdateFeedPayload", @@ -86671,6 +87950,71 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "feed_invitations", + "description": null, + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "FeedInvitationConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "get_send_email_notifications", "description": null, diff --git a/test/controllers/graphql_controller_12_test.rb b/test/controllers/graphql_controller_12_test.rb new file mode 100644 index 0000000000..7de2edc196 --- /dev/null +++ b/test/controllers/graphql_controller_12_test.rb @@ -0,0 +1,178 @@ +require_relative '../test_helper' +require 'error_codes' +require 'sidekiq/testing' + +class GraphqlController12Test < ActionController::TestCase + def setup + @controller = Api::V1::GraphqlController.new + TestDynamicAnnotationTables.load! + + @u = create_user + @t = create_team + create_team_user team: @t, user: @u, role: 'admin' + end + + def teardown + User.unstub(:current) + Team.unstub(:current) + User.current = nil + Team.current = nil + end + + test "should list feed invitations for a feed" do + f = create_feed team: @t + fi = create_feed_invitation feed: f + create_feed_invitation + + authenticate_with_user(@u) + query = 'query { feed(id: "' + f.id.to_s + '") { feed_invitations(first: 10) { edges { node { id, dbid } } } } }' + post :create, params: { query: query } + assert_response :success + assert_equal [fi.id], JSON.parse(@response.body)['data']['feed']['feed_invitations']['edges'].collect{ |edge| edge['node']['dbid'] } + end + + test "should list feed invitations for a user" do + fi = create_feed_invitation email: @u.email + create_feed_invitation + + authenticate_with_user(@u) + query = 'query { me { feed_invitations(first: 10) { edges { node { id, dbid } } } } }' + post :create, params: { query: query } + assert_response :success + assert_equal [fi.id], JSON.parse(@response.body)['data']['me']['feed_invitations']['edges'].collect{ |edge| edge['node']['dbid'] } + end + + test "should list teams for a feed" do + t2 = create_team + f = create_feed team: @t + create_feed_team feed: f, team: t2 + create_feed_team + + authenticate_with_user(@u) + query = 'query { feed(id: "' + f.id.to_s + '") { teams(first: 10) { edges { node { id, dbid } } } } }' + post :create, params: { query: query } + assert_response :success + assert_equal [@t.id, t2.id].sort, JSON.parse(@response.body)['data']['feed']['teams']['edges'].collect{ |edge| edge['node']['dbid'] }.sort + end + + test "should create feed invitation" do + f = create_feed team: @t + + authenticate_with_user(@u) + query = 'mutation { createFeedInvitation(input: { feed_id: ' + f.id.to_s + ', email: "' + random_email + '"}) { feed_invitation { id } } }' + assert_difference 'FeedInvitation.count' do + post :create, params: { query: query } + end + assert_response :success + end + + test "should not create feed invitation" do + f = create_feed + + authenticate_with_user(@u) + query = 'mutation { createFeedInvitation(input: { feed_id: ' + f.id.to_s + ', email: "' + random_email + '"}) { feed_invitation { id } } }' + assert_no_difference 'FeedInvitation.count' do + post :create, params: { query: query } + end + assert_response 400 + end + + test "should destroy feed invitation" do + f = create_feed team: @t + fi = create_feed_invitation feed: f + + authenticate_with_user(@u) + query = 'mutation { destroyFeedInvitation(input: { id: "' + fi.graphql_id + '" }) { deletedId } }' + assert_difference 'FeedInvitation.count', -1 do + post :create, params: { query: query } + end + assert_response :success + end + + test "should not destroy feed invitation" do + fi = create_feed_invitation + + authenticate_with_user(@u) + query = 'mutation { destroyFeedInvitation(input: { id: "' + fi.graphql_id + '" }) { deletedId } }' + assert_no_difference 'FeedInvitation.count' do + post :create, params: { query: query } + end + assert_response 400 + end + + test "should accept feed invitation" do + fi = create_feed_invitation email: @u.email + + authenticate_with_user(@u) + query = 'mutation { acceptFeedInvitation(input: { id: ' + fi.id.to_s + ', team_id: ' + @t.id.to_s + ' }) { success } }' + assert_difference 'FeedTeam.count' do + post :create, params: { query: query } + end + assert_response :success + assert JSON.parse(@response.body).dig('data', 'acceptFeedInvitation', 'success') + end + + test "should not accept feed invitation if it's not the same email" do + fi = create_feed_invitation + + authenticate_with_user(@u) + query = 'mutation { acceptFeedInvitation(input: { id: ' + fi.id.to_s + ', team_id: ' + @t.id.to_s + ' }) { success } }' + assert_no_difference 'FeedTeam.count' do + post :create, params: { query: query } + end + assert_response :success + assert_nil JSON.parse(@response.body).dig('data', 'acceptFeedInvitation', 'success') + end + + test "should not accept feed invitation if it's not a member of the target workspace" do + fi = create_feed_invitation email: @u.email + + authenticate_with_user(@u) + query = 'mutation { acceptFeedInvitation(input: { id: ' + fi.id.to_s + ', team_id: ' + create_team.id.to_s + ' }) { success } }' + assert_no_difference 'FeedTeam.count' do + post :create, params: { query: query } + end + assert_response :success + assert !JSON.parse(@response.body).dig('data', 'acceptFeedInvitation', 'success') + end + + test "should reject feed invitation" do + fi = create_feed_invitation email: @u.email + + authenticate_with_user(@u) + query = 'mutation { rejectFeedInvitation(input: { id: ' + fi.id.to_s + ' }) { success } }' + post :create, params: { query: query } + assert_response :success + assert JSON.parse(@response.body).dig('data', 'rejectFeedInvitation', 'success') + end + + test "should not reject feed invitation if it's not the same email" do + fi = create_feed_invitation + + authenticate_with_user(@u) + query = 'mutation { rejectFeedInvitation(input: { id: ' + fi.id.to_s + ' }) { success } }' + post :create, params: { query: query } + assert_response :success + assert !JSON.parse(@response.body).dig('data', 'rejectFeedInvitation', 'success') + end + + test "should read feed invitation" do + fi = create_feed_invitation email: @u.email + + authenticate_with_user(@u) + query = 'query { feed_invitation(id: ' + fi.id.to_s + ') { id } }' + post :create, params: { query: query } + assert_response :success + assert_not_nil JSON.parse(@response.body).dig('data', 'feed_invitation') + end + + test "should not read feed invitation" do + fi = create_feed_invitation + + authenticate_with_user(@u) + query = 'query { feed_invitation(id: ' + fi.id.to_s + ') { dbid } }' + post :create, params: { query: query } + assert_response :success + assert_nil JSON.parse(@response.body).dig('data', 'feed_invitation') + end +end diff --git a/test/models/ability_test.rb b/test/models/ability_test.rb index c6fb07215b..e8b52f786b 100644 --- a/test/models/ability_test.rb +++ b/test/models/ability_test.rb @@ -1287,4 +1287,22 @@ def teardown assert ability.cannot?(:destroy, tn2) end end + + test "permissions for feed invitation" do + t = create_team + u = create_user + create_team_user user: u, team: t, role: 'admin' + f = create_feed team: t + fi1 = create_feed_invitation feed: f + fi2 = create_feed_invitation + with_current_user_and_team(u, t) do + ability = Ability.new + assert ability.can?(:create, fi1) + assert ability.can?(:update, fi1) + assert ability.can?(:destroy, fi1) + assert ability.cannot?(:create, fi2) + assert ability.cannot?(:update, fi2) + assert ability.cannot?(:destroy, fi2) + end + end end diff --git a/test/models/feed_invitation_test.rb b/test/models/feed_invitation_test.rb new file mode 100644 index 0000000000..d66cc7ca8e --- /dev/null +++ b/test/models/feed_invitation_test.rb @@ -0,0 +1,66 @@ +require_relative '../test_helper' + +class FeedInvitationTest < ActiveSupport::TestCase + def setup + end + + def teardown + end + + test 'should create feed invitation' do + assert_difference 'FeedInvitation.count' do + create_feed_invitation + end + end + + test 'should not create feed invitation without feed' do + assert_no_difference 'FeedInvitation.count' do + assert_raises ActiveRecord::RecordInvalid do + create_feed_invitation feed: nil + end + end + end + + test 'should not create feed invitation without user' do + assert_no_difference 'FeedInvitation.count' do + assert_raises ActiveRecord::RecordInvalid do + create_feed_invitation user: nil + end + end + end + + test 'should not create feed invitation without email' do + assert_no_difference 'FeedInvitation.count' do + assert_raises ActiveRecord::RecordInvalid do + create_feed_invitation email: nil + end + end + end + + test 'should not create feed invitation with invalid email' do + assert_no_difference 'FeedInvitation.count' do + assert_raises ActiveRecord::RecordInvalid do + create_feed_invitation email: random_string + end + end + end + + test 'should belong to feed and user' do + fi = create_feed_invitation + assert_kind_of Feed, fi.feed + assert_kind_of User, fi.user + end + + test 'should accept and reject invitation' do + f = create_feed + t = create_team + fi = create_feed_invitation feed: f + assert_equal 'invited', fi.reload.state + assert_difference "FeedTeam.where(feed_id: #{f.id}, team_id: #{t.id}).count" do + fi.accept!(t.id) + end + assert_equal 'accepted', fi.reload.state + fi.reject! + assert_equal 'rejected', fi.reload.state + end +end From 9b2a90d7086d45f10f39471756ea73f9f1cad6e5 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 17 Oct 2023 10:57:41 -0300 Subject: [PATCH 03/54] A couple of small fixes for the conversation history * Return `media_url` for Smooch WhatsApp template * Set tipline messages with event `status_change` Reference: CV2-3853. --- app/models/concerns/smooch_messages.rb | 2 +- app/models/tipline_message.rb | 3 +++ test/models/bot/smooch_test.rb | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index 4dd71298fe..313ac656d2 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -420,7 +420,7 @@ def send_message_on_status_change(pm_id, status, request_actor_session_id = nil) self.get_installation(self.installation_setting_id_keys, data['app_id']) if self.config.blank? message = parent.team.get_status_message_for_language(status, data['language']) unless message.blank? - response = self.send_message_to_user(data['authorId'], message) + response = self.send_message_to_user(data['authorId'], message, {}, false, true, 'status_change') self.save_smooch_response(response, parent, data['received'].to_i, 'fact_check_status', data['language'], { message: message }) requestors_count += 1 end diff --git a/app/models/tipline_message.rb b/app/models/tipline_message.rb index 46d968289f..1999d5af35 100644 --- a/app/models/tipline_message.rb +++ b/app/models/tipline_message.rb @@ -20,8 +20,11 @@ def media_url if self.direction == 'incoming' media_url = payload.dig('messages', 0, 'mediaUrl') elsif self.direction == 'outgoing' + # WhatsApp Cloud API template header = payload.dig('override', 'whatsapp', 'payload', 'interactive', 'header') media_url = header[header['type']]['link'] unless header.nil? + # WhatsApp template on Smooch + media_url ||= payload.dig('text').to_s.match(/header_image=\[\[([^\]]+)\]\]/).to_a.last end media_url || payload['mediaUrl'] end diff --git a/test/models/bot/smooch_test.rb b/test/models/bot/smooch_test.rb index 34eaf55de5..c587c90d6d 100644 --- a/test/models/bot/smooch_test.rb +++ b/test/models/bot/smooch_test.rb @@ -737,7 +737,7 @@ def teardown }.to_json assert Bot::Smooch.run(payload) pm = ProjectMedia.last - Bot::Smooch.stubs(:send_message_to_user).with(uid, 'Custom').once + Bot::Smooch.stubs(:send_message_to_user).with(uid, 'Custom', {}, false, true, 'status_change').once s = pm.annotations.where(annotation_type: 'verification_status').last.load s.status = '2' s.save! From 5397dd68f614864b580ac6b75a6e20a961ef3c3e Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:07:16 -0300 Subject: [PATCH 04/54] Block tipline users when they send more than X messages in 24 hours "X" is defined by a configuration key but has a default value. Reference: CV2-3860. --- app/models/tipline_message.rb | 12 ++++++++++++ config/config.yml.example | 7 +++++++ test/models/tipline_message_test.rb | 22 ++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/app/models/tipline_message.rb b/app/models/tipline_message.rb index 1999d5af35..2586554ac5 100644 --- a/app/models/tipline_message.rb +++ b/app/models/tipline_message.rb @@ -6,6 +6,8 @@ class TiplineMessage < ApplicationRecord validates_presence_of :team, :uid, :platform, :language, :direction, :sent_at, :payload, :state validates_inclusion_of :state, in: ['sent', 'received', 'delivered'] + after_commit :verify_user_rate_limit, on: :create + def save_ignoring_duplicate! begin self.save! @@ -29,6 +31,16 @@ def media_url media_url || payload['mediaUrl'] end + private + + def verify_user_rate_limit + rate_limit = CheckConfig.get('tipline_user_max_messages_per_day', 1500, :integer) + # Block tipline user when they have sent more than X messages in 24 hours + if self.state == 'received' && TiplineMessage.where(uid: self.uid, created_at: Time.now.ago(1.day)..Time.now, state: 'received').count > rate_limit + Bot::Smooch.block_user(self.uid) + end + end + class << self def from_smooch_payload(msg, payload, event = nil, language = nil) msg = msg.with_indifferent_access diff --git a/config/config.yml.example b/config/config.yml.example index 63a205f99e..e9f89e4976 100644 --- a/config/config.yml.example +++ b/config/config.yml.example @@ -258,6 +258,13 @@ development: &default otel_traces_sampler: otel_custom_sampling_rate: + # Rate limit for tipline submissions, tipline users are blocked after reaching this limit + # + # OPTIONAL + # When not set, a default number will be used. + # + tipline_user_max_messages_per_day: 1500 + test: <<: *default checkdesk_base_url_private: http://api:3000 diff --git a/test/models/tipline_message_test.rb b/test/models/tipline_message_test.rb index 979a1a8dce..7d7029a37d 100644 --- a/test/models/tipline_message_test.rb +++ b/test/models/tipline_message_test.rb @@ -223,4 +223,26 @@ def setup assert_equal url, create_tipline_message(direction: 'incoming', payload: incoming_payload).media_url assert_equal url, create_tipline_message(direction: 'outgoing', payload: outgoing_payload).media_url end + + test "should block user when rate limit is reached" do + uid = random_string + assert !Bot::Smooch.user_blocked?(uid) + stub_configs({ 'tipline_user_max_messages_per_day' => 2 }) do + # User sent a message + create_tipline_message uid: uid, state: 'received' + assert !Bot::Smooch.user_blocked?(uid) + # User sent a message + create_tipline_message uid: uid, state: 'received' + assert !Bot::Smooch.user_blocked?(uid) + # Another user sent a message + create_tipline_message state: 'received' + assert !Bot::Smooch.user_blocked?(uid) + # User received a message + create_tipline_message uid: uid, state: 'delivered' + assert !Bot::Smooch.user_blocked?(uid) + # User sent a message and is now over rate limit, so should be blocked + create_tipline_message uid: uid, state: 'received' + assert Bot::Smooch.user_blocked?(uid) + end + end end From eb6a97c3a49792492840d8c7f033e888fa487bf5 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 18 Oct 2023 11:31:33 -0700 Subject: [PATCH 05/54] CV2-3827 restructure expectation on callback (#1699) --- app/models/concerns/alegre_webhooks.rb | 2 +- test/controllers/webhooks_controller_test.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/concerns/alegre_webhooks.rb b/app/models/concerns/alegre_webhooks.rb index 4b71b2d582..a0ce16106d 100644 --- a/app/models/concerns/alegre_webhooks.rb +++ b/app/models/concerns/alegre_webhooks.rb @@ -12,7 +12,7 @@ def valid_request?(request) def webhook(request) begin - doc_id = request.params.dig('data', 'requested', 'body', 'id') + doc_id = request.params.dig('requested', 'id') raise 'Unexpected params format' if doc_id.blank? redis = Redis.new(REDIS_CONFIG) key = "alegre:webhook:#{doc_id}" diff --git a/test/controllers/webhooks_controller_test.rb b/test/controllers/webhooks_controller_test.rb index e809d0001d..fa28aeacab 100644 --- a/test/controllers/webhooks_controller_test.rb +++ b/test/controllers/webhooks_controller_test.rb @@ -237,12 +237,12 @@ def setup CheckSentry.expects(:notify).never redis = Redis.new(REDIS_CONFIG) redis.del('foo') - payload = { 'action' => 'audio', 'data' => { 'requested' => { 'body' => { 'id' => 'foo', 'context' => { 'project_media_id' => random_number } } } } } + payload = { 'action' => 'audio', 'requested' => { 'id' => 'foo', 'context' => { 'project_media_id' => random_number } } } assert_nil redis.lpop('alegre:webhook:foo') post :index, params: { name: :alegre, token: CheckConfig.get('alegre_token') }.merge(payload) response = JSON.parse(redis.lpop('alegre:webhook:foo')) - assert_equal 'foo', response.dig('data', 'requested', 'body', 'id') + assert_equal 'foo', response.dig('requested', 'id') travel_to Time.now.since(2.days) assert_nil redis.lpop('alegre:webhook:foo') From 9e08de34404eff3f04cfd54a2da2392e858d168e Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 18 Oct 2023 21:41:33 +0300 Subject: [PATCH 06/54] CV2-3776: implement a custom pagination for TiplineMessageType (#1697) * CV2-3776: implement a custom pagination for graphql query * CV2-3776: fix after/before * CV2-3776: fix indentation * CV2-3776: fix tests and indentation * CV2-3776: fix sort and add more tests --- .../mutations/tipline_messages_pagination.rb | 50 +++++++++++++++++++ app/graph/types/query_type.rb | 1 + app/graph/types/team_type.rb | 2 +- app/graph/types/tipline_message_type.rb | 5 ++ lib/relay.idl | 1 + public/relay.json | 14 ++++++ test/controllers/graphql_controller_9_test.rb | 41 +++++++++++---- 7 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 app/graph/mutations/tipline_messages_pagination.rb diff --git a/app/graph/mutations/tipline_messages_pagination.rb b/app/graph/mutations/tipline_messages_pagination.rb new file mode 100644 index 0000000000..4de538ee16 --- /dev/null +++ b/app/graph/mutations/tipline_messages_pagination.rb @@ -0,0 +1,50 @@ +class TiplineMessagesPagination < GraphQL::Pagination::ArrayConnection + def cursor_for(item) + encode(item.id.to_i.to_s) + end + + def load_nodes + @nodes ||= begin + sliced_nodes = if before && after + end_idx = index_from_cursor(before) + start_idx = index_from_cursor(after) + items.where(id: start_idx..end_idx) + elsif before + end_idx = index_from_cursor(before) + items.where('id < ?', end_idx) + elsif after + start_idx = index_from_cursor(after) + items.where('id > ?', start_idx) + else + items + end + + @has_previous_page = if last + # There are items preceding the ones in this result + sliced_nodes.count > last + elsif after + # We've paginated into the Array a bit, there are some behind us + index_from_cursor(after) > items.map(&:id).min + else + false + end + + @has_next_page = if first + # There are more items after these items + sliced_nodes.count > first + elsif before + # The original array is longer than the `before` index + index_from_cursor(before) < items.map(&:id).max + else + false + end + + limited_nodes = sliced_nodes + + limited_nodes = limited_nodes.first(first) if first + limited_nodes = limited_nodes.last(last) if last + + limited_nodes + end + end +end diff --git a/app/graph/types/query_type.rb b/app/graph/types/query_type.rb index efe650cca3..a340647923 100644 --- a/app/graph/types/query_type.rb +++ b/app/graph/types/query_type.rb @@ -215,6 +215,7 @@ def dynamic_annotation_field(query:, only_cache: nil) feed request feed_invitation + tipline_message ].each do |type| field type, "#{type.to_s.camelize}Type", diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index dbd44828fb..2cfb2973c1 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -301,6 +301,6 @@ def shared_teams end def tipline_messages(uid:) - object.tipline_messages.where(uid: uid).order('sent_at DESC') + TiplineMessagesPagination.new(object.tipline_messages.where(uid: uid).order('id ASC')) end end diff --git a/app/graph/types/tipline_message_type.rb b/app/graph/types/tipline_message_type.rb index 7bc364551c..88b1384878 100644 --- a/app/graph/types/tipline_message_type.rb +++ b/app/graph/types/tipline_message_type.rb @@ -16,8 +16,13 @@ class TiplineMessageType < DefaultObject field :team, TeamType, null: true field :sent_at, GraphQL::Types::String, null: true, camelize: false field :media_url, GraphQL::Types::String, null: true + field :cursor, GraphQL::Types::String, null: true def sent_at object.sent_at.to_i.to_s end + + def cursor + GraphQL::Schema::Base64Encoder.encode(object.id.to_i.to_s) + end end diff --git a/lib/relay.idl b/lib/relay.idl index 737442f109..6dd2bb852e 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13423,6 +13423,7 @@ TiplineMessage type """ type TiplineMessage implements Node { created_at: String + cursor: String dbid: Int direction: String event: String diff --git a/public/relay.json b/public/relay.json index 52af15b27d..24b6813dcd 100644 --- a/public/relay.json +++ b/public/relay.json @@ -70085,6 +70085,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "cursor", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "dbid", "description": null, diff --git a/test/controllers/graphql_controller_9_test.rb b/test/controllers/graphql_controller_9_test.rb index 3ae958b198..a984c86860 100644 --- a/test/controllers/graphql_controller_9_test.rb +++ b/test/controllers/graphql_controller_9_test.rb @@ -430,7 +430,7 @@ def setup data = JSON.parse(@response.body)['data']['team']['tipline_messages'] results = data['edges'].to_a.collect{ |e| e['node']['dbid'] } assert_equal 1, results.size - assert_equal tp3_uid.id, results[0] + assert_equal tp1_uid.id, results[0] page_info = data['pageInfo'] assert page_info['hasNextPage'] @@ -444,7 +444,7 @@ def setup assert_equal tp2_uid.id, results[0] page_info = data['pageInfo'] assert page_info['hasNextPage'] - + id_cursor_2 = page_info['endCursor'] # Page 3 query = 'query read { team(slug: "test") { tipline_messages(first: 1, after: "' + page_info['endCursor'] + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }' post :create, params: { query: query } @@ -452,25 +452,25 @@ def setup data = JSON.parse(@response.body)['data']['team']['tipline_messages'] results = data['edges'].to_a.collect{ |e| e['node']['dbid'] } assert_equal 1, results.size - assert_equal tp1_uid.id, results[0] + assert_equal tp3_uid.id, results[0] page_info = data['pageInfo'] assert !page_info['hasNextPage'] # paginate using specific message id tp4_uid = create_tipline_message team_id: t.id, uid: uid - query = 'query read { team(slug: "test") { tipline_messages(uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid }, cursor } } } }' + tp5_uid = create_tipline_message team_id: t.id, uid: uid + tp6_uid = create_tipline_message team_id: t.id, uid: uid + query = 'query read { tipline_message(id: "' + tp4_uid.id.to_s + '") { cursor } }' post :create, params: { query: query } assert_response :success - data = JSON.parse(@response.body)['data']['team']['tipline_messages'] - id_cursor = {} - data['edges'].to_a.each{ |e| id_cursor[e['node']['dbid']] = e['cursor'] } - # Start with tp4_uid message - query = 'query read { team(slug: "test") { tipline_messages(first: 1, after: "' + id_cursor[tp4_uid.id] + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }' + id_cursor_4 = JSON.parse(@response.body)['data']['tipline_message']['cursor'] + # Start with tp4_uid message and use after + query = 'query read { team(slug: "test") { tipline_messages(first: 1, after: "' + id_cursor_4 + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }' post :create, params: { query: query } assert_response :success data = JSON.parse(@response.body)['data']['team']['tipline_messages'] results = data['edges'].to_a.collect{ |e| e['node']['dbid'] } assert_equal 1, results.size - assert_equal tp3_uid.id, results[0] + assert_equal tp5_uid.id, results[0] page_info = data['pageInfo'] assert page_info['hasNextPage'] # Next page @@ -480,9 +480,28 @@ def setup data = JSON.parse(@response.body)['data']['team']['tipline_messages'] results = data['edges'].to_a.collect{ |e| e['node']['dbid'] } assert_equal 1, results.size - assert_equal tp2_uid.id, results[0] + assert_equal tp6_uid.id, results[0] + page_info = data['pageInfo'] + assert page_info['hasPreviousPage'] + assert !page_info['hasNextPage'] + # Start with tp4_uid message and use before + query = 'query read { team(slug: "test") { tipline_messages(last: 1, before: "' + id_cursor_4 + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }' + post :create, params: { query: query } + assert_response :success + data = JSON.parse(@response.body)['data']['team']['tipline_messages'] + results = data['edges'].to_a.collect{ |e| e['node']['dbid'] } + assert_equal 1, results.size + assert_equal tp3_uid.id, results[0] page_info = data['pageInfo'] assert page_info['hasNextPage'] + # User after and before + query = 'query read { team(slug: "test") { tipline_messages(after: "' + id_cursor_2 + '", before: "' + id_cursor_4 + '", uid:"'+ uid +'") { pageInfo { endCursor, startCursor, hasPreviousPage, hasNextPage } edges { node { dbid } } } } }' + post :create, params: { query: query } + assert_response :success + data = JSON.parse(@response.body)['data']['team']['tipline_messages'] + results = data['edges'].to_a.collect{ |e| e['node']['dbid'] } + assert_equal 3, results.size + assert_equal [tp2_uid.id, tp3_uid.id, tp4_uid.id], results.sort end protected From 29f6be34c78b2e00c613132e8939d70564840f96 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Fri, 20 Oct 2023 01:37:06 +0300 Subject: [PATCH 07/54] CV2-3777: expose TiplineRequestType (#1692) * CV2-3777: expose TiplineRequestType * CV2-3777: add sent_at(report and report corrections) fields * CV2-3777: get smooch report sent at fields and add more tests * CV2-3777: add permission check for TiplineRequest * CV2-3777: add smooch_request_type field * CV2-3777: add missing tests --- app/graph/types/project_media_type.rb | 2 +- app/graph/types/tipline_request_type.rb | 19 + app/models/bot/smooch.rb | 14 +- app/models/concerns/smooch_fields.rb | 18 + ...mooch_sent_fields_to_smooch_annotations.rb | 12 + lib/relay.idl | 102 ++-- public/relay.json | 547 ++++++++++++++---- test/models/bot/smooch_7_test.rb | 41 ++ test/test_helper.rb | 4 +- 9 files changed, 591 insertions(+), 168 deletions(-) create mode 100644 app/graph/types/tipline_request_type.rb create mode 100644 db/migrate/20231011090947_add_smooch_sent_fields_to_smooch_annotations.rb diff --git a/app/graph/types/project_media_type.rb b/app/graph/types/project_media_type.rb index f92dd09bd2..9c9e75d00e 100644 --- a/app/graph/types/project_media_type.rb +++ b/app/graph/types/project_media_type.rb @@ -193,7 +193,7 @@ def comments end field :requests, - DynamicAnnotationFieldType.connection_type, + TiplineRequestType.connection_type, null: true def requests diff --git a/app/graph/types/tipline_request_type.rb b/app/graph/types/tipline_request_type.rb new file mode 100644 index 0000000000..22f41f2c2e --- /dev/null +++ b/app/graph/types/tipline_request_type.rb @@ -0,0 +1,19 @@ +class TiplineRequestType < DefaultObject + description "TiplineRequest type" + + implements GraphQL::Types::Relay::Node + + field :dbid, GraphQL::Types::Int, null: true + field :value_json, JsonStringType, null: true + field :annotation, AnnotationType, null: true + field :annotation_id, GraphQL::Types::Int, null: true + field :associated_graphql_id, GraphQL::Types::String, null: true + field :smooch_user_slack_channel_url, GraphQL::Types::String, null: true + field :smooch_user_external_identifier, GraphQL::Types::String, null: true + field :smooch_report_received_at, GraphQL::Types::Int, null: true + field :smooch_report_update_received_at, GraphQL::Types::Int, null: true + field :smooch_user_request_language, GraphQL::Types::String, null: true + field :smooch_report_sent_at, GraphQL::Types::Int, null: true + field :smooch_report_correction_sent_at, GraphQL::Types::Int, null: true + field :smooch_request_type, GraphQL::Types::String, null: true +end diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb index 9f4057bd29..5f31425dd0 100644 --- a/app/models/bot/smooch.rb +++ b/app/models/bot/smooch.rb @@ -940,23 +940,33 @@ def self.send_report_to_users(pm, action) parent.get_deduplicated_smooch_annotations.each do |annotation| data = JSON.parse(annotation.load.get_field_value('smooch_data')) self.get_installation(self.installation_setting_id_keys, data['app_id']) if self.config.blank? - self.send_correction_to_user(data, parent, annotation.created_at, last_published_at, action, report.get_field_value('published_count').to_i) unless self.config['smooch_disabled'] + self.send_correction_to_user(data, parent, annotation, last_published_at, action, report.get_field_value('published_count').to_i) unless self.config['smooch_disabled'] end end - def self.send_correction_to_user(data, pm, subscribed_at, last_published_at, action, published_count = 0) + def self.send_correction_to_user(data, pm, annotation, last_published_at, action, published_count = 0) + subscribed_at = annotation.created_at self.get_platform_from_message(data) uid = data['authorId'] lang = data['language'] + field_name = '' # User received a report before if subscribed_at.to_i < last_published_at.to_i && published_count > 0 if ['publish', 'republish_and_resend'].include?(action) + field_name = 'smooch_report_correction_sent_at' self.send_report_to_user(uid, data, pm, lang, 'fact_check_report_updated', self.get_string(:report_updated, lang)) end # First report else + field_name = 'smooch_report_sent_at' self.send_report_to_user(uid, data, pm, lang, 'fact_check_report') end + unless field_name.blank? + annotation = annotation.load + annotation.skip_check_ability = true + annotation.set_fields = { "#{field_name}": Time.now.to_i }.to_json + annotation.save! + end end def self.send_report_to_user(uid, data, pm, lang = 'en', fallback_template = nil, pre_message = nil) diff --git a/app/models/concerns/smooch_fields.rb b/app/models/concerns/smooch_fields.rb index bd776a709f..05f6cb89a9 100644 --- a/app/models/concerns/smooch_fields.rb +++ b/app/models/concerns/smooch_fields.rb @@ -65,6 +65,24 @@ def smooch_report_update_received_at end end + def smooch_report_sent_at + Concurrent::Future.execute(executor: CheckGraphql::POOL) do + begin self.annotation.load.get_field_value('smooch_report_sent_at').to_i rescue nil end + end + end + + def smooch_report_correction_sent_at + Concurrent::Future.execute(executor: CheckGraphql::POOL) do + begin self.annotation.load.get_field_value('smooch_report_correction_sent_at').to_i rescue nil end + end + end + + def smooch_request_type + Concurrent::Future.execute(executor: CheckGraphql::POOL) do + begin self.annotation.load.get_field_value('smooch_request_type') rescue nil end + end + end + def smooch_user_request_language Concurrent::Future.execute(executor: CheckGraphql::POOL) do return '' unless self.field_name == 'smooch_data' diff --git a/db/migrate/20231011090947_add_smooch_sent_fields_to_smooch_annotations.rb b/db/migrate/20231011090947_add_smooch_sent_fields_to_smooch_annotations.rb new file mode 100644 index 0000000000..51bddf7827 --- /dev/null +++ b/db/migrate/20231011090947_add_smooch_sent_fields_to_smooch_annotations.rb @@ -0,0 +1,12 @@ +require 'sample_data' +include SampleData +class AddSmoochSentFieldsToSmoochAnnotations < ActiveRecord::Migration[6.1] + def change + t = DynamicAnnotation::FieldType.where(field_type: 'timestamp').last || create_field_type(field_type: 'timestamp', label: 'Timestamp') + at = DynamicAnnotation::AnnotationType.where(annotation_type: 'smooch').last + unless at.nil? + create_field_instance annotation_type_object: at, name: 'smooch_report_correction_sent_at', label: 'Report correction sent time', field_type_object: t, optional: true + create_field_instance annotation_type_object: at, name: 'smooch_report_sent_at', label: 'Report sent time', field_type_object: t, optional: true + end + end +end diff --git a/lib/relay.idl b/lib/relay.idl index 6dd2bb852e..a5ad34e0b5 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -4685,42 +4685,6 @@ type DynamicAnnotationField implements Node { value_json: JsonStringType } -""" -The connection type for DynamicAnnotationField. -""" -type DynamicAnnotationFieldConnection { - """ - A list of edges. - """ - edges: [DynamicAnnotationFieldEdge] - - """ - A list of nodes. - """ - nodes: [DynamicAnnotationField] - - """ - Information to aid in pagination. - """ - pageInfo: PageInfo! - totalCount: Int -} - -""" -An edge in a connection. -""" -type DynamicAnnotationFieldEdge { - """ - A cursor for use in pagination. - """ - cursor: String! - - """ - The item at the end of the edge. - """ - node: DynamicAnnotationField -} - """ The connection type for Dynamic. """ @@ -11465,7 +11429,7 @@ type ProjectMedia implements Node { Returns the last _n_ elements from the list. """ last: Int - ): DynamicAnnotationFieldConnection + ): TiplineRequestConnection requests_count: Int share_count: Int show_warning_cover: Boolean @@ -11788,6 +11752,11 @@ type Query { """ team(id: ID, random: String, slug: String): Team + """ + Information about the tipline_message with given id + """ + tipline_message(id: ID!): TiplineMessage + """ Information about the user with given id """ @@ -13548,6 +13517,65 @@ type TiplineNewsletterEdge { node: TiplineNewsletter } +""" +TiplineRequest type +""" +type TiplineRequest implements Node { + annotation: Annotation + annotation_id: Int + associated_graphql_id: String + created_at: String + dbid: Int + id: ID! + permissions: String + smooch_report_correction_sent_at: Int + smooch_report_received_at: Int + smooch_report_sent_at: Int + smooch_report_update_received_at: Int + smooch_request_type: String + smooch_user_external_identifier: String + smooch_user_request_language: String + smooch_user_slack_channel_url: String + updated_at: String + value_json: JsonStringType +} + +""" +The connection type for TiplineRequest. +""" +type TiplineRequestConnection { + """ + A list of edges. + """ + edges: [TiplineRequestEdge] + + """ + A list of nodes. + """ + nodes: [TiplineRequest] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! + totalCount: Int +} + +""" +An edge in a connection. +""" +type TiplineRequestEdge { + """ + A cursor for use in pagination. + """ + cursor: String! + + """ + The item at the end of the edge. + """ + node: TiplineRequest +} + """ TiplineResource type """ diff --git a/public/relay.json b/public/relay.json index 24b6813dcd..d10ba20745 100644 --- a/public/relay.json +++ b/public/relay.json @@ -26446,132 +26446,6 @@ "enumValues": null, "possibleTypes": null }, - { - "kind": "OBJECT", - "name": "DynamicAnnotationFieldConnection", - "description": "The connection type for DynamicAnnotationField.", - "fields": [ - { - "name": "edges", - "description": "A list of edges.", - "args": [ - - ], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "DynamicAnnotationFieldEdge", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [ - - ], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "DynamicAnnotationField", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [ - - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": null, - "args": [ - - ], - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "DynamicAnnotationFieldEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination.", - "args": [ - - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge.", - "args": [ - - ], - "type": { - "kind": "OBJECT", - "name": "DynamicAnnotationField", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - - ], - "enumValues": null, - "possibleTypes": null - }, { "kind": "OBJECT", "name": "DynamicConnection", @@ -55111,6 +54985,11 @@ "name": "TiplineNewsletter", "ofType": null }, + { + "kind": "OBJECT", + "name": "TiplineRequest", + "ofType": null + }, { "kind": "OBJECT", "name": "TiplineResource", @@ -60191,7 +60070,7 @@ ], "type": { "kind": "OBJECT", - "name": "DynamicAnnotationFieldConnection", + "name": "TiplineRequestConnection", "ofType": null }, "isDeprecated": false, @@ -61984,6 +61863,35 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "tipline_message", + "description": "Information about the tipline_message with given id", + "args": [ + { + "name": "id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "TiplineMessage", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "user", "description": "Information about the user with given id", @@ -71004,6 +70912,391 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "TiplineRequest", + "description": "TiplineRequest type", + "fields": [ + { + "name": "annotation", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Annotation", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "annotation_id", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "associated_graphql_id", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "dbid", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "permissions", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "smooch_report_correction_sent_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "smooch_report_received_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "smooch_report_sent_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "smooch_report_update_received_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "smooch_request_type", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "smooch_user_external_identifier", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "smooch_user_request_language", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "smooch_user_slack_channel_url", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updated_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "value_json", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "Node", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "TiplineRequestConnection", + "description": "The connection type for TiplineRequest.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TiplineRequestEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TiplineRequest", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "totalCount", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "TiplineRequestEdge", + "description": "An edge in a connection.", + "fields": [ + { + "name": "cursor", + "description": "A cursor for use in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "node", + "description": "The item at the end of the edge.", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "TiplineRequest", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "TiplineResource", diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 234e523472..db022afd1b 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -529,4 +529,45 @@ def teardown assert_equal 2, pm.positive_tipline_search_results_count end end + + test "should save report and report correction sent at " do + messages = [ + { + '_id': random_string, + authorId: random_string, + type: 'text', + text: random_string + } + ] + payload = { + trigger: 'message:appUser', + app: { + '_id': @app_id + }, + version: 'v1.1', + messages: messages, + appUser: { + '_id': random_string, + 'conversationStarted': true + } + }.to_json + Bot::Smooch.run(payload) + sleep 1 + pm = ProjectMedia.last + r = create_report(pm) + publish_report(pm, {}, r) + r = Dynamic.find(r.id) + r.set_fields = { state: 'paused' }.to_json + r.action = 'pause' + r.save! + r = Dynamic.find(r.id) + r.set_fields = { state: 'published' }.to_json + r.action = 'republish_and_resend' + r.save! + a = pm.annotations('smooch').last.load + smooch_data = a.get_field('smooch_data') + assert_not_nil smooch_data.smooch_report_sent_at + assert_not_nil smooch_data.smooch_report_correction_sent_at + assert_not_nil smooch_data.smooch_request_type + end end diff --git a/test/test_helper.rb b/test/test_helper.rb index a7888e70da..38473d9cdd 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -850,7 +850,9 @@ def setup_smooch_bot(menu = false, extra_settings = {}) 'Data' => ['JSON', false], 'Report Received' => ['Timestamp', true], 'Request Type' => ['Text', true], - 'Resource Id' => ['Text', true] + 'Resource Id' => ['Text', true], + 'Report correction sent at' => ['Timestamp', true], + 'Report sent at' => ['Timestamp', true], }) create_annotation_type_and_fields('Smooch Response', { 'Data' => ['JSON', true] }) create_annotation_type annotation_type: 'reverse_image', label: 'Reverse Image' From 0cab2aa6e7dffb3081c9857d4e2a39fdb9c263d1 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Fri, 20 Oct 2023 22:35:40 -0300 Subject: [PATCH 08/54] Update tagger bot name and description and make the rake task idempotent * Ticket CV2-3877: Update tagger bot name and description and make the rake task idempotent * Update 202306040000_add_tagger_bot_user.rake Tweak bot description --------- Co-authored-by: Scott Hale --- .../202306040000_add_tagger_bot_user.rake | 81 +++++++++---------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/lib/tasks/migrate/202306040000_add_tagger_bot_user.rake b/lib/tasks/migrate/202306040000_add_tagger_bot_user.rake index 4c1bf6504c..be6d3f9168 100644 --- a/lib/tasks/migrate/202306040000_add_tagger_bot_user.rake +++ b/lib/tasks/migrate/202306040000_add_tagger_bot_user.rake @@ -1,47 +1,42 @@ namespace :check do - namespace :migrate do - desc "Create the Bot user Tagger for the Tagger bot" - task create_tagger_user: :environment do - RequestStore.store[:skip_notifications] = true + namespace :migrate do + desc 'Create the bot user Tagger for the Tagger bot' + task create_tagger_user: :environment do + RequestStore.store[:skip_notifications] = true - if !BotUser.get_user("tagger").nil? - puts "Tagger user already exists, exiting" - return - end + meedan_team = Team.where(slug: 'meedan').last || Team.new(name: 'Meedan', slug: 'meedan') + meedan_team.skip_notifications = true + meedan_team.skip_clear_cache = true + meedan_team.skip_check_ability = true + meedan_team.save! if meedan_team.id.nil? + + Team.current = meedan_team + tb = BotUser.get_user('tagger') || BotUser.new + tb.login = 'tagger' + tb.name = 'Tagger (in beta)' + tb.set_description 'Automatically tags new items based on existing similar items. This feature is in beta, please consult with the Meedan support team before enabling it.' + File.open(File.join(Rails.root, 'public', 'tagger.png')) do |f| + tb.image = f + end + tb.set_role 'editor' + tb.set_version '0.0.1' + tb.set_source_code_url 'https://github.com/meedan/check-api/blob/develop/app/models/bot/tagger.rb' + tb.set_team_author_id meedan_team.id + tb.set_events [{ 'event' => 'create_project_media', 'graphql' => 'dbid, title, description, type' }] + tb.set_settings [ + { name: 'auto_tag_prefix', label: 'Emoji prefix', description: 'Emoji to be placed in front of autotags', type: 'string', default: '⚡' }, + { name: 'threshold', label: 'threshold', description: 'Search similarity threshold (0-100)', type: 'integer', default: 70 }, + { name: 'ignore_autotags', label: 'Ignore auto-tags?', description:'If enabled, autotags will not be considered in finding the most common tag', type: 'boolean', default: false }, + { name: 'minimum_count', label: 'Minimum count', description:'Minimum number of times a tag must appear to be applied', type: 'integer', default: 0 } + ] + tb.set_approved true + tb['settings']['listed'] = true # Appear on the integrations tab of Check Web + tb.save! + + Team.current = nil + RequestStore.store[:skip_notifications] = false - meedan_team = Team.where(slug: 'meedan').last || Team.new(name: 'Meedan', slug: 'meedan') - meedan_team.skip_notifications = true - meedan_team.skip_clear_cache = true - meedan_team.skip_check_ability = true - meedan_team.save! - - Team.current = meedan_team - tb = BotUser.new - tb.login = 'tagger' - tb.name = 'Tagger' - tb.set_description 'Add tags to items automatically based on similar items.' - File.open(File.join(Rails.root, 'public', 'tagger.png')) do |f| - tb.image = f - end - tb.set_role 'editor' - tb.set_version '0.0.1' - tb.set_source_code_url 'https://github.com/meedan/check-api/blob/develop/app/models/bot/tagger.rb' - tb.set_team_author_id meedan_team.id - tb.set_events [{"event"=>"create_project_media", "graphql"=>"dbid, title, description, type"}] - tb.set_settings [ - { name: 'auto_tag_prefix', label: 'Emoji prefix', description: 'Emoji to be placed in front of autotags', type: 'string', default: '⚡' }, - { name: 'threshold', label: 'threshold', description: 'Search similarity threshold (0-100)', type: 'integer', default: 70 }, - { name: 'ignore_autotags', label: 'Ignore auto-tags?', description:'If enabled, autotags will not be considered in finding the most common tag', type: 'boolean', default: false }, - { name: 'minimum_count', label: 'Minimum count', description:'Minimum number of times a tag must appear to be applied', type: 'integer', default: 0 } - ] - tb.set_approved true - tb['settings']['listed']=true #Appear on the integrations tab of Check Web - tb.save! - - Team.current = nil - RequestStore.store[:skip_notifications] = false - - puts "Tagger user successfully created" - end + puts 'Tagger bot successfully saved.' end -end \ No newline at end of file + end +end From df05a19ccdd1a425e222d021236249561019c06f Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Sun, 22 Oct 2023 19:28:30 -0300 Subject: [PATCH 09/54] Support Instagram tiplines Allow Instagram profiles to be connected to tiplines, through Smooch. * oAuth flow (authorization flow, reusing most of the logic implemented for Messenger, that connects an Instagram profile with a Check tipline, through Smooch) * Report settings (allow the Instagram tipline to be advertised in reports) * Search for items with Instagram requests * Support buttons and menus on tipline interactions Reference: CV2-3727. --- app/controllers/api/v1/admin_controller.rb | 19 +++++++++++---- .../concerns/facebook_authentication.rb | 2 +- app/lib/check_channels.rb | 4 +++- app/models/bot/smooch.rb | 2 +- app/models/concerns/smooch_fields.rb | 2 +- app/models/concerns/smooch_menus.rb | 6 ++--- config/initializers/report_designer.rb | 7 ++++-- config/locales/en.yml | 4 ++-- config/locales/es.yml | 4 ++-- config/locales/pt.yml | 4 ++-- config/routes.rb | 3 ++- .../report-design-default-image-template.html | 17 +++++++++++--- test/controllers/admin_controller_test.rb | 23 ++++++++++++++++--- 13 files changed, 71 insertions(+), 26 deletions(-) diff --git a/app/controllers/api/v1/admin_controller.rb b/app/controllers/api/v1/admin_controller.rb index 42cc7a2fd2..b9e208a761 100644 --- a/app/controllers/api/v1/admin_controller.rb +++ b/app/controllers/api/v1/admin_controller.rb @@ -1,5 +1,5 @@ class Api::V1::AdminController < Api::V1::BaseApiController - before_action :authenticate_from_token!, except: [:add_publisher_to_project, :save_twitter_credentials_for_smooch_bot, :save_facebook_credentials_for_smooch_bot] + before_action :authenticate_from_token!, except: [:add_publisher_to_project, :save_twitter_credentials_for_smooch_bot, :save_messenger_credentials_for_smooch_bot, :save_instagram_credentials_for_smooch_bot] # GET /api/admin/project/add_publisher?token=:project-token def add_publisher_to_project @@ -51,8 +51,19 @@ def save_twitter_credentials_for_smooch_bot render template: 'message', formats: :html, status: status end - # GET /api/admin/smooch_bot/:bot-installation-id/authorize/facebook?token=:bot-installation-token - def save_facebook_credentials_for_smooch_bot + # GET /api/admin/smooch_bot/:bot-installation-id/authorize/messenger?token=:bot-installation-token + def save_messenger_credentials_for_smooch_bot + self.save_facebook_credentials_for_smooch_bot('messenger') + end + + # GET /api/admin/smooch_bot/:bot-installation-id/authorize/instagram?token=:bot-installation-token + def save_instagram_credentials_for_smooch_bot + self.save_facebook_credentials_for_smooch_bot('instagram') + end + + private + + def save_facebook_credentials_for_smooch_bot(platform) # "platform" is either "instagram" or "messenger" tbi = TeamBotInstallation.find(params[:id]) auth = session['check.facebook.authdata'] status = nil @@ -68,7 +79,7 @@ def save_facebook_credentials_for_smooch_bot 'appSecret' => CheckConfig.get('smooch_facebook_app_secret'), 'pageAccessToken' => pages[0]['access_token'] } - tbi.smooch_add_integration('messenger', params) + tbi.smooch_add_integration(platform, params) @message = I18n.t(:smooch_facebook_success) status = 200 end diff --git a/app/controllers/concerns/facebook_authentication.rb b/app/controllers/concerns/facebook_authentication.rb index 06a8882321..29a142d9d8 100644 --- a/app/controllers/concerns/facebook_authentication.rb +++ b/app/controllers/concerns/facebook_authentication.rb @@ -5,7 +5,7 @@ def setup_facebook # pages_manage_metadata is for Facebook API > 7 # manage_pages is for Facebook API < 7 # An error will be displayed for Facebook users that are admins of the Facebook app, but should be transparent for other users - request.env['omniauth.strategy'].options[:scope] = 'pages_manage_metadata,manage_pages,pages_messaging' if params[:context] == 'smooch' + request.env['omniauth.strategy'].options[:scope] = 'pages_manage_metadata,manage_pages,pages_messaging,instagram_basic,instagram_manage_messages' if params[:context] == 'smooch' prefix = facebook_context == 'smooch' ? 'smooch_' : '' request.env['omniauth.strategy'].options[:client_id] = CheckConfig.get("#{prefix}facebook_app_id") request.env['omniauth.strategy'].options[:client_secret] = CheckConfig.get("#{prefix}facebook_app_secret") diff --git a/app/lib/check_channels.rb b/app/lib/check_channels.rb index ee7a3a2551..8e17a0de16 100644 --- a/app/lib/check_channels.rb +++ b/app/lib/check_channels.rb @@ -14,6 +14,7 @@ def self.all_channels "TELEGRAM" => TELEGRAM, "VIBER" => VIBER, "LINE" => LINE, + "INSTAGRAM" => INSTAGRAM, }, "WEB_FORM" => WEB_FORM, "SHARED_DATABASE" => SHARED_DATABASE @@ -30,7 +31,8 @@ def self.all_channels TELEGRAM = 8 VIBER = 9 LINE = 10 - TIPLINE = [WHATSAPP, MESSENGER, TWITTER, TELEGRAM, VIBER, LINE] + INSTAGRAM = 13 + TIPLINE = [WHATSAPP, MESSENGER, TWITTER, TELEGRAM, VIBER, LINE, INSTAGRAM] WEB_FORM = 11 SHARED_DATABASE = 12 ALL = [MANUAL, FETCH, BROWSER_EXTENSION, API, ZAPIER, WEB_FORM, SHARED_DATABASE] + TIPLINE diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb index 5f31425dd0..d07fa4dc67 100644 --- a/app/models/bot/smooch.rb +++ b/app/models/bot/smooch.rb @@ -10,7 +10,7 @@ class CapiUnhandledMessageWarning < MessageDeliveryError; end MESSAGE_BOUNDARY = "\u2063" - SUPPORTED_INTEGRATION_NAMES = { 'whatsapp' => 'WhatsApp', 'messenger' => 'Facebook Messenger', 'twitter' => 'Twitter', 'telegram' => 'Telegram', 'viber' => 'Viber', 'line' => 'LINE' } + SUPPORTED_INTEGRATION_NAMES = { 'whatsapp' => 'WhatsApp', 'messenger' => 'Facebook Messenger', 'twitter' => 'Twitter', 'telegram' => 'Telegram', 'viber' => 'Viber', 'line' => 'LINE', 'instagram' => 'Instagram' } SUPPORTED_INTEGRATIONS = SUPPORTED_INTEGRATION_NAMES.keys SUPPORTED_TRIGGER_MAPPING = { 'message:appUser' => :incoming, 'message:delivery:channel' => :outgoing } diff --git a/app/models/concerns/smooch_fields.rb b/app/models/concerns/smooch_fields.rb index 05f6cb89a9..51ea089455 100644 --- a/app/models/concerns/smooch_fields.rb +++ b/app/models/concerns/smooch_fields.rb @@ -35,7 +35,7 @@ def smooch_user_external_identifier case user[:platform] when 'whatsapp' user[:displayName] - when 'telegram' + when 'telegram', 'instagram' '@' + user[:raw][:username].to_s when 'messenger', 'viber', 'line' user[:externalId] diff --git a/app/models/concerns/smooch_menus.rb b/app/models/concerns/smooch_menus.rb index 49d5b53efa..08b4b2001d 100644 --- a/app/models/concerns/smooch_menus.rb +++ b/app/models/concerns/smooch_menus.rb @@ -97,7 +97,7 @@ def send_message_to_user_with_main_menu_appended(uid, text, workflow, language, end end - if ['Telegram', 'Viber', 'Facebook Messenger', 'LINE'].include?(self.request_platform) + if ['Telegram', 'Viber', 'Facebook Messenger', 'LINE', 'Instagram'].include?(self.request_platform) actions = [] main.each do |section| section[:rows].each do |row| @@ -248,7 +248,7 @@ def format_fallback_text_menu_from_options(text, options, extra) fallback << self.format_fallback_text_menu_option(option, :value, :label) end - if ['Telegram', 'Viber', 'Facebook Messenger', 'LINE'].include?(self.request_platform) + if ['Telegram', 'Viber', 'Facebook Messenger', 'LINE', 'Instagram'].include?(self.request_platform) actions = [] options.each do |option| actions << { @@ -286,7 +286,7 @@ def ask_for_language_confirmation(_workflow, language, uid, with_text = true) } end text = text.join("\n\n") - if ['Telegram', 'Viber', 'Facebook Messenger', 'LINE'].include?(self.request_platform) + if ['Telegram', 'Viber', 'Facebook Messenger', 'LINE', 'Instagram'].include?(self.request_platform) text = '🌐​' unless with_text self.send_message_to_user_with_single_section_menu(uid, text, options, self.get_string('languages', language)) else diff --git a/config/initializers/report_designer.rb b/config/initializers/report_designer.rb index 6662ffa454..5fe96de943 100644 --- a/config/initializers/report_designer.rb +++ b/config/initializers/report_designer.rb @@ -74,9 +74,10 @@ def report_design_text_footer(language) twitter: 'Twitter: twitter.com/', telegram: 'Telegram: t.me/', viber: 'Viber: ', - line: 'LINE: ' + line: 'LINE: ', + instagram: 'Instagram: instagram.com/' } - [:signature, :whatsapp, :facebook, :twitter, :telegram, :viber, :line].each do |field| + [:signature, :whatsapp, :facebook, :twitter, :telegram, :viber, :line, :instagram].each do |field| value = self.report_design_team_setting_value(field.to_s, language) footer << "#{prefixes[field]}#{value}" unless value.blank? end @@ -126,6 +127,7 @@ def report_design_placeholders(language) facebook = self.report_design_team_setting_value('facebook', language) twitter = self.report_design_team_setting_value('twitter', language) telegram = self.report_design_team_setting_value('telegram', language) + instagram = self.report_design_team_setting_value('instagram', language) { title: self.report_design_field_value('headline'), status: self.report_design_field_value('status_label'), @@ -135,6 +137,7 @@ def report_design_placeholders(language) facebook: facebook.blank? ? nil : "m.me/#{facebook}", twitter: twitter.blank? ? nil : "@#{twitter}", telegram: telegram.blank? ? nil : "t.me/#{telegram}", + instagram: instagram.blank? ? nil : "instagram.com/#{instagram}", viber: self.report_design_team_setting_value('viber', language), line: self.report_design_team_setting_value('line', language) } diff --git a/config/locales/en.yml b/config/locales/en.yml index 54b9d6e54b..68704aed0d 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -629,8 +629,8 @@ en: bot_request_url_invalid: Invalid bot URL smooch_facebook_success: | Success! - Your tipline is now connected to Facebook Messenger. - Any message received by this Facebook Page will be handled by the tipline. + Your tipline is now connected to the profile. + Any message received by this profile will be handled by the tipline. Please reload your tipline settings page to see the new changes. smooch_twitter_success: | Success! diff --git a/config/locales/es.yml b/config/locales/es.yml index 2dd82b5429..e26f5754d9 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -564,8 +564,8 @@ es: bot_request_url_invalid: El URL del bot es inválido smooch_facebook_success: | ¡Éxito! - Tu Tipline está ahora connectada a Facebook Messenger. - Cualquier mensaje recibido por esta Página de Facebook va a ser manejado por la Tipline. + Tu Tipline está ahora connectada. + Cualquier mensaje recibido va a ser manejado por la Tipline. smooch_twitter_success: | ¡Éxito! Tu Tipline está ahora connectada a Twitter. diff --git a/config/locales/pt.yml b/config/locales/pt.yml index 4b3636e3c2..e1521cab2a 100644 --- a/config/locales/pt.yml +++ b/config/locales/pt.yml @@ -562,8 +562,8 @@ pt: bot_request_url_invalid: A URL do bot é inválida smooch_facebook_success: | Sucesso! - Sua Tipline agora está conectada ao Facebook Messenger. - Todas as mensagens recebidas por esta página do Facebook será gerenciada pela Tipline. + Sua Tipline agora está conectada. + Todas as mensagens recebidas por este perfil será gerenciada pela Tipline. Por favor, recarregue a página de configurações da sua Tipline para ver as novas mudanças. smooch_twitter_success: | Sucesso! diff --git a/config/routes.rb b/config/routes.rb index f2771d4b72..e4bc0fa4e2 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -34,7 +34,8 @@ match '/admin/project/:id/add_publisher/:provider' => 'admin#add_publisher_to_project', via: [:get] match '/admin/user/slack' => 'admin#slack_user', via: [:get] match '/admin/smooch_bot/:id/authorize/twitter' => 'admin#save_twitter_credentials_for_smooch_bot', via: [:get] - match '/admin/smooch_bot/:id/authorize/facebook' => 'admin#save_facebook_credentials_for_smooch_bot', via: [:get] + match '/admin/smooch_bot/:id/authorize/messenger' => 'admin#save_messenger_credentials_for_smooch_bot', via: [:get] + match '/admin/smooch_bot/:id/authorize/instagram' => 'admin#save_instagram_credentials_for_smooch_bot', via: [:get] match '/project_medias/:id/oembed' => 'project_medias#oembed', via: [:get], defaults: { format: :json } match '/webhooks/:name' => 'webhooks#index', via: [:post, :get], defaults: { format: :json } devise_for :users, controllers: { invitations: 'api/v1/invitations', sessions: 'api/v1/sessions', registrations: 'api/v1/registrations', omniauth_callbacks: 'api/v1/omniauth_callbacks', confirmations: 'api/v1/confirmations' } diff --git a/public/report-design-default-image-template.html b/public/report-design-default-image-template.html index 96da77c0c7..3cb76591c7 100644 --- a/public/report-design-default-image-template.html +++ b/public/report-design-default-image-template.html @@ -113,7 +113,7 @@ right: 20px; } - #url.translucid, #date.translucid, #whatsapp.translucid, #facebook.translucid, #twitter.translucid, #telegram.translucid, #viber.translucid, #line .translucid { + #url.translucid, #date.translucid, #whatsapp.translucid, #facebook.translucid, #twitter.translucid, #telegram.translucid, #viber.translucid, #line.translucid, #instagram.translucid { padding: 5px 10px; } @@ -145,7 +145,7 @@ background: rgba(0, 0, 0, 0.75); } - .dark #description, .dark #url, .dark #date, .dark #whatsapp, .dark #facebook, .dark #twitter, .dark #telegram, .dark #viber, .dark #line { + .dark #description, .dark #url, .dark #date, .dark #whatsapp, .dark #facebook, .dark #twitter, .dark #telegram, .dark #viber, .dark #line, .dark #instagram { color: white; } @@ -165,7 +165,7 @@ filter: invert(1); } - #whatsapp, #facebook, #twitter, #telegram, #viber, #line { + #whatsapp, #facebook, #twitter, #telegram, #viber, #line, #instagram { display: flex; } @@ -192,6 +192,10 @@ #line .social-icon { background-color: #00b900; } + + #instagram .social-icon { + background-color: #E1306C; + } @@ -251,6 +255,13 @@ +
+
+ +
+
diff --git a/test/controllers/admin_controller_test.rb b/test/controllers/admin_controller_test.rb index 9f0ffd2e91..7cc336f89d 100644 --- a/test/controllers/admin_controller_test.rb +++ b/test/controllers/admin_controller_test.rb @@ -93,7 +93,7 @@ def setup b = create_team_bot login: 'smooch' tbi = create_team_bot_installation session['check.facebook.authdata'] = { 'token' => '123456', 'secret' => '654321' } - get :save_facebook_credentials_for_smooch_bot, params: { id: tbi.id, token: random_string } + get :save_messenger_credentials_for_smooch_bot, params: { id: tbi.id, token: random_string } assert_response 401 end @@ -108,7 +108,7 @@ def setup tbi.set_smooch_authorization_token = t tbi.save! session['check.facebook.authdata'] = { 'token' => '123456', 'secret' => '654321' } - get :save_facebook_credentials_for_smooch_bot, params: { id: tbi.id, token: t } + get :save_messenger_credentials_for_smooch_bot, params: { id: tbi.id, token: t } assert_response 400 Bot::Smooch.unstub(:smooch_api_client) SmoochApi::IntegrationApi.any_instance.unstub(:create_integration) @@ -125,7 +125,24 @@ def setup tbi.set_smooch_authorization_token = t tbi.save! session['check.facebook.authdata'] = { 'token' => '123456', 'secret' => '654321' } - get :save_facebook_credentials_for_smooch_bot, params: { id: tbi.id, token: t } + get :save_messenger_credentials_for_smooch_bot, params: { id: tbi.id, token: t } + assert_response :success + Bot::Smooch.unstub(:smooch_api_client) + SmoochApi::IntegrationApi.any_instance.unstub(:create_integration) + end + + test "should connect Instagram profile to Smooch bot" do + Bot::Smooch.stubs(:smooch_api_client).returns(nil) + SmoochApi::IntegrationApi.any_instance.expects(:create_integration).once + WebMock.stub_request(:get, /graph\.facebook\.com\/me\/accounts/).to_return(body: { data: [{ access_token: random_string }] }.to_json, status: 200) + b = create_team_bot login: 'smooch' + t = random_string + tbi = create_team_bot_installation + tbi = TeamBotInstallation.find(tbi.id) + tbi.set_smooch_authorization_token = t + tbi.save! + session['check.facebook.authdata'] = { 'token' => '123456', 'secret' => '654321' } + get :save_instagram_credentials_for_smooch_bot, params: { id: tbi.id, token: t } assert_response :success Bot::Smooch.unstub(:smooch_api_client) SmoochApi::IntegrationApi.any_instance.unstub(:create_integration) From 8e5c935ed0c443ecc2d2606299b67bb86bd5359c Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 23 Oct 2023 08:53:34 -0700 Subject: [PATCH 10/54] CV2-3827 explicitly request callback (#1700) * CV2-3827 explicitly request callback * CV2-3827 more tweaks to fix integration * fix fixture * fix fixtures' * Update webhooks_controller_test.rb --- app/models/concerns/alegre_similarity.rb | 1 + app/models/concerns/alegre_webhooks.rb | 4 ++-- test/controllers/webhooks_controller_test.rb | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/models/concerns/alegre_similarity.rb b/app/models/concerns/alegre_similarity.rb index bc5d6faf61..b5187cafbd 100644 --- a/app/models/concerns/alegre_similarity.rb +++ b/app/models/concerns/alegre_similarity.rb @@ -184,6 +184,7 @@ def send_to_media_similarity_index(pm) url: self.media_file_url(pm), context: self.get_context(pm), match_across_content_types: true, + requires_callback: true } self.request_api( 'post', diff --git a/app/models/concerns/alegre_webhooks.rb b/app/models/concerns/alegre_webhooks.rb index a0ce16106d..326bd4297c 100644 --- a/app/models/concerns/alegre_webhooks.rb +++ b/app/models/concerns/alegre_webhooks.rb @@ -12,14 +12,14 @@ def valid_request?(request) def webhook(request) begin - doc_id = request.params.dig('requested', 'id') + doc_id = request.params.dig('data', 'requested', 'id') raise 'Unexpected params format' if doc_id.blank? redis = Redis.new(REDIS_CONFIG) key = "alegre:webhook:#{doc_id}" redis.lpush(key, request.params.to_json) redis.expire(key, 1.day.to_i) rescue StandardError => e - CheckSentry.notify(AlegreCallbackError.new(e.message), { alegre_response: request.params }) + CheckSentry.notify(AlegreCallbackError.new(e.message), params: { alegre_response: request.params }) end end end diff --git a/test/controllers/webhooks_controller_test.rb b/test/controllers/webhooks_controller_test.rb index fa28aeacab..87142f294d 100644 --- a/test/controllers/webhooks_controller_test.rb +++ b/test/controllers/webhooks_controller_test.rb @@ -237,15 +237,15 @@ def setup CheckSentry.expects(:notify).never redis = Redis.new(REDIS_CONFIG) redis.del('foo') - payload = { 'action' => 'audio', 'requested' => { 'id' => 'foo', 'context' => { 'project_media_id' => random_number } } } + id = random_number + payload = { 'action' => 'audio', 'data' => {'requested' => { 'id' => 'foo', 'context' => { 'project_media_id' => id } }} } assert_nil redis.lpop('alegre:webhook:foo') post :index, params: { name: :alegre, token: CheckConfig.get('alegre_token') }.merge(payload) response = JSON.parse(redis.lpop('alegre:webhook:foo')) - assert_equal 'foo', response.dig('requested', 'id') - - travel_to Time.now.since(2.days) - assert_nil redis.lpop('alegre:webhook:foo') + assert_equal 'foo', response.dig('data', 'requested', 'id') + expectation = {"action"=>"index", "data"=>{"requested"=>{"context"=>{"project_media_id"=>id.to_s}, "id"=>"foo"}}, "token"=>"test", "name"=>"alegre", "controller"=>"api/v1/webhooks"} + assert_equal expectation, response end test "should report error if can't process Alegre webhook" do From 41d08222217e59bb021b5e00d3c31ddf0e915a8e Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 23 Oct 2023 19:09:02 -0300 Subject: [PATCH 11/54] Ticket CV2-3727: Testing if we really need the instagram_basic permission --- app/controllers/concerns/facebook_authentication.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/concerns/facebook_authentication.rb b/app/controllers/concerns/facebook_authentication.rb index 29a142d9d8..8b61035e64 100644 --- a/app/controllers/concerns/facebook_authentication.rb +++ b/app/controllers/concerns/facebook_authentication.rb @@ -5,7 +5,7 @@ def setup_facebook # pages_manage_metadata is for Facebook API > 7 # manage_pages is for Facebook API < 7 # An error will be displayed for Facebook users that are admins of the Facebook app, but should be transparent for other users - request.env['omniauth.strategy'].options[:scope] = 'pages_manage_metadata,manage_pages,pages_messaging,instagram_basic,instagram_manage_messages' if params[:context] == 'smooch' + request.env['omniauth.strategy'].options[:scope] = 'pages_manage_metadata,manage_pages,pages_messaging,instagram_manage_messages' if params[:context] == 'smooch' prefix = facebook_context == 'smooch' ? 'smooch_' : '' request.env['omniauth.strategy'].options[:client_id] = CheckConfig.get("#{prefix}facebook_app_id") request.env['omniauth.strategy'].options[:client_secret] = CheckConfig.get("#{prefix}facebook_app_secret") From d520b8f518e7ab94bb9701c314a24c3c149b73c9 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Oct 2023 11:02:06 -0300 Subject: [PATCH 12/54] Delete `FeedTeam`'s when deleting a `Feed` * Delete `FeedTeam`'s when deleting a `Feed`. Reference: CV2-3801. --- app/models/feed.rb | 2 +- test/models/feed_test.rb | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/models/feed.rb b/app/models/feed.rb index eaed8392dd..73fde75666 100755 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -4,7 +4,7 @@ class Feed < ApplicationRecord check_settings has_many :requests - has_many :feed_teams + has_many :feed_teams, dependent: :destroy has_many :teams, through: :feed_teams has_many :feed_invitations belongs_to :user, optional: true diff --git a/test/models/feed_test.rb b/test/models/feed_test.rb index 5be1deebbb..311677d51f 100755 --- a/test/models/feed_test.rb +++ b/test/models/feed_test.rb @@ -161,4 +161,20 @@ def setup CheckSearch.any_instance.unstub(:medias) end + + test "should delete feed teams when feed is deleted" do + f = create_feed + f.teams << create_team + ft = create_feed_team team: create_team, feed: f + assert_no_difference 'Feed.count' do + assert_difference 'FeedTeam.count', -1 do + ft.destroy! + end + end + assert_difference 'Feed.count', -1 do + assert_difference 'FeedTeam.count', -1 do + f.destroy! + end + end + end end From 34bf7f86c9000033fecb2b1d4de8c3bcc2e96478 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Tue, 24 Oct 2023 17:54:55 +0300 Subject: [PATCH 13/54] CV2-3800: filter feed by organization (#1703) --- app/models/feed.rb | 10 ++++----- app/models/feed_team.rb | 4 ---- lib/check_search.rb | 6 ++++-- test/controllers/elastic_search_9_test.rb | 25 +++++++++++++++++++++++ 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/app/models/feed.rb b/app/models/feed.rb index 73fde75666..588f9fdc94 100755 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -34,12 +34,12 @@ def filters end # Filters defined by each team - def get_team_filters + def get_team_filters(feed_team_ids = nil) filters = [] - self.feed_teams.each do |ft| - if ft.sharing_enabled? - filters << ft.filters.to_h.reject{ |k, _v| PROHIBITED_FILTERS.include?(k.to_s) }.merge({ 'team_id' => ft.team_id }) - end + conditions = { shared: true } + conditions[:team_id] = feed_team_ids unless feed_team_ids.blank? + self.feed_teams.where(conditions).find_each do |ft| + filters << ft.filters.to_h.reject{ |k, _v| PROHIBITED_FILTERS.include?(k.to_s) }.merge({ 'team_id' => ft.team_id }) end filters end diff --git a/app/models/feed_team.rb b/app/models/feed_team.rb index d6386faae6..1f79262ff7 100644 --- a/app/models/feed_team.rb +++ b/app/models/feed_team.rb @@ -13,10 +13,6 @@ def requests_filters=(filters) self.send(:set_requests_filters, filters) end - def sharing_enabled? - self.shared - end - def filters self.saved_search&.filters.to_h end diff --git a/lib/check_search.rb b/lib/check_search.rb index 33de85cda6..822a477765 100644 --- a/lib/check_search.rb +++ b/lib/check_search.rb @@ -61,7 +61,9 @@ def initialize(options, file = nil, team_id = Team.current&.id) def team_condition(team_id = nil) if feed_query? - FeedTeam.where(feed_id: @feed.id, team_id: Team.current&.id).last.shared ? @feed.team_ids : [0] # Invalidate the query if the current team is not sharing content + feed_teams = @options['feed_team_ids'].blank? ? @feed.team_ids : (@feed.team_ids & @options['feed_team_ids']) + is_shared = FeedTeam.where(feed_id: @feed.id, team_id: Team.current&.id, shared: true).last + is_shared ? feed_teams : [0] # Invalidate the query if the current team is not sharing content else team_id || Team.current&.id end @@ -746,7 +748,7 @@ def hit_es_for_range_filter def build_feed_conditions return {} unless feed_query? conditions = [] - @feed.get_team_filters.each do |filters| + @feed.get_team_filters(@options['feed_team_ids']).each do |filters| team_id = filters['team_id'].to_i conditions << CheckSearch.new(filters.to_json, nil, team_id).medias_query end diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb index c799fc6cb1..b34af3239a 100644 --- a/test/controllers/elastic_search_9_test.rb +++ b/test/controllers/elastic_search_9_test.rb @@ -264,5 +264,30 @@ def setup assert_equal [pm1.id, pm3.id, pm2.id], result.medias.map(&:id) end + test "should filter feed by organization" do + f = create_feed + t1 = create_team ; f.teams << t1 + t2 = create_team ; f.teams << t2 + t3 = create_team ; f.teams << t3 + FeedTeam.update_all(shared: true) + pm1 = create_project_media team: t1, disable_es_callbacks: false + pm2 = create_project_media team: t2, disable_es_callbacks: false + pm3 = create_project_media team: t3, disable_es_callbacks: false + sleep 2 + u = create_user + create_team_user team: t1, user: u, role: 'admin' + with_current_user_and_team(u, t1) do + query = { feed_id: f.id } + result = CheckSearch.new(query.to_json) + assert_equal [pm1.id, pm2.id, pm3.id], result.medias.map(&:id).sort + query[:feed_team_ids] = [t1.id, t3.id] + result = CheckSearch.new(query.to_json) + assert_equal [pm1.id, pm3.id], result.medias.map(&:id).sort + query[:feed_team_ids] = [t2.id] + result = CheckSearch.new(query.to_json) + assert_equal [pm2.id], result.medias.map(&:id) + end + end + # Please add new tests to test/controllers/elastic_search_10_test.rb end From 4d5f5f0c5ed8a6dd9c71c3664265f7bc79eb1d9f Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:13:52 -0300 Subject: [PATCH 14/54] Change how tipline search result feedback is stored Previously: When users received tipline search results and voted on them as relevant or chose not to vote, a request was generated and linked to the items returned in the search results. No new media or items were created. Now: When users receive tipline search results and cast votes on them as relevant or abstain from voting, a request is generated and associated with newly created media and items. These new items are linked to the search results through suggestion relationships, unless identical media already exist, in which case the previous behavior remains unchanged. Additionally: - To prevent redundancy, refrain from sending reports to users when their suggestions are accepted, as they have already received the report through the search results. - When confirming a match, delete any suggestion relationship between the same items, but exclusively when the suggestion was initiated by a bot. - When confirming a suggestion initiated by the Smooch Bot, remove other suggestions made by the Smooch Bot for the same item. - Introduce new database indexes to enhance performance. --- app/models/bot/smooch.rb | 2 ++ app/models/concerns/smooch_messages.rb | 8 ++++++- app/models/relationship.rb | 23 ++++++++++++++----- ...756_update_relationship_target_id_index.rb | 9 ++++++++ db/schema.rb | 7 ++++-- test/models/bot/smooch_6_test.rb | 6 ++--- test/models/relationship_2_test.rb | 13 ----------- test/models/relationship_test.rb | 7 +++--- 8 files changed, 47 insertions(+), 28 deletions(-) create mode 100644 db/migrate/20231023002756_update_relationship_target_id_index.rb diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb index d07fa4dc67..e85a593db2 100644 --- a/app/models/bot/smooch.rb +++ b/app/models/bot/smooch.rb @@ -63,6 +63,8 @@ def suggestion_accepted? def self.inherit_status_and_send_report(rid) relationship = Relationship.find_by_id(rid) + # A relationship created by the Smooch Bot is related to search results, so the user has already received the report as a search result + return if relationship&.user && relationship.user == BotUser.smooch_user unless relationship.nil? target = relationship.target parent = relationship.source diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index 313ac656d2..a77a2edf09 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -338,8 +338,14 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req if ['default_requests', 'timeout_requests', 'irrelevant_search_result_requests'].include?(request_type) message['archived'] = ['default_requests', 'irrelevant_search_result_requests'].include?(request_type) ? self.default_archived_flag : CheckArchivedFlags::FlagCodes::UNCONFIRMED annotated = self.create_project_media_from_message(message) - elsif ['menu_options_requests', 'relevant_search_result_requests', 'timeout_search_requests', 'resource_requests'].include?(request_type) + elsif ['menu_options_requests', 'resource_requests'].include?(request_type) annotated = annotated_obj + elsif ['relevant_search_result_requests', 'timeout_search_requests'].include?(request_type) + message['archived'] = (request_type == 'relevant_search_result_requests' ? self.default_archived_flag : CheckArchivedFlags::FlagCodes::UNCONFIRMED) + annotated = self.create_project_media_from_message(message) + if annotated != annotated_obj && annotated.is_a?(ProjectMedia) + Relationship.create!(relationship_type: Relationship.suggested_type, source: annotated_obj, target: annotated, user: BotUser.smooch_user) + end end return if annotated.nil? diff --git a/app/models/relationship.rb b/app/models/relationship.rb index 210d604a93..4d7cf33256 100644 --- a/app/models/relationship.rb +++ b/app/models/relationship.rb @@ -14,12 +14,13 @@ class Relationship < ApplicationRecord before_validation :set_confirmed, if: :is_being_confirmed?, on: :update before_validation :set_cluster, if: :is_being_confirmed?, on: :update validate :relationship_type_is_valid, :items_are_from_the_same_team - validate :target_not_pulished_report, on: :create + validate :target_not_published_report, on: :create validate :similar_item_exists, on: :create, if: proc { |r| r.is_suggested? } validate :cant_be_related_to_itself validates :relationship_type, uniqueness: { scope: [:source_id, :target_id], message: :already_exists }, on: :create - before_create :destroy_suggest_item, if: proc { |r| r.is_confirmed? } + before_create :destroy_same_suggested_item, if: proc { |r| r.is_confirmed? } + before_update :destroy_other_suggested_items, if: proc { |r| r.is_confirmed? } after_create :move_to_same_project_as_main, prepend: true after_create :point_targets_to_new_source, :update_counters, prepend: true after_update :reset_counters, prepend: true @@ -242,7 +243,7 @@ def items_are_from_the_same_team end end - def target_not_pulished_report + def target_not_published_report unless self.target.nil? state = self.target.get_annotations('report_design').last&.load&.get_field_value('state') errors.add(:base, I18n.t(:target_is_published)) if state == 'published' @@ -323,10 +324,20 @@ def move_to_same_project_as_main end end - def destroy_suggest_item - # Check if same item already exists as a suggested item + def destroy_same_suggested_item + # Check if same item already exists as a suggested item by a bot Relationship.where(source_id: self.source_id, target_id: self.target_id) - .where('relationship_type = ?', Relationship.suggested_type.to_yaml).destroy_all + .joins(:user).where('users.type' => 'BotUser') + .where('relationship_type = ?', Relationship.suggested_type.to_yaml) + .destroy_all + end + + def destroy_other_suggested_items + # If created by Smooch Bot, destroy other suggestions to the same media + Relationship.where(target_id: self.target_id, user: BotUser.smooch_user) + .where('relationship_type = ?', Relationship.suggested_type.to_yaml) + .where('id != ?', self.id) + .destroy_all end def cant_be_related_to_itself diff --git a/db/migrate/20231023002756_update_relationship_target_id_index.rb b/db/migrate/20231023002756_update_relationship_target_id_index.rb new file mode 100644 index 0000000000..7200426eb2 --- /dev/null +++ b/db/migrate/20231023002756_update_relationship_target_id_index.rb @@ -0,0 +1,9 @@ +class UpdateRelationshipTargetIdIndex < ActiveRecord::Migration[6.1] + def change + remove_index :relationships, name: 'index_relationships_on_target_id' + add_index :relationships, :target_id + add_index :relationships, :source_id + add_index :relationships, [:target_id, :relationship_type] + add_index :relationships, [:source_id, :relationship_type] + end +end diff --git a/db/schema.rb b/db/schema.rb index 14b6c60131..a90ecf3918 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_10_15_203900) do +ActiveRecord::Schema.define(version: 2023_10_23_002756) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -516,8 +516,11 @@ t.datetime "updated_at", null: false t.index "LEAST(source_id, target_id), GREATEST(source_id, target_id)", name: "relationships_least_greatest_idx", unique: true t.index ["relationship_type"], name: "index_relationships_on_relationship_type" + t.index ["source_id", "relationship_type"], name: "index_relationships_on_source_id_and_relationship_type" t.index ["source_id", "target_id", "relationship_type"], name: "relationship_index", unique: true - t.index ["target_id"], name: "index_relationships_on_target_id", unique: true + t.index ["source_id"], name: "index_relationships_on_source_id" + t.index ["target_id", "relationship_type"], name: "index_relationships_on_target_id_and_relationship_type" + t.index ["target_id"], name: "index_relationships_on_target_id" end create_table "requests", force: :cascade do |t| diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index 4d98d7ba06..161c375a18 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -223,7 +223,7 @@ def send_message_outside_24_hours_window(template, pm = nil) Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' assert_state 'search_result' - assert_difference 'Dynamic.count + ProjectMedia.count' do + assert_difference 'Dynamic.where(annotation_type: "smooch").count + ProjectMedia.count + Relationship.where(relationship_type: Relationship.suggested_type).count', 3 do send_message '1' end assert_state 'main' @@ -240,7 +240,7 @@ def send_message_outside_24_hours_window(template, pm = nil) Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar foo bar foo bar', '1' assert_state 'search_result' - assert_difference 'Dynamic.count + ProjectMedia.count' do + assert_difference 'Dynamic.where(annotation_type: "smooch").count + ProjectMedia.count + Relationship.where(relationship_type: Relationship.suggested_type).count', 3 do send_message '1' end assert_state 'main' @@ -261,7 +261,7 @@ def send_message_outside_24_hours_window(template, pm = nil) Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Image here', '1' assert_state 'search_result' - assert_difference 'Dynamic.count + ProjectMedia.count' do + assert_difference 'Dynamic.where(annotation_type: "smooch").count + ProjectMedia.count + Relationship.where(relationship_type: Relationship.suggested_type).count', 3 do send_message '1' end assert_state 'main' diff --git a/test/models/relationship_2_test.rb b/test/models/relationship_2_test.rb index 9b356b4872..093073f1df 100644 --- a/test/models/relationship_2_test.rb +++ b/test/models/relationship_2_test.rb @@ -361,17 +361,4 @@ def setup r.destroy! assert_queries(0, '=') { assert_nil pm2.confirmed_as_similar_by_name } end - - test "should not save duplicate values for target_id column" do - t = create_team - pm1 = create_project_media team: t - pm2 = create_project_media team: t - pm3 = create_project_media team: t - assert_nothing_raised do - create_relationship source_id: pm1.id, target_id: pm3.id - end - assert_raises 'ActiveRecord::RecordNotUnique' do - create_relationship source_id: pm2.id, target_id: pm3.id - end - end end diff --git a/test/models/relationship_test.rb b/test/models/relationship_test.rb index 84bbcc9981..61acaa14a7 100644 --- a/test/models/relationship_test.rb +++ b/test/models/relationship_test.rb @@ -80,14 +80,15 @@ def setup test "should remove suggested relation when same items added as similar" do team = create_team + b = create_bot name: 'Alegre', login: 'alegre' s = create_project_media team: team t = create_project_media team: team - r = create_relationship source_id: s.id, target_id: t.id, relationship_type: Relationship.suggested_type - r2 = create_relationship source_id: s.id, target_id: t.id, relationship_type: Relationship.confirmed_type + r = create_relationship source_id: s.id, target_id: t.id, relationship_type: Relationship.suggested_type, user: b + r2 = create_relationship source_id: s.id, target_id: t.id, relationship_type: Relationship.confirmed_type, user: b assert_nil Relationship.where(id: r.id).last assert_not_nil Relationship.where(id: r2.id).last assert_raises ActiveRecord::RecordInvalid do - create_relationship source_id: s.id, target_id: t.id, relationship_type: Relationship.suggested_type + create_relationship source_id: s.id, target_id: t.id, relationship_type: Relationship.suggested_type, user: b end end From be25e601121f3134bc7d1181317bb09d2e0cc46c Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 25 Oct 2023 19:39:25 +0300 Subject: [PATCH 15/54] CV2-3798 shared feed invite emails (#1704) * CV2-3798: send feed invitation mail * CV2-3798: adjust mail template and fix tests * CV2-3798: fix check logo for other templates --- app/mailers/feed_invitation_mailer.rb | 16 ++ app/models/feed_invitation.rb | 6 + .../feed_invitation_mailer/notify.html.erb | 140 ++++++++++++++++++ .../feed_invitation_mailer/notify.text.erb | 25 ++++ app/views/shared/_header.html.erb | 7 +- config/locales/en.yml | 5 + test/mailers/feed_invitation_mailer_test.rb | 12 ++ test/models/feed_invitation_test.rb | 10 ++ 8 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 app/mailers/feed_invitation_mailer.rb create mode 100644 app/views/feed_invitation_mailer/notify.html.erb create mode 100644 app/views/feed_invitation_mailer/notify.text.erb create mode 100644 test/mailers/feed_invitation_mailer_test.rb diff --git a/app/mailers/feed_invitation_mailer.rb b/app/mailers/feed_invitation_mailer.rb new file mode 100644 index 0000000000..c63229c13c --- /dev/null +++ b/app/mailers/feed_invitation_mailer.rb @@ -0,0 +1,16 @@ +class FeedInvitationMailer < ApplicationMailer + layout nil + + def notify(record) + @recipient = record.email + @user = record.user + @feed = record.feed + @direction = ApplicationMailer.set_template_direction + @due_at = record.created_at + CheckConfig.get('feed_invitation_due_to', 30).to_i.days + subject = I18n.t("mails_notifications.feed_invitation.subject", user: @user.name, feed: @feed.name) + attachments.inline['check_logo.png'] = File.read("#{Rails.root}/public/images/check.svg") + @logo_url = attachments['check_logo.png'].url + Rails.logger.info "Sending a feed invitation e-mail to #{@recipient}" + mail(to: @recipient, email_type: 'feed_invitation', subject: subject) + end +end diff --git a/app/models/feed_invitation.rb b/app/models/feed_invitation.rb index 69b3b7b223..12eb3e8c79 100644 --- a/app/models/feed_invitation.rb +++ b/app/models/feed_invitation.rb @@ -8,6 +8,8 @@ class FeedInvitation < ApplicationRecord validates_presence_of :email, :feed, :user validates :email, format: { with: URI::MailTo::EMAIL_REGEXP } + after_create :send_feed_invitation_mail + def accept!(team_id) feed_team = FeedTeam.new(feed_id: self.feed_id, team_id: team_id, shared: true) feed_team.skip_check_ability = true @@ -24,4 +26,8 @@ def reject! def set_user self.user ||= User.current end + + def send_feed_invitation_mail + FeedInvitationMailer.delay.notify(self) + end end diff --git a/app/views/feed_invitation_mailer/notify.html.erb b/app/views/feed_invitation_mailer/notify.html.erb new file mode 100644 index 0000000000..83d4e30910 --- /dev/null +++ b/app/views/feed_invitation_mailer/notify.html.erb @@ -0,0 +1,140 @@ +<%= render "shared/header" %> + + + + + +
+ + + + +
 
+
+ + +
+
+
+ <%= I18n.t(:"mails_notifications.feed_invitation.hello", email: @recipient) %> +
+ + + + +
 
+
+ <%= I18n.t("mails_notifications.invitation.title") %> +
+ + + + +
 
+
+
+ <%= I18n.t(:"mails_notifications.feed_invitation.body", name: @user.name, email: @user.email) %> +
+
+ "<%= @feed.name %>" +
+
+
+ + + + + +
 
+ + + + +
+ + + + + +
+ + + + + +
+ + <%= + link_to(I18n.t('mails_notifications.feed_invitation.view_button'), + '#', + :style => "text-decoration: none !important;color: #fff !important;" + ) + %> + + + <%= image_tag("https://images.ctfassets.net/g118h5yoccvd/#{@direction[:arrow]}", width: "7", alt: "arrow-icon", style: "-ms-interpolation-mode: bicubic; border: 0 none; height: auto; line-height: 100%; outline: none; text-decoration: none;") %> +
+
+ + + + + +
 
+ +
+
+ <% if @due_at %> + <%= I18n.t("devise.mailer.invitation_instructions.accept_until", due_date: l(@due_at, format: :'devise.mailer.invitation_instructions.accept_until_format')) %> +
+ <% end %> + + <%= t("devise.mailer.invitation_instructions.ignore") %> + +
+
+ +
+ + + + +
 
+ + +<%= render "shared/footer" %> diff --git a/app/views/feed_invitation_mailer/notify.text.erb b/app/views/feed_invitation_mailer/notify.text.erb new file mode 100644 index 0000000000..11551bb641 --- /dev/null +++ b/app/views/feed_invitation_mailer/notify.text.erb @@ -0,0 +1,25 @@ +<%= I18n.t(:"mails_notifications.feed_invitation.hello", email: @recipient) %> +========================================== +<%= I18n.t("mails_notifications.invitation.title") %> +========================================== + +<%= + raw I18n.t(:"mails_notifications.feed_invitation.body", + name: @user.name, + email: @user.email, + feed: @feed.name + ) + %> + +<%= I18n.t('mails_notifications.feed_invitation.view_button') %> + +<% if @due_at %> + <%= t("devise.mailer.invitation_instructions.accept_until", due_date: l(@due_at, format: :'devise.mailer.invitation_instructions.accept_until_format')) %> +<% end %> + +<%= t("devise.mailer.invitation_instructions.ignore") %> + +... + +<%= strip_tags I18n.t("mails_notifications.copyright_html", app_name: CheckConfig.get('app_name')) %> +https://meedan.com diff --git a/app/views/shared/_header.html.erb b/app/views/shared/_header.html.erb index 60a293f71b..c23ad5ca7b 100644 --- a/app/views/shared/_header.html.erb +++ b/app/views/shared/_header.html.erb @@ -173,7 +173,12 @@ 		- diff --git a/config/locales/en.yml b/config/locales/en.yml index 68704aed0d..200415dd83 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -460,6 +460,11 @@ en: rejected_title: Request Denied rejected_text: Sorry, your request to join workspace %{team} on %{app_name} was not approved. + feed_invitation: + subject: "%{user} invited your organization to contribute to %{feed}" + hello: Hello %{email} + body: "%{name}, %{email} has invited your organization to contribute to a Check Shared Feed " + view_button: View invitation mail_security: device_subject: 'Security alert: New login to %{app_name} from %{browser} on %{platform}' ip_subject: 'Security alert: New or unusual %{app_name} login' diff --git a/test/mailers/feed_invitation_mailer_test.rb b/test/mailers/feed_invitation_mailer_test.rb new file mode 100644 index 0000000000..25c957fad3 --- /dev/null +++ b/test/mailers/feed_invitation_mailer_test.rb @@ -0,0 +1,12 @@ +require "test_helper" + +class FeedInvitationMailerTest < ActionMailer::TestCase + test "should notify about feed invitation" do + fi = create_feed_invitation + email = FeedInvitationMailer.notify(fi) + assert_emails 1 do + email.deliver_now + end + assert_equal [fi.email], email.to + end +end diff --git a/test/models/feed_invitation_test.rb b/test/models/feed_invitation_test.rb index d66cc7ca8e..a6c03b2329 100644 --- a/test/models/feed_invitation_test.rb +++ b/test/models/feed_invitation_test.rb @@ -63,4 +63,14 @@ def teardown fi.reject! assert_equal 'rejected', fi.reload.state end + + test "should send email after create feed invitation" do + u = create_user + f = create_feed + Sidekiq::Extensions::DelayedMailer.clear + Sidekiq::Testing.fake! do + FeedInvitation.create!({ email: random_email, feed: f, user: u, state: :invited }) + assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size + end + end end From 563aa39b534a31efb5e3d4e951ea440f4bba2a54 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Thu, 26 Oct 2023 01:04:39 -0300 Subject: [PATCH 16/54] Adding GraphQL mutations to add and remove NLU keywords to/from tipline menu options Only super-admins can use these mutations. Reference: CV2-3709. --- app/graph/mutations/nlu_mutations.rb | 41 +++ app/graph/types/mutation_type.rb | 3 + app/lib/smooch_nlu.rb | 2 +- app/lib/smooch_nlu_menus.rb | 7 +- lib/relay.idl | 62 ++++ public/relay.json | 314 ++++++++++++++++++ .../controllers/graphql_controller_10_test.rb | 66 ++++ 7 files changed, 491 insertions(+), 4 deletions(-) create mode 100644 app/graph/mutations/nlu_mutations.rb diff --git a/app/graph/mutations/nlu_mutations.rb b/app/graph/mutations/nlu_mutations.rb new file mode 100644 index 0000000000..0a98b8b239 --- /dev/null +++ b/app/graph/mutations/nlu_mutations.rb @@ -0,0 +1,41 @@ +module NluMutations + class ToggleKeywordInTiplineMenu < Mutations::BaseMutation + argument :language, GraphQL::Types::String, required: true + argument :keyword, GraphQL::Types::String, required: true + argument :menu, GraphQL::Types::String, required: true # "main" or "secondary" + argument :menu_option_index, GraphQL::Types::Int, required: true # zero-based... the order is the same displayed in the tipline and in the tipline settings page + + field :success, GraphQL::Types::Boolean, null: true + + def resolve(language:, menu:, menu_option_index:, keyword:) + begin + if User.current.is_admin + nlu = SmoochNlu.new(Team.current.slug) + nlu.enable! + if toggle == :add + nlu.add_keyword_to_menu_option(language, menu, menu_option_index, keyword) + elsif toggle == :remove + nlu.remove_keyword_from_menu_option(language, menu, menu_option_index, keyword) + end + { success: true } + else + { success: false } + end + rescue + { success: false } + end + end + end + + class AddKeywordToTiplineMenu < ToggleKeywordInTiplineMenu + def toggle + :add + end + end + + class RemoveKeywordFromTiplineMenu < ToggleKeywordInTiplineMenu + def toggle + :remove + end + end +end diff --git a/app/graph/types/mutation_type.rb b/app/graph/types/mutation_type.rb index ffc94151fb..353cdbf0f9 100644 --- a/app/graph/types/mutation_type.rb +++ b/app/graph/types/mutation_type.rb @@ -158,4 +158,7 @@ class MutationType < BaseObject field :destroyTiplineResource, mutation: TiplineResourceMutations::Destroy field :sendTiplineMessage, mutation: TiplineMessageMutations::Send + + field :addNluKeywordToTiplineMenu, mutation: NluMutations::AddKeywordToTiplineMenu + field :removeNluKeywordFromTiplineMenu, mutation: NluMutations::RemoveKeywordFromTiplineMenu end diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index cf7b3e1b76..7144e9cc73 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -5,7 +5,7 @@ class SmoochBotNotInstalledError < ::ArgumentError # FIXME: Make it more flexible # FIXME: Once we support paraphrase-multilingual-mpnet-base-v2 make it the only model used ALEGRE_MODELS_AND_THRESHOLDS = { - # Bot::Alegre::ELASTICSEARCH_MODEL => 0.8, Sometimes this is easier for local development + # Bot::Alegre::ELASTICSEARCH_MODEL => 0.8 # , Sometimes this is easier for local development Bot::Alegre::OPENAI_ADA_MODEL => 0.8, Bot::Alegre::MEAN_TOKENS_MODEL => 0.6 } diff --git a/app/lib/smooch_nlu_menus.rb b/app/lib/smooch_nlu_menus.rb index 3857f1b6e0..236483e6a4 100644 --- a/app/lib/smooch_nlu_menus.rb +++ b/app/lib/smooch_nlu_menus.rb @@ -13,7 +13,7 @@ def remove_keyword_from_menu_option(language, menu, menu_option_index, keyword) update_menu_option_keywords(language, menu, menu_option_index, keyword, 'remove') end - def list_menu_keywords(languages = nil, menus = nil) + def list_menu_keywords(languages = nil, menus = nil, include_empty = true) if languages.nil? languages = @smooch_bot_installation.get_smooch_workflows.map { |w| w['smooch_workflow_language'] } elsif languages.is_a? String @@ -33,12 +33,13 @@ def list_menu_keywords(languages = nil, menus = nil) output[language][menu] = [] i = 0 workflow.fetch("smooch_state_#{menu}",{}).fetch('smooch_menu_options', []).each do |option| + keywords = option.dig('smooch_menu_option_nlu_keywords').to_a output[language][menu] << { 'index' => i, 'title' => option.dig('smooch_menu_option_label'), - 'keywords' => option.dig('smooch_menu_option_nlu_keywords').to_a, + 'keywords' => keywords, 'id' => option.dig('smooch_menu_option_id'), - } + } if include_empty || !keywords.blank? i += 1 end end diff --git a/lib/relay.idl b/lib/relay.idl index a5ad34e0b5..21d3f58ea7 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -257,6 +257,31 @@ type AddFilesToTaskPayload { task: Task } +""" +Autogenerated input type of AddKeywordToTiplineMenu +""" +input AddKeywordToTiplineMenuInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + keyword: String! + language: String! + menu: String! + menuOptionIndex: Int! +} + +""" +Autogenerated return type of AddKeywordToTiplineMenu +""" +type AddKeywordToTiplineMenuPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + success: Boolean +} + type Annotation implements Node { annotated_id: String annotated_type: String @@ -8911,6 +8936,12 @@ type MutationType { """ input: AddFilesToTaskInput! ): AddFilesToTaskPayload + addNluKeywordToTiplineMenu( + """ + Parameters for AddKeywordToTiplineMenu + """ + input: AddKeywordToTiplineMenuInput! + ): AddKeywordToTiplineMenuPayload """ Allow multiple items to be marked as read or unread. @@ -9737,6 +9768,12 @@ type MutationType { """ input: RemoveFilesFromTaskInput! ): RemoveFilesFromTaskPayload + removeNluKeywordFromTiplineMenu( + """ + Parameters for RemoveKeywordFromTiplineMenu + """ + input: RemoveKeywordFromTiplineMenuInput! + ): RemoveKeywordFromTiplineMenuPayload replaceProjectMedia( """ Parameters for ReplaceProjectMedia @@ -11858,6 +11895,31 @@ type RemoveFilesFromTaskPayload { task: Task } +""" +Autogenerated input type of RemoveKeywordFromTiplineMenu +""" +input RemoveKeywordFromTiplineMenuInput { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + keyword: String! + language: String! + menu: String! + menuOptionIndex: Int! +} + +""" +Autogenerated return type of RemoveKeywordFromTiplineMenu +""" +type RemoveKeywordFromTiplineMenuPayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + success: Boolean +} + """ Autogenerated input type of ReplaceProjectMedia """ diff --git a/public/relay.json b/public/relay.json index d10ba20745..4d93e551ed 100644 --- a/public/relay.json +++ b/public/relay.json @@ -1073,6 +1073,134 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "INPUT_OBJECT", + "name": "AddKeywordToTiplineMenuInput", + "description": "Autogenerated input type of AddKeywordToTiplineMenu", + "fields": null, + "inputFields": [ + { + "name": "language", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "keyword", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "menu", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "menuOptionIndex", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "AddKeywordToTiplineMenuPayload", + "description": "Autogenerated return type of AddKeywordToTiplineMenu", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "success", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "Annotation", @@ -48407,6 +48535,35 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "addNluKeywordToTiplineMenu", + "description": null, + "args": [ + { + "name": "input", + "description": "Parameters for AddKeywordToTiplineMenu", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "AddKeywordToTiplineMenuInput", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "AddKeywordToTiplineMenuPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "bulkProjectMediaMarkRead", "description": "Allow multiple items to be marked as read or unread.", @@ -52380,6 +52537,35 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "removeNluKeywordFromTiplineMenu", + "description": null, + "args": [ + { + "name": "input", + "description": "Parameters for RemoveKeywordFromTiplineMenu", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "RemoveKeywordFromTiplineMenuInput", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "RemoveKeywordFromTiplineMenuPayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "replaceProjectMedia", "description": null, @@ -62364,6 +62550,134 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "INPUT_OBJECT", + "name": "RemoveKeywordFromTiplineMenuInput", + "description": "Autogenerated input type of RemoveKeywordFromTiplineMenu", + "fields": null, + "inputFields": [ + { + "name": "language", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "keyword", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "menu", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "menuOptionIndex", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "RemoveKeywordFromTiplineMenuPayload", + "description": "Autogenerated return type of RemoveKeywordFromTiplineMenu", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "success", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "INPUT_OBJECT", "name": "ReplaceProjectMediaInput", diff --git a/test/controllers/graphql_controller_10_test.rb b/test/controllers/graphql_controller_10_test.rb index 4d256a7024..35c4c331fb 100644 --- a/test/controllers/graphql_controller_10_test.rb +++ b/test/controllers/graphql_controller_10_test.rb @@ -808,4 +808,70 @@ def setup assert_response :success assert !JSON.parse(@response.body)['data']['sendTiplineMessage']['success'] end + + test "should add NLU keyword to tipline menu option" do + SmoochNlu.any_instance.stubs(:enable!).once + SmoochNlu.any_instance.stubs(:add_keyword_to_menu_option).once + SmoochNlu.any_instance.stubs(:remove_keyword_from_menu_option).never + u = create_user is_admin: true + t = create_team + b = create_team_bot name: 'Smooch', login: 'smooch', set_approved: true + b.install_to!(t) + authenticate_with_user(u) + + query = "mutation { addNluKeywordToTiplineMenu(input: { language: \"en\", menu: \"main\", menuOptionIndex: 0, keyword: \"Foo bar\" }) { success } }" + post :create, params: { query: query, team: t.slug } + + assert_response :success + assert JSON.parse(@response.body)['data']['addNluKeywordToTiplineMenu']['success'] + end + + test "should remove NLU keyword from tipline menu option" do + SmoochNlu.any_instance.stubs(:enable!).once + SmoochNlu.any_instance.stubs(:add_keyword_to_menu_option).never + SmoochNlu.any_instance.stubs(:remove_keyword_from_menu_option).once + u = create_user is_admin: true + t = create_team + b = create_team_bot name: 'Smooch', login: 'smooch', set_approved: true + b.install_to!(t) + authenticate_with_user(u) + + query = "mutation { removeNluKeywordFromTiplineMenu(input: { language: \"en\", menu: \"main\", menuOptionIndex: 0, keyword: \"Foo bar\" }) { success } }" + post :create, params: { query: query, team: t.slug } + + assert_response :success + assert JSON.parse(@response.body)['data']['removeNluKeywordFromTiplineMenu']['success'] + end + + test "should not change tipline menu option NLU keywords if it's not a super-admin" do + SmoochNlu.any_instance.stubs(:enable!).never + SmoochNlu.any_instance.stubs(:add_keyword_to_menu_option).never + SmoochNlu.any_instance.stubs(:remove_keyword_from_menu_option).never + u = create_user is_admin: false + t = create_team + b = create_team_bot name: 'Smooch', login: 'smooch', set_approved: true + b.install_to!(t) + authenticate_with_user(u) + + query = "mutation { addNluKeywordToTiplineMenu(input: { language: \"en\", menu: \"main\", menuOptionIndex: 0, keyword: \"Foo bar\" }) { success } }" + post :create, params: { query: query, team: t.slug } + + assert_response :success + assert !JSON.parse(@response.body)['data']['addNluKeywordToTiplineMenu']['success'] + end + + test "should not change tipline menu option NLU keywords if tipline is not installed" do + SmoochNlu.any_instance.stubs(:enable!).never + SmoochNlu.any_instance.stubs(:add_keyword_to_menu_option).never + SmoochNlu.any_instance.stubs(:remove_keyword_from_menu_option).never + u = create_user is_admin: true + t = create_team + authenticate_with_user(u) + + query = "mutation { addNluKeywordToTiplineMenu(input: { language: \"en\", menu: \"main\", menuOptionIndex: 0, keyword: \"Foo bar\" }) { success } }" + post :create, params: { query: query, team: t.slug } + + assert_response :success + assert !JSON.parse(@response.body)['data']['addNluKeywordToTiplineMenu']['success'] + end end From f1167b5a38fad178aabc56483081f24f7bfbb4e5 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Thu, 26 Oct 2023 17:14:53 +0300 Subject: [PATCH 17/54] CV2-3798: update mail logo for all mails templates (#1708) --- app/mailers/application_mailer.rb | 2 ++ app/mailers/devise_mailer.rb | 12 +++++++++--- app/mailers/feed_invitation_mailer.rb | 3 --- public/images/checklogo.png | Bin 0 -> 6499 bytes 4 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 public/images/checklogo.png diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb index 1f8b23b819..a9b627c780 100644 --- a/app/mailers/application_mailer.rb +++ b/app/mailers/application_mailer.rb @@ -30,6 +30,8 @@ def mail(options={}) return if options[:to].blank? options[:to] = [options[:to]].flatten.collect{ |to| to.gsub(/[\u200B-\u200D\uFEFF]/, '') } @direction = ApplicationMailer.set_template_direction + attachments.inline['checklogo.png'] = File.read('public/images/checklogo.png') + @logo_url = attachments['checklogo.png'].url super(options) end diff --git a/app/mailers/devise_mailer.rb b/app/mailers/devise_mailer.rb index cd692ed1e4..2c651e30c9 100644 --- a/app/mailers/devise_mailer.rb +++ b/app/mailers/devise_mailer.rb @@ -5,7 +5,7 @@ class DeviseMailer < Devise::Mailer def confirmation_instructions(record, token, opts={}) @host = CheckConfig.get('checkdesk_base_url') @client_host = CheckConfig.get('checkdesk_client') - @direction = ApplicationMailer.set_template_direction + self.set_direction_and_logo opts[:subject] = I18n.t(:mail_account_confirmation, app_name: CheckConfig.get('app_name')) super end @@ -13,7 +13,7 @@ def confirmation_instructions(record, token, opts={}) def reset_password_instructions(record, token, opts={}) @host = CheckConfig.get('checkdesk_base_url') @title = I18n.t("mails_notifications.reset_password.title") - @direction = ApplicationMailer.set_template_direction + self.set_direction_and_logo opts[:subject] = I18n.t('devise.mailer.reset_password_instructions.subject', app_name: CheckConfig.get('app_name')) super end @@ -29,10 +29,16 @@ def invitation_instructions(record, token, opts={}) @invited_type = @invited_text.blank? ? 'default' : 'custom' @due_at = opts[:due_at] @title = I18n.t("mails_notifications.invitation.title") - @direction = ApplicationMailer.set_template_direction + self.set_direction_and_logo tu = record.team_users.where(team_id: @team.id).last opts[:to] = tu.invitation_email unless tu.nil? opts[:subject] = I18n.t(:'devise.mailer.invitation_instructions.subject', user: @invited_by, team: @team.name) super end + + def set_direction_and_logo + @direction = ApplicationMailer.set_template_direction + attachments.inline['checklogo.png'] = File.read('public/images/checklogo.png') + @logo_url = attachments['checklogo.png'].url + end end diff --git a/app/mailers/feed_invitation_mailer.rb b/app/mailers/feed_invitation_mailer.rb index c63229c13c..82412342b4 100644 --- a/app/mailers/feed_invitation_mailer.rb +++ b/app/mailers/feed_invitation_mailer.rb @@ -5,11 +5,8 @@ def notify(record) @recipient = record.email @user = record.user @feed = record.feed - @direction = ApplicationMailer.set_template_direction @due_at = record.created_at + CheckConfig.get('feed_invitation_due_to', 30).to_i.days subject = I18n.t("mails_notifications.feed_invitation.subject", user: @user.name, feed: @feed.name) - attachments.inline['check_logo.png'] = File.read("#{Rails.root}/public/images/check.svg") - @logo_url = attachments['check_logo.png'].url Rails.logger.info "Sending a feed invitation e-mail to #{@recipient}" mail(to: @recipient, email_type: 'feed_invitation', subject: subject) end diff --git a/public/images/checklogo.png b/public/images/checklogo.png new file mode 100644 index 0000000000000000000000000000000000000000..9de2cf35ba46345eec3387653f511d3cdfcb01bb GIT binary patch literal 6499 zcmV-p8Jy;cP)_HMxq(M=D<3rD)U8I-7(HKSBpgD+^RachDoxIGL45Lx3}hlAq6xjp-yD7Z?6%h+7UthSdfwYFp49J$CL$tQCa-G7FF#%xB;{bO zP_N7yG7%9GPK9yr$>hA@K(Ed~CL*HF8#hyzwc@}*EZAgCL_}29j2o#HpMDptArlc% zXZ2)>%SMA(u!c-TL><<{Xw*jx7|29K)KNVo$-9UJ1DS}3I;dBmND&hTG7%AVOt10& z;%OMjL`2jfJ;aKEOhiN-5pNg^1~L&5b%5zJMvj;@F>*x8{9Fa<+giq(o~UW&wv^22qn^y%mz!8JtCxOwlzsJ4 z@4$qd-+hd=u!ii++z2Vk8{r-MReoQ)3&7@`)F5+nd?>#*H%kU0BAUyp{rbop?FeJR zKwgNg^~&e_o;jpT!bjixhCOcB*Obpi0{TA6l57qlBARMEwP|%U+B21#9BsluT{D?L z?3?3``Q;OHNVCuY2zI{Yb^7MGC$Deb#{=6>L_|bqjKes&j>Dwa+d|tgkf*XCEa5B1 zHW0=6ks9+o^S+)il!=H=d$s6?^c|aUyNkAAAir`Zg!_n73WhQf(J7}^;JWnA$kd`* zIPj+9fh+^ES`fl#r6@!BfgMmrMATWYEr+>!e0_;Y|F1c_EMFw(QIuOK_)Ep03 zLnflGn7N-Z133;I3gU{?kRO1Er~_t(v0xxap-Vwr>2pvQ5p}^_VJsNP&9Q-$BAoGV zOeoiu&dwYM_Rm9+L0x?i5iPORMZQ@Axp+LPSxAm@ z$EAGMUzpf@B1JjJ`Ca%cFvzs0>*js6=5wn|Y)x66t8d`-sc4_#qyRMDV55yc>%7jK zlWMEZY3D`G!Dsf@d3AJg(-C{ls(Y?Ff78yH_|$&QadJ)N%Y02OgN4FaFp$Hs{lvH7 zM}{(hhjngJ`*$+?+oAl&)?f!HcZbrHqAZZ?AaWChlpmv9&?H3-=HIDWB_6eTJIXWX zBSl$ob84ZoTXtqR*|Kx#b_Fv(qnxuoPH%JS`fpY{sg2JWmb}o{HVJAf9%HSQfh-an ztX_>jknN%#1L%^Q@s`gAye&k?d+$TnwM8Ggg?)0@v}ImzIL|lQpIHhaJ~rmaQ9e9( z+YmZ7fCnf8xT?wdR&Ce$u&u#t3$t?01_-y1Plg7zIkn68_EC?#*G)9Q-sdu~RP*zZ zLeb6IVJsNP{@4%w33N#a?;#NLGe|IL4#1LH{6f2n`3_?nJkomMyDOnGM5LkIP?$`U zndAPv1*s<4>qnNf^-+jh)I*f#?IApCV;Fmy(UO_trnFGZJ%DQ(4(jqsvw1o<$Mujw zEbQxDhu(m%K}?H**mc3Bz!WNo#lRf7VJsNP`s@ckfR5Z8@_{$<`PkQ$Qmk4aqhS%O3!Wl^wY(?JL#(@n zr6|wSy;ury_MAyUJLctVQ=AlLQgochARbKq59crzu?O<#>B{&x`sDKXYw=>@$Br|Q zwQ9vV${^0sAREZHpyT}}SLTXrUGT=%@}6iF7$ANFb3Cyj*cjW4FW1fv+OfGyit@bO zY4V@zOnT?K92`VeAdtbHg;jeLN1lK#+J zq3;XZiV?CqLt75Tf_)qMjBOCh`_wFmND%K#`vaAlai!@RvebM(|Lb);yZbwM@h@LD zVSH(75DTdxzx?i)Tcu^9}U;gXY@cG}aZ9M6L>(KZs8xNcO(!KIEUm z&7t`@*Jpi`<7ny(^*)arr#F5OYdF2ozfaOSat)3=EypS{l)4*p>&q~0VzS8!d-QaGI(Oo-vBx*y!GpjMz>9!DfCnexom2k zzcU4x9C}i;XprN?GOh#Y$>F#tO3*MCbppBJARY@)J5J#)2l-ZwklaobSG=+}u{6Eq z&@;e-6xT&pKSviy%Kl!Brptxzb9>Hqpy#gndAt>3Yyn0BS!2$BMPp?>?m^sd-=ho?_5E0C7QR@ChRlzBWtkM zqA^%(-mVpwhv=ioYQ>cd^{vgZ5RXc&81c}!-qNP~joTQ+B8t?LAnxI~+{k$)!R2dj zL>b5Y5AM{Uqyqa264p(^2;Y*nm*(aV$e%^8K_GRBUng9T4Q`a+)}u? z9cDj{;L`K{z&6FYKJ;T;OJ3en3OB)n;0IJ zA;Opix#`lHe>N}9fN&5fh%2&Y`3^h^nr^j%xYA%Co9?jP@BaK(IpVes(#D4+`ILid z2^_?g+e-V+PZccLoPxO0Od#i9T)E=C^B(RGe%5aEF^uUvDI6t%wwGK&eKgmv{lM2e z>L97662z6jfqYzG^V-8a#7*PcKbeB4p5UWjiCQX!r`@T%Cz@&yR{{mH3F57Q&C{-d zzZT<;Dd&;t0)@MMkOqb^(fqc3?A`1&&AbG0#RhWkP5s$fiTSr}5DyVf4tI@Awy!CL zJFbYPgUzLjb3SKnDm*WV%pBLDbG_fw{`|a!vDYB3n9#iRy~|kn_9bXZE+CX6rt;A;E`X z8`CbBORXrqRsY_AMfPxZ^xcYBPp$7>58#;(9 z$tBZl{fAdDdUlNC!(;RHXuiK^Lb&(VDptO^YMPCDTkC)M$q>!R;ZEklt;Ie{Q}F_3 z80+{Hv?z^xOuCh#x`yq&*@v6VZaNWz^EP?dGRI@M=ah`DzNYcaRSn~&f_PHz&6V*{ zj4q*gKEmk52*ry6#c{cIu7odE%=caCq4)I_Bv;D4lSju%lR-S5=3qY59PWLo+lYTT zyZ7MkfKx-%xSYov<%6wja=43ejK>Xgybm{WJ!NR#xw|dN(w63VE6vq+;HFxg>n0qj zg>ei)JSpj!zF!;6vosyV6EqfSmm6sZaqGh!YsGqU)VCH;zdLh#6lxrWQx-Ok+SbCg}WGcylE?!n(>;CX&$k& ztun4HGaE6DsJ8E=4|h45wY5MnN~?}W3)^jT{!%z_4r2zfs)x{VJK|{*S|boMkd_Q? z`sgOt!n53?F*Q}+fPx0M9hFeMXphu9~ZBa|&Yyu`YXIBW?N@JdTO!ukESPnEpu3wqDv~Yz+adfo|pxs7IwZoV} z?9C(SkYV>4HWbwPUUKpjDZeSB|nqoWp@~bM*t%uy7bN zi2br7Q#0t0&uHH~Bvtx?>N2k{eJLq)4B15OMFux1B587!L4xLZqHtXJjI)EdGPrYU z-gIRLRSuvdLpV7Ug1DmCTrC6hcCes@!k9rEjArR_Tv98_{Xc_}hnWrH0rY5;Ug->5 z2y|_x^Rj^cB5V_(Tjm%F(te5n?1M#CxEVkX2X6Uw7w#T3+%+PLIHYhl@3@2^^zPN5=gvM0!k9rEmMvXsJQ&YP%^`t&&(}>aA#Arc)V#?nU0mrYq&>K+ zx<-OGyV_P5jV_sd-sv&+UeyXAId_jkv|`!&U-KTF`*oV@MD1Trk7*eHJIh^+kdAD;Xd@Suh=uiK2Uj9+_8ne6rRaz*lTY99gEhChS0TR z@68m-Y}b(lgtS_v(Bq@>&?K$q>bbjaE?f?iLEId>R<&FZ`|xMO9&(e)g?C_&&rlWw z&aN_)PJDgaewDxP`}&agz`o)zW&R&72x?++NnaWEgE z43pPrdTX

&%|VoWBC~Z3S$4o63%Kby?X*`q?)Z`3oMeIeB0|fSyOlb z{pZuQI`7SfoyIOe7s*ggL6ZV=jw#Kx_WV?boF`dpT0YAZ{w#c-*p`ew>RI@juFLU# zPTDi?`PN0>FtujPAdZUEj^(CG(M`j|aRGK(m^?K4(7*q0*yFsGm}%rayp5A^@j1Y0 zw{9Upxh8d@)^vf@Gp(%_rMddUi&hAGEsPn&ak0(SvOQNTKNCke*gPFv zpkS#sCFmm#!}GS=o0sA)Ky8s@$p=+>Xs$kY`q!xeTu*)BawC@yc@Z*RE@me%mN726B7aw6$>x|>{z(RLgd~v7AvL{a8B7=CTC+9eWhpr4D9>5fXEtcHOun``Nbl)(y4JQ@ahHR* zA`cRC4AA}oBRR^Od^-uZ$UN0xYea3n-sJka8S+|Nl7e%F@*d65aXy=?dn=b#ZlFaN z#HZP2>o(r()v8+8d6TzCi?27S4PR?w&{2yE&N<2uW{cGX+d80e9Z7TbtG$m+Gj;JU zT7p45-Nem)4e>uDs}Pn4;|^=X?Q`aO%MdTQwuUILwZ>asq#$DsIcKqlG7HMHX*E~B z+WXjqagw417{s%kc}T`~!)GYxax#RMguULRmRuDAWBE*ToD}%BVyo4i3FQp&$WYdV z*Lja#%$^&qsix9g-A_LI;QLPz`P^a#aea12rasEGeESHeTJBKRB7WG^1JQ1)SJ(Zf zYX)}57ncnPBrt;vyt3;eOSf0w9Ma;JBAn`4%i30(v;Bs0-kQ8ehatn)4qkc(^#r@7 zXR)=~x8d$n3FF2WH@yghSkJ^F`+bx*LJFO=e&r_4QQp`e*{=sIS$CT)2v)~C(A(iF zM|q8Xin2DBEz7kwagWMT6==NNqilkXOVnY^AP&Qp%;i`+b4-&@e3j#np-c0!VUwb~ zzEbNL6iS_N>g7u1&{9a}%f;#6(2YbxHM$XCM6jeu437Kfy4FiHL}7%~%|-9BVCnJr@Qs5fM@8 zRT!`TG|xw)L~G&e3o(d^h=@+~CV#X37oR*F72}%}dU&s9 zC?A?y_FYrk-T@I25luBSjD^&aalF!B0p6Qu`jadoa$FRTaCJ0X-^sak7ZDNB1?CFl zNl$muH9TM407e_8#{6pQ#nDKnMfDDF_2_^BOhiOPL_|cb@f)FU%(hJHem?*J002ov JPDHLkV1goiusr|( literal 0 HcmV?d00001 From 4231d049154bb4890c21d20c58a26d7ae49df8d7 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Oct 2023 13:49:38 -0300 Subject: [PATCH 18/54] Trying to avoid menu being sent before resources and greeting Mostly for Instagram. Just increasing one second before sending the main menu after greetings and resources. Also adding one more permission required by Instagram. Fixes CV2-3727. --- app/controllers/concerns/facebook_authentication.rb | 2 +- app/models/concerns/smooch_menus.rb | 2 +- app/models/concerns/smooch_resources.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/concerns/facebook_authentication.rb b/app/controllers/concerns/facebook_authentication.rb index 8b61035e64..e958197c1d 100644 --- a/app/controllers/concerns/facebook_authentication.rb +++ b/app/controllers/concerns/facebook_authentication.rb @@ -5,7 +5,7 @@ def setup_facebook # pages_manage_metadata is for Facebook API > 7 # manage_pages is for Facebook API < 7 # An error will be displayed for Facebook users that are admins of the Facebook app, but should be transparent for other users - request.env['omniauth.strategy'].options[:scope] = 'pages_manage_metadata,manage_pages,pages_messaging,instagram_manage_messages' if params[:context] == 'smooch' + request.env['omniauth.strategy'].options[:scope] = 'pages_manage_metadata,manage_pages,pages_messaging,instagram_manage_messages,instagram_basic' if params[:context] == 'smooch' prefix = facebook_context == 'smooch' ? 'smooch_' : '' request.env['omniauth.strategy'].options[:client_id] = CheckConfig.get("#{prefix}facebook_app_id") request.env['omniauth.strategy'].options[:client_secret] = CheckConfig.get("#{prefix}facebook_app_secret") diff --git a/app/models/concerns/smooch_menus.rb b/app/models/concerns/smooch_menus.rb index 08b4b2001d..7e0e186420 100644 --- a/app/models/concerns/smooch_menus.rb +++ b/app/models/concerns/smooch_menus.rb @@ -303,7 +303,7 @@ def send_greeting(uid, workflow) text = self.get_custom_string('smooch_message_smooch_bot_greetings', workflow['smooch_workflow_language']) image = workflow['smooch_greeting_image'] if workflow['smooch_greeting_image'] =~ /^https?:\/\// image.blank? || image == 'none' ? self.send_message_to_user(uid, text) : self.send_message_to_user(uid, text, { 'type' => 'image', 'mediaUrl' => image }) - sleep 2 # Give it some time, so the main menu message is sent after the greetings + sleep 3 # Give it some time, so the main menu message is sent after the greetings end end end diff --git a/app/models/concerns/smooch_resources.rb b/app/models/concerns/smooch_resources.rb index 33078e918c..0a72f27fa4 100644 --- a/app/models/concerns/smooch_resources.rb +++ b/app/models/concerns/smooch_resources.rb @@ -14,7 +14,7 @@ def send_resource_to_user(uid, workflow, resource_uuid, language) type = 'video' if type == 'audio' # Audio gets converted to video with a cover type = 'file' if type == 'video' && RequestStore.store[:smooch_bot_provider] == 'ZENDESK' # Smooch doesn't support video self.send_message_to_user(uid, message, { 'type' => type, 'mediaUrl' => CheckS3.rewrite_url(resource.header_media_url) }, false, true, 'resource') - sleep 2 # Wait a couple of seconds before sending the main menu + sleep 3 # Wait a few seconds before sending the main menu self.send_message_for_state(uid, workflow, 'main', language) else preview_url = (resource.header_type == 'link_preview') From da031923284db8573bb7dc67643b93088d5d0908 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Fri, 27 Oct 2023 16:28:08 -0300 Subject: [PATCH 19/54] Exposing new fields on GraphQL API for shared feeds invitations - Adding feed_teams connection to Feed and Team - Adding feed_team (get by database ID) to the root query type - Allow feed owner to delete a FeedTeam Reference: CV2-3900. --- app/graph/types/feed_type.rb | 1 + app/graph/types/query_type.rb | 1 + app/graph/types/team_type.rb | 1 + app/models/ability.rb | 3 + lib/relay.idl | 68 ++++++++++ public/relay.json | 240 ++++++++++++++++++++++++++++++++++ test/models/ability_test.rb | 30 +++++ 7 files changed, 344 insertions(+) diff --git a/app/graph/types/feed_type.rb b/app/graph/types/feed_type.rb index 7e5593c0e4..702faaba3b 100644 --- a/app/graph/types/feed_type.rb +++ b/app/graph/types/feed_type.rb @@ -44,4 +44,5 @@ def requests(**args) field :feed_invitations, FeedInvitationType.connection_type, null: false field :teams, TeamType.connection_type, null: false + field :feed_teams, FeedTeamType.connection_type, null: false end diff --git a/app/graph/types/query_type.rb b/app/graph/types/query_type.rb index a340647923..52d20a3f4a 100644 --- a/app/graph/types/query_type.rb +++ b/app/graph/types/query_type.rb @@ -215,6 +215,7 @@ def dynamic_annotation_field(query:, only_cache: nil) feed request feed_invitation + feed_team tipline_message ].each do |type| field type, diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index 2cfb2973c1..7c28ff6350 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -293,6 +293,7 @@ def shared_teams field :saved_searches, SavedSearchType.connection_type, null: true field :project_groups, ProjectGroupType.connection_type, null: true field :feeds, FeedType.connection_type, null: true + field :feed_teams, FeedTeamType.connection_type, null: false field :tipline_newsletters, TiplineNewsletterType.connection_type, null: true field :tipline_resources, TiplineResourceType.connection_type, null: true diff --git a/app/models/ability.rb b/app/models/ability.rb index 84f21cd749..f0526a34a2 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -85,6 +85,9 @@ def editor_perms end can [:create, :update, :read, :destroy], [Account, Source, TiplineNewsletter, TiplineResource, Feed, FeedTeam], :team_id => @context_team.id can [:create, :update, :destroy], FeedInvitation, { feed: { team_id: @context_team.id } } + can :destroy, FeedTeam do |obj| + obj.team.id == @context_team.id || obj.feed.team.id == @context_team.id + end can [:cud], AccountSource, source: { team: { team_users: { team_id: @context_team.id }}} %w(annotation comment dynamic task tag).each do |annotation_type| can [:cud], annotation_type.classify.constantize do |obj| diff --git a/lib/relay.idl b/lib/relay.idl index 21d3f58ea7..2646cb4eac 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -8430,6 +8430,27 @@ type Feed implements Node { """ last: Int ): FeedInvitationConnection! + feed_teams( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): FeedTeamConnection! filters: JsonStringType id: ID! licenses: [Int] @@ -8611,6 +8632,27 @@ type FeedTeam implements Node { updated_at: String } +""" +The connection type for FeedTeam. +""" +type FeedTeamConnection { + """ + A list of edges. + """ + edges: [FeedTeamEdge] + + """ + A list of nodes. + """ + nodes: [FeedTeam] + + """ + Information to aid in pagination. + """ + pageInfo: PageInfo! + totalCount: Int +} + """ An edge in a connection. """ @@ -11690,6 +11732,11 @@ type Query { """ feed_invitation(id: ID!): FeedInvitation + """ + Information about the feed_team with given id + """ + feed_team(id: ID!): FeedTeam + """ Find whether a team exists """ @@ -12875,6 +12922,27 @@ type Team implements Node { description: String dynamic_search_fields_json_schema: JsonStringType feed(dbid: Int!): Feed + feed_teams( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): FeedTeamConnection! feeds( """ Returns the elements in the list that come after the specified cursor. diff --git a/public/relay.json b/public/relay.json index 4d93e551ed..b4e4c56506 100644 --- a/public/relay.json +++ b/public/relay.json @@ -45752,6 +45752,71 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "feed_teams", + "description": null, + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "FeedTeamConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "filters", "description": null, @@ -46944,6 +47009,87 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "FeedTeamConnection", + "description": "The connection type for FeedTeam.", + "fields": [ + { + "name": "edges", + "description": "A list of edges.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "FeedTeamEdge", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "nodes", + "description": "A list of nodes.", + "args": [ + + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "FeedTeam", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "pageInfo", + "description": "Information to aid in pagination.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "PageInfo", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "totalCount", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "FeedTeamEdge", @@ -61529,6 +61675,35 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "feed_team", + "description": "Information about the feed_team with given id", + "args": [ + { + "name": "id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "FeedTeam", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "find_public_team", "description": "Find whether a team exists", @@ -67337,6 +67512,71 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "feed_teams", + "description": null, + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "FeedTeamConnection", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "feeds", "description": null, diff --git a/test/models/ability_test.rb b/test/models/ability_test.rb index e8b52f786b..cdef014064 100644 --- a/test/models/ability_test.rb +++ b/test/models/ability_test.rb @@ -1305,4 +1305,34 @@ def teardown assert ability.cannot?(:destroy, fi2) end end + + test "permissions for feed team" do + t1 = create_team + t2 = create_team + t3 = create_team + u1 = create_user + u2 = create_user + u3 = create_user + create_team_user user: u1, team: t1, role: 'admin' + create_team_user user: u2, team: t2, role: 'admin' + create_team_user user: u3, team: t3, role: 'admin' + f = create_feed team: t1 + ft2 = create_feed_team feed: f, team: t2 + ft3 = create_feed_team feed: f, team: t3 + with_current_user_and_team(u1, t1) do + ability = Ability.new + assert ability.can?(:destroy, ft2) + assert ability.can?(:destroy, ft3) + end + with_current_user_and_team(u2, t2) do + ability = Ability.new + assert ability.can?(:destroy, ft2) + assert ability.cannot?(:destroy, ft3) + end + with_current_user_and_team(u3, t3) do + ability = Ability.new + assert ability.cannot?(:destroy, ft2) + assert ability.can?(:destroy, ft3) + end + end end From 322c5db7403e439d0be6810684d649b9d50f632b Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Sat, 28 Oct 2023 00:31:12 -0300 Subject: [PATCH 20/54] Fixing race condition when storing tipline media and requests Add some more interval when creating requests and medias for tipline requests just to try to avoid some race conditions. Also adding a new field and unique index so we don't have two requests related to the same message. Reference: CV2-765. --- app/models/concerns/smooch_messages.rb | 17 ++-- app/models/concerns/smooch_search.rb | 11 ++- ...31026162554_add_smooch_message_id_field.rb | 11 +++ db/schema.rb | 3 +- test/mailers/security_mailer_test.rb | 7 ++ test/models/bot/smooch_2_test.rb | 2 +- test/models/bot/smooch_4_test.rb | 10 +-- test/models/bot/smooch_7_test.rb | 78 ++++++++++--------- test/models/bot/smooch_8_test.rb | 26 +++++++ test/models/login_activity_test.rb | 5 +- test/test_helper.rb | 2 +- 11 files changed, 116 insertions(+), 56 deletions(-) create mode 100644 db/migrate/20231026162554_add_smooch_message_id_field.rb create mode 100644 test/models/bot/smooch_8_test.rb diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index a77a2edf09..a8df490b0f 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -286,7 +286,7 @@ def handle_bundle_messages(type, list, last, app_id, annotated, send_message = t if ['timeout_requests', 'menu_options_requests', 'resource_requests', 'relevant_search_result_requests', 'timeout_search_requests'].include?(type) key = "smooch:banned:#{bundle['authorId']}" if Rails.cache.read(key).nil? - [annotated].flatten.uniq.each { |a| self.save_message_later(bundle, app_id, type, a) } + [annotated].flatten.uniq.each_with_index { |a, i| self.save_message_later(bundle, app_id, type, a, i * 10) } end end end @@ -317,12 +317,12 @@ def supported_message?(message) ret end - def save_message_later(message, app_id, request_type = 'default_requests', annotated = nil) + def save_message_later(message, app_id, request_type = 'default_requests', annotated = nil, interval = 0) mapping = { 'siege' => 'siege' } queue = RequestStore.store[:smooch_bot_queue].to_s queue = queue.blank? ? 'smooch' : (mapping[queue] || 'smooch') type = (message['type'] == 'text' && !message['text'][/https?:\/\/[^\s]+/, 0].blank?) ? 'link' : message['type'] - SmoochWorker.set(queue: queue).perform_in(1.second, message.to_json, type, app_id, request_type, YAML.dump(annotated)) + SmoochWorker.set(queue: queue).perform_in(1.second + interval.seconds, message.to_json, type, app_id, request_type, YAML.dump(annotated)) end def default_archived_flag @@ -344,7 +344,7 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req message['archived'] = (request_type == 'relevant_search_result_requests' ? self.default_archived_flag : CheckArchivedFlags::FlagCodes::UNCONFIRMED) annotated = self.create_project_media_from_message(message) if annotated != annotated_obj && annotated.is_a?(ProjectMedia) - Relationship.create!(relationship_type: Relationship.suggested_type, source: annotated_obj, target: annotated, user: BotUser.smooch_user) + Relationship.create(relationship_type: Relationship.suggested_type, source: annotated_obj, target: annotated, user: BotUser.smooch_user) end end @@ -370,8 +370,9 @@ def create_smooch_request(annotated, message, app_id, author) fields = { smooch_data: message.merge({ app_id: app_id }).to_json } result = self.smooch_api_get_messages(app_id, message['authorId']) fields[:smooch_conversation_id] = result.conversation.id unless result.nil? || result.conversation.nil? + fields[:smooch_message_id] = message['_id'] self.create_smooch_annotations(annotated, author, fields) - # update channel values for ProjectMedia items + # Update channel values for ProjectMedia items if annotated.class.name == 'ProjectMedia' channel_value = self.get_smooch_channel(message) unless channel_value.blank? @@ -407,7 +408,11 @@ def create_smooch_annotations(annotated, author, fields, attach_to = false) a.skip_notifications = true a.disable_es_callbacks = Rails.env.to_s == 'test' a.set_fields = fields.to_json - a.save! + begin + a.save! + rescue ActiveRecord::RecordNotUnique + Rails.logger.info('[Smooch Bot] Not storing tipline request because it already exists.') + end User.current = current_user end diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index 083f000ad0..7a652eb8f0 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -247,15 +247,16 @@ def send_search_results_to_whatsapp_user(uid, reports, app_id) # Expires after the time to give feedback is expired Rails.cache.write("smooch:user_search_bundle:#{uid}:#{search_id}", self.list_of_bundled_messages_from_user(uid), expires_in: 20.minutes) self.clear_user_bundled_messages(uid) - reports.each do |report| + reports.each_with_index do |report, i| text = report.report_design_text if report.report_design_field_value('use_text_message') image_url = report.report_design_image_url if report.report_design_field_value('use_visual_card') options = [{ - value: { project_media_id: report.annotated_id, keyword: 'search_result_is_relevant', search_id: search_id }.to_json, + value: { project_media_id: report.annotated_id, keyword: 'search_result_is_relevant', search_id: search_id, index: i }.to_json, label: '👍' }] self.send_message_to_user_with_buttons(uid, text || '-', options, image_url, 'search_result') # "text" is mandatory for WhatsApp interactive messages - self.delay_for(15.minutes, { queue: 'smooch_priority' }).timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, report.annotated_id) + # Schedule each timeout with some interval between them just to avoid race conditions that could create duplicate media + self.delay_for((15 + i).minutes, { queue: 'smooch_priority' }).timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, report.annotated_id) end end @@ -313,8 +314,10 @@ def search_result_button_click_callback(message, uid, app_id, workflow, language result = ProjectMedia.find(payload['project_media_id']) bundle = Rails.cache.read("smooch:user_search_bundle:#{uid}:#{payload['search_id']}").to_a unless bundle.empty? + # Give some interval just to avoid race conditions if the user gives feedback too fast + interval = payload['index'].to_i * 10 Rails.cache.write("smooch:user_search_bundle:#{uid}:#{payload['search_id']}:#{result.id}", Time.now.to_i) # Store that the user has given feedback to this search result - self.delay_for(1.seconds, { queue: 'smooch', retry: false }).bundle_messages(uid, message['_id'], app_id, 'relevant_search_result_requests', [result], true, bundle) + self.delay_for((interval + 1).seconds, { queue: 'smooch', retry: false }).bundle_messages(uid, message['_id'], app_id, 'relevant_search_result_requests', [result], true, bundle) self.send_final_message_to_user(uid, self.get_custom_string('search_result_is_relevant', language), workflow, language) end end diff --git a/db/migrate/20231026162554_add_smooch_message_id_field.rb b/db/migrate/20231026162554_add_smooch_message_id_field.rb new file mode 100644 index 0000000000..0d4fb0f6bd --- /dev/null +++ b/db/migrate/20231026162554_add_smooch_message_id_field.rb @@ -0,0 +1,11 @@ +class AddSmoochMessageIdField < ActiveRecord::Migration[6.1] + require 'sample_data' + include SampleData + + def change + at = DynamicAnnotation::AnnotationType.where(annotation_type: 'smooch').last + ft = DynamicAnnotation::FieldType.where(field_type: 'text').last || create_field_type(field_type: 'text', label: 'Text') + create_field_instance annotation_type_object: at, name: 'smooch_message_id', label: 'Message Id', field_type_object: ft, optional: true + execute %{CREATE UNIQUE INDEX smooch_request_message_id_unique_id ON dynamic_annotation_fields (value) WHERE field_name = 'smooch_message_id' AND value <> '' AND value <> '""'} + end +end diff --git a/db/schema.rb b/db/schema.rb index a90ecf3918..eecd1ba949 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_10_23_002756) do +ActiveRecord::Schema.define(version: 2023_10_26_162554) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -276,6 +276,7 @@ t.index ["field_type"], name: "index_dynamic_annotation_fields_on_field_type" t.index ["value"], name: "fetch_unique_id", unique: true, where: "(((field_name)::text = 'external_id'::text) AND (value <> ''::text) AND (value <> '\"\"'::text))" t.index ["value"], name: "index_status", where: "((field_name)::text = 'verification_status_status'::text)" + t.index ["value"], name: "smooch_request_message_id_unique_id", unique: true, where: "(((field_name)::text = 'smooch_message_id'::text) AND (value <> ''::text) AND (value <> '\"\"'::text))" t.index ["value"], name: "smooch_user_unique_id", unique: true, where: "(((field_name)::text = 'smooch_user_id'::text) AND (value <> ''::text) AND (value <> '\"\"'::text))" t.index ["value"], name: "translation_request_id", unique: true, where: "((field_name)::text = 'translation_request_id'::text)" t.index ["value_json"], name: "index_dynamic_annotation_fields_on_value_json", using: :gin diff --git a/test/mailers/security_mailer_test.rb b/test/mailers/security_mailer_test.rb index 988932d729..6804e1861c 100644 --- a/test/mailers/security_mailer_test.rb +++ b/test/mailers/security_mailer_test.rb @@ -1,6 +1,13 @@ require_relative '../test_helper' class SecurityMailerTest < ActionMailer::TestCase + def setup + WebMock.stub_request(:get, /ipinfo\.io/).to_return(body: { country: 'US', city: 'San Francisco' }.to_json, status: 200) + end + + def teardown + end + test "should send security notification" do user = create_user la = create_login_activity user: user, success: true diff --git a/test/models/bot/smooch_2_test.rb b/test/models/bot/smooch_2_test.rb index 30434fab9e..baf4c53e38 100644 --- a/test/models/bot/smooch_2_test.rb +++ b/test/models/bot/smooch_2_test.rb @@ -99,7 +99,7 @@ def teardown s.save! child = create_project_media project: @project - create_dynamic_annotation annotation_type: 'smooch', annotated: child, set_fields: { smooch_data: { app_id: @app_id, authorId: random_string, language: 'en' }.to_json }.to_json + create_dynamic_annotation annotation_type: 'smooch', annotated: child, set_fields: { smooch_message_id: random_string, smooch_data: { app_id: @app_id, authorId: random_string, language: 'en' }.to_json }.to_json r = create_relationship source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type s = child.annotations.where(annotation_type: 'verification_status').last.load assert_equal 'verified', s.status diff --git a/test/models/bot/smooch_4_test.rb b/test/models/bot/smooch_4_test.rb index 1d619bc543..92358f3da7 100644 --- a/test/models/bot/smooch_4_test.rb +++ b/test/models/bot/smooch_4_test.rb @@ -376,15 +376,15 @@ def teardown u = create_user is_admin: true t = create_team with_current_user_and_team(u, t) do - d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_data: { 'authorId' => whatsapp_uid }.to_json }.to_json + d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_message_id: random_string, smooch_data: { 'authorId' => whatsapp_uid }.to_json }.to_json assert_equal '+55 12 3456-7890', d.get_field('smooch_data').smooch_user_external_identifier - d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_data: { 'authorId' => twitter_uid }.to_json }.to_json + d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_message_id: random_string, smooch_data: { 'authorId' => twitter_uid }.to_json }.to_json assert_equal '@foobar', d.get_field('smooch_data').smooch_user_external_identifier - d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_data: { 'authorId' => facebook_uid }.to_json }.to_json + d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_message_id: random_string, smooch_data: { 'authorId' => facebook_uid }.to_json }.to_json assert_equal '123456', d.get_field('smooch_data').smooch_user_external_identifier - d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_data: { 'authorId' => telegram_uid }.to_json }.to_json + d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_message_id: random_string, smooch_data: { 'authorId' => telegram_uid }.to_json }.to_json assert_equal '@barfoo', d.get_field('smooch_data').smooch_user_external_identifier - d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_data: { 'authorId' => other_uid }.to_json }.to_json + d = create_dynamic_annotation annotation_type: 'smooch', set_fields: { smooch_message_id: random_string, smooch_data: { 'authorId' => other_uid }.to_json }.to_json assert_equal '', d.get_field('smooch_data').smooch_user_external_identifier end end diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index db022afd1b..7fb40d4fc0 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -471,43 +471,47 @@ def teardown pm = create_project_media team: @team, quote: text, disable_es_callbacks: false text2 = random_string pm2 = create_project_media team: @team, quote: text2, disable_es_callbacks: false - message = { - type: 'text', - text: text, - role: 'appUser', - received: 1573082583.219, - name: random_string, - authorId: random_string, - '_id': random_string, - source: { - originalMessageId: random_string, - originalMessageTimestamp: 1573082582, - type: 'whatsapp', - integrationId: random_string - }, - } - Bot::Smooch.save_message(message.to_json, @app_id, nil, 'relevant_search_result_requests', pm) - Bot::Smooch.save_message(message.to_json, @app_id, nil, 'relevant_search_result_requests', pm) - Bot::Smooch.save_message(message.to_json, @app_id, nil, 'timeout_search_requests', pm) - Bot::Smooch.save_message(message.to_json, @app_id, nil, 'irrelevant_search_result_requests') - Bot::Smooch.save_message(message.to_json, @app_id, nil, 'irrelevant_search_result_requests') - message = { - type: 'text', - text: text2, - role: 'appUser', - received: 1573082583.219, - name: random_string, - authorId: random_string, - '_id': random_string, - source: { - originalMessageId: random_string, - originalMessageTimestamp: 1573082582, - type: 'whatsapp', - integrationId: random_string - }, - } - Bot::Smooch.save_message(message.to_json, @app_id, nil, 'relevant_search_result_requests', pm2) - Bot::Smooch.save_message(message.to_json, @app_id, nil, 'irrelevant_search_result_requests') + message = lambda do + { + type: 'text', + text: text, + role: 'appUser', + received: 1573082583.219, + name: random_string, + authorId: random_string, + '_id': random_string, + source: { + originalMessageId: random_string, + originalMessageTimestamp: 1573082582, + type: 'whatsapp', + integrationId: random_string + } + } + end + Bot::Smooch.save_message(message.call.to_json, @app_id, nil, 'relevant_search_result_requests', pm) + Bot::Smooch.save_message(message.call.to_json, @app_id, nil, 'relevant_search_result_requests', pm) + Bot::Smooch.save_message(message.call.to_json, @app_id, nil, 'timeout_search_requests', pm) + Bot::Smooch.save_message(message.call.to_json, @app_id, nil, 'irrelevant_search_result_requests') + Bot::Smooch.save_message(message.call.to_json, @app_id, nil, 'irrelevant_search_result_requests') + message = lambda do + { + type: 'text', + text: text2, + role: 'appUser', + received: 1573082583.219, + name: random_string, + authorId: random_string, + '_id': random_string, + source: { + originalMessageId: random_string, + originalMessageTimestamp: 1573082582, + type: 'whatsapp', + integrationId: random_string + }, + } + end + Bot::Smooch.save_message(message.call.to_json, @app_id, nil, 'relevant_search_result_requests', pm2) + Bot::Smooch.save_message(message.call.to_json, @app_id, nil, 'irrelevant_search_result_requests') # Verify cached field assert_equal 5, pm.tipline_search_results_count assert_equal 2, pm.positive_tipline_search_results_count diff --git a/test/models/bot/smooch_8_test.rb b/test/models/bot/smooch_8_test.rb new file mode 100644 index 0000000000..9e2b985700 --- /dev/null +++ b/test/models/bot/smooch_8_test.rb @@ -0,0 +1,26 @@ +require_relative '../../test_helper' +require 'sidekiq/testing' + +class Bot::Smooch8Test < ActiveSupport::TestCase + def setup + end + + def teardown + end + + test "should not store duplicated Smooch requests" do + create_annotation_type_and_fields('Smooch', { + 'Data' => ['JSON', false], + 'Message Id' => ['Text', false] + }) + + pm = create_project_media + fields = { 'smooch_message_id' => random_string, 'smooch_data' => '{}' } + assert_difference 'Annotation.count' do + Bot::Smooch.create_smooch_annotations(pm, nil, fields) + end + assert_no_difference 'Annotation.count' do + Bot::Smooch.create_smooch_annotations(pm, nil, fields) + end + end +end diff --git a/test/models/login_activity_test.rb b/test/models/login_activity_test.rb index a32328fc92..54a9fdb723 100644 --- a/test/models/login_activity_test.rb +++ b/test/models/login_activity_test.rb @@ -2,9 +2,12 @@ class LoginActivityTest < ActiveSupport::TestCase def setup - super require 'sidekiq/testing' Sidekiq::Testing.inline! + WebMock.stub_request(:get, /ipinfo\.io/).to_return(body: { country: 'US', city: 'San Francisco' }.to_json, status: 200) + end + + def teardown end test "should create login activity" do diff --git a/test/test_helper.rb b/test/test_helper.rb index 38473d9cdd..695d08ad75 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -852,7 +852,7 @@ def setup_smooch_bot(menu = false, extra_settings = {}) 'Request Type' => ['Text', true], 'Resource Id' => ['Text', true], 'Report correction sent at' => ['Timestamp', true], - 'Report sent at' => ['Timestamp', true], + 'Report sent at' => ['Timestamp', true] }) create_annotation_type_and_fields('Smooch Response', { 'Data' => ['JSON', true] }) create_annotation_type annotation_type: 'reverse_image', label: 'Reverse Image' From 134a663a1091607ea40f3ff337011349a151a465 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:19:08 -0300 Subject: [PATCH 21/54] Tipline: NLU disambiguation If NLU identifies that two options are very similar, present both to the user and let them choose. Reference: CV2-3710. --- app/lib/smooch_nlu.rb | 13 +++++--- app/lib/smooch_nlu_menus.rb | 34 ++++++++++++++++----- app/models/bot/smooch.rb | 6 ++-- app/models/concerns/tipline_resource_nlu.rb | 2 +- config/config.yml.example | 1 + config/tipline_strings.yml | 1 + test/lib/smooch_nlu_test.rb | 4 +-- test/models/bot/smooch_6_test.rb | 25 ++++++++++----- 8 files changed, 61 insertions(+), 25 deletions(-) diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index 7144e9cc73..c88a483f4a 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -5,9 +5,9 @@ class SmoochBotNotInstalledError < ::ArgumentError # FIXME: Make it more flexible # FIXME: Once we support paraphrase-multilingual-mpnet-base-v2 make it the only model used ALEGRE_MODELS_AND_THRESHOLDS = { - # Bot::Alegre::ELASTICSEARCH_MODEL => 0.8 # , Sometimes this is easier for local development - Bot::Alegre::OPENAI_ADA_MODEL => 0.8, - Bot::Alegre::MEAN_TOKENS_MODEL => 0.6 + # Bot::Alegre::ELASTICSEARCH_MODEL => 0.8 # Sometimes this is easier for local development + # Bot::Alegre::OPENAI_ADA_MODEL => 0.8 # Not in use right now + Bot::Alegre::PARAPHRASE_MULTILINGUAL_MODEL => 0.6 } include SmoochNluMenus @@ -54,6 +54,11 @@ def update_keywords(language, keywords, keyword, operation, doc_id, context) keywords end + # If NLU matches two results that have at least this distance between them, they are both presented to the user for disambiguation + def self.disambiguation_threshold + CheckConfig.get('nlu_disambiguation_threshold', 0.11, :float).to_f + end + def self.alegre_matches_from_message(message, language, context, alegre_result_key) # FIXME: Raise exception if not in a tipline context (so, if Bot::Smooch.config is nil) matches = [] @@ -86,7 +91,7 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k # Second approach is to sort the results from best to worst sorted_options = response['result'].to_a.sort_by{ |result| result['_score'] }.reverse - ranked_options = sorted_options.map{ |o| o.dig('_source', 'context', alegre_result_key) } + ranked_options = sorted_options.map{ |o| { 'key' => o.dig('_source', 'context', alegre_result_key), 'score' => o['_score'] } } matches = ranked_options # FIXME: Deal with ties (i.e., where two options have an equal _score or count) diff --git a/app/lib/smooch_nlu_menus.rb b/app/lib/smooch_nlu_menus.rb index 236483e6a4..c27408e8dd 100644 --- a/app/lib/smooch_nlu_menus.rb +++ b/app/lib/smooch_nlu_menus.rb @@ -67,20 +67,38 @@ def update_menu_option_keywords(language, menu, menu_option_index, keyword, oper end module ClassMethods - def menu_option_from_message(message, language, options) - return nil if options.blank? - option = nil + def menu_options_from_message(message, language, options) + return [{ 'smooch_menu_option_value' => 'main_state' }] if message == 'cancel_nlu' + return [] if options.blank? context = { context: ALEGRE_CONTEXT_KEY_MENU } matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'menu_option_id') - # Select the top menu option that exists in `options` + # Select the top two menu options that exists in `options` + top_options = [] matches.each do |r| - option = options.find{ |o| !o['smooch_menu_option_id'].blank? && o['smooch_menu_option_id'] == r } - break unless option.nil? + option = options.find { |o| !o['smooch_menu_option_id'].blank? && o['smooch_menu_option_id'] == r['key'] } + top_options << { 'option' => option, 'score' => r['score'] } if !option.nil? && (top_options.empty? || (top_options.first['score'] - r['score']) <= SmoochNlu.disambiguation_threshold) + break if top_options.size == 2 + end + Rails.logger.info("[Smooch NLU] [Menu Option From Message] Menu options: #{top_options.inspect} | Message: #{message}") + top_options.collect{ |o| o['option'] } + end + + def process_menu_options(uid, options, message, language, workflow, app_id) + + if options.size == 1 + Bot::Smooch.process_menu_option_value(options.first['smooch_menu_option_value'], options.first, message, language, workflow, app_id) + # Disambiguation + else + buttons = options.collect do |option| + { + value: { keyword: option['smooch_menu_option_keyword'] }.to_json, + label: option['smooch_menu_option_label'] + } + end.concat([{ value: { keyword: 'cancel_nlu' }.to_json, label: Bot::Smooch.get_string('main_state_button_label', language, 20) }]) + Bot::Smooch.send_message_to_user_with_buttons(uid, Bot::Smooch.get_string('nlu_disambiguation', language), buttons) end - Rails.logger.info("[Smooch NLU] [Menu Option From Message] Menu option: #{option} | Message: #{message}") - option end end end diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb index e85a593db2..228aafc185 100644 --- a/app/models/bot/smooch.rb +++ b/app/models/bot/smooch.rb @@ -575,9 +575,9 @@ def self.process_menu_option(message, state, app_id) end # ...if nothing is matched, try using the NLU feature if state != 'query' - option = SmoochNlu.menu_option_from_message(typed, language, options) - unless option.nil? - self.process_menu_option_value(option['smooch_menu_option_value'], option, message, language, workflow, app_id) + options = SmoochNlu.menu_options_from_message(typed, language, options) + unless options.blank? + SmoochNlu.process_menu_options(uid, options, message, language, workflow, app_id) return true end resource = TiplineResource.resource_from_message(typed, language) diff --git a/app/models/concerns/tipline_resource_nlu.rb b/app/models/concerns/tipline_resource_nlu.rb index d3efb60e53..5d92c469d2 100644 --- a/app/models/concerns/tipline_resource_nlu.rb +++ b/app/models/concerns/tipline_resource_nlu.rb @@ -36,7 +36,7 @@ def resource_from_message(message, language) context = { context: ALEGRE_CONTEXT_KEY_RESOURCE } - matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'resource_id') + matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'resource_id').collect{ |m| m['key'] } # Select the top resource that exists resource_id = matches.find { |id| TiplineResource.where(id: id).exists? } Rails.logger.info("[Smooch NLU] [Resource From Message] Resource ID: #{resource_id} | Message: #{message}") diff --git a/config/config.yml.example b/config/config.yml.example index e9f89e4976..4d5602cbb4 100644 --- a/config/config.yml.example +++ b/config/config.yml.example @@ -44,6 +44,7 @@ development: &default text_cluster_similarity_threshold: 0.9 similarity_media_file_url_host: '' min_number_of_words_for_tipline_submit_shortcut: 10 + nlu_disambiguation_threshold: 0.11 # Localization # diff --git a/config/tipline_strings.yml b/config/tipline_strings.yml index 9599a683dc..3845091311 100644 --- a/config/tipline_strings.yml +++ b/config/tipline_strings.yml @@ -261,6 +261,7 @@ en: subscribed: You are currently subscribed to our newsletter. unsubscribe_button_label: Unsubscribe unsubscribed: You are currently not subscribed to our newsletter. + nlu_disambiguation: "Choose one of the options below:" fil: add_more_details_state_button_label: Dagdagan pa ask_if_ready_state_button_label: Kanselahin diff --git a/test/lib/smooch_nlu_test.rb b/test/lib/smooch_nlu_test.rb index 3c1a1eac48..3f1b4cbaf4 100644 --- a/test/lib/smooch_nlu_test.rb +++ b/test/lib/smooch_nlu_test.rb @@ -110,7 +110,7 @@ def create_team_with_smooch_bot_installed team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).disable! Bot::Smooch.get_installation('smooch_id', 'test') - assert_nil SmoochNlu.menu_option_from_message('I want to subscribe to the newsletter', 'en', @menu_options) + assert_equal [], SmoochNlu.menu_options_from_message('I want to subscribe to the newsletter', 'en', @menu_options) end test 'should return a menu option if NLU is enabled' do @@ -120,6 +120,6 @@ def create_team_with_smooch_bot_installed team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).enable! Bot::Smooch.get_installation('smooch_id', 'test') - assert_not_nil SmoochNlu.menu_option_from_message('I want to subscribe to the newsletter', 'en', @menu_options) + assert_not_nil SmoochNlu.menu_options_from_message('I want to subscribe to the newsletter', 'en', @menu_options) end end diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index 161c375a18..4f809069f8 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -752,22 +752,23 @@ def send_message_outside_24_hours_window(template, pm = nil) end test 'should process menu option using NLU' do - # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "newsletter" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /newsletter/ }.returns(true) - # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "newsletter" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && (z[:text] =~ /newsletter/).nil? }.returns({ 'result' => [] }) + # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "want" + Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns(true) + # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "want" + Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords for the newsletter menu option nlu = SmoochNlu.new(@team.slug) nlu.enable! + nlu.add_keyword_to_menu_option('en', 'main', 1, 'I want to query') nlu.add_keyword_to_menu_option('en', 'main', 2, 'I want to subscribe to the newsletter') nlu.add_keyword_to_menu_option('en', 'main', 2, 'I want to unsubscribe from the newsletter') reload_tipline_settings query_option_id = @installation.get_smooch_workflows[0]['smooch_state_main']['smooch_menu_options'][1]['smooch_menu_option_id'] subscription_option_id = @installation.get_smooch_workflows[0]['smooch_state_main']['smooch_menu_options'][2]['smooch_menu_option_id'] - # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "newsletter" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ + # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" + Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns({ 'result' => [ { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, { '_score' => 0.2, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } ]}) @@ -775,11 +776,21 @@ def send_message_outside_24_hours_window(template, pm = nil) # Sending a message about the newsletter should take to the newsletter state, as per configurations done above send_message 'hello', '1' # Sends a first message and confirms language as English assert_state 'main' - send_message 'Can I subscribe to the newsletter?' + send_message 'I want to subscribe to the newsletter?' assert_state 'subscription' send_message '2' # Keep subscription assert_state 'main' + # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" + Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns({ 'result' => [ + { '_score' => 0.96, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, + { '_score' => 0.91, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } + ]}) + + # Sending a message that returns more than one option (disambiguation) + send_message 'I want to subscribe to the newsletter?' + assert_state 'main' + # After disabling NLU nlu.disable! reload_tipline_settings From 380daf65ff60b69d9ed340c8936ff4623654c559 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:19:55 -0300 Subject: [PATCH 22/54] Reverting: No more individual feedback to tipline search results Reverting this PR: #1687. Reference: CV2-3856. --- app/models/bot/smooch.rb | 5 -- app/models/concerns/smooch_menus.rb | 8 +-- app/models/concerns/smooch_search.rb | 83 +++------------------------- test/models/bot/smooch_6_test.rb | 49 ---------------- 4 files changed, 9 insertions(+), 136 deletions(-) diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb index 228aafc185..d07e7746ac 100644 --- a/app/models/bot/smooch.rb +++ b/app/models/bot/smooch.rb @@ -383,11 +383,6 @@ def self.parse_message_based_on_state(message, app_id) return true end - if self.clicked_on_search_result_button?(message) - self.search_result_button_click_callback(message, uid, app_id, workflow, language) - return true - end - case state when 'waiting_for_message' self.bundle_message(message) diff --git a/app/models/concerns/smooch_menus.rb b/app/models/concerns/smooch_menus.rb index 7e0e186420..9a7296a871 100644 --- a/app/models/concerns/smooch_menus.rb +++ b/app/models/concerns/smooch_menus.rb @@ -166,7 +166,7 @@ def get_custom_string(key, language, truncate_at = 1024) label.to_s.truncate(truncate_at) end - def send_message_to_user_with_buttons(uid, text, options, image_url = nil, event = nil) + def send_message_to_user_with_buttons(uid, text, options, event = nil) buttons = [] options.each_with_index do |option, i| buttons << { @@ -196,12 +196,6 @@ def send_message_to_user_with_buttons(uid, text, options, image_url = nil, event } } } - extra[:override][:whatsapp][:payload][:interactive][:header] = { - type: 'image', - image: { - link: CheckS3.rewrite_url(image_url) - } - } unless image_url.blank? extra, fallback = self.format_fallback_text_menu_from_options(text, options, extra) self.send_message_to_user(uid, fallback.join("\n"), extra, false, true, event) end diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index 7a652eb8f0..1abc860431 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -19,15 +19,10 @@ def search(app_id, uid, language, message, team_id, workflow) self.bundle_messages(uid, '', app_id, 'default_requests', nil, true) self.send_final_message_to_user(uid, self.get_custom_string('search_no_results', language), workflow, language, 'no_results') else - self.send_search_results_to_user(uid, results, team_id, platform, app_id) - # For WhatsApp, each search result goes with a button where the user can give feedback individually, so, reset the conversation right away - if platform == 'WhatsApp' - sm.reset - else - sm.go_to_search_result - self.save_search_results_for_user(uid, results.map(&:id)) - self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, 1) - end + self.send_search_results_to_user(uid, results, team_id) + sm.go_to_search_result + self.save_search_results_for_user(uid, results.map(&:id)) + self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, 1) end rescue StandardError => e self.handle_search_error(uid, e, language) @@ -220,48 +215,17 @@ def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, results end - def send_search_results_to_user(uid, results, team_id, platform, app_id) + def send_search_results_to_user(uid, results, team_id) team = Team.find(team_id) + redis = Redis.new(REDIS_CONFIG) language = self.get_user_language(uid) - reports = results.collect{ |r| r.get_dynamic_annotation('report_design') }.reject{ |r| r.blank? } + reports = results.collect{ |r| r.get_dynamic_annotation('report_design') } # Get reports languages - reports_language = reports.map{ |r| r.report_design_field_value('language') }.uniq + reports_language = reports.map{|r| r&.report_design_field_value('language')}.uniq if team.get_languages.to_a.size > 1 && !reports_language.include?(language) self.send_message_to_user(uid, self.get_string(:no_results_in_language, language).gsub('%{language}', CheckCldr.language_code_to_name(language, language)), {}, false, true, 'no_results') sleep 1 end - if platform == 'WhatsApp' - self.send_search_results_to_whatsapp_user(uid, reports, app_id) - else - self.send_search_results_to_non_whatsapp_user(uid, reports) - end - end - - def generate_search_id - SecureRandom.hex - end - - def send_search_results_to_whatsapp_user(uid, reports, app_id) - search_id = self.generate_search_id - # Cache the current bundle of messages from this user related to this search, so a request can be created correctly - # Expires after the time to give feedback is expired - Rails.cache.write("smooch:user_search_bundle:#{uid}:#{search_id}", self.list_of_bundled_messages_from_user(uid), expires_in: 20.minutes) - self.clear_user_bundled_messages(uid) - reports.each_with_index do |report, i| - text = report.report_design_text if report.report_design_field_value('use_text_message') - image_url = report.report_design_image_url if report.report_design_field_value('use_visual_card') - options = [{ - value: { project_media_id: report.annotated_id, keyword: 'search_result_is_relevant', search_id: search_id, index: i }.to_json, - label: '👍' - }] - self.send_message_to_user_with_buttons(uid, text || '-', options, image_url, 'search_result') # "text" is mandatory for WhatsApp interactive messages - # Schedule each timeout with some interval between them just to avoid race conditions that could create duplicate media - self.delay_for((15 + i).minutes, { queue: 'smooch_priority' }).timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, report.annotated_id) - end - end - - def send_search_results_to_non_whatsapp_user(uid, reports) - redis = Redis.new(REDIS_CONFIG) reports.each do |report| response = nil response = self.send_message_to_user(uid, report.report_design_text, {}, false, true, 'search_result') if report.report_design_field_value('use_text_message') @@ -290,36 +254,5 @@ def ask_for_feedback_when_all_search_results_are_received(app_id, language, work self.delay_for(1.second, { queue: 'smooch_priority' }).ask_for_feedback_when_all_search_results_are_received(app_id, language, workflow, uid, platform, attempts + 1) if attempts < max # Try for 20 seconds end end - - def timeout_if_no_feedback_is_given_to_search_result(app_id, uid, search_id, pmid) - key = "smooch:user_search_bundle:#{uid}:#{search_id}:#{pmid}" - if Rails.cache.read(key).nil? # User gave no feedback for the search result - bundle = Rails.cache.read("smooch:user_search_bundle:#{uid}:#{search_id}").to_a - self.delay_for(1.seconds, { queue: 'smooch', retry: false }).bundle_messages(uid, nil, app_id, 'timeout_search_requests', [ProjectMedia.find(pmid)], true, bundle) - else - Rails.cache.delete(key) # User gave feedback to search result - end - end - - def clicked_on_search_result_button?(message) - begin - JSON.parse(message['payload'])['keyword'] == 'search_result_is_relevant' - rescue - false - end - end - - def search_result_button_click_callback(message, uid, app_id, workflow, language) - payload = JSON.parse(message['payload']) - result = ProjectMedia.find(payload['project_media_id']) - bundle = Rails.cache.read("smooch:user_search_bundle:#{uid}:#{payload['search_id']}").to_a - unless bundle.empty? - # Give some interval just to avoid race conditions if the user gives feedback too fast - interval = payload['index'].to_i * 10 - Rails.cache.write("smooch:user_search_bundle:#{uid}:#{payload['search_id']}:#{result.id}", Time.now.to_i) # Store that the user has given feedback to this search result - self.delay_for((interval + 1).seconds, { queue: 'smooch', retry: false }).bundle_messages(uid, message['_id'], app_id, 'relevant_search_result_requests', [result], true, bundle) - self.send_final_message_to_user(uid, self.get_custom_string('search_result_is_relevant', language), workflow, language) - end - end end end diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index 4f809069f8..2ad590b041 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -891,53 +891,4 @@ def send_message_outside_24_hours_window(template, pm = nil) end end end - - test 'should generate a unique ID for search results' do - assert_not_equal Bot::Smooch.generate_search_id, Bot::Smooch.generate_search_id - end - - test 'should give individual feedback to search result on WhatsApp' do - search_id = random_string - Bot::Smooch.stubs(:generate_search_id).returns(search_id) - pm1 = create_project_media(team: @team) - pm2 = create_project_media(team: @team) - publish_report(pm1, {}, nil, { language: 'en', use_visual_card: true }) - publish_report(pm2, {}, nil, { language: 'en', use_visual_card: false }) - CheckSearch.any_instance.stubs(:medias).returns([pm1, pm2]) - Sidekiq::Testing.inline! do - send_message 'hello', '1', '1', random_string - end - assert_state 'ask_if_ready' - Rails.cache.write("smooch:user_search_bundle:#{@uid}:#{search_id}:#{pm1.id}", Time.now.to_i) # User gave feedback to one result - Sidekiq::Testing.fake! do - send_message_to_smooch_bot('1', @uid, { 'source' => { 'type' => 'whatsapp' } }) - end - assert_difference 'DynamicAnnotation::Field.where(field_name: "smooch_request_type", value: "timeout_search_requests").count' do - Sidekiq::Worker.drain_all - end - end - - test 'should click on button to evaluate search result on WhatsApp' do - search_id = random_string - pm = create_project_media(team: @team) - bundle = [ - { - '_id': random_string, - authorId: @uid, - type: 'text', - text: random_string - }.to_json - ] - Rails.cache.write("smooch:user_search_bundle:#{@uid}:#{search_id}", bundle) - Sidekiq::Testing.inline! do - assert_difference 'DynamicAnnotation::Field.where(field_name: "smooch_request_type", value: "relevant_search_result_requests").count' do - payload = { - 'keyword' => 'search_result_is_relevant', - 'project_media_id' => pm.id, - 'search_id' => search_id - } - send_message_to_smooch_bot('Thumbs up', @uid, { 'source' => { 'type' => 'whatsapp' }, 'payload' => payload.to_json }) - end - end - end end From 253851fe9e4f405582f01f9ef8503b7574adf0ef Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 31 Oct 2023 01:13:45 -0300 Subject: [PATCH 23/54] Updating copy for NLU disambiguation. Reference: CV2-3709. --- app/lib/smooch_nlu_menus.rb | 2 +- config/tipline_strings.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/lib/smooch_nlu_menus.rb b/app/lib/smooch_nlu_menus.rb index c27408e8dd..b1c14e26d0 100644 --- a/app/lib/smooch_nlu_menus.rb +++ b/app/lib/smooch_nlu_menus.rb @@ -96,7 +96,7 @@ def process_menu_options(uid, options, message, language, workflow, app_id) value: { keyword: option['smooch_menu_option_keyword'] }.to_json, label: option['smooch_menu_option_label'] } - end.concat([{ value: { keyword: 'cancel_nlu' }.to_json, label: Bot::Smooch.get_string('main_state_button_label', language, 20) }]) + end.concat([{ value: { keyword: 'cancel_nlu' }.to_json, label: Bot::Smooch.get_string('nlu_cancel', language, 20) }]) Bot::Smooch.send_message_to_user_with_buttons(uid, Bot::Smooch.get_string('nlu_disambiguation', language), buttons) end end diff --git a/config/tipline_strings.yml b/config/tipline_strings.yml index 3845091311..7d5f0a63a8 100644 --- a/config/tipline_strings.yml +++ b/config/tipline_strings.yml @@ -262,6 +262,7 @@ en: unsubscribe_button_label: Unsubscribe unsubscribed: You are currently not subscribed to our newsletter. nlu_disambiguation: "Choose one of the options below:" + nlu_cancel: "Back to menu" fil: add_more_details_state_button_label: Dagdagan pa ask_if_ready_state_button_label: Kanselahin From ce4c8c1cd4129af6997dccf89864dc29a57a36bd Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 31 Oct 2023 02:10:02 -0300 Subject: [PATCH 24/54] Fixing flaky test --- app/graph/mutations/smooch_bot_mutations.rb | 4 +- .../controllers/graphql_controller_11_test.rb | 60 ------------------- test/controllers/graphql_controller_6_test.rb | 23 +++++++ ...mooch_add_slack_channel_url_worker_test.rb | 18 ++++++ 4 files changed, 42 insertions(+), 63 deletions(-) delete mode 100644 test/controllers/graphql_controller_11_test.rb create mode 100644 test/workers/smooch_add_slack_channel_url_worker_test.rb diff --git a/app/graph/mutations/smooch_bot_mutations.rb b/app/graph/mutations/smooch_bot_mutations.rb index c30f6da515..ae766eeab7 100644 --- a/app/graph/mutations/smooch_bot_mutations.rb +++ b/app/graph/mutations/smooch_bot_mutations.rb @@ -16,9 +16,7 @@ def resolve(id:, set_fields:) if annotation.nil? raise ActiveRecord::RecordNotFound else - unless annotation.ability.can?(:update, annotation) - raise "No permission to update #{annotation.class.name}" - end + raise "No permission to update #{annotation.class.name}" unless annotation.ability.can?(:update, annotation) SmoochAddSlackChannelUrlWorker.perform_in( 1.second, id, diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb deleted file mode 100644 index 890f7e3d2f..0000000000 --- a/test/controllers/graphql_controller_11_test.rb +++ /dev/null @@ -1,60 +0,0 @@ -require_relative '../test_helper' -require 'error_codes' -require 'sidekiq/testing' - -class GraphqlController11Test < ActionController::TestCase - def setup - @controller = Api::V1::GraphqlController.new - TestDynamicAnnotationTables.load! - end - - def teardown - end - - test "should set Smooch user Slack channel URL in background" do - Sidekiq::Worker.clear_all - u = create_user - t = create_team - create_team_user team: t, user: u, role: 'admin' - p = create_project team: t - author_id = random_string - set_fields = { smooch_user_data: { id: author_id }.to_json, smooch_user_app_id: 'fake', smooch_user_id: 'fake' }.to_json - d = create_dynamic_annotation annotated: p, annotation_type: 'smooch_user', set_fields: set_fields - authenticate_with_token - url = random_url - query = 'mutation { updateDynamicAnnotationSmoochUser(input: { clientMutationId: "1", id: "' + d.graphql_id + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url + '\"}" }) { project { dbid } } }' - - Sidekiq::Testing.fake! do - post :create, params: { query: query } - assert_response :success - end - Sidekiq::Worker.drain_all - assert_equal url, Dynamic.find(d.id).get_field_value('smooch_user_slack_channel_url') - - # Check that cache key exists - key = "SmoochUserSlackChannelUrl:Team:#{d.team_id}:#{author_id}" - assert_equal url, Rails.cache.read(key) - - # Test using a new mutation `smoochBotAddSlackChannelUrl` - url2 = random_url - query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "' + d.id.to_s + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }' - Sidekiq::Testing.fake! do - post :create, params: { query: query } - assert_response :success - end - assert Sidekiq::Worker.jobs.size > 0 - assert_equal url, d.reload.get_field_value('smooch_user_slack_channel_url') - - # Execute job and check that URL was set - Sidekiq::Worker.drain_all - assert_equal url2, d.get_field_value('smooch_user_slack_channel_url') - - # Check that cache key exists - assert_equal url2, Rails.cache.read(key) - - # Call mutation with non existing ID - query = 'mutation { smoochBotAddSlackChannelUrl(input: { clientMutationId: "1", id: "99999", set_fields: "{\"smooch_user_slack_channel_url\":\"' + url2 + '\"}" }) { annotation { dbid } } }' - post :create, params: { query: query } - assert_response :success - end -end diff --git a/test/controllers/graphql_controller_6_test.rb b/test/controllers/graphql_controller_6_test.rb index 8e91f89dd9..68ab5b2904 100644 --- a/test/controllers/graphql_controller_6_test.rb +++ b/test/controllers/graphql_controller_6_test.rb @@ -276,4 +276,27 @@ def teardown assert_equal 1, response['item_navigation_offset'] end + test "should set Smooch user Slack channel URL" do + u = create_user + t = create_team + p = create_project team: t + create_team_user team: t, user: u, role: 'admin' + set_fields = { smooch_user_data: { id: random_string }.to_json, smooch_user_app_id: 'fake', smooch_user_id: 'fake' }.to_json + d = create_dynamic_annotation annotated: p, annotation_type: 'smooch_user', set_fields: set_fields + authenticate_with_token + query = 'mutation { smoochBotAddSlackChannelUrl(input: { id: "' + d.id.to_s + '", set_fields: "{\"smooch_user_slack_channel_url\":\"' + random_url + '\"}" }) { annotation { dbid } } }' + post :create, params: { query: query } + assert_response :success + end + + test "should not set Smooch user Slack channel URL" do + u = create_user + t = create_team + p = create_project team: t + create_team_user team: t, user: u, role: 'admin' + authenticate_with_token + query = 'mutation { smoochBotAddSlackChannelUrl(input: { id: "0", set_fields: "{\"smooch_user_slack_channel_url\":\"' + random_url + '\"}" }) { annotation { dbid } } }' + post :create, params: { query: query } + assert_response :success + end end diff --git a/test/workers/smooch_add_slack_channel_url_worker_test.rb b/test/workers/smooch_add_slack_channel_url_worker_test.rb new file mode 100644 index 0000000000..3630259c0e --- /dev/null +++ b/test/workers/smooch_add_slack_channel_url_worker_test.rb @@ -0,0 +1,18 @@ +require_relative '../test_helper' + +class SmoochAddSlackChannelUrlWorkerTest < ActiveSupport::TestCase + def setup + end + + def teardown + end + + test 'should set fields for annotation if annotation exists' do + create_annotation_type_and_fields('Smooch User', { 'Slack Channel URL' => ['Text'] }) + a = create_dynamic_annotation annotation_type: 'smooch_user' + assert_nil a.reload.get_field_value('smooch_user_slack_channel_url') + url = random_url + SmoochAddSlackChannelUrlWorker.new.perform(a.id, { smooch_user_slack_channel_url: url }.to_json) + assert_equal url, a.reload.get_field_value('smooch_user_slack_channel_url') + end +end From 256f67db5d8956d5f7c26d93b905375d507feda2 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Tue, 31 Oct 2023 11:33:06 +0200 Subject: [PATCH 25/54] CV2-3611-3806: generate invitation link and handle query empty feed organization (#1714) * CV2-3611-3806: generate invitation link and handle query empty feed organization --- app/mailers/feed_invitation_mailer.rb | 5 ++++- app/models/feed.rb | 2 +- app/models/feed_invitation.rb | 2 +- app/views/feed_invitation_mailer/notify.html.erb | 4 ++-- lib/check_search.rb | 6 +++--- test/controllers/elastic_search_9_test.rb | 3 +++ test/mailers/feed_invitation_mailer_test.rb | 3 ++- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/app/mailers/feed_invitation_mailer.rb b/app/mailers/feed_invitation_mailer.rb index 82412342b4..7ef466a585 100644 --- a/app/mailers/feed_invitation_mailer.rb +++ b/app/mailers/feed_invitation_mailer.rb @@ -1,11 +1,14 @@ class FeedInvitationMailer < ApplicationMailer layout nil - def notify(record) + def notify(record_id, team_id) + record = FeedInvitation.find_by_id record_id + team = Team.find_by_id team_id @recipient = record.email @user = record.user @feed = record.feed @due_at = record.created_at + CheckConfig.get('feed_invitation_due_to', 30).to_i.days + @accept_feed_url = "#{CheckConfig.get('checkdesk_client')}/#{team.slug}/feed/#{record.id}/accept_invitation" subject = I18n.t("mails_notifications.feed_invitation.subject", user: @user.name, feed: @feed.name) Rails.logger.info "Sending a feed invitation e-mail to #{@recipient}" mail(to: @recipient, email_type: 'feed_invitation', subject: subject) diff --git a/app/models/feed.rb b/app/models/feed.rb index 588f9fdc94..e94130b66c 100755 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -37,7 +37,7 @@ def filters def get_team_filters(feed_team_ids = nil) filters = [] conditions = { shared: true } - conditions[:team_id] = feed_team_ids unless feed_team_ids.blank? + conditions[:team_id] = feed_team_ids if feed_team_ids.is_a?(Array) self.feed_teams.where(conditions).find_each do |ft| filters << ft.filters.to_h.reject{ |k, _v| PROHIBITED_FILTERS.include?(k.to_s) }.merge({ 'team_id' => ft.team_id }) end diff --git a/app/models/feed_invitation.rb b/app/models/feed_invitation.rb index 12eb3e8c79..4075b3f7ee 100644 --- a/app/models/feed_invitation.rb +++ b/app/models/feed_invitation.rb @@ -28,6 +28,6 @@ def set_user end def send_feed_invitation_mail - FeedInvitationMailer.delay.notify(self) + FeedInvitationMailer.delay.notify(self.id, Team.current&.id) end end diff --git a/app/views/feed_invitation_mailer/notify.html.erb b/app/views/feed_invitation_mailer/notify.html.erb index 83d4e30910..643c9d2c4b 100644 --- a/app/views/feed_invitation_mailer/notify.html.erb +++ b/app/views/feed_invitation_mailer/notify.html.erb @@ -83,8 +83,8 @@ <%= link_to(I18n.t('mails_notifications.feed_invitation.view_button'), - '#', - :style => "text-decoration: none !important;color: #fff !important;" + @accept_feed_url, + :style => "text-decoration: none !important;color: #fff !important;" ) %> diff --git a/lib/check_search.rb b/lib/check_search.rb index 822a477765..8c76012a8c 100644 --- a/lib/check_search.rb +++ b/lib/check_search.rb @@ -61,7 +61,7 @@ def initialize(options, file = nil, team_id = Team.current&.id) def team_condition(team_id = nil) if feed_query? - feed_teams = @options['feed_team_ids'].blank? ? @feed.team_ids : (@feed.team_ids & @options['feed_team_ids']) + feed_teams = @options['feed_team_ids'].is_a?(Array) ? (@feed.team_ids & @options['feed_team_ids']) : @feed.team_ids is_shared = FeedTeam.where(feed_id: @feed.id, team_id: Team.current&.id, shared: true).last is_shared ? feed_teams : [0] # Invalidate the query if the current team is not sharing content else @@ -233,7 +233,7 @@ def clusterized_feed_query? def get_pg_results_for_media custom_conditions = {} core_conditions = {} - core_conditions['team_id'] = @options['team_id'] unless @options['team_id'].blank? + core_conditions['team_id'] = @options['team_id'] if @options['team_id'].is_a?(Array) # Add custom conditions for array values { 'project_id' => 'projects', 'user_id' => 'users', 'source_id' => 'sources', 'read' => 'read', 'unmatched' => 'unmatched' @@ -289,7 +289,7 @@ def medias_query(include_related_items = self.should_include_related_items?) core_conditions = [] custom_conditions = [] core_conditions << { terms: { get_search_field => @options['project_media_ids'] } } unless @options['project_media_ids'].blank? - core_conditions << { terms: { team_id: [@options['team_id']].flatten } } unless @options['team_id'].blank? + core_conditions << { terms: { team_id: [@options['team_id']].flatten } } if @options['team_id'].is_a?(Array) core_conditions << { terms: { archived: @options['archived'] } } custom_conditions << { terms: { read: @options['read'].map(&:to_i) } } if @options.has_key?('read') custom_conditions << { terms: { cluster_teams: @options['cluster_teams'] } } if @options.has_key?('cluster_teams') diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb index b34af3239a..3c9c87f229 100644 --- a/test/controllers/elastic_search_9_test.rb +++ b/test/controllers/elastic_search_9_test.rb @@ -286,6 +286,9 @@ def setup query[:feed_team_ids] = [t2.id] result = CheckSearch.new(query.to_json) assert_equal [pm2.id], result.medias.map(&:id) + query[:feed_team_ids] = [] + result = CheckSearch.new(query.to_json) + assert_empty result.medias.map(&:id) end end diff --git a/test/mailers/feed_invitation_mailer_test.rb b/test/mailers/feed_invitation_mailer_test.rb index 25c957fad3..565d8235f8 100644 --- a/test/mailers/feed_invitation_mailer_test.rb +++ b/test/mailers/feed_invitation_mailer_test.rb @@ -3,7 +3,8 @@ class FeedInvitationMailerTest < ActionMailer::TestCase test "should notify about feed invitation" do fi = create_feed_invitation - email = FeedInvitationMailer.notify(fi) + t = create_team + email = FeedInvitationMailer.notify(fi.id, t.id) assert_emails 1 do email.deliver_now end From 20242b4de4000b6137dbfa1ab90abcc2198a1c34 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:00:29 -0300 Subject: [PATCH 26/54] Fixing feed query Make sure that for the feed creator the filters are also set along with the other feed participantes, not over. For example, imagine that A, B, C and D are the filters for each team participating in a feed, where A are the filters for the feed creator. Previously, the query was: A and (B or C or D). But it should be: (A or B or C or D). --- app/models/feed.rb | 8 +++++--- test/controllers/graphql_controller_6_test.rb | 5 +---- test/models/bot/smooch_5_test.rb | 5 ++--- test/models/feed_test.rb | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/app/models/feed.rb b/app/models/feed.rb index e94130b66c..f1ba5245fe 100755 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -24,13 +24,13 @@ class Feed < ApplicationRecord # Filters for the whole feed: applies to all data from all teams def get_feed_filters - filters = self.filters.to_h.reject{ |k, _v| PROHIBITED_FILTERS.include?(k.to_s) } + filters = {} filters.merge!({ 'report_status' => ['published'] }) if self.published filters end def filters - self.saved_search&.filters.to_h + {} end # Filters defined by each team @@ -39,7 +39,9 @@ def get_team_filters(feed_team_ids = nil) conditions = { shared: true } conditions[:team_id] = feed_team_ids if feed_team_ids.is_a?(Array) self.feed_teams.where(conditions).find_each do |ft| - filters << ft.filters.to_h.reject{ |k, _v| PROHIBITED_FILTERS.include?(k.to_s) }.merge({ 'team_id' => ft.team_id }) + filter = ft.filters + filter = self.saved_search&.filters.to_h if self.team_id == ft.team_id + filters << filter.to_h.reject{ |k, _v| PROHIBITED_FILTERS.include?(k.to_s) }.merge({ 'team_id' => ft.team_id }) end filters end diff --git a/test/controllers/graphql_controller_6_test.rb b/test/controllers/graphql_controller_6_test.rb index 68ab5b2904..da89a1a89e 100644 --- a/test/controllers/graphql_controller_6_test.rb +++ b/test/controllers/graphql_controller_6_test.rb @@ -176,18 +176,15 @@ def teardown u = create_user create_team_user(team: t1, user: u, role: 'editor') authenticate_with_user(u) - f_ss = create_saved_search team_id: t1.id, filters: { keyword: 'banana' } + f_ss = create_saved_search team_id: t1.id, filters: { keyword: 'apple' } f = create_feed team_id: t1.id f.teams = [t1, t2] f.saved_search = f_ss f.save! - # Team 1 content to be shared - ft1_ss = create_saved_search team_id: t1.id, filters: { keyword: 'apple' } ft1 = FeedTeam.where(feed: f, team: t1).last ft1.shared = false - ft1.saved_search = ft1_ss ft1.save! pm1a = create_project_media quote: 'I like apple and banana', team: t1 pm1b = create_project_media quote: 'I like orange and banana', team: t1 diff --git a/test/models/bot/smooch_5_test.rb b/test/models/bot/smooch_5_test.rb index 96ade17ebc..21239339ec 100644 --- a/test/models/bot/smooch_5_test.rb +++ b/test/models/bot/smooch_5_test.rb @@ -48,9 +48,8 @@ def teardown FeedTeam.update_all(shared: true) f1.teams << t3 ft_ss = create_saved_search team_id: t1.id, filters: { keyword: 'Bar' } - ft = FeedTeam.where(feed: f1, team: t1).last - ft.saved_search = ft_ss - ft.save! + f1.saved_search = ft_ss + f1.save! u = create_bot_user [t1, t2, t3, t4].each { |t| TeamUser.create!(user: u, team: t, role: 'editor') } alegre_results = {} diff --git a/test/models/feed_test.rb b/test/models/feed_test.rb index 311677d51f..13623d0abc 100755 --- a/test/models/feed_test.rb +++ b/test/models/feed_test.rb @@ -85,7 +85,7 @@ def setup ss = create_saved_search team: t, filters: { foo: 'bar' } Team.stubs(:current).returns(t) f = create_feed saved_search: ss - assert_equal 'bar', f.reload.filters['foo'] + assert_equal({}, f.reload.filters) Team.unstub(:current) end From 912087979742e529aa7f5e0a3ad3f54d8370b439 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Tue, 31 Oct 2023 21:31:56 +0200 Subject: [PATCH 27/54] CV2-3798: fix tests (#1718) --- app/models/feed_invitation.rb | 4 ++-- test/mailers/feed_invitation_mailer_test.rb | 2 ++ test/models/feed_invitation_test.rb | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/models/feed_invitation.rb b/app/models/feed_invitation.rb index 4075b3f7ee..d2aa1b06f8 100644 --- a/app/models/feed_invitation.rb +++ b/app/models/feed_invitation.rb @@ -8,7 +8,7 @@ class FeedInvitation < ApplicationRecord validates_presence_of :email, :feed, :user validates :email, format: { with: URI::MailTo::EMAIL_REGEXP } - after_create :send_feed_invitation_mail + after_create :send_feed_invitation_mail, if: proc { |_x| Team.current.present? } def accept!(team_id) feed_team = FeedTeam.new(feed_id: self.feed_id, team_id: team_id, shared: true) @@ -28,6 +28,6 @@ def set_user end def send_feed_invitation_mail - FeedInvitationMailer.delay.notify(self.id, Team.current&.id) + FeedInvitationMailer.delay.notify(self.id, Team.current.id) end end diff --git a/test/mailers/feed_invitation_mailer_test.rb b/test/mailers/feed_invitation_mailer_test.rb index 565d8235f8..acaa9b05d9 100644 --- a/test/mailers/feed_invitation_mailer_test.rb +++ b/test/mailers/feed_invitation_mailer_test.rb @@ -4,10 +4,12 @@ class FeedInvitationMailerTest < ActionMailer::TestCase test "should notify about feed invitation" do fi = create_feed_invitation t = create_team + Team.stubs(:current).returns(t) email = FeedInvitationMailer.notify(fi.id, t.id) assert_emails 1 do email.deliver_now end assert_equal [fi.email], email.to + Team.unstub(:current) end end diff --git a/test/models/feed_invitation_test.rb b/test/models/feed_invitation_test.rb index a6c03b2329..d3020579f9 100644 --- a/test/models/feed_invitation_test.rb +++ b/test/models/feed_invitation_test.rb @@ -67,10 +67,13 @@ def teardown test "should send email after create feed invitation" do u = create_user f = create_feed + t = create_team + Team.stubs(:current).returns(t) Sidekiq::Extensions::DelayedMailer.clear Sidekiq::Testing.fake! do FeedInvitation.create!({ email: random_email, feed: f, user: u, state: :invited }) assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size end + Team.unstub(:current) end end From 5e601a424b9df2453c54f959a025a38020b6b10b Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Thu, 2 Nov 2023 12:24:07 +0200 Subject: [PATCH 28/54] CV2-3694: Audit 'Updated' column (#1715) * CV2-3694: remove extra action that update updated_at column * CV2-3694: restrict update_at to current user * CV2-3694: fix tests --- app/models/annotations/dynamic.rb | 2 +- app/models/annotations/task.rb | 2 +- app/models/assignment.rb | 5 +- app/models/concerns/annotation_base.rb | 4 +- .../dynamic_annotation_field_concern.rb | 28 +++-- test/controllers/elastic_search_2_test.rb | 21 ++-- test/controllers/elastic_search_4_test.rb | 119 +++++++++--------- test/models/request_test.rb | 3 + 8 files changed, 98 insertions(+), 86 deletions(-) diff --git a/app/models/annotations/dynamic.rb b/app/models/annotations/dynamic.rb index 956e7615db..27e2343418 100644 --- a/app/models/annotations/dynamic.rb +++ b/app/models/annotations/dynamic.rb @@ -139,7 +139,7 @@ def handle_elasticsearch_response(op) op = self.annotation_type =~ /choice/ ? 'update' : op keys = %w(id team_task_id value field_type fieldset date_value numeric_value) self.add_update_nested_obj({ op: op, pm_id: pm.id, nested_key: 'task_responses', keys: keys }) - self.update_recent_activity(pm) + self.update_recent_activity(pm) if User.current.present? end end end diff --git a/app/models/annotations/task.rb b/app/models/annotations/task.rb index 8af414393a..95bee5e061 100644 --- a/app/models/annotations/task.rb +++ b/app/models/annotations/task.rb @@ -213,7 +213,7 @@ def add_update_elasticsearch_task(op = 'create') pm = self.project_media data = { 'team_task_id' => self.team_task_id, 'fieldset' => self.fieldset } self.add_update_nested_obj({ op: op, pm_id: pm.id, nested_key: 'task_responses', keys: data.keys, data: data }) - self.update_recent_activity(pm) + self.update_recent_activity(pm) if User.current.present? end end diff --git a/app/models/assignment.rb b/app/models/assignment.rb index af5d8d91e3..d0b30fed64 100644 --- a/app/models/assignment.rb +++ b/app/models/assignment.rb @@ -133,12 +133,9 @@ def apply_rules_and_actions def update_elasticsearch_assignment if ['Annotation', 'Dynamic'].include?(self.assigned_type) && self.assigned.annotation_type == 'verification_status' pm = self.assigned.annotated - # update updated_at for ProjectMedia for recent_activity sort - updated_at = Time.now - pm.update_columns(updated_at: updated_at) # Update ES uids = Assignment.where(assigned_type: self.assigned_type, assigned_id: self.assigned_id).map(&:user_id) - data = { 'assigned_user_ids' => uids, 'updated_at' => updated_at.utc } + data = { 'assigned_user_ids' => uids } pm.update_elasticsearch_doc(data.keys, data, pm.id) end end diff --git a/app/models/concerns/annotation_base.rb b/app/models/concerns/annotation_base.rb index e1295fd01d..be19354ea4 100644 --- a/app/models/concerns/annotation_base.rb +++ b/app/models/concerns/annotation_base.rb @@ -111,8 +111,8 @@ def touch_annotated ApplicationRecord.connection_pool.with_connection do annotated.save!(validate: false) end - elsif annotated.is_a?(ProjectMedia) - if ['report_design', 'tag', 'archiver', 'language'].include?(self.annotation_type) + elsif annotated.is_a?(ProjectMedia) && User.current.present? + if ['report_design', 'tag', 'archiver'].include?(self.annotation_type) self.update_recent_activity(annotated) end end diff --git a/app/models/workflow/concerns/dynamic_annotation_field_concern.rb b/app/models/workflow/concerns/dynamic_annotation_field_concern.rb index b4c7784b60..6ce6a592e8 100644 --- a/app/models/workflow/concerns/dynamic_annotation_field_concern.rb +++ b/app/models/workflow/concerns/dynamic_annotation_field_concern.rb @@ -19,13 +19,13 @@ def status def index_on_es_background obj = self&.annotation&.annotated if !obj.nil? && obj.class.name == 'ProjectMedia' - updated_at = Time.now - # Update PG - obj.update_columns(updated_at: updated_at) - data = { - self.annotation_type => { method: 'value', klass: self.class.name, id: self.id }, - 'updated_at' => updated_at.utc - } + data = { self.annotation_type => { method: 'value', klass: self.class.name, id: self.id } } + if User.current.present? + updated_at = Time.now + # Update PG + obj.update_columns(updated_at: updated_at) + data['updated_at'] = updated_at.utc + end self.update_elasticsearch_doc(data.keys, data, obj.id) end end @@ -34,13 +34,17 @@ def index_on_es_foreground return if self.disable_es_callbacks || RequestStore.store[:disable_es_callbacks] obj = self&.annotation&.annotated if !obj.nil? && obj.class.name == 'ProjectMedia' - updated_at = Time.now - obj.update_columns(updated_at: updated_at) + data = { self.annotation_type => self.value } + if User.current.present? + updated_at = Time.now + obj.update_columns(updated_at: updated_at) + data['updated_at'] = updated_at.utc + end options = { - keys: [self.annotation_type, 'updated_at'], - data: { self.annotation_type => self.value, 'updated_at' => updated_at.utc }, + keys: data.keys, + data: data, pm_id: obj.id, - doc_id: Base64.encode64("#{obj.class.name}/#{obj.id}") + doc_id: Base64.encode64("#{obj.class.name}/#{obj.id}") } self.update_elasticsearch_doc_bg(options) end diff --git a/test/controllers/elastic_search_2_test.rb b/test/controllers/elastic_search_2_test.rb index 1f6919a084..bdc5bf27be 100644 --- a/test/controllers/elastic_search_2_test.rb +++ b/test/controllers/elastic_search_2_test.rb @@ -92,16 +92,19 @@ def setup test "should add or destroy es for annotations in background" do Sidekiq::Testing.fake! t = create_team - p = create_project team: t - pm = create_project_media project: p, disable_es_callbacks: false - # add tag - ElasticSearchWorker.clear - t = create_tag annotated: pm, disable_es_callbacks: false - assert_equal 2, ElasticSearchWorker.jobs.size - # destroy tag - ElasticSearchWorker.clear - t.destroy + u = create_user + create_team_user team: t, user: u, role: 'admin' + with_current_user_and_team(u, t) do + pm = create_project_media team: t, disable_es_callbacks: false + # add tag + ElasticSearchWorker.clear + t = create_tag annotated: pm, disable_es_callbacks: false + assert_equal 2, ElasticSearchWorker.jobs.size + # destroy tag + ElasticSearchWorker.clear + t.destroy assert_equal 1, ElasticSearchWorker.jobs.size + end end test "should update status in background" do diff --git a/test/controllers/elastic_search_4_test.rb b/test/controllers/elastic_search_4_test.rb index 0c2c68e267..f23d2488c7 100644 --- a/test/controllers/elastic_search_4_test.rb +++ b/test/controllers/elastic_search_4_test.rb @@ -56,47 +56,50 @@ def setup test "should sort results by recent activities and recent added" do t = create_team - p = create_project team: t - quote = 'search_sort' - m1 = create_claim_media quote: 'search_sort' - m2 = create_claim_media quote: 'search_sort' - m3 = create_claim_media quote: 'search_sort' - pm1 = create_project_media project: p, media: m1, disable_es_callbacks: false - pm2 = create_project_media project: p, media: m2, disable_es_callbacks: false - pm3 = create_project_media project: p, media: m3, disable_es_callbacks: false - create_tag tag: 'search_sort', annotated: pm1, disable_es_callbacks: false - sleep 5 - # sort with keywords - Team.current = t - result = CheckSearch.new({keyword: 'search_sort', projects: [p.id]}.to_json) - assert_equal [pm3.id, pm2.id, pm1.id], result.medias.map(&:id) - result = CheckSearch.new({keyword: 'search_sort', projects: [p.id], sort: 'recent_activity'}.to_json) - assert_equal [pm1.id, pm3.id, pm2.id], result.medias.map(&:id) - # sort with keywords and tags - create_tag tag: 'sorts', annotated: pm3, disable_es_callbacks: false - create_tag tag: 'sorts', annotated: pm2, disable_es_callbacks: false - sleep 5 - result = CheckSearch.new({tags: ["sorts"], projects: [p.id], sort: 'recent_activity'}.to_json) - assert_equal [pm2.id, pm3.id], result.medias.map(&:id).sort - result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], projects: [p.id], sort: 'recent_activity'}.to_json) - assert_equal [pm2.id, pm3.id], result.medias.map(&:id) - create_status status: 'verified', annotated: pm3, disable_es_callbacks: false - create_status status: 'verified', annotated: pm2, disable_es_callbacks: false - create_status status: 'verified', annotated: pm1, disable_es_callbacks: false - create_status status: 'false', annotated: pm1, disable_es_callbacks: false - sleep 5 - # sort with keywords, tags and status - result = CheckSearch.new({verification_status: ["verified"], projects: [p.id], sort: 'recent_activity'}.to_json) - assert_equal [pm2.id, pm3.id], result.medias.map(&:id) - result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], verification_status: ["verified"], projects: [p.id], sort: 'recent_activity'}.to_json) - assert_equal [pm2.id, pm3.id], result.medias.map(&:id) - result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], verification_status: ["verified"], projects: [p.id]}.to_json) - assert_equal [pm3.id, pm2.id], result.medias.map(&:id) - # sort asc and desc by created_date - result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], projects: [p.id], sort: 'recent_added'}.to_json) - assert_equal [pm3.id, pm2.id], result.medias.map(&:id) - result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], projects: [p.id], sort: 'recent_added', sort_type: 'asc'}.to_json) - assert_equal [pm2.id, pm3.id], result.medias.map(&:id) + u = create_user + create_team_user team: t, user: u, role: 'admin' + with_current_user_and_team(u, t) do + p = create_project team: t + quote = 'search_sort' + m1 = create_claim_media quote: 'search_sort' + m2 = create_claim_media quote: 'search_sort' + m3 = create_claim_media quote: 'search_sort' + pm1 = create_project_media project: p, media: m1, disable_es_callbacks: false + pm2 = create_project_media project: p, media: m2, disable_es_callbacks: false + pm3 = create_project_media project: p, media: m3, disable_es_callbacks: false + create_tag tag: 'search_sort', annotated: pm1, disable_es_callbacks: false + sleep 2 + # sort with keywords + result = CheckSearch.new({keyword: 'search_sort', projects: [p.id]}.to_json) + assert_equal [pm3.id, pm2.id, pm1.id], result.medias.map(&:id) + result = CheckSearch.new({keyword: 'search_sort', projects: [p.id], sort: 'recent_activity'}.to_json) + assert_equal [pm1.id, pm3.id, pm2.id], result.medias.map(&:id) + # sort with keywords and tags + create_tag tag: 'sorts', annotated: pm3, disable_es_callbacks: false + create_tag tag: 'sorts', annotated: pm2, disable_es_callbacks: false + sleep 2 + result = CheckSearch.new({tags: ["sorts"], projects: [p.id], sort: 'recent_activity'}.to_json) + assert_equal [pm2.id, pm3.id], result.medias.map(&:id).sort + result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], projects: [p.id], sort: 'recent_activity'}.to_json) + assert_equal [pm2.id, pm3.id], result.medias.map(&:id) + create_status status: 'verified', annotated: pm3, disable_es_callbacks: false + create_status status: 'verified', annotated: pm2, disable_es_callbacks: false + create_status status: 'verified', annotated: pm1, disable_es_callbacks: false + create_status status: 'false', annotated: pm1, disable_es_callbacks: false + sleep 2 + # sort with keywords, tags and status + result = CheckSearch.new({verification_status: ["verified"], projects: [p.id], sort: 'recent_activity'}.to_json) + assert_equal [pm2.id, pm3.id], result.medias.map(&:id) + result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], verification_status: ["verified"], projects: [p.id], sort: 'recent_activity'}.to_json) + assert_equal [pm2.id, pm3.id], result.medias.map(&:id) + result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], verification_status: ["verified"], projects: [p.id]}.to_json) + assert_equal [pm3.id, pm2.id], result.medias.map(&:id) + # sort asc and desc by created_date + result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], projects: [p.id], sort: 'recent_added'}.to_json) + assert_equal [pm3.id, pm2.id], result.medias.map(&:id) + result = CheckSearch.new({keyword: 'search_sort', tags: ["sorts"], projects: [p.id], sort: 'recent_added', sort_type: 'asc'}.to_json) + assert_equal [pm2.id, pm3.id], result.medias.map(&:id) + end end test "should search annotations for multiple projects" do @@ -161,22 +164,24 @@ def setup test "should include tag and status in recent activity sort" do RequestStore.store[:skip_cached_field_update] = false t = create_team - p = create_project team: t - pm1 = create_project_media project: p, disable_es_callbacks: false - pm2 = create_project_media project: p, disable_es_callbacks: false - pm3 = create_project_media project: p, disable_es_callbacks: false - create_status annotated: pm1, status: 'in_progress', disable_es_callbacks: false - sleep 2 - Team.current = t - result = CheckSearch.new({projects: [p.id], sort: "recent_activity"}.to_json) - assert_equal [pm1.id, pm3.id, pm2.id], result.medias.map(&:id) - create_tag annotated: pm3, tag: 'in_progress', disable_es_callbacks: false - sleep 1 - result = CheckSearch.new({projects: [p.id], sort: "recent_activity"}.to_json) - assert_equal [pm3.id, pm1.id, pm2.id], result.medias.map(&:id) - # should sort by recent activity with project and status filters - result = CheckSearch.new({projects: [p.id], verification_status: ['in_progress'], sort: "recent_activity"}.to_json) - assert_equal 1, result.project_medias.count + u = create_user + create_team_user team: t, user: u, role: 'admin' + with_current_user_and_team(u, t) do + pm1 = create_project_media team: t, disable_es_callbacks: false + pm2 = create_project_media team: t, disable_es_callbacks: false + pm3 = create_project_media team: t, disable_es_callbacks: false + create_status annotated: pm1, status: 'in_progress', disable_es_callbacks: false + sleep 2 + result = CheckSearch.new({sort: "recent_activity"}.to_json) + assert_equal [pm1.id, pm3.id, pm2.id], result.medias.map(&:id) + create_tag annotated: pm3, tag: 'in_progress', disable_es_callbacks: false + sleep 1 + result = CheckSearch.new({sort: "recent_activity"}.to_json) + assert_equal [pm3.id, pm1.id, pm2.id], result.medias.map(&:id) + # should sort by recent activity with project and status filters + result = CheckSearch.new({verification_status: ['in_progress'], sort: "recent_activity"}.to_json) + assert_equal 1, result.project_medias.count + end end test "should always hit ElasticSearch" do diff --git a/test/models/request_test.rb b/test/models/request_test.rb index 29d4da4edf..9b0a079352 100644 --- a/test/models/request_test.rb +++ b/test/models/request_test.rb @@ -247,6 +247,7 @@ def setup test "should cache team names that fact-checked a request" do Bot::Alegre.stubs(:request_api).returns({}) RequestStore.store[:skip_cached_field_update] = false + u = create_user is_admin: true f = create_feed t1 = create_team t2 = create_team name: 'Foo' @@ -259,6 +260,7 @@ def setup FeedTeam.update_all(shared: true) f.teams << t5 m = create_uploaded_image + User.stubs(:current).returns(u) r = create_request feed: f, media: m assert_equal '', r.reload.fact_checked_by assert_equal 0, r.reload.fact_checked_by_count @@ -278,6 +280,7 @@ def setup assert_equal 'Bar, Foo, Test', r.reload.fact_checked_by assert_equal 3, r.reload.fact_checked_by_count Bot::Alegre.unstub(:request_api) + User.unstub(:current) end test "should return if there is a subscription for a request" do From 33ca4df665a4749ecbbe6faac9a2ce14ec029765 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Sat, 4 Nov 2023 05:58:01 +0200 Subject: [PATCH 29/54] Testing refactor (#1719) * Fix testing time be reduce sleep time and remove duplicate creation for objects * Fix tests --- lib/sample_data.rb | 61 ++++++++++++++----- test/controllers/elastic_search_2_test.rb | 2 +- test/controllers/elastic_search_3_test.rb | 2 +- test/controllers/elastic_search_4_test.rb | 2 +- test/controllers/elastic_search_5_test.rb | 6 +- test/controllers/elastic_search_6_test.rb | 10 +-- test/controllers/elastic_search_test.rb | 4 +- .../controllers/graphql_controller_10_test.rb | 2 +- test/controllers/graphql_controller_2_test.rb | 4 +- test/controllers/graphql_controller_6_test.rb | 6 +- test/controllers/graphql_controller_8_test.rb | 8 +-- test/controllers/graphql_controller_test.rb | 4 +- test/models/bot/smooch_3_test.rb | 2 +- test/models/bot/smooch_5_test.rb | 4 +- test/models/bot/smooch_7_test.rb | 4 +- test/models/feed_team_test.rb | 2 +- test/models/team_2_test.rb | 7 ++- 17 files changed, 80 insertions(+), 50 deletions(-) diff --git a/lib/sample_data.rb b/lib/sample_data.rb index ce4e68b4e9..674dd501f8 100644 --- a/lib/sample_data.rb +++ b/lib/sample_data.rb @@ -42,7 +42,7 @@ def create_api_key(options = {}) def create_saved_search(options = {}) ss = SavedSearch.new - ss.team = create_team + ss.team = options[:team] || create_team ss.title = random_string ss.filters = {} options.each do |key, value| @@ -54,7 +54,7 @@ def create_saved_search(options = {}) def create_project_group(options = {}) pg = ProjectGroup.new - pg.team = create_team + pg.team = options[:team] || create_team pg.title = random_string options.each do |key, value| pg.send("#{key}=", value) if pg.respond_to?("#{key}=") @@ -210,7 +210,11 @@ def create_comment(options = {}) end def create_tag(options = {}) - options = { tag: random_string(50), annotator: create_user, disable_es_callbacks: true }.merge(options) + options = { + tag: random_string(50), + annotator: options[:annotator] || create_user, + disable_es_callbacks: true + }.merge(options) unless options.has_key?(:annotated) t = options[:team] || create_team p = create_project team: t @@ -227,7 +231,11 @@ def create_tag(options = {}) # Verification status def create_status(options = {}) create_verification_status_stuff if User.current.nil? - options = { status: 'in_progress', annotator: create_user, disable_es_callbacks: true }.merge(options) + options = { + status: 'in_progress', + annotator: options[:annotator] || create_user, + disable_es_callbacks: true + }.merge(options) unless options.has_key?(:annotated) t = options[:team] || create_team p = create_project team: t @@ -262,7 +270,10 @@ def create_flag(options = {}) 'racy': 4, 'spam': 5 } - options = { set_fields: { flags: flags }.to_json, annotator: create_user }.merge(options) + options = { + set_fields: { flags: flags }.to_json, + annotator: options[:annotator] || create_user + }.merge(options) unless options.has_key?(:annotated) t = options[:team] || create_team p = create_project team: t @@ -718,7 +729,7 @@ def create_team_bot(options = {}) name: random_string, set_description: random_string, set_request_url: random_url, - team_author_id: create_team.id, + team_author_id: options[:team_author_id] || create_team.id, set_events: [{ event: 'create_project_media', graphql: nil }] }.merge(options) @@ -752,7 +763,10 @@ def create_team_bot_installation(options = {}) def create_tag_text(options = {}) tt = TagText.new - options = { text: random_string, team_id: create_team.id }.merge(options) + options = { + text: random_string, + team_id: options[:team_id] || create_team.id + }.merge(options) options.each do |key, value| tt.send("#{key}=", value) if tt.respond_to?("#{key}=") end @@ -762,7 +776,12 @@ def create_tag_text(options = {}) def create_team_task(options = {}) tt = TeamTask.new - options = { label: random_string, team_id: create_team.id, task_type: 'free_text', fieldset: 'tasks' }.merge(options) + options = { + label: random_string, + team_id: options[:team_id] || create_team.id, + task_type: 'free_text', + fieldset: 'tasks' + }.merge(options) options.each do |key, value| tt.send("#{key}=", value) if tt.respond_to?("#{key}=") end @@ -795,7 +814,7 @@ def create_tipline_resource(options = {}) tr.content_type = 'rss' tr.language = 'en' tr.number_of_articles = random_number - tr.team = create_team + tr.team = options[:team] || create_team options.each do |key, value| tr.send("#{key}=", value) if tr.respond_to?("#{key}=") end @@ -806,7 +825,7 @@ def create_tipline_resource(options = {}) def create_tipline_message(options = {}) TiplineMessage.create!({ uid: random_string, - team_id: create_team.id, + team_id: options[:team_id] || create_team.id, language: 'en', platform: 'WhatsApp', direction: :incoming, @@ -820,7 +839,7 @@ def create_tipline_message(options = {}) def create_tipline_subscription(options = {}) TiplineSubscription.create!({ uid: random_string, - team_id: create_team.id, + team_id: options[:team_id] || create_team.id, language: 'en', platform: 'WhatsApp' }.merge(options)) @@ -859,13 +878,18 @@ def create_feed(options = {}) def create_feed_team(options = {}) FeedTeam.create!({ - feed: create_feed, - team: create_team + feed: options[:feed] || create_feed, + team: options[:team] || create_team }.merge(options)) end def create_request(options = {}) - Request.create!({ content: random_string, request_type: 'text', feed: create_feed, media: create_valid_media }.merge(options)) + Request.create!({ + content: random_string, + request_type: 'text', + feed: options[:feed] || create_feed, + media: options[:media] || create_valid_media + }.merge(options)) end def create_project_media_request(options = {}) @@ -909,7 +933,7 @@ def create_tipline_newsletter(options = {}) footer: 'Test', language: 'en', enabled: true, - team: create_team + team: options[:team] || create_team }.merge(options)) unless options[:header_file].blank? File.open(File.join(Rails.root, 'test', 'data', options[:header_file])) do |f| @@ -1102,6 +1126,11 @@ def create_blocked_tipline_user(options = {}) end def create_feed_invitation(options = {}) - FeedInvitation.create!({ email: random_email, feed: create_feed, user: create_user, state: :invited }.merge(options)) + FeedInvitation.create!({ + email: random_email, + feed: options[:feed] || create_feed, + user: options[:user] || create_user, + state: :invited + }.merge(options)) end end diff --git a/test/controllers/elastic_search_2_test.rb b/test/controllers/elastic_search_2_test.rb index bdc5bf27be..18fd40fabb 100644 --- a/test/controllers/elastic_search_2_test.rb +++ b/test/controllers/elastic_search_2_test.rb @@ -81,7 +81,7 @@ def setup pm2.refresh_media = true pm2.save! end - sleep 3 + sleep 2 ms2 = $repository.find(get_es_id(pm2)) assert_equal 'overridden_title', ms2['title'] ms = $repository.find(get_es_id(pm)) diff --git a/test/controllers/elastic_search_3_test.rb b/test/controllers/elastic_search_3_test.rb index 78fa2521b0..77a1db3a20 100644 --- a/test/controllers/elastic_search_3_test.rb +++ b/test/controllers/elastic_search_3_test.rb @@ -136,7 +136,7 @@ def setup pm2 = create_project_media project: p, media: m2, disable_es_callbacks: false create_tag tag: 'test', annotated: pm, disable_es_callbacks: false create_tag tag: 'Test', annotated: pm2, disable_es_callbacks: false - sleep 5 + sleep 2 # search by tags result = CheckSearch.new({tags: ['test']}.to_json, nil, t.id) assert_equal [pm.id, pm2.id].sort, result.medias.map(&:id).sort diff --git a/test/controllers/elastic_search_4_test.rb b/test/controllers/elastic_search_4_test.rb index f23d2488c7..bd8a9c0a35 100644 --- a/test/controllers/elastic_search_4_test.rb +++ b/test/controllers/elastic_search_4_test.rb @@ -130,7 +130,7 @@ def setup create_tag tag: 'iron maiden', annotated: pm, disable_es_callbacks: false pm2 = create_project_media project: p, disable_es_callbacks: false create_tag tag: 'iron', annotated: pm2, disable_es_callbacks: false - sleep 5 + sleep 2 Team.current = t result = CheckSearch.new({tags: ['iron maiden']}.to_json) assert_equal [pm.id], result.medias.map(&:id) diff --git a/test/controllers/elastic_search_5_test.rb b/test/controllers/elastic_search_5_test.rb index 6edfc48e63..ee541edd20 100644 --- a/test/controllers/elastic_search_5_test.rb +++ b/test/controllers/elastic_search_5_test.rb @@ -172,14 +172,14 @@ def setup m = create_valid_media Sidekiq::Testing.inline! do pm = create_project_media media: m, disable_es_callbacks: false - sleep 5 + sleep 2 ms = $repository.find(get_es_id(pm)) assert_equal 'undetermined', ms['verification_status'] # update status s = pm.get_annotations('verification_status').last.load s.status = 'verified' s.save! - sleep 5 + sleep 2 ms = $repository.find(get_es_id(pm)) assert_equal 'verified', ms['verification_status'] end @@ -222,7 +222,7 @@ def setup m = create_valid_media pm = create_project_media project: p, media: m, disable_es_callbacks: false assert_equal 'foo-bar', pm.last_verification_status - sleep 5 + sleep 2 result = CheckSearch.new({verification_status: ['foo']}.to_json, nil, t.id) assert_empty result.medias result = CheckSearch.new({verification_status: ['bar']}.to_json, nil, t.id) diff --git a/test/controllers/elastic_search_6_test.rb b/test/controllers/elastic_search_6_test.rb index 88ba6cf6db..a50c69c868 100644 --- a/test/controllers/elastic_search_6_test.rb +++ b/test/controllers/elastic_search_6_test.rb @@ -21,15 +21,15 @@ def setup Time.stubs(:now).returns(Time.new(2019, 05, 19, 13, 00)) pm1 = create_project_media project: p, quote: 'Test A', disable_es_callbacks: false - sleep 5 + sleep 2 Time.stubs(:now).returns(Time.new(2019, 05, 20, 13, 00)) pm2 = create_project_media project: p, quote: 'Test B', disable_es_callbacks: false - sleep 5 + sleep 2 Time.stubs(:now).returns(Time.new(2019, 05, 21, 13, 00)) pm3 = create_project_media project: p, quote: 'Test C', disable_es_callbacks: false - sleep 5 + sleep 2 Time.unstub(:now) @@ -77,11 +77,11 @@ def setup Time.stubs(:now).returns(Time.new(2019, 05, 19, 13, 00)) pm1 = create_project_media project: p, quote: 'claim a', disable_es_callbacks: false - sleep 5 + sleep 2 Time.stubs(:now).returns(Time.new(2019, 05, 20, 13, 00)) pm2 = create_project_media project: p, quote: 'claim b', disable_es_callbacks: false - sleep 5 + sleep 2 Time.unstub(:now) # Missing start_time, end_time and timezone diff --git a/test/controllers/elastic_search_test.rb b/test/controllers/elastic_search_test.rb index 441e054313..bff98d6e63 100644 --- a/test/controllers/elastic_search_test.rb +++ b/test/controllers/elastic_search_test.rb @@ -201,7 +201,7 @@ def setup create_tag tag: 'sports', annotated: pm2, disable_es_callbacks: false create_tag tag: 'newtag', annotated: pm2, disable_es_callbacks: false create_tag tag: 'news', annotated: pm, disable_es_callbacks: false - sleep 5 + sleep 2 Team.current = t # search by status result = CheckSearch.new({verification_status: ['false']}.to_json) @@ -235,7 +235,7 @@ def setup m = create_valid_media pm = create_project_media project: p, media: m, disable_es_callbacks: false create_tag tag: 'two Words', annotated: pm, disable_es_callbacks: false - sleep 5 + sleep 2 Team.current = t # search by tags result = CheckSearch.new({tags: ['two Words']}.to_json) diff --git a/test/controllers/graphql_controller_10_test.rb b/test/controllers/graphql_controller_10_test.rb index 35c4c331fb..ee659a048d 100644 --- a/test/controllers/graphql_controller_10_test.rb +++ b/test/controllers/graphql_controller_10_test.rb @@ -189,7 +189,7 @@ def setup pm2 = create_project_media disable_es_callbacks: false, project: p create_dynamic_annotation annotation_type: 'language', annotated: pm2, set_fields: { language: 'pt' }.to_json, disable_es_callbacks: false - sleep 5 + sleep 2 query = 'query CheckSearch { search(query: "{\"language\":[\"en\"]}") { id,medias(first:20){edges{node{dbid}}}}}'; post :create, params: { query: query, team: 'team' } assert_response :success diff --git a/test/controllers/graphql_controller_2_test.rb b/test/controllers/graphql_controller_2_test.rb index c88ac69ef8..929898e4fc 100644 --- a/test/controllers/graphql_controller_2_test.rb +++ b/test/controllers/graphql_controller_2_test.rb @@ -247,7 +247,7 @@ def setup m.disable_es_callbacks = false m.response = { annotation_type: 'task_response_free_text', set_fields: { response_free_text: 'C' }.to_json }.to_json m.save! - sleep 5 + sleep 2 m = pm1.get_annotations('task').map(&:load).select{ |t| t.team_task_id == tt2.id }.last m.disable_es_callbacks = false @@ -261,7 +261,7 @@ def setup m.disable_es_callbacks = false m.response = { annotation_type: 'task_response_free_text', set_fields: { response_free_text: 'A' }.to_json }.to_json m.save! - sleep 5 + sleep 2 authenticate_with_user(u) diff --git a/test/controllers/graphql_controller_6_test.rb b/test/controllers/graphql_controller_6_test.rb index da89a1a89e..4dd43fada3 100644 --- a/test/controllers/graphql_controller_6_test.rb +++ b/test/controllers/graphql_controller_6_test.rb @@ -176,7 +176,7 @@ def teardown u = create_user create_team_user(team: t1, user: u, role: 'editor') authenticate_with_user(u) - f_ss = create_saved_search team_id: t1.id, filters: { keyword: 'apple' } + f_ss = create_saved_search team: t1, filters: { keyword: 'apple' } f = create_feed team_id: t1.id f.teams = [t1, t2] f.saved_search = f_ss @@ -190,7 +190,7 @@ def teardown pm1b = create_project_media quote: 'I like orange and banana', team: t1 # Team 2 content to be shared - ft2_ss = create_saved_search team_id: t2.id, filters: { keyword: 'orange' } + ft2_ss = create_saved_search team: t2, filters: { keyword: 'orange' } ft2 = FeedTeam.where(feed: f, team: t2).last ft2.shared = true ft2.saved_search = ft2_ss @@ -199,7 +199,7 @@ def teardown pm2b = create_project_media quote: 'I love orange and banana', team: t2 # Wait for content to be indexed in ElasticSearch - sleep 5 + sleep 2 query = 'query CheckSearch { search(query: "{\"keyword\":\"and\",\"feed_id\":' + f.id.to_s + '}") { medias(first: 20) { edges { node { dbid } } } } }' # Can't see anything until content is shared diff --git a/test/controllers/graphql_controller_8_test.rb b/test/controllers/graphql_controller_8_test.rb index 3c4034fdb9..837207a20c 100644 --- a/test/controllers/graphql_controller_8_test.rb +++ b/test/controllers/graphql_controller_8_test.rb @@ -239,7 +239,7 @@ def setup t1 = create_team private: true create_team_user(user: u, team: t1, role: 'admin') f = create_feed team_id: t1.id - ss = create_saved_search team_id: t1.id + ss = create_saved_search team: t1 assert_not f.reload.published query = "mutation { updateFeed(input: { id: \"#{f.graphql_id}\", published: true, saved_search_id: #{ss.id} }) { feed { published, saved_search_id } } }" post :create, params: { query: query, team: t1.slug } @@ -256,7 +256,7 @@ def setup t1 = create_team private: true create_team_user(user: u, team: t1, role: 'admin') t2 = create_team private: true - ss = create_saved_search team_id: t1.id + ss = create_saved_search team: t1 f = create_feed f.teams << t1 f.teams << t2 @@ -622,7 +622,7 @@ def setup assert_equal 'id2', pm2.status end assert_not_equal [], t.reload.get_media_verification_statuses[:statuses].select{ |s| s[:id] == 'id2' } - sleep 5 + sleep 2 assert_equal [pm2.id], CheckSearch.new({ verification_status: ['id2'] }.to_json, nil, t.id).medias.map(&:id) assert_equal [], CheckSearch.new({ verification_status: ['id3'] }.to_json, nil, t.id).medias.map(&:id) assert_equal 'published', r1.reload.get_field_value('state') @@ -642,7 +642,7 @@ def setup assert_equal 'id1', pm1.status assert_equal 'id3', pm2.status end - sleep 5 + sleep 2 assert_equal [], CheckSearch.new({ verification_status: ['id2'] }.to_json, nil, t.id).medias.map(&:id) assert_equal [pm2.id], CheckSearch.new({ verification_status: ['id3'] }.to_json, nil, t.id).medias.map(&:id) assert_equal [], t.reload.get_media_verification_statuses[:statuses].select{ |s| s[:id] == 'id2' } diff --git a/test/controllers/graphql_controller_test.rb b/test/controllers/graphql_controller_test.rb index 512244115a..9d60020cf6 100644 --- a/test/controllers/graphql_controller_test.rb +++ b/test/controllers/graphql_controller_test.rb @@ -56,7 +56,7 @@ def setup assert_equal 'id2', pm2.status end assert_not_equal [], t.reload.get_media_verification_statuses[:statuses].select{ |s| s[:id] == 'id2' } - sleep 5 + sleep 2 assert_equal [pm2.id], CheckSearch.new({ verification_status: ['id2'] }.to_json, nil, t.id).medias.map(&:id) assert_equal [], CheckSearch.new({ verification_status: ['id3'] }.to_json, nil, t.id).medias.map(&:id) assert_equal 'published', r1.reload.get_field_value('state') @@ -76,7 +76,7 @@ def setup assert_equal 'id1', pm1.status assert_equal 'id3', pm2.status end - sleep 5 + sleep 2 assert_equal [], CheckSearch.new({ verification_status: ['id2'] }.to_json, nil, t.id).medias.map(&:id) assert_equal [pm2.id], CheckSearch.new({ verification_status: ['id3'] }.to_json, nil, t.id).medias.map(&:id) assert_equal [], t.reload.get_media_verification_statuses[:statuses].select{ |s| s[:id] == 'id2' } diff --git a/test/models/bot/smooch_3_test.rb b/test/models/bot/smooch_3_test.rb index 567e8f9729..33f6de1a98 100644 --- a/test/models/bot/smooch_3_test.rb +++ b/test/models/bot/smooch_3_test.rb @@ -403,7 +403,7 @@ def teardown pm1 = create_project_media quote: 'A segurança das urnas está provada.', team: t pm2 = create_project_media quote: 'Segurança pública é tema de debate.', team: t [pm1, pm2].each { |pm| publish_report(pm) } - sleep 3 # Wait for ElasticSearch to index content + sleep 2 # Wait for ElasticSearch to index content [ 'Segurança das urnas', diff --git a/test/models/bot/smooch_5_test.rb b/test/models/bot/smooch_5_test.rb index 21239339ec..4a59d5e7cd 100644 --- a/test/models/bot/smooch_5_test.rb +++ b/test/models/bot/smooch_5_test.rb @@ -42,12 +42,12 @@ def teardown t4 = create_team pm4a = create_project_media quote: 'Test 4', team: t4 # Should not be in search results (team is not part of feed) pm4b = create_project_media media: l, team: t4 # Should not be in search results by URL - ss = create_saved_search team_id: t1.id, filters: { show: ['claims', 'weblink'] } + ss = create_saved_search team: t1, filters: { show: ['claims', 'weblink'] } f1 = create_feed team_id: t1.id, published: true f1.teams << t2 FeedTeam.update_all(shared: true) f1.teams << t3 - ft_ss = create_saved_search team_id: t1.id, filters: { keyword: 'Bar' } + ft_ss = create_saved_search team: t1, filters: { keyword: 'Bar' } f1.saved_search = ft_ss f1.save! u = create_bot_user diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 7fb40d4fc0..33df4b88b6 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -364,7 +364,7 @@ def teardown t = create_team pm = create_project_media quote: '🤣 word', team: t publish_report(pm) - sleep 3 # Wait for ElasticSearch to index content + sleep 2 # Wait for ElasticSearch to index content [ '🤣', #Direct match @@ -394,7 +394,7 @@ def teardown pm2 = create_project_media quote: 'Foo Bar Test', team: t pm3 = create_project_media quote: 'Foo Bar Test Testing', team: t [pm1, pm2, pm3].each { |pm| publish_report(pm) } - sleep 3 # Wait for ElasticSearch to index content + sleep 2 # Wait for ElasticSearch to index content assert_equal [pm1.id, pm2.id, pm3.id], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Foo Bar', [t.id]).to_a.map(&:id) end diff --git a/test/models/feed_team_test.rb b/test/models/feed_team_test.rb index d84b7da054..9447674589 100644 --- a/test/models/feed_team_test.rb +++ b/test/models/feed_team_test.rb @@ -38,7 +38,7 @@ def setup test "should get filters" do t = create_team - ss = create_saved_search team_id: t.id, filters: { foo: 'bar'} + ss = create_saved_search team: t, filters: { foo: 'bar'} ft = create_feed_team team_id: t.id, saved_search_id: ss.id assert_equal 'bar', ft.reload.filters['foo'] end diff --git a/test/models/team_2_test.rb b/test/models/team_2_test.rb index 5307c2b55c..87ec2681d7 100644 --- a/test/models/team_2_test.rb +++ b/test/models/team_2_test.rb @@ -962,7 +962,8 @@ def setup test "should return team tasks" do t = create_team - create_team_task team_id: t.id + 1 + t2 = create_team + create_team_task team_id: t2.id assert t.auto_tasks().empty? tt = create_team_task team_id: t.id assert_equal [tt], t.auto_tasks() @@ -1120,13 +1121,13 @@ def setup s = pm1.last_status_obj s.status = 'in_progress' s.save! - sleep 5 + sleep 2 result = $repository.find(get_es_id(pm1)) assert_equal p1.id, result['project_id'] assert_equal 0, p0.reload.medias_count assert_equal 1, p1.reload.medias_count pm2 = create_project_media project: p0, disable_es_callbacks: false - sleep 5 + sleep 2 assert_equal p1.id, pm1.reload.project_id assert_equal p0.id, pm2.reload.project_id end From fbdb4d2983328862bfceb86ec7fa482240b48358 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Mon, 6 Nov 2023 12:29:03 +0200 Subject: [PATCH 30/54] CV2-3912: search claim description (#1721) --- app/models/concerns/smooch_search.rb | 2 +- test/models/bot/smooch_7_test.rb | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index 1abc860431..42eb220830 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -200,7 +200,7 @@ def should_restrict_by_language?(team_ids) end def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, feed_id = nil, language = nil) - search_fields = %w(title description fact_check_title fact_check_summary extracted_text url claim_description_content') + search_fields = %w(title description fact_check_title fact_check_summary extracted_text url claim_description_content) filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: 3 } filters.merge!({ fc_language: [language] }) if should_restrict_by_language?(team_ids) filters.merge!({ sort: 'score' }) if words.size > 1 # We still want to be able to return the latest fact-checks if a meaninful query is not passed diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 33df4b88b6..2626bef893 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -574,4 +574,20 @@ def teardown assert_not_nil smooch_data.smooch_report_correction_sent_at assert_not_nil smooch_data.smooch_request_type end + + test "should include claim_description_content in smooch search" do + WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + RequestStore.store[:skip_cached_field_update] = false + t = create_team + m = create_uploaded_image + pm = create_project_media team: t, media: m, disable_es_callbacks: false + query = "Claim content" + results = Bot::Smooch.search_by_keywords_for_similar_published_fact_checks(query.split(), nil, [t.id]) + assert_empty results + cd = create_claim_description project_media: pm, description: query + publish_report(pm) + assert_equal query, pm.claim_description_content + results = Bot::Smooch.search_by_keywords_for_similar_published_fact_checks(query.split(), nil, [t.id]) + assert_equal [pm.id], results.map(&:id) + end end From 558bcd3476daf309979c5b1502d2a73517603c49 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:34:59 -0300 Subject: [PATCH 31/54] =?UTF-8?q?3884=20=E2=80=93=20Improve=20seeds=20scri?= =?UTF-8?q?pt:=20create=20more=20Tipline=20requests=20(#1722)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * create claims with 1 or 15 requests Notes: - We want 15 requests so that "load more" appears - We also want variation, so for now that means 1 request or 15 or 0 - I wanted to add the requests to the claims I created in the beginning of the script, but because those claims' project medias don't have the channel, we don't get them in the inbox. I don't think I should add channel to all project medias, so for now I'm creating new claims and project medias * add begin/rescue when creating links with requests we don't want the whole script to fail if Pender is not running --- db/seeds.rb | 206 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 123 insertions(+), 83 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index f1702bfc51..cb6f2894cc 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -32,7 +32,8 @@ 'https://meedan.com/post/what-is-gendered-health-misinformation-and-why-is-it-an-equity-problem-worth', 'https://meedan.com/post/the-case-for-a-public-health-approach-to-moderate-health-misinformation', ], - quotes: ['Garlic can help you fight covid', 'Tea with garlic is a covid treatment', 'If you have covid you should eat garlic', 'Are you allergic to garlic?', 'Vampires can\'t eat garlic'] + quotes: ['Garlic can help you fight covid', 'Tea with garlic is a covid treatment', 'If you have covid you should eat garlic', 'Are you allergic to garlic?', 'Vampires can\'t eat garlic'], + tipline_claims: Array.new(9) { Faker::Lorem.paragraph(sentence_count: 10) } } def open_file(file) @@ -75,6 +76,109 @@ def create_claim_description(user, project, team) ClaimDescription.create!(description: Faker::Company.catch_phrase, context: Faker::Lorem.sentence, user: user, project_media: create_blank(project, team)) end +def create_tipline_project_media(project, team, media) + ProjectMedia.create!(project: project, team: team, media: media, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }) +end + +def create_tipline_user_and_data(project_media, team) + tipline_user_name = Faker::Name.first_name.downcase + tipline_user_surname = Faker::Name.last_name + tipline_text = Faker::Lorem.paragraph(sentence_count: 10) + phone = [ Faker::PhoneNumber.phone_number, Faker::PhoneNumber.cell_phone, Faker::PhoneNumber.cell_phone_in_e164, Faker::PhoneNumber.phone_number_with_country_code, Faker::PhoneNumber.cell_phone_with_country_code].sample + uid = random_string + + # Tipline user + smooch_user_data = { + 'id': uid, + 'raw': { + '_id': uid, + 'givenName': tipline_user_name, + 'surname': tipline_user_surname, + 'signedUpAt': Time.now.to_s, + 'properties': {}, + 'conversationStarted': true, + 'clients': [ + { + 'id': random_string, + 'status': 'active', + 'externalId': phone, + 'active': true, + 'lastSeen': Time.now.to_s, + 'platform': 'whatsapp', + 'integrationId': random_string, + 'displayName': phone, + 'raw': { + 'profile': { + 'name': tipline_user_name + }, + 'from': phone + } + } + ], + 'pendingClients': [] + }, + 'identifier': random_string, + 'app_name': random_string + } + + fields = { + smooch_user_id: uid, + smooch_user_app_id: random_string, + smooch_user_data: smooch_user_data.to_json + } + + Dynamic.create!(annotation_type: 'smooch_user', annotated: team, annotator: BotUser.smooch_user, set_fields: fields.to_json) + + # Tipline request + smooch_data = { + 'role': 'appUser', + 'source': { + 'type': 'whatsapp', + 'id': random_string, + 'integrationId': random_string, + 'originalMessageId': random_string, + 'originalMessageTimestamp': Time.now.to_i + }, + 'authorId': uid, + 'name': tipline_user_name, + '_id': random_string, + 'type': 'text', + 'received': Time.now.to_f, + 'text': tipline_text, + 'language': 'en', + 'mediaUrl': nil, + 'mediaSize': 0, + 'archived': 3, + 'app_id': random_string + } + + fields = { + smooch_request_type: 'default_requests', + smooch_data: smooch_data.to_json + } + + Dynamic.create!(annotation_type: 'smooch', annotated: project_media, annotator: BotUser.smooch_user, set_fields: fields.to_json) +end + +def create_tipline_requests(team, project, user, data_instances, model_string) + tipline_pm_arr = [] + model = Object.const_get(model_string) + data_instances[0..5].each do |data_instance| + case model_string + when 'Claim' + media = model.create!(user_id: user.id, quote: data_instance) + when 'Link' + media = model.create!(user_id: user.id, url: data_instance+"?timestamp=#{Time.now.to_f}") + else + media = model.create!(user_id: user.id, file: open_file(data_instance)) + end + project_media = create_tipline_project_media(project, team, media) + tipline_pm_arr.push(project_media) + end + tipline_pm_arr[0..2].each {|pm| create_tipline_user_and_data(pm, team)} + tipline_pm_arr[3..5].each {|pm| 15.times {create_tipline_user_and_data(pm, team)}} +end + puts "If you want to create a new user: press 1 then enter" puts "If you want to add more data to an existing user: press 2 then enter" print ">> " @@ -165,88 +269,24 @@ def create_claim_description(user, project, team) Relationship.create!(source_id: project_medias_for_audio[0].id, target_id: project_medias_for_audio[1].id, relationship_type: Relationship.confirmed_type) puts 'Making Tipline requests...' - 9.times do - claim_media = Claim.create!(user_id: user.id, quote: Faker::Lorem.paragraph(sentence_count: 10)) - project_media = ProjectMedia.create!(project: project, team: team, media: claim_media, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }) - - tipline_user_name = Faker::Name.first_name.downcase - tipline_user_surname = Faker::Name.last_name - tipline_text = Faker::Lorem.paragraph(sentence_count: 10) - phone = [ Faker::PhoneNumber.phone_number, Faker::PhoneNumber.cell_phone, Faker::PhoneNumber.cell_phone_in_e164, Faker::PhoneNumber.phone_number_with_country_code, Faker::PhoneNumber.cell_phone_with_country_code].sample - uid = random_string - - # Tipline user - smooch_user_data = { - 'id': uid, - 'raw': { - '_id': uid, - 'givenName': tipline_user_name, - 'surname': tipline_user_surname, - 'signedUpAt': Time.now.to_s, - 'properties': {}, - 'conversationStarted': true, - 'clients': [ - { - 'id': random_string, - 'status': 'active', - 'externalId': phone, - 'active': true, - 'lastSeen': Time.now.to_s, - 'platform': 'whatsapp', - 'integrationId': random_string, - 'displayName': phone, - 'raw': { - 'profile': { - 'name': tipline_user_name - }, - 'from': phone - } - } - ], - 'pendingClients': [] - }, - 'identifier': random_string, - 'app_name': random_string - } - - fields = { - smooch_user_id: uid, - smooch_user_app_id: random_string, - smooch_user_data: smooch_user_data.to_json - } - - Dynamic.create!(annotation_type: 'smooch_user', annotated: team, annotator: BotUser.smooch_user, set_fields: fields.to_json) - - # Tipline request - smooch_data = { - 'role': 'appUser', - 'source': { - 'type': 'whatsapp', - 'id': random_string, - 'integrationId': random_string, - 'originalMessageId': random_string, - 'originalMessageTimestamp': Time.now.to_i - }, - 'authorId': uid, - 'name': tipline_user_name, - '_id': random_string, - 'type': 'text', - 'received': Time.now.to_f, - 'text': tipline_text, - 'language': 'en', - 'mediaUrl': nil, - 'mediaSize': 0, - 'archived': 3, - 'app_id': random_string - } - - fields = { - smooch_request_type: 'default_requests', - smooch_data: smooch_data.to_json - } - - a = Dynamic.create!(annotation_type: 'smooch', annotated: project_media, annotator: BotUser.smooch_user, set_fields: fields.to_json) - end + puts 'Making Tipline requests: Claims...' + create_tipline_requests(team, project, user, data[:tipline_claims], 'Claim') + + puts 'Making Tipline requests: Links...' + begin + create_tipline_requests(team, project, user, data[:link_media_links], 'Link') + rescue + puts "Couldn't create Links. Other medias will still be created. \nIn order to create Links make sure Pender is running." + end + + puts 'Making Tipline requests: Audios...' + create_tipline_requests(team, project, user, data[:audios], 'UploadedAudio') + + puts 'Making Tipline requests: Images...' + create_tipline_requests(team, project, user, data[:images], 'UploadedImage') + + puts 'Making Tipline requests: Videos...' + create_tipline_requests(team, project, user, data[:videos], 'UploadedVideo') add_claim_descriptions_and_fact_checks(user) From 7770a061b22bc3f3a7399277c8ddf181c8983d31 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Thu, 9 Nov 2023 08:51:12 +0200 Subject: [PATCH 32/54] CV2-3940: fix default language for duplicated team (#1720) --- app/models/concerns/team_duplication.rb | 1 + test/models/team_test.rb | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/app/models/concerns/team_duplication.rb b/app/models/concerns/team_duplication.rb index 4af04a7b68..642700e497 100644 --- a/app/models/concerns/team_duplication.rb +++ b/app/models/concerns/team_duplication.rb @@ -45,6 +45,7 @@ def self.modify_settings(old_team, new_team) new_list_columns = old_team.get_list_columns.to_a.collect{|lc| lc.include?("task_value_") ? "task_value_#{team_task_map[lc.split("_").last.to_i]}" : lc} new_team.set_list_columns = new_list_columns unless new_list_columns.blank? new_team.set_languages = old_team.get_languages + new_team.set_language = old_team.get_language new_team end diff --git a/test/models/team_test.rb b/test/models/team_test.rb index 98655c9c88..fb85ad27ba 100644 --- a/test/models/team_test.rb +++ b/test/models/team_test.rb @@ -984,6 +984,16 @@ def setup assert_equal ['alegre'], tbi.map(&:user).map(&:login) end + test "should duplicate team with non english default language" do + t1 = create_team + t1.set_languages = ['fr'] + t1.set_language = 'fr' + t1.save! + t2 = Team.duplicate(t1) + assert_equal ['fr'], t2.get_languages + assert_equal 'fr', t2.get_language + end + test "should delete team and partition" do t = create_team assert_difference 'Team.count', -1 do From 77e6ca01a2473830e8e2cca7fce49dfa70a76922 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:31:13 -0300 Subject: [PATCH 33/54] =?UTF-8?q?3884=20=E2=80=93=20Improve=20seeds=20scri?= =?UTF-8?q?pt:=20create=20more=20suggestions=20(#1723)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * create more suggestions - One item of each media type with a confirmed relationship - One item of each media type with a confirmed relationship and 5 suggested relationships Changed some smaller details: - Changed quotes to use faker: we want to more easily create multiple claims - Refactored add_claim_descriptions_and_fact_checks: helped creating the relationships - Updated when we add fact checks: the way I was doint it before it meant, for example when we created requests: we would only have fact checks for the items with 15 requests. this way we have more variety, we have one fact check for 1, 15 or 7 requests - Update to create relationships with tipline requests - Updated which items we add 1/15/20 requests, this way it 'looks more random', and we know we are adding requests to the source in the relationship --- db/seeds.rb | 127 +++++++++++++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 56 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index cb6f2894cc..bf7d4950aa 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -32,16 +32,28 @@ 'https://meedan.com/post/what-is-gendered-health-misinformation-and-why-is-it-an-equity-problem-worth', 'https://meedan.com/post/the-case-for-a-public-health-approach-to-moderate-health-misinformation', ], - quotes: ['Garlic can help you fight covid', 'Tea with garlic is a covid treatment', 'If you have covid you should eat garlic', 'Are you allergic to garlic?', 'Vampires can\'t eat garlic'], - tipline_claims: Array.new(9) { Faker::Lorem.paragraph(sentence_count: 10) } + claims: Array.new(9) { Faker::Lorem.paragraph(sentence_count: 10) } } def open_file(file) File.open(File.join(Rails.root, 'test', 'data', file)) end -def create_project_medias(user, project, team, n_medias = 9) - Media.last(n_medias).each { |media| ProjectMedia.create!(user_id: user.id, project: project, team: team, media: media) } +def create_media(user, data, model_string) + model = Object.const_get(model_string) + case model_string + when 'Claim' + media = model.create!(user_id: user.id, quote: data) + when 'Link' + media = model.create!(user_id: user.id, url: data+"?timestamp=#{Time.now.to_f}") + else + media = model.create!(user_id: user.id, file: open_file(data)) + end + media +end + +def create_project_medias(user, project, team, data) + data.map { |media| ProjectMedia.create!(user_id: user.id, project: project, team: team, media: media) } end def humanize_link(link) @@ -53,9 +65,9 @@ def create_description(project_media) Media.last.type == "Link" ? humanize_link(Media.find(project_media.media_id).url) : Faker::Company.catch_phrase end -def add_claim_descriptions_and_fact_checks(user,n_project_medias = 6, n_claim_descriptions = 3) - ProjectMedia.last(n_project_medias).each { |project_media| ClaimDescription.create!(description: create_description(project_media), context: Faker::Lorem.sentence, user: user, project_media: project_media) } - ClaimDescription.last(n_claim_descriptions).each { |claim_description| FactCheck.create!(summary: Faker::Company.catch_phrase, title: Faker::Company.name, user: user, claim_description: claim_description) } +def add_claim_descriptions_and_fact_checks(user, project_medias) + claim_descriptions = project_medias.map { |project_media| ClaimDescription.create!(description: create_description(project_media), context: Faker::Lorem.sentence, user: user, project_media: project_media) } + claim_descriptions.values_at(0,3,8).each { |claim_description| FactCheck.create!(summary: Faker::Company.catch_phrase, title: Faker::Company.name, user: user, claim_description: claim_description) } end def fact_check_attributes(fact_check_link, user, project, team) @@ -76,8 +88,15 @@ def create_claim_description(user, project, team) ClaimDescription.create!(description: Faker::Company.catch_phrase, context: Faker::Lorem.sentence, user: user, project_media: create_blank(project, team)) end -def create_tipline_project_media(project, team, media) - ProjectMedia.create!(project: project, team: team, media: media, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }) +def create_relationship(project_medias) + Relationship.create!(source_id: project_medias[0].id, target_id: project_medias[1].id, relationship_type: Relationship.confirmed_type) + Relationship.create!(source_id: project_medias[2].id, target_id: project_medias[3].id, relationship_type: Relationship.confirmed_type) + + project_medias[4..9].each { |pm| Relationship.create!(source_id: project_medias[2].id, target_id: pm.id, relationship_type: Relationship.suggested_type)} +end + +def create_tipline_project_media(user, project, team, media) + ProjectMedia.create!(user_id: user.id, project: project, team: team, media: media, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }) end def create_tipline_user_and_data(project_media, team) @@ -162,21 +181,18 @@ def create_tipline_user_and_data(project_media, team) def create_tipline_requests(team, project, user, data_instances, model_string) tipline_pm_arr = [] - model = Object.const_get(model_string) - data_instances[0..5].each do |data_instance| - case model_string - when 'Claim' - media = model.create!(user_id: user.id, quote: data_instance) - when 'Link' - media = model.create!(user_id: user.id, url: data_instance+"?timestamp=#{Time.now.to_f}") - else - media = model.create!(user_id: user.id, file: open_file(data_instance)) - end - project_media = create_tipline_project_media(project, team, media) + + data_instances.each do |data_instance| + media = create_media(user, data_instance, model_string) + project_media = create_tipline_project_media(user, project, team, media) tipline_pm_arr.push(project_media) end - tipline_pm_arr[0..2].each {|pm| create_tipline_user_and_data(pm, team)} - tipline_pm_arr[3..5].each {|pm| 15.times {create_tipline_user_and_data(pm, team)}} + add_claim_descriptions_and_fact_checks(user, tipline_pm_arr) + create_relationship(tipline_pm_arr) + + tipline_pm_arr.values_at(0,3,6).each {|pm| create_tipline_user_and_data(pm, team)} + tipline_pm_arr.values_at(1,4,7).each {|pm| 15.times {create_tipline_user_and_data(pm, team)}} + tipline_pm_arr.values_at(2,5,8).each {|pm| 20.times {create_tipline_user_and_data(pm, team)}} end puts "If you want to create a new user: press 1 then enter" @@ -218,59 +234,60 @@ def create_tipline_requests(team, project, user, data_instances, model_string) puts 'Making Medias...' puts 'Making Medias and Project Medias: Claims...' - 9.times { Claim.create!(user_id: user.id, quote: Faker::Quotes::Shakespeare.hamlet_quote) } - create_project_medias(user, project, team) - add_claim_descriptions_and_fact_checks(user) + claims = data[:claims].map { |data| create_media(user, data, 'Claim')} + claim_project_medias = create_project_medias(user, project, team, claims) + add_claim_descriptions_and_fact_checks(user, claim_project_medias) puts 'Making Medias and Project Medias: Links...' begin - data[:link_media_links].each { |link_media_link| Link.create!(user_id: user.id, url: link_media_link+"?timestamp=#{Time.now.to_f}") } - create_project_medias(user, project, team) - add_claim_descriptions_and_fact_checks(user) + links = data[:link_media_links].map { |data| create_media(user, data, 'Link')} + link_project_medias = create_project_medias(user, project, team, links) + add_claim_descriptions_and_fact_checks(user, link_project_medias) rescue puts "Couldn't create Links. Other medias will still be created. \nIn order to create Links make sure Pender is running." end puts 'Making Medias and Project Medias: Audios...' - data[:audios].each { |audio| UploadedAudio.create!(user_id: user.id, file: open_file(audio)) } - create_project_medias(user, project, team) - add_claim_descriptions_and_fact_checks(user) + audios = data[:audios].map { |data| create_media(user, data, 'UploadedAudio')} + audio_project_medias = create_project_medias(user, project, team, audios) + add_claim_descriptions_and_fact_checks(user, audio_project_medias) puts 'Making Medias and Project Medias: Images...' - data[:images].each { |image| UploadedImage.create!(user_id: user.id, file: open_file(image))} - create_project_medias(user, project, team) - add_claim_descriptions_and_fact_checks(user) + images = data[:images].map { |data| create_media(user, data, 'UploadedImage')} + image_project_medias = create_project_medias(user, project, team, images) + add_claim_descriptions_and_fact_checks(user, image_project_medias) puts 'Making Medias and Project Medias: Videos...' - data[:videos].each { |video| UploadedVideo.create!(user_id: user.id, file: open_file(video)) } - create_project_medias(user, project, team) - add_claim_descriptions_and_fact_checks(user) + videos = data[:videos].map { |data| create_media(user, data, 'UploadedVideo')} + video_project_medias = create_project_medias(user, project, team, videos) + add_claim_descriptions_and_fact_checks(user, video_project_medias) puts 'Making Claim Descriptions and Fact Checks: Imported Fact Checks...' - data[:fact_check_links].each { |fact_check_link| create_fact_check(fact_check_attributes(fact_check_link, user, project, team)) } + data[:fact_check_links].map { |fact_check_link| create_fact_check(fact_check_attributes(fact_check_link, user, project, team)) } - puts 'Making Relationship between Claims...' - project_medias_for_relationship_claims = [] - relationship_claims = data[:quotes].map { |quote| Claim.create!(user_id: user.id, quote: quote) } - relationship_claims.each { |claim| project_medias_for_relationship_claims.push(ProjectMedia.create!(user_id: user.id, project: project, team: team, media: claim))} + puts 'Making Relationship...' + puts 'Making Relationship: Claims / Confirmed Type and Suggested Type...' + create_relationship(claim_project_medias) - Relationship.create!(source_id: project_medias_for_relationship_claims[0].id, target_id: project_medias_for_relationship_claims[1].id, relationship_type: Relationship.confirmed_type) - Relationship.create!(source_id: project_medias_for_relationship_claims[0].id, target_id: project_medias_for_relationship_claims[2].id, relationship_type: Relationship.confirmed_type) - Relationship.create!(source_id: project_medias_for_relationship_claims[3].id, target_id: project_medias_for_relationship_claims[4].id, relationship_type: Relationship.suggested_type) + puts 'Making Relationship: Links / Suggested Type...' + begin + create_relationship(link_project_medias) + rescue + puts "Couldn't create Links. Other medias will still be created. \nIn order to create Links make sure Pender is running." + end + + puts 'Making Relationship: Audios / Confirmed Type and Suggested Type...' + create_relationship(audio_project_medias) - puts 'Making Relationship between Images...' - project_medias_for_images = [] - 2.times { project_medias_for_images.push(ProjectMedia.create!(user_id: user.id, project: project, team: team, media: UploadedImage.create!(user_id: user.id, file: File.open(File.join(Rails.root, 'test', 'data', 'rails.png'))))) } - Relationship.create!(source_id: project_medias_for_images[0].id, target_id: project_medias_for_images[1].id, relationship_type: Relationship.confirmed_type) + puts 'Making Relationship: Images / Confirmed Type and Suggested Type...' + create_relationship(image_project_medias) - puts 'Making Relationship between Audios...' - project_medias_for_audio = [] - 2.times { project_medias_for_audio.push(ProjectMedia.create!(user_id: user.id, project: project, team: team, media: UploadedAudio.create!(user_id: user.id, file: File.open(File.join(Rails.root, 'test', 'data', 'rails.mp3'))))) } - Relationship.create!(source_id: project_medias_for_audio[0].id, target_id: project_medias_for_audio[1].id, relationship_type: Relationship.confirmed_type) + puts 'Making Relationship: Videos / Confirmed Type and Suggested Type...' + create_relationship(video_project_medias) puts 'Making Tipline requests...' puts 'Making Tipline requests: Claims...' - create_tipline_requests(team, project, user, data[:tipline_claims], 'Claim') + create_tipline_requests(team, project, user, data[:claims], 'Claim') puts 'Making Tipline requests: Links...' begin @@ -288,8 +305,6 @@ def create_tipline_requests(team, project, user, data_instances, model_string) puts 'Making Tipline requests: Videos...' create_tipline_requests(team, project, user, data[:videos], 'UploadedVideo') - add_claim_descriptions_and_fact_checks(user) - if answer == "1" puts "Created — user: #{data[:user_name]} — email: #{user.email} — password : #{data[:user_password]}" elsif answer == "2" From d62395845deca8d40dfda59d440e47d3d18174eb Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Thu, 9 Nov 2023 18:35:21 +0200 Subject: [PATCH 34/54] CV2-3804: delete FeedInvitation when feed deleted (#1724) --- app/models/feed.rb | 2 +- test/models/feed_test.rb | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/app/models/feed.rb b/app/models/feed.rb index f1ba5245fe..02dbd70220 100755 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -6,7 +6,7 @@ class Feed < ApplicationRecord has_many :requests has_many :feed_teams, dependent: :destroy has_many :teams, through: :feed_teams - has_many :feed_invitations + has_many :feed_invitations, dependent: :destroy belongs_to :user, optional: true belongs_to :saved_search, optional: true belongs_to :team, optional: true diff --git a/test/models/feed_test.rb b/test/models/feed_test.rb index 13623d0abc..2df160090c 100755 --- a/test/models/feed_test.rb +++ b/test/models/feed_test.rb @@ -162,7 +162,7 @@ def setup CheckSearch.any_instance.unstub(:medias) end - test "should delete feed teams when feed is deleted" do + test "should delete feed teams and invitation when feed is deleted" do f = create_feed f.teams << create_team ft = create_feed_team team: create_team, feed: f @@ -176,5 +176,12 @@ def setup f.destroy! end end + f = create_feed + create_feed_invitation feed: f + assert_difference 'Feed.count', -1 do + assert_difference 'FeedInvitation.count', -1 do + f.destroy! + end + end end end From c5e498d929da24eba0a09687150257062a8c0715 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Mon, 13 Nov 2023 20:36:40 +0200 Subject: [PATCH 35/54] CV2-3694: update `updated_at` column after delete tags (#1726) * CV2-3694: trigger update column after delete tags * CV2-3694: fix tests --- app/models/annotations/tag.rb | 8 +++++++- test/controllers/elastic_search_2_test.rb | 2 +- test/controllers/elastic_search_4_test.rb | 13 ++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/app/models/annotations/tag.rb b/app/models/annotations/tag.rb index 02d9e3943e..4d8d15355d 100644 --- a/app/models/annotations/tag.rb +++ b/app/models/annotations/tag.rb @@ -119,7 +119,13 @@ def add_update_es_tags(op) end def destroy_elasticsearch_tag - destroy_es_items('tags', 'destroy_doc_nested', self.annotated_id) if self.annotated_type == 'ProjectMedia' + if self.annotated_type == 'ProjectMedia' + destroy_es_items('tags', 'destroy_doc_nested', self.annotated_id) + if User.current.present? + pm = self.annotated + pm.update_recent_activity(pm) + end + end end def update_tags_count diff --git a/test/controllers/elastic_search_2_test.rb b/test/controllers/elastic_search_2_test.rb index 18fd40fabb..963da34f48 100644 --- a/test/controllers/elastic_search_2_test.rb +++ b/test/controllers/elastic_search_2_test.rb @@ -103,7 +103,7 @@ def setup # destroy tag ElasticSearchWorker.clear t.destroy - assert_equal 1, ElasticSearchWorker.jobs.size + assert_equal 2, ElasticSearchWorker.jobs.size end end diff --git a/test/controllers/elastic_search_4_test.rb b/test/controllers/elastic_search_4_test.rb index bd8a9c0a35..5276dc01cb 100644 --- a/test/controllers/elastic_search_4_test.rb +++ b/test/controllers/elastic_search_4_test.rb @@ -174,13 +174,24 @@ def setup sleep 2 result = CheckSearch.new({sort: "recent_activity"}.to_json) assert_equal [pm1.id, pm3.id, pm2.id], result.medias.map(&:id) - create_tag annotated: pm3, tag: 'in_progress', disable_es_callbacks: false + tag_3 = create_tag annotated: pm3, tag: 'in_progress', disable_es_callbacks: false sleep 1 result = CheckSearch.new({sort: "recent_activity"}.to_json) assert_equal [pm3.id, pm1.id, pm2.id], result.medias.map(&:id) # should sort by recent activity with project and status filters result = CheckSearch.new({verification_status: ['in_progress'], sort: "recent_activity"}.to_json) assert_equal 1, result.project_medias.count + tag_1 = create_tag annotated: pm1, tag: 'in_progress', disable_es_callbacks: false + tag_2 = create_tag annotated: pm2, tag: 'in_progress', disable_es_callbacks: false + sleep 1 + result = CheckSearch.new({sort: "recent_activity"}.to_json) + assert_equal [pm2.id, pm1.id, pm3.id], result.medias.map(&:id) + tag_3.destroy! + sleep 1 + result = CheckSearch.new({sort: "recent_activity"}.to_json) + assert_equal [pm3.id, pm2.id, pm1.id], result.medias.map(&:id) + result = CheckSearch.new({tags: ['in_progress'], sort: "recent_activity"}.to_json) + assert_equal [pm2.id, pm1.id], result.medias.map(&:id) end end From 30db221bcce8bfef978f9c9298a2531a923612d9 Mon Sep 17 00:00:00 2001 From: Darius Kazemi Date: Wed, 15 Nov 2023 10:34:54 -0800 Subject: [PATCH 36/54] Changing the email url to match check-web #1728, CV2-3806 --- app/mailers/feed_invitation_mailer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mailers/feed_invitation_mailer.rb b/app/mailers/feed_invitation_mailer.rb index 7ef466a585..c6f21c4f55 100644 --- a/app/mailers/feed_invitation_mailer.rb +++ b/app/mailers/feed_invitation_mailer.rb @@ -8,7 +8,7 @@ def notify(record_id, team_id) @user = record.user @feed = record.feed @due_at = record.created_at + CheckConfig.get('feed_invitation_due_to', 30).to_i.days - @accept_feed_url = "#{CheckConfig.get('checkdesk_client')}/#{team.slug}/feed/#{record.id}/accept_invitation" + @accept_feed_url = "#{CheckConfig.get('checkdesk_client')}/#{team.slug}/feed-invitation/#{record.id}" subject = I18n.t("mails_notifications.feed_invitation.subject", user: @user.name, feed: @feed.name) Rails.logger.info "Sending a feed invitation e-mail to #{@recipient}" mail(to: @recipient, email_type: 'feed_invitation', subject: subject) From d0eff3abf30146ffa5adc5f3e3c376ad8444a406 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:54:02 -0300 Subject: [PATCH 37/54] A couple of updates for the GraphQL feed API * `feed_team`: Adding the ability to query by `feedId` and `teamSlug` in addition to `id` * `feed_invitation`: Adding the ability to query by `feedId` (and then the current logged in user email is assumed) in addition to `id` Reference: CV2-3801. --- app/graph/types/query_type.rb | 23 ++++++- lib/relay.idl | 8 +-- public/relay.json | 60 ++++++++++++++----- .../controllers/graphql_controller_12_test.rb | 60 +++++++++++++++++++ 4 files changed, 129 insertions(+), 22 deletions(-) diff --git a/app/graph/types/query_type.rb b/app/graph/types/query_type.rb index 52d20a3f4a..5e9b9b7589 100644 --- a/app/graph/types/query_type.rb +++ b/app/graph/types/query_type.rb @@ -202,6 +202,27 @@ def dynamic_annotation_field(query:, only_cache: nil) end end + field :feed_invitation, FeedInvitationType, description: 'Information about a feed invitation, given its database ID or feed database ID (and then the current user email is used)', null: true do + argument :id, GraphQL::Types::Int, required: false + argument :feed_id, GraphQL::Types::Int, required: false + end + + def feed_invitation(id: nil, feed_id: nil) + feed_invitation_id = id || FeedInvitation.where(feed_id: feed_id, email: User.current.email).last&.id + GraphqlCrudOperations.load_if_can(FeedInvitation, feed_invitation_id, context) + end + + field :feed_team, FeedTeamType, description: 'Information about a feed team, given its database ID or the combo feed database ID plus team slug', null: true do + argument :id, GraphQL::Types::Int, required: false + argument :feed_id, GraphQL::Types::Int, required: false + argument :team_slug, GraphQL::Types::String, required: false + end + + def feed_team(id: nil, feed_id: nil, team_slug: nil) + feed_team_id = id || FeedTeam.where(feed_id: feed_id, team_id: Team.find_by_slug(team_slug).id).last&.id + GraphqlCrudOperations.load_if_can(FeedTeam, feed_team_id, context) + end + # Getters by ID %i[ source @@ -214,8 +235,6 @@ def dynamic_annotation_field(query:, only_cache: nil) cluster feed request - feed_invitation - feed_team tipline_message ].each do |type| field type, diff --git a/lib/relay.idl b/lib/relay.idl index 2646cb4eac..d5556a348c 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -11728,14 +11728,14 @@ type Query { feed(id: ID!): Feed """ - Information about the feed_invitation with given id + Information about a feed invitation, given its database ID or feed database ID (and then the current user email is used) """ - feed_invitation(id: ID!): FeedInvitation + feed_invitation(feedId: Int, id: Int): FeedInvitation """ - Information about the feed_team with given id + Information about a feed team, given its database ID or the combo feed database ID plus team slug """ - feed_team(id: ID!): FeedTeam + feed_team(feedId: Int, id: Int, teamSlug: String): FeedTeam """ Find whether a team exists diff --git a/public/relay.json b/public/relay.json index b4e4c56506..ebd37659d2 100644 --- a/public/relay.json +++ b/public/relay.json @@ -61648,19 +61648,27 @@ }, { "name": "feed_invitation", - "description": "Information about the feed_invitation with given id", + "description": "Information about a feed invitation, given its database ID or feed database ID (and then the current user email is used)", "args": [ { "name": "id", "description": null, "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feedId", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null }, "defaultValue": null, "isDeprecated": false, @@ -61677,19 +61685,39 @@ }, { "name": "feed_team", - "description": "Information about the feed_team with given id", + "description": "Information about a feed team, given its database ID or the combo feed database ID plus team slug", "args": [ { "name": "id", "description": null, "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "feedId", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "teamSlug", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null }, "defaultValue": null, "isDeprecated": false, diff --git a/test/controllers/graphql_controller_12_test.rb b/test/controllers/graphql_controller_12_test.rb index 7de2edc196..96423ddb66 100644 --- a/test/controllers/graphql_controller_12_test.rb +++ b/test/controllers/graphql_controller_12_test.rb @@ -175,4 +175,64 @@ def teardown assert_response :success assert_nil JSON.parse(@response.body).dig('data', 'feed_invitation') end + + test "should read feed invitation based on feed ID and current user email" do + fi = create_feed_invitation email: @u.email + + authenticate_with_user(@u) + query = 'query { feed_invitation(feedId: ' + fi.feed_id.to_s + ') { id } }' + post :create, params: { query: query } + assert_response :success + assert_not_nil JSON.parse(@response.body).dig('data', 'feed_invitation') + end + + test "should not read feed invitation based on feed ID and current user email" do + fi = create_feed_invitation + + authenticate_with_user(@u) + query = 'query { feed_invitation(feedId: ' + fi.feed_id.to_s + ') { id } }' + post :create, params: { query: query } + assert_response :success + assert_nil JSON.parse(@response.body).dig('data', 'feed_invitation') + end + + test "should read feed team" do + ft = create_feed_team team: @t + + authenticate_with_user(@u) + query = 'query { feed_team(id: ' + ft.id.to_s + ') { id } }' + post :create, params: { query: query } + assert_response :success + assert_not_nil JSON.parse(@response.body).dig('data', 'feed_team') + end + + test "should not read feed team" do + ft = create_feed_team + + authenticate_with_user(@u) + query = 'query { feed_team(id: ' + ft.id.to_s + ') { id } }' + post :create, params: { query: query } + assert_response :success + assert_nil JSON.parse(@response.body).dig('data', 'feed_team') + end + + test "should read feed team based on feed ID and team slug" do + ft = create_feed_team team: @t + + authenticate_with_user(@u) + query = 'query { feed_team(feedId: ' + ft.feed_id.to_s + ', teamSlug: "' + @t.slug + '") { id } }' + post :create, params: { query: query } + assert_response :success + assert_not_nil JSON.parse(@response.body).dig('data', 'feed_team') + end + + test "should not read feed team based on feed ID and team slug" do + ft = create_feed_team + + authenticate_with_user(@u) + query = 'query { feed_team(feedId: ' + ft.feed_id.to_s + ', teamSlug: "' + ft.team.slug + '") { id } }' + post :create, params: { query: query } + assert_response :success + assert_nil JSON.parse(@response.body).dig('data', 'feed_team') + end end From 462fe742519853125c51db3d0fd5fb9adf30ecd5 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:21:43 -0300 Subject: [PATCH 38/54] Ticket CV2-3904: Graceful error when cannot send custom message to tipline user --- app/models/concerns/smooch_messages.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index a8df490b0f..6b09835376 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -461,7 +461,7 @@ def send_custom_message_to_user(team, uid, timestamp, message, language) date = I18n.l(Time.at(timestamp), locale: language, format: :short) message = self.format_template_message('custom_message', [date, message.to_s.gsub(/\s+/, ' ')], nil, message, language, nil, true) if platform == 'WhatsApp' response = self.send_message_to_user(uid, message, {}, false, true, 'custom_message') - success = (response.code.to_i < 400) + success = (response && response.code.to_i < 400) success end end From c92330aaac62cb58fc86a8233d983085374b6166 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 17 Nov 2023 21:37:39 -0300 Subject: [PATCH 39/54] Ticket CV2-3948: Fixing how number of tipline newsletters delivered are calculated --- lib/check_statistics.rb | 2 +- test/lib/check_statistics_test.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/check_statistics.rb b/lib/check_statistics.rb index 1556fa62ca..8f9ba19d66 100644 --- a/lib/check_statistics.rb +++ b/lib/check_statistics.rb @@ -195,7 +195,7 @@ def get_statistics(start_date, end_date, team_id, platform, language, tracing_at CheckTracer.in_span('CheckStatistics#newsletters_delivered', attributes: tracing_attributes) do # Number of newsletters effectively delivered, accounting for user errors for each platform - statistics[:newsletters_delivered] = TiplineMessage.where(created_at: start_date..end_date, team_id: team_id, platform: platform_name, language: language, direction: 'outgoing', event: 'newsletter').count + statistics[:newsletters_delivered] = TiplineMessage.where(created_at: start_date..end_date, team_id: team_id, platform: platform_name, language: language, direction: 'outgoing', state: 'delivered', event: 'newsletter').count end CheckTracer.in_span('CheckStatistics#whatsapp_conversations', attributes: tracing_attributes) do diff --git a/test/lib/check_statistics_test.rb b/test/lib/check_statistics_test.rb index 96e9c36a5f..a6e8479db1 100644 --- a/test/lib/check_statistics_test.rb +++ b/test/lib/check_statistics_test.rb @@ -74,4 +74,12 @@ def teardown WebMock.stub_request(:get, @url).to_return(status: 400, body: { error: 'Error' }.to_json) assert_nil CheckStatistics.number_of_whatsapp_conversations(create_team.id, @from, @to) end + + test 'should calculate number of delivered newsletters' do + WebMock.stub_request(:get, /graph\.facebook\.com/).to_return(status: 400, body: { error: 'Error' }.to_json) + create_tipline_message team_id: @team.id, event: 'newsletter', direction: :outgoing, state: 'sent' + create_tipline_message team_id: @team.id, event: 'newsletter', direction: :outgoing, state: 'delivered' + data = CheckStatistics.get_statistics(Time.now.yesterday, Time.now.tomorrow, @team.id, 'whatsapp', 'en') + assert_equal 1, data[:newsletters_delivered] + end end From 401d89aeecd38aded24953a385269666e771fa0d Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Mon, 20 Nov 2023 07:48:12 -0300 Subject: [PATCH 40/54] =?UTF-8?q?3884=20=E2=80=93=20Improve=20seeds:=20cre?= =?UTF-8?q?ate=20feed=20(#1727)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a feed with published items (randomized verified and false items). Important: Notes - Run with elasticsearch_log set to 0, so it doesn't output the logs: `elasticsearch_log=0 bundle exec rake db:seed` - It can take some time for the items to appear in the published list and on the feed - If there are any issues with the Feed not displaying the clickable items (items with published_article_url): log into the rails console, clear the cache (run `Rails.cache.clear`), and refresh the browser --- db/seeds.rb | 217 +++++++++++++++++++++++++++------------------------- 1 file changed, 111 insertions(+), 106 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index bf7d4950aa..54c248001c 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -3,13 +3,11 @@ Rails.env.development? || raise('To run the seeds file you should be in the development environment') -Rails.cache.clear - data = { team_name: Faker::Company.name, user_name: Faker::Name.first_name.downcase, user_password: Faker::Internet.password(min_length: 8), - link_media_links: [ + 'Link' => [ 'https://meedan.com/post/addressing-misinformation-across-countries-a-pioneering-collaboration-between-taiwan-factcheck-center-vera-files', 'https://meedan.com/post/entre-becos-a-women-led-hyperlocal-newsletter-from-the-peripheries-of-brazil', 'https://meedan.com/post/check-global-launches-independent-media-response-fund-tackles-on-climate-misinformation', @@ -20,11 +18,20 @@ 'https://meedan.com/post/2022-french-elections-meedan-software-supported-agence-france-presse', 'https://meedan.com/post/how-to-write-longform-git-commits-for-better-software-development', 'https://meedan.com/post/welcome-smriti-singh-our-research-intern', - + 'https://meedan.com/post/countdown-to-u-s-2024-meedan-coalition-to-exchange-critical-election-information-with-overlooked-voters', + 'https://meedan.com/post/a-statement-on-the-israel-gaza-war-by-meedans-ceo', + 'https://meedan.com/post/resources-to-capture-critical-evidence-from-the-israel-gaza-war', + 'https://meedan.com/post/turkeys-largest-fact-checking-group-debunks-election-related-disinformation', + 'https://meedan.com/post/meedan-joins-diverse-cohort-of-partners-committed-to-partnership-on-ais-responsible-practices-for-synthetic-media', + 'https://meedan.com/post/nurturing-equity-diversity-and-inclusion-meedans-people-first-approach', + 'https://meedan.com/post/students-find-top-spreader-of-climate-misinformation-is-most-read-online-news-publisher-in-egypt', + 'https://meedan.com/post/new-e-course-on-the-fundamentals-of-climate-and-environmental-reporting-in-africa', + 'https://meedan.com/post/annual-report-2022', + 'https://meedan.com/post/meedan-joins-partnership-on-ais-ai-and-media-integrity-steering-committee' ], - audios: ['e-item.mp3', 'rails.mp3', 'with_cover.mp3', 'with_cover.ogg', 'with_cover.wav', 'e-item.mp3', 'rails.mp3', 'with_cover.mp3', 'with_cover.ogg'], - images: ['large-image.jpg', 'maçã.png', 'rails-photo.jpg', 'rails.png', 'rails2.png', 'ruby-big.png', 'ruby-small.png', 'ruby-big.png', 'ruby-small.png'], - videos: ['d-item.mp4', 'rails.mp4', 'd-item.mp4', 'rails.mp4', 'd-item.mp4', 'rails.mp4', 'd-item.mp4', 'rails.mp4', 'd-item.mp4'], + 'UploadedAudio' => ['e-item.mp3', 'rails.mp3', 'with_cover.mp3', 'with_cover.ogg', 'with_cover.wav']*4, + 'UploadedImage' => ['large-image.jpg', 'maçã.png', 'rails-photo.jpg', 'rails.png', 'ruby-small.png']*4, + 'UploadedVideo' => ['d-item.mp4', 'rails.mp4', 'd-item.mp4', 'rails.mp4', 'd-item.mp4']*4, fact_check_links: [ 'https://meedan.com/post/welcome-haramoun-hamieh-our-program-manager-for-nawa', 'https://meedan.com/post/strengthening-fact-checking-with-media-literacy-technology-and-collaboration', @@ -32,7 +39,7 @@ 'https://meedan.com/post/what-is-gendered-health-misinformation-and-why-is-it-an-equity-problem-worth', 'https://meedan.com/post/the-case-for-a-public-health-approach-to-moderate-health-misinformation', ], - claims: Array.new(9) { Faker::Lorem.paragraph(sentence_count: 10) } + 'Claim' => Array.new(20) { Faker::Lorem.paragraph(sentence_count: 10) }, } def open_file(file) @@ -65,9 +72,26 @@ def create_description(project_media) Media.last.type == "Link" ? humanize_link(Media.find(project_media.media_id).url) : Faker::Company.catch_phrase end -def add_claim_descriptions_and_fact_checks(user, project_medias) - claim_descriptions = project_medias.map { |project_media| ClaimDescription.create!(description: create_description(project_media), context: Faker::Lorem.sentence, user: user, project_media: project_media) } - claim_descriptions.values_at(0,3,8).each { |claim_description| FactCheck.create!(summary: Faker::Company.catch_phrase, title: Faker::Company.name, user: user, claim_description: claim_description) } +def create_claim_descriptions(user, project_medias) + project_medias.map { |project_media| ClaimDescription.create!(description: create_description(project_media), context: Faker::Lorem.sentence, user: user, project_media: project_media) } +end + +def create_fact_checks(user, claim_descriptions) + claim_descriptions.each { |claim_description| FactCheck.create!(summary: Faker::Company.catch_phrase, title: Faker::Company.name, user: user, claim_description: claim_description, language: 'en') } +end + +def verify_fact_check_and_publish_report(project_media, url = '') + status = ['verified', 'false'].sample + + verification_status = project_media.last_status_obj + verification_status.status = status + verification_status.save! + + report_design = project_media.get_dynamic_annotation('report_design') + report_design.set_fields = { status_label: status, state: 'published' }.to_json + report_design.data[:options][:published_article_url] = url + report_design.action = 'publish' + report_design.save! end def fact_check_attributes(fact_check_link, user, project, team) @@ -76,7 +100,7 @@ def fact_check_attributes(fact_check_link, user, project, team) url: fact_check_link, title: Faker::Company.name, user: user, - claim_description: create_claim_description(user, project, team) + claim_description: create_claim_description_for_imported_fact_check(user, project, team) } end @@ -84,19 +108,20 @@ def create_blank(project, team) ProjectMedia.create!(project: project, team: team, media: Blank.create!, channel: { main: CheckChannels::ChannelCodes::FETCH }) end -def create_claim_description(user, project, team) +def create_claim_description_for_imported_fact_check(user, project, team) ClaimDescription.create!(description: Faker::Company.catch_phrase, context: Faker::Lorem.sentence, user: user, project_media: create_blank(project, team)) end -def create_relationship(project_medias) - Relationship.create!(source_id: project_medias[0].id, target_id: project_medias[1].id, relationship_type: Relationship.confirmed_type) - Relationship.create!(source_id: project_medias[2].id, target_id: project_medias[3].id, relationship_type: Relationship.confirmed_type) - - project_medias[4..9].each { |pm| Relationship.create!(source_id: project_medias[2].id, target_id: pm.id, relationship_type: Relationship.suggested_type)} +def create_confirmed_relationship(parent, child) + Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type) +end + +def create_suggested_relationship(parent, children) + children.each { |child| Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.suggested_type)} end -def create_tipline_project_media(user, project, team, media) - ProjectMedia.create!(user_id: user.id, project: project, team: team, media: media, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }) +def create_project_medias_with_channel(user, project, team, data) + data.map { |media| ProjectMedia.create!(user_id: user.id, project: project, team: team, media: media, channel: { main: CheckChannels::ChannelCodes::WHATSAPP })} end def create_tipline_user_and_data(project_media, team) @@ -179,31 +204,24 @@ def create_tipline_user_and_data(project_media, team) Dynamic.create!(annotation_type: 'smooch', annotated: project_media, annotator: BotUser.smooch_user, set_fields: fields.to_json) end -def create_tipline_requests(team, project, user, data_instances, model_string) - tipline_pm_arr = [] - - data_instances.each do |data_instance| - media = create_media(user, data_instance, model_string) - project_media = create_tipline_project_media(user, project, team, media) - tipline_pm_arr.push(project_media) - end - add_claim_descriptions_and_fact_checks(user, tipline_pm_arr) - create_relationship(tipline_pm_arr) - - tipline_pm_arr.values_at(0,3,6).each {|pm| create_tipline_user_and_data(pm, team)} - tipline_pm_arr.values_at(1,4,7).each {|pm| 15.times {create_tipline_user_and_data(pm, team)}} - tipline_pm_arr.values_at(2,5,8).each {|pm| 20.times {create_tipline_user_and_data(pm, team)}} +def create_tipline_requests(team, project_medias, x_times) + project_medias.each {|pm| x_times.times {create_tipline_user_and_data(pm, team)}} end +###################### +# 0. Start the script puts "If you want to create a new user: press 1 then enter" puts "If you want to add more data to an existing user: press 2 then enter" print ">> " answer = STDIN.gets.chomp ActiveRecord::Base.transaction do + # 1. Creating what we need for the workspace + # We create a user, team and project OR we fetch one if answer == "1" puts 'Making Team / Workspace...' - team = create_team(name: Faker::Company.name) + team = create_team(name: "#{data[:team_name]} / Feed Creator") + team.set_language('en') puts 'Making User...' user = create_user(name: data[:user_name], login: data[:user_name], password: data[:user_password], password_confirmation: data[:user_password], email: Faker::Internet.safe_email(name: data[:user_name]), is_admin: true) @@ -222,7 +240,7 @@ def create_tipline_requests(team, project, user, data_instances, model_string) user = User.find_by(email: email) if user.team_users.first.nil? - team = create_team(name: Faker::Company.name) + team = create_team(name: data[:team_name]) project = create_project(title: team.name, team_id: team.id, user: user, description: '') create_team_user(team: team, user: user, role: 'admin') else @@ -232,82 +250,69 @@ def create_tipline_requests(team, project, user, data_instances, model_string) end end - puts 'Making Medias...' - puts 'Making Medias and Project Medias: Claims...' - claims = data[:claims].map { |data| create_media(user, data, 'Claim')} - claim_project_medias = create_project_medias(user, project, team, claims) - add_claim_descriptions_and_fact_checks(user, claim_project_medias) - - puts 'Making Medias and Project Medias: Links...' - begin - links = data[:link_media_links].map { |data| create_media(user, data, 'Link')} - link_project_medias = create_project_medias(user, project, team, links) - add_claim_descriptions_and_fact_checks(user, link_project_medias) - rescue - puts "Couldn't create Links. Other medias will still be created. \nIn order to create Links make sure Pender is running." + # 2. Creating Items in different states + # 2.1 Create medias: claims, audios, images, videos and links + media_data_collection = [ data['Claim'], data['UploadedAudio'], data['UploadedImage'], data['UploadedVideo'], data['Link']] + media_data_collection.each do |media_data| + begin + media_type = data.key(media_data) + puts "Making #{media_type}..." + puts "#{media_type}: Making Medias and Project Medias..." + medias = media_data.map { |individual_data| create_media(user, individual_data, media_type)} + project_medias = create_project_medias_with_channel(user, project, team, medias) + + puts "#{media_type}: Making Claim Descriptions and Fact Checks..." + # add claim description to all items, don't add fact checks to all + claim_descriptions = create_claim_descriptions(user, project_medias) + claim_descriptions_for_fact_checks = claim_descriptions[0..10] + create_fact_checks(user, claim_descriptions_for_fact_checks) + + puts "#{media_type}: Making Relationship: Confirmed Type and Suggested Type..." + # because we want a lot of state variety between items, we are not creating relationships for 7..13 + # send parent and child index + create_confirmed_relationship(project_medias[0], project_medias[1]) + create_confirmed_relationship(project_medias[2], project_medias[3]) + create_confirmed_relationship(project_medias[4], project_medias[5]) + create_confirmed_relationship(project_medias[6], project_medias[1]) + # send parent and children + create_suggested_relationship(project_medias[6], project_medias[14..19]) + + puts "#{media_type}: Making Tipline requests..." + # we want different ammounts of requests, so we send the item and the number of requests that should be created + # we jump between numbers so it looks more real in the UI (instead of all 1 requests, then all 15 etc) + create_tipline_requests(team, project_medias.values_at(0,3,6,9,12,15,18), 1) + create_tipline_requests(team, project_medias.values_at(1,4,7,10,13,16,19), 15) + create_tipline_requests(team, project_medias.values_at(2,5,8,11,14,17), 17) + + puts "#{media_type}: Publishing Reports..." + # we want some published items to have and some to not have published_article_url, because they behave differently in the feed + # we send the published_article_url when we want one + project_medias[7..8].each { |pm| verify_fact_check_and_publish_report(pm, "https://www.thespruceeats.com/step-by-step-basic-cake-recipe-304553?timestamp=#{Time.now.to_f}")} + project_medias[9..10].each { |pm| verify_fact_check_and_publish_report(pm)} + rescue StandardError => e + if media_type != 'Link' + raise e + else + puts "Couldn't create Links. Other medias will still be created. \nIn order to create Links make sure Pender is running." + end + end end - - puts 'Making Medias and Project Medias: Audios...' - audios = data[:audios].map { |data| create_media(user, data, 'UploadedAudio')} - audio_project_medias = create_project_medias(user, project, team, audios) - add_claim_descriptions_and_fact_checks(user, audio_project_medias) - - puts 'Making Medias and Project Medias: Images...' - images = data[:images].map { |data| create_media(user, data, 'UploadedImage')} - image_project_medias = create_project_medias(user, project, team, images) - add_claim_descriptions_and_fact_checks(user, image_project_medias) - - puts 'Making Medias and Project Medias: Videos...' - videos = data[:videos].map { |data| create_media(user, data, 'UploadedVideo')} - video_project_medias = create_project_medias(user, project, team, videos) - add_claim_descriptions_and_fact_checks(user, video_project_medias) - - puts 'Making Claim Descriptions and Fact Checks: Imported Fact Checks...' + + # 2.2 Create medias: imported Fact Checks + puts 'Making Imported Fact Checks...' data[:fact_check_links].map { |fact_check_link| create_fact_check(fact_check_attributes(fact_check_link, user, project, team)) } - puts 'Making Relationship...' - puts 'Making Relationship: Claims / Confirmed Type and Suggested Type...' - create_relationship(claim_project_medias) - - puts 'Making Relationship: Links / Suggested Type...' - begin - create_relationship(link_project_medias) - rescue - puts "Couldn't create Links. Other medias will still be created. \nIn order to create Links make sure Pender is running." - end - - puts 'Making Relationship: Audios / Confirmed Type and Suggested Type...' - create_relationship(audio_project_medias) - - puts 'Making Relationship: Images / Confirmed Type and Suggested Type...' - create_relationship(image_project_medias) - - puts 'Making Relationship: Videos / Confirmed Type and Suggested Type...' - create_relationship(video_project_medias) - - puts 'Making Tipline requests...' - puts 'Making Tipline requests: Claims...' - create_tipline_requests(team, project, user, data[:claims], 'Claim') - - puts 'Making Tipline requests: Links...' - begin - create_tipline_requests(team, project, user, data[:link_media_links], 'Link') - rescue - puts "Couldn't create Links. Other medias will still be created. \nIn order to create Links make sure Pender is running." - end - - puts 'Making Tipline requests: Audios...' - create_tipline_requests(team, project, user, data[:audios], 'UploadedAudio') - - puts 'Making Tipline requests: Images...' - create_tipline_requests(team, project, user, data[:images], 'UploadedImage') - - puts 'Making Tipline requests: Videos...' - create_tipline_requests(team, project, user, data[:videos], 'UploadedVideo') + # 3. Create Shared feed + puts 'Making Shared Feed' + saved_search = SavedSearch.create!(title: "#{user.name}'s list #{random_number}", team: team, filters: {created_by: user}) + Feed.create!(name: "Feed Test: #{Faker::Alphanumeric.alpha(number: 10)}", user: user, team: team, published: true, saved_search: saved_search, licenses: [1]) + # 4. Return user information if answer == "1" - puts "Created — user: #{data[:user_name]} — email: #{user.email} — password : #{data[:user_password]}" + puts "Created user: name: #{data[:user_name]} — email: #{user.email} — password : #{data[:user_password]}" elsif answer == "2" puts "Data added to user: #{user.email}" end end + +Rails.cache.clear From 9b62e07177144abf786909310d2e41d2ea837c4b Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Mon, 20 Nov 2023 19:07:14 +0200 Subject: [PATCH 41/54] CV2-3897: ignore reaction request (#1731) --- app/models/concerns/smooch_capi.rb | 1 + test/models/concerns/smooch_capi_test.rb | 30 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/app/models/concerns/smooch_capi.rb b/app/models/concerns/smooch_capi.rb index 469ce56d51..cc50bd13fe 100644 --- a/app/models/concerns/smooch_capi.rb +++ b/app/models/concerns/smooch_capi.rb @@ -5,6 +5,7 @@ module SmoochCapi module ClassMethods def should_ignore_capi_request?(request) + return true if request.params.dig('entry', 0, 'changes', 0, 'value', 'messages', 0, 'type') == 'reaction' event = request.params.dig('entry', 0, 'changes', 0, 'value', 'statuses', 0, 'status').to_s ['read', 'sent'].include?(event) end diff --git a/test/models/concerns/smooch_capi_test.rb b/test/models/concerns/smooch_capi_test.rb index 791e18a7e0..2a9f9d3277 100644 --- a/test/models/concerns/smooch_capi_test.rb +++ b/test/models/concerns/smooch_capi_test.rb @@ -296,4 +296,34 @@ def teardown Bot::Smooch.unblock_user(@uid) assert !Bot::Smooch.user_blocked?(@uid) end + + test "should ignore reaction message" do + request = OpenStruct.new(params: { + "object": "whatsapp_business_account", + "entry": [ + { + "id": "121774707564591", + "changes": [ + { + "value": { + "messages": [ + { + "from": "201060407981", + "timestamp": "1700484874", + "type": "reaction", + "reaction": { + "message_id": "wamid.HBgMMjAxMDYwNDA3OTgxFQIAERgSNjFDMEI3NkY4QzA2Mzc4Q0MwAA==", + "emoji": "👍" + } + } + ] + }, + "field": "messages" + } + ] + } + ], + }.with_indifferent_access) + assert Bot::Smooch.should_ignore_capi_request?(request) + end end From a32436c3531fd085daf74733855d7518ae675618 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:16:38 -0300 Subject: [PATCH 42/54] =?UTF-8?q?3993=20=E2=80=93=20Improvements=20for=20C?= =?UTF-8?q?heck=20API=20seed=20script:=20Add=20an=20item=20with=20multiple?= =?UTF-8?q?=20medias=20(#1732)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add item with many confirmed relationships So the center column on the item page has a good size list to check functionality against https://github.com/meedan/check-api/pull/1722#issuecomment-1798729043 --- db/seeds.rb | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index 54c248001c..68d9a0867a 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -112,8 +112,13 @@ def create_claim_description_for_imported_fact_check(user, project, team) ClaimDescription.create!(description: Faker::Company.catch_phrase, context: Faker::Lorem.sentence, user: user, project_media: create_blank(project, team)) end -def create_confirmed_relationship(parent, child) - Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type) +def create_confirmed_relationship(parent, children) + if children.class == Array + children.each { |child| Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type)} + else + child = children + Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type) + end end def create_suggested_relationship(parent, children) @@ -276,7 +281,18 @@ def create_tipline_requests(team, project_medias, x_times) create_confirmed_relationship(project_medias[6], project_medias[1]) # send parent and children create_suggested_relationship(project_medias[6], project_medias[14..19]) - + + puts "#{media_type}: Making Relationship: Create item with many confirmed relationships" + # so the center column on the item page has a good size list to check functionality against + # https://github.com/meedan/check-api/pull/1722#issuecomment-1798729043 + # create the children we need for the relationship + confirmed_children_media = ['Claim', 'UploadedAudio', 'UploadedImage', 'UploadedVideo', 'Link'].flat_map do |media_type| + data[media_type][0..1].map { |data| create_media(user, data , media_type)} + end + confirmed_children_project_medias = create_project_medias(user, project, team, confirmed_children_media) + # send parent and children + create_confirmed_relationship(project_medias[0], confirmed_children_project_medias) + puts "#{media_type}: Making Tipline requests..." # we want different ammounts of requests, so we send the item and the number of requests that should be created # we jump between numbers so it looks more real in the UI (instead of all 1 requests, then all 15 etc) From 76b8d386a7ce24ed4ec54e2e0f0cd275d76e2746 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:10:58 -0300 Subject: [PATCH 43/54] update create_confirmed_relationship (#1734) --- db/seeds.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index 68d9a0867a..c35ca45c33 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -113,12 +113,7 @@ def create_claim_description_for_imported_fact_check(user, project, team) end def create_confirmed_relationship(parent, children) - if children.class == Array - children.each { |child| Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type)} - else - child = children - Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type) - end + [children].flatten.each { |child| Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type) } end def create_suggested_relationship(parent, children) From 6f75aa11fa5dbd422ae580e542daa4db8f4dbb89 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 22 Nov 2023 00:02:18 +0200 Subject: [PATCH 44/54] Fix sentry issues (#1733) * CV2-3829: fix sentry error * CV2-3987: fix sentry error * Fix CC --- app/models/annotations/dynamic.rb | 7 ++++--- app/models/bot/keep.rb | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/models/annotations/dynamic.rb b/app/models/annotations/dynamic.rb index 27e2343418..663619ab45 100644 --- a/app/models/annotations/dynamic.rb +++ b/app/models/annotations/dynamic.rb @@ -132,8 +132,9 @@ def handle_elasticsearch_response(op) # Index response for team tasks or free text tasks if task&.annotated_type == 'ProjectMedia' && (task.team_task_id || self.annotation_type == 'task_response_free_text') pm = task.project_media + return if pm.nil? if op == 'destroy' - handle_destroy_response(task, pm) + handle_destroy_response(task, pm.id) else # OP will be update for choices tasks as it's already created in TASK model(add_elasticsearch_task) op = self.annotation_type =~ /choice/ ? 'update' : op @@ -189,14 +190,14 @@ def handle_annotated_by(op) end end - def handle_destroy_response(task, pm) + def handle_destroy_response(task, pm_id) # destroy choice should reset the answer to nil to keep search for ANY/NON value in ES # so it'll be update action for choice # otherwise delete the field from ES if self.annotation_type =~ /choice/ task.add_update_elasticsearch_task('update') else - task.destroy_es_items('task_responses', 'destroy_doc_nested', pm.id) + task.destroy_es_items('task_responses', 'destroy_doc_nested', pm_id) end end diff --git a/app/models/bot/keep.rb b/app/models/bot/keep.rb index f56c929372..e319c986a2 100644 --- a/app/models/bot/keep.rb +++ b/app/models/bot/keep.rb @@ -137,7 +137,7 @@ def create_archive_annotation(type) return if self.should_skip_create_archive_annotation?(type) data = begin JSON.parse(self.media.metadata_annotation.get_field_value('metadata_value')) rescue self.media.pender_data end - return unless data.has_key?('archives') + return unless data&.has_key?('archives') a = Dynamic.where(annotation_type: 'archiver', annotated_type: self.class_name, annotated_id: self.id).last if a.nil? a = Dynamic.new From 64d2c088d39fbdb70d7d9c33df88b0d725d5bd36 Mon Sep 17 00:00:00 2001 From: Darius Kazemi Date: Tue, 21 Nov 2023 14:47:13 -0800 Subject: [PATCH 45/54] Updating feed invitation mailer url --- app/mailers/feed_invitation_mailer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mailers/feed_invitation_mailer.rb b/app/mailers/feed_invitation_mailer.rb index c6f21c4f55..4b1d88f42c 100644 --- a/app/mailers/feed_invitation_mailer.rb +++ b/app/mailers/feed_invitation_mailer.rb @@ -8,7 +8,7 @@ def notify(record_id, team_id) @user = record.user @feed = record.feed @due_at = record.created_at + CheckConfig.get('feed_invitation_due_to', 30).to_i.days - @accept_feed_url = "#{CheckConfig.get('checkdesk_client')}/#{team.slug}/feed-invitation/#{record.id}" + @accept_feed_url = "#{CheckConfig.get('checkdesk_client')}/check/feed/#{@feed.id}/invitation" subject = I18n.t("mails_notifications.feed_invitation.subject", user: @user.name, feed: @feed.name) Rails.logger.info "Sending a feed invitation e-mail to #{@recipient}" mail(to: @recipient, email_type: 'feed_invitation', subject: subject) From 8ec1653f4c15a450225dcec576de69f36bada55b Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 22 Nov 2023 15:40:41 +0200 Subject: [PATCH 46/54] CV2-4028: fix url strip (#1737) --- config/initializers/field_validators.rb | 3 ++- test/models/dynamic_annotation/field_test.rb | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/initializers/field_validators.rb b/config/initializers/field_validators.rb index 75d9a53705..50056915f0 100644 --- a/config/initializers/field_validators.rb +++ b/config/initializers/field_validators.rb @@ -16,7 +16,8 @@ def field_validator_type_url errormsg = I18n.t(:url_invalid_value) urls = self.value urls.each do |item| - url = URI.parse(item['url'].strip!) + item['url'] = item['url'].strip + url = URI.parse(item['url']) errors.add(:base, errormsg + ' ' + url.to_s) unless url.is_a?(URI::HTTP) && !url.host.nil? end end diff --git a/test/models/dynamic_annotation/field_test.rb b/test/models/dynamic_annotation/field_test.rb index e72abb7022..ce938c6d3e 100644 --- a/test/models/dynamic_annotation/field_test.rb +++ b/test/models/dynamic_annotation/field_test.rb @@ -263,6 +263,9 @@ def field_formatter_name_response f = create_field field_name: 'url', value: [{ 'url' => ' https://archive.org/web/ ' }] end assert_equal 'https://archive.org/web/', f.reload.value[0]['url'] + assert_nothing_raised do + create_field field_name: 'url', value: [{ 'url' => random_url }] + end end test "should ignore permission check for changing status if previous value is empty" do From 991af1e52051deb10301a78bce2f4d29f0d5cb2b Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Wed, 22 Nov 2023 15:04:31 -0300 Subject: [PATCH 47/54] =?UTF-8?q?3994=20=E2=80=93=20Improve=20seeds=20file?= =?UTF-8?q?:=20add=20variation=20to=20requests=20(#1735)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip: add variation to requests add: - timeout_search_requests: Search result no feedback (fields / smooch_request_type) - relevant_search_result_requests: Search result positive feedback (fields / smooch_request_type) - different sources (source / type) - add random tipline message links --- db/seeds.rb | 56 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index c35ca45c33..5145c59c5b 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -80,20 +80,6 @@ def create_fact_checks(user, claim_descriptions) claim_descriptions.each { |claim_description| FactCheck.create!(summary: Faker::Company.catch_phrase, title: Faker::Company.name, user: user, claim_description: claim_description, language: 'en') } end -def verify_fact_check_and_publish_report(project_media, url = '') - status = ['verified', 'false'].sample - - verification_status = project_media.last_status_obj - verification_status.status = status - verification_status.save! - - report_design = project_media.get_dynamic_annotation('report_design') - report_design.set_fields = { status_label: status, state: 'published' }.to_json - report_design.data[:options][:published_article_url] = url - report_design.action = 'publish' - report_design.save! -end - def fact_check_attributes(fact_check_link, user, project, team) { summary: Faker::Company.catch_phrase, @@ -125,9 +111,22 @@ def create_project_medias_with_channel(user, project, team, data) end def create_tipline_user_and_data(project_media, team) + tipline_message_data = { + link: 'https://www.nytimes.com/interactive/2023/09/28/world/europe/russia-ukraine-war-map-front-line.html', + audio: "#{random_url}/wnHkwjykxOqU3SMWpEpuVzSa.oga", + video: "#{random_url}/AOVFpYOfMm_ssRUizUQhJHDD.mp4", + image: "#{random_url}/bOoeoeV9zNA51ecial0eWDG6.jpeg", + facebook: 'https://www.facebook.com/boomlive/posts/pfbid0ZoZPYTQAAmrrPR2XmpZ2BCPED1UgozxFGxSQiH68Aa6BF1Cvx2uWHyHrFrAwK7RPl', + instagram: 'https://www.instagram.com/p/CxsV1Gcskk8/?img_index=1', + tiktok: 'https://www.tiktok.com/@235flavien/video/7271360629615758597?_r=1&_t=8fFCIWTDWVt', + twitter: 'https://twitter.com/VietFactCheck/status/1697642909883892175', + youtube: 'https://www.youtube.com/watch?v=4EIHB-DG_JA', + text: Faker::Lorem.paragraph(sentence_count: 10) + } + tipline_user_name = Faker::Name.first_name.downcase tipline_user_surname = Faker::Name.last_name - tipline_text = Faker::Lorem.paragraph(sentence_count: 10) + tipline_message = tipline_message_data.values.sample((1..10).to_a.sample).join(' ') phone = [ Faker::PhoneNumber.phone_number, Faker::PhoneNumber.cell_phone, Faker::PhoneNumber.cell_phone_in_e164, Faker::PhoneNumber.phone_number_with_country_code, Faker::PhoneNumber.cell_phone_with_country_code].sample uid = random_string @@ -170,14 +169,14 @@ def create_tipline_user_and_data(project_media, team) smooch_user_app_id: random_string, smooch_user_data: smooch_user_data.to_json } - + Dynamic.create!(annotation_type: 'smooch_user', annotated: team, annotator: BotUser.smooch_user, set_fields: fields.to_json) # Tipline request smooch_data = { 'role': 'appUser', 'source': { - 'type': 'whatsapp', + 'type': ['whatsapp', 'telegram', 'messenger'].sample, 'id': random_string, 'integrationId': random_string, 'originalMessageId': random_string, @@ -188,17 +187,18 @@ def create_tipline_user_and_data(project_media, team) '_id': random_string, 'type': 'text', 'received': Time.now.to_f, - 'text': tipline_text, + 'text': tipline_message, 'language': 'en', 'mediaUrl': nil, 'mediaSize': 0, 'archived': 3, 'app_id': random_string } - + fields = { - smooch_request_type: 'default_requests', - smooch_data: smooch_data.to_json + smooch_request_type: ['default_requests', 'timeout_search_requests', 'relevant_search_result_requests'].sample, + smooch_data: smooch_data.to_json, + smooch_report_received: [Time.now.to_i, nil].sample } Dynamic.create!(annotation_type: 'smooch', annotated: project_media, annotator: BotUser.smooch_user, set_fields: fields.to_json) @@ -208,6 +208,20 @@ def create_tipline_requests(team, project_medias, x_times) project_medias.each {|pm| x_times.times {create_tipline_user_and_data(pm, team)}} end +def verify_fact_check_and_publish_report(project_media, url = '') + status = ['verified', 'false'].sample + + verification_status = project_media.last_status_obj + verification_status.status = status + verification_status.save! + + report_design = project_media.get_dynamic_annotation('report_design') + report_design.set_fields = { status_label: status, state: 'published' }.to_json + report_design.data[:options][:published_article_url] = url + report_design.action = 'publish' + report_design.save! +end + ###################### # 0. Start the script puts "If you want to create a new user: press 1 then enter" From 342d5f739503f4d2eba5936c62a244097a2a93c1 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Wed, 22 Nov 2023 18:23:50 -0300 Subject: [PATCH 48/54] Small changes to feed permissions * Expose FeedTeam and FeedInvitation permissions * Invalidate team permissions cache * Do not return feed invitations to users that are not admins in the feed creator organization Reference: CV2-4027. --- app/graph/types/feed_type.rb | 7 +++++++ app/lib/check_permissions.rb | 4 ++-- app/models/ability.rb | 1 + test/models/ability_test.rb | 6 +++++- test/models/team_2_test.rb | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/app/graph/types/feed_type.rb b/app/graph/types/feed_type.rb index 702faaba3b..71fe8b9d24 100644 --- a/app/graph/types/feed_type.rb +++ b/app/graph/types/feed_type.rb @@ -43,6 +43,13 @@ def requests(**args) end field :feed_invitations, FeedInvitationType.connection_type, null: false + + def feed_invitations + ability = context[:ability] || Ability.new + return FeedInvitation.none unless ability.can?(:read_feed_invitations, object) + object.feed_invitations + end + field :teams, TeamType.connection_type, null: false field :feed_teams, FeedTeamType.connection_type, null: false end diff --git a/app/lib/check_permissions.rb b/app/lib/check_permissions.rb index 762d5b8a8e..159ec8bee4 100644 --- a/app/lib/check_permissions.rb +++ b/app/lib/check_permissions.rb @@ -54,7 +54,7 @@ def permissions(ability = nil, klass = self.class) if self.class.name == 'Team' role = User.current.role(self) role ||= 'authenticated' - cache_key = "team_permissions_#{self.private.to_i}_#{role}_role" + cache_key = "team_permissions_#{self.private.to_i}_#{role}_role_202311221222" perms = Rails.cache.read(cache_key) if Rails.cache.exist?(cache_key) end if perms.blank? @@ -77,7 +77,7 @@ def set_custom_permissions(ability = nil) def get_create_permissions { - 'Team' => [Project, Account, TeamUser, User, TagText, ProjectMedia, TiplineNewsletter, Feed], + 'Team' => [Project, Account, TeamUser, User, TagText, ProjectMedia, TiplineNewsletter, Feed, FeedTeam, FeedInvitation], 'Account' => [Media, Link, Claim], 'Media' => [ProjectMedia, Comment, Tag, Dynamic, Task], 'Link' => [ProjectMedia, Comment, Tag, Dynamic, Task], diff --git a/app/models/ability.rb b/app/models/ability.rb index f0526a34a2..5c4c7e6ece 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -59,6 +59,7 @@ def admin_perms can [:update, :destroy], TeamUser, team_id: @context_team.id can :duplicate, Team, :id => @context_team.id can :set_privacy, Project, :team_id => @context_team.id + can :read_feed_invitations, Feed, :team_id => @context_team.id end def editor_perms diff --git a/test/models/ability_test.rb b/test/models/ability_test.rb index cdef014064..a4a1def3fe 100644 --- a/test/models/ability_test.rb +++ b/test/models/ability_test.rb @@ -714,7 +714,8 @@ def teardown "bulk_create Tag", "bulk_update ProjectMedia", "create TagText", "read Team", "update Team", "destroy Team", "empty Trash", "create Project", "create Account", "create TeamUser", "create User", "create ProjectMedia", "invite Members", "not_spam ProjectMedia", "restore ProjectMedia", "confirm ProjectMedia", "update ProjectMedia", "duplicate Team", "create Feed", - "manage TagText", "manage TeamTask", "set_privacy Project", "update Relationship", "destroy Relationship", "create TiplineNewsletter" + "manage TagText", "manage TeamTask", "set_privacy Project", "update Relationship", "destroy Relationship", "create TiplineNewsletter", + "create FeedInvitation", "create FeedTeam" ] project_perms = [ "read Project", "update Project", "destroy Project", "create Source", "create Media", "create ProjectMedia", @@ -1323,16 +1324,19 @@ def teardown ability = Ability.new assert ability.can?(:destroy, ft2) assert ability.can?(:destroy, ft3) + assert ability.can?(:destroy, f) end with_current_user_and_team(u2, t2) do ability = Ability.new assert ability.can?(:destroy, ft2) assert ability.cannot?(:destroy, ft3) + assert ability.cannot?(:destroy, f) end with_current_user_and_team(u3, t3) do ability = Ability.new assert ability.cannot?(:destroy, ft2) assert ability.can?(:destroy, ft3) + assert ability.cannot?(:destroy, f) end end end diff --git a/test/models/team_2_test.rb b/test/models/team_2_test.rb index 87ec2681d7..5736dcb8bf 100644 --- a/test/models/team_2_test.rb +++ b/test/models/team_2_test.rb @@ -292,7 +292,7 @@ def setup "destroy Team", "empty Trash", "create Project", "create ProjectMedia", "create Account", "create TeamUser", "create User", "invite Members", "not_spam ProjectMedia", "restore ProjectMedia", "confirm ProjectMedia", "update ProjectMedia", "duplicate Team", "manage TagText", "manage TeamTask", "set_privacy Project", "update Relationship", - "destroy Relationship", "create TiplineNewsletter", "create Feed" + "destroy Relationship", "create TiplineNewsletter", "create Feed", "create FeedTeam", "create FeedInvitation" ].sort # load permissions as owner From 65200752bb330691e44f09965df9c88243172afa Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Thu, 23 Nov 2023 12:24:48 -0300 Subject: [PATCH 49/54] Do not crash if link metadata contains invalid JSON when parsing Keep request Do not crash if link metadata contains invalid JSON when parsing Keep request. References: CV2-3966. --- app/mailers/feed_invitation_mailer.rb | 3 +-- app/models/bot/keep.rb | 16 ++++++++++------ app/models/feed_invitation.rb | 2 +- test/mailers/feed_invitation_mailer_test.rb | 9 +++------ test/models/bot/keep_test.rb | 13 +++++++++++++ 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/app/mailers/feed_invitation_mailer.rb b/app/mailers/feed_invitation_mailer.rb index 4b1d88f42c..ff5540fdb5 100644 --- a/app/mailers/feed_invitation_mailer.rb +++ b/app/mailers/feed_invitation_mailer.rb @@ -1,9 +1,8 @@ class FeedInvitationMailer < ApplicationMailer layout nil - def notify(record_id, team_id) + def notify(record_id) record = FeedInvitation.find_by_id record_id - team = Team.find_by_id team_id @recipient = record.email @user = record.user @feed = record.feed diff --git a/app/models/bot/keep.rb b/app/models/bot/keep.rb index e319c986a2..f9ad2f553c 100644 --- a/app/models/bot/keep.rb +++ b/app/models/bot/keep.rb @@ -60,6 +60,15 @@ def self.valid_request?(request) end end + def self.save_archive_information(link, response, payload) + m = link.metadata_annotation + data = begin JSON.parse(m.get_field_value('metadata_value')) rescue {} end + data['archives'] ||= {} + data['archives'][payload['type']] = response + m.set_fields = { metadata_value: data.to_json }.to_json + m.save! + end + def self.webhook(request) payload = JSON.parse(request.raw_post) if payload['url'] @@ -71,12 +80,7 @@ def self.webhook(request) else type = Bot::Keep.archiver_to_annotation_type(payload['type']) response = Bot::Keep.set_response_based_on_pender_data(type, payload) || { error: true } - m = link.metadata_annotation - data = JSON.parse(m.get_field_value('metadata_value')) - data['archives'] ||= {} - data['archives'][payload['type']] = response - m.set_fields = { metadata_value: data.to_json }.to_json - m.save! + Bot::Keep.save_archive_information(link, response, payload) project_media = ProjectMedia.where(media_id: link.id) raise ObjectNotReadyError.new('ProjectMedia') unless project_media.count > 0 diff --git a/app/models/feed_invitation.rb b/app/models/feed_invitation.rb index d2aa1b06f8..3b983386cc 100644 --- a/app/models/feed_invitation.rb +++ b/app/models/feed_invitation.rb @@ -28,6 +28,6 @@ def set_user end def send_feed_invitation_mail - FeedInvitationMailer.delay.notify(self.id, Team.current.id) + FeedInvitationMailer.delay.notify(self.id) end end diff --git a/test/mailers/feed_invitation_mailer_test.rb b/test/mailers/feed_invitation_mailer_test.rb index acaa9b05d9..2d6a3e5644 100644 --- a/test/mailers/feed_invitation_mailer_test.rb +++ b/test/mailers/feed_invitation_mailer_test.rb @@ -1,15 +1,12 @@ -require "test_helper" +require_relative '../test_helper' class FeedInvitationMailerTest < ActionMailer::TestCase - test "should notify about feed invitation" do + test 'should notify about feed invitation' do fi = create_feed_invitation - t = create_team - Team.stubs(:current).returns(t) - email = FeedInvitationMailer.notify(fi.id, t.id) + email = FeedInvitationMailer.notify(fi.id) assert_emails 1 do email.deliver_now end assert_equal [fi.email], email.to - Team.unstub(:current) end end diff --git a/test/models/bot/keep_test.rb b/test/models/bot/keep_test.rb index d7242454a4..ca406648e6 100644 --- a/test/models/bot/keep_test.rb +++ b/test/models/bot/keep_test.rb @@ -212,4 +212,17 @@ def fake_request(url) assert_match /Archiver annotation for ProjectMedia not found/, error.message end + + test ".webhook archiving doesn't raise exception if metadata contains special characters" do + url = 'https://example.com/foo' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + pender_response = '{"type":"media","data":{"url":"' + url + '","type":"item"}}' + WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: pender_response) + pm = create_project_media url: url + create_dynamic_annotation annotated: pm, annotation_type: 'archiver', set_fields: {}.to_json + Dynamic.any_instance.stubs(:get_field_value).with('metadata_value').returns('invalid JSON') + assert_nothing_raised do + Bot::Keep.webhook(fake_request(url)) + end + end end From 4ad28558457380f4e0abf63bcbc8e78b5d977a08 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Fri, 24 Nov 2023 15:41:26 -0300 Subject: [PATCH 50/54] When a user leaves a feed, any previous feed invitation they received for that feed should be deleted This way, they can be invited again in the future. Fixes: CV2-3802. --- app/models/feed_team.rb | 7 +++++++ test/models/feed_team_test.rb | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/app/models/feed_team.rb b/app/models/feed_team.rb index 1f79262ff7..23971ba034 100644 --- a/app/models/feed_team.rb +++ b/app/models/feed_team.rb @@ -8,6 +8,8 @@ class FeedTeam < ApplicationRecord validates_presence_of :team_id, :feed_id validate :saved_search_belongs_to_feed_team + after_destroy :delete_invitations + def requests_filters=(filters) filters = filters.is_a?(String) ? JSON.parse(filters) : filters self.send(:set_requests_filters, filters) @@ -24,4 +26,9 @@ def saved_search_belongs_to_feed_team errors.add(:saved_search_id, I18n.t(:"errors.messages.invalid_feed_saved_search_value")) if self.team_id != self.saved_search.team_id end end + + def delete_invitations + # Delete invitations to that feed when a user leaves a feed so they can be invited again in the future + FeedInvitation.where(email: User.current.email, feed_id: self.feed_id).delete_all unless User.current.blank? + end end diff --git a/test/models/feed_team_test.rb b/test/models/feed_team_test.rb index 9447674589..36a67aded1 100644 --- a/test/models/feed_team_test.rb +++ b/test/models/feed_team_test.rb @@ -65,4 +65,19 @@ def setup ft.save! assert_equal 'bar', ft.reload.get_requests_filters[:foo] end + + test "should delete invitations when leaving feed" do + u = create_user + ft = create_feed_team + create_team_user user: u, team: ft.team, role: 'admin' + create_feed_invitation feed: ft.feed + fi = create_feed_invitation email: u.email, feed: ft.feed + assert_not_nil FeedInvitation.find_by_id(fi.id) + User.current = u + assert_difference 'FeedInvitation.count', -1 do + ft.destroy! + end + User.current = nil + assert_nil FeedInvitation.find_by_id(fi.id) + end end From 3b2bb356eb5f1f9ee2c43c46173eef183bff2bb4 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Fri, 24 Nov 2023 18:04:03 -0300 Subject: [PATCH 51/54] If a user declines a feed invitation, they can be invited again Fixes CV2-3802. --- app/models/ability.rb | 2 +- app/models/feed_invitation.rb | 3 ++- lib/check_basic_abilities.rb | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/models/ability.rb b/app/models/ability.rb index 5c4c7e6ece..ced37dbdaf 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -85,7 +85,7 @@ def editor_perms obj.annotation.team&.id == @context_team.id end can [:create, :update, :read, :destroy], [Account, Source, TiplineNewsletter, TiplineResource, Feed, FeedTeam], :team_id => @context_team.id - can [:create, :update, :destroy], FeedInvitation, { feed: { team_id: @context_team.id } } + can [:create, :update], FeedInvitation, { feed: { team_id: @context_team.id } } can :destroy, FeedTeam do |obj| obj.team.id == @context_team.id || obj.feed.team.id == @context_team.id end diff --git a/app/models/feed_invitation.rb b/app/models/feed_invitation.rb index 3b983386cc..c7c2ec49fd 100644 --- a/app/models/feed_invitation.rb +++ b/app/models/feed_invitation.rb @@ -18,7 +18,8 @@ def accept!(team_id) end def reject! - self.update_column(:state, :rejected) + # self.update_column(:state, :rejected) + self.destroy! end private diff --git a/lib/check_basic_abilities.rb b/lib/check_basic_abilities.rb index 8b5691d720..5d57449858 100644 --- a/lib/check_basic_abilities.rb +++ b/lib/check_basic_abilities.rb @@ -125,8 +125,8 @@ def extra_perms_for_all_users !(@user.cached_teams & obj.feed.team_ids).empty? end - can :read, FeedInvitation do |obj| - @user.email == obj.email || @user.id == obj.user_id + can [:read, :destroy], FeedInvitation do |obj| + @user.email == obj.email || @user.id == obj.user_id || TeamUser.where(user_id: @user.id, team_id: obj.feed.team_id, role: 'admin').exists? end end From 031f43120eaf5c411def7b9bdcb4ec62927b1b86 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 24 Nov 2023 20:19:48 -0300 Subject: [PATCH 52/54] Fixing test --- test/models/feed_invitation_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/models/feed_invitation_test.rb b/test/models/feed_invitation_test.rb index d3020579f9..02d3b36d59 100644 --- a/test/models/feed_invitation_test.rb +++ b/test/models/feed_invitation_test.rb @@ -61,7 +61,7 @@ def teardown end assert_equal 'accepted', fi.reload.state fi.reject! - assert_equal 'rejected', fi.reload.state + assert_nil FeedInvitation.find_by_id(fi.id) end test "should send email after create feed invitation" do From cd835a51653db50ed550541f8491e3f4f68f4218 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Sun, 26 Nov 2023 09:43:57 -0300 Subject: [PATCH 53/54] Reverting conversation history sort order until pagination is implemented on the frontend side References: CV2-3776 and CV2-3703. --- .../mutations/tipline_messages_pagination.rb | 96 +++++++++---------- app/graph/types/team_type.rb | 2 +- app/graph/types/tipline_message_type.rb | 6 +- test/controllers/graphql_controller_9_test.rb | 21 ++++ 4 files changed, 73 insertions(+), 52 deletions(-) diff --git a/app/graph/mutations/tipline_messages_pagination.rb b/app/graph/mutations/tipline_messages_pagination.rb index 4de538ee16..1b68fe42b0 100644 --- a/app/graph/mutations/tipline_messages_pagination.rb +++ b/app/graph/mutations/tipline_messages_pagination.rb @@ -1,50 +1,50 @@ class TiplineMessagesPagination < GraphQL::Pagination::ArrayConnection - def cursor_for(item) - encode(item.id.to_i.to_s) - end - - def load_nodes - @nodes ||= begin - sliced_nodes = if before && after - end_idx = index_from_cursor(before) - start_idx = index_from_cursor(after) - items.where(id: start_idx..end_idx) - elsif before - end_idx = index_from_cursor(before) - items.where('id < ?', end_idx) - elsif after - start_idx = index_from_cursor(after) - items.where('id > ?', start_idx) - else - items - end - - @has_previous_page = if last - # There are items preceding the ones in this result - sliced_nodes.count > last - elsif after - # We've paginated into the Array a bit, there are some behind us - index_from_cursor(after) > items.map(&:id).min - else - false - end - - @has_next_page = if first - # There are more items after these items - sliced_nodes.count > first - elsif before - # The original array is longer than the `before` index - index_from_cursor(before) < items.map(&:id).max - else - false - end - - limited_nodes = sliced_nodes - - limited_nodes = limited_nodes.first(first) if first - limited_nodes = limited_nodes.last(last) if last - - limited_nodes - end - end +# def cursor_for(item) +# encode(item.id.to_i.to_s) +# end +# +# def load_nodes +# @nodes ||= begin +# sliced_nodes = if before && after +# end_idx = index_from_cursor(before) +# start_idx = index_from_cursor(after) +# items.where(id: start_idx..end_idx) +# elsif before +# end_idx = index_from_cursor(before) +# items.where('id < ?', end_idx) +# elsif after +# start_idx = index_from_cursor(after) +# items.where('id > ?', start_idx) +# else +# items +# end +# +# @has_previous_page = if last +# # There are items preceding the ones in this result +# sliced_nodes.count > last +# elsif after +# # We've paginated into the Array a bit, there are some behind us +# index_from_cursor(after) > items.map(&:id).min +# else +# false +# end +# +# @has_next_page = if first +# # There are more items after these items +# sliced_nodes.count > first +# elsif before +# # The original array is longer than the `before` index +# index_from_cursor(before) < items.map(&:id).max +# else +# false +# end +# +# limited_nodes = sliced_nodes +# +# limited_nodes = limited_nodes.first(first) if first +# limited_nodes = limited_nodes.last(last) if last +# +# limited_nodes +# end +# end end diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index 7c28ff6350..2b42dbe43c 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -302,6 +302,6 @@ def shared_teams end def tipline_messages(uid:) - TiplineMessagesPagination.new(object.tipline_messages.where(uid: uid).order('id ASC')) + TiplineMessagesPagination.new(object.tipline_messages.where(uid: uid).order('sent_at DESC')) end end diff --git a/app/graph/types/tipline_message_type.rb b/app/graph/types/tipline_message_type.rb index 88b1384878..545dbaa9f7 100644 --- a/app/graph/types/tipline_message_type.rb +++ b/app/graph/types/tipline_message_type.rb @@ -22,7 +22,7 @@ def sent_at object.sent_at.to_i.to_s end - def cursor - GraphQL::Schema::Base64Encoder.encode(object.id.to_i.to_s) - end + # def cursor + # GraphQL::Schema::Base64Encoder.encode(object.id.to_i.to_s) + # end end diff --git a/test/controllers/graphql_controller_9_test.rb b/test/controllers/graphql_controller_9_test.rb index a984c86860..f44b72d287 100644 --- a/test/controllers/graphql_controller_9_test.rb +++ b/test/controllers/graphql_controller_9_test.rb @@ -409,7 +409,28 @@ def setup assert_equal "Not Found", JSON.parse(@response.body)['errors'][0]['message'] end + test "should get latest tipline messages" do + t = create_team slug: 'test', private: true + u = create_user + create_team_user user: u, team: t, role: 'admin' + uid = random_string + + tm1 = create_tipline_message team_id: t.id, uid: uid, sent_at: Time.now.ago(2.hours) + tm2 = create_tipline_message team_id: t.id, uid: uid, sent_at: Time.now.ago(1.hour) + + authenticate_with_user(u) + + query = 'query latest { team(slug: "test") { tipline_messages(first: 1, uid:"' + uid + '") { edges { node { dbid } } } } }' + post :create, params: { query: query } + assert_response :success + data = JSON.parse(@response.body)['data']['team']['tipline_messages'] + results = data['edges'].to_a.collect{ |e| e['node']['dbid'] } + assert_equal 1, results.size + assert_equal tm2.id, results[0] + end + test "should paginate tipline messages" do + skip 'Pagination disabled in this commit' t = create_team slug: 'test', private: true u = create_user create_team_user user: u, team: t, role: 'admin' From 651feb9dba1d66afa8b0efc1f07daac081fce0fd Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 27 Nov 2023 06:19:43 -0800 Subject: [PATCH 54/54] CV2-3828 First pass on check api integration updates for audio on alegre (#1725) * CV2-3828 First pass on check api integration updates for audio on alegre * change request name to avoid clash * fix * more fixes * remove test that is no longer needed * resolve last few fixture issues * more excising of request_api * remove more query_or_body examples * remove more query_or_body examples * more fixes on fixtures * not sure how contract tests work, but the 'query' field needs to be converted to 'body' in order for this to work * resolve errors and back down to sending encoded www forms * drop down to queries again * fix one more stray query * tweak request method to accommodate passive / uniform query setting * resolve more broken tests * updates to tests * reset contract test * force embedded data to json on get requests * force embedded data to json on get requests * json-encode content inside query params * only set header for posts * allow deletes to use post bodies for the moment * move all GET routes to POSTs * fix pathing issue * more post change fixes * fix more to_json tests * fix more to_json tests * resolve some more stragglers * final fixes on post rewrite * Update alegre_v2.rb per PR discussion * Update alegre_similarity.rb per PR discussion --- app/lib/smooch_nlu.rb | 4 +- app/models/bot/alegre.rb | 71 +-- app/models/concerns/alegre_similarity.rb | 31 +- app/models/concerns/alegre_v2.rb | 242 ++++++++ app/models/request.rb | 8 +- app/workers/reindex_alegre_workspace.rb | 2 +- spec/pacts/check_api-alegre.json | 71 ++- test/contract/alegre_contract_test.rb | 41 +- test/controllers/elastic_search_9_test.rb | 16 +- test/controllers/feeds_controller_test.rb | 12 +- test/controllers/graphql_controller_8_test.rb | 14 +- test/controllers/reports_controller_test.rb | 12 +- test/lib/smooch_nlu_test.rb | 14 +- test/models/bot/alegre_2_test.rb | 112 ++-- test/models/bot/alegre_3_test.rb | 66 +- test/models/bot/alegre_test.rb | 18 +- test/models/bot/alegre_v2_test.rb | 582 ++++++++++++++++++ test/models/bot/smooch_6_test.rb | 23 +- test/models/request_test.rb | 49 +- test/workers/reindex_alegre_workspace_test.rb | 6 +- 20 files changed, 1091 insertions(+), 303 deletions(-) create mode 100644 app/models/concerns/alegre_v2.rb create mode 100644 test/models/bot/alegre_v2_test.rb diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index c88a483f4a..2af050bc4d 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -50,7 +50,7 @@ def update_keywords(language, keywords, keyword, operation, doc_id, context) alegre_params = common_alegre_params.merge({ quiet: true }) end # FIXME: Add error handling and better logging - Bot::Alegre.request_api(alegre_operation, '/text/similarity/', alegre_params) if alegre_operation && alegre_params + Bot::Alegre.request(alegre_operation, '/text/similarity/', alegre_params) if alegre_operation && alegre_params keywords end @@ -79,7 +79,7 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k language: language, }.merge(context) } - response = Bot::Alegre.request_api('get', '/text/similarity/', params) + response = Bot::Alegre.request('post', '/text/similarity/search/', params) # One approach would be to take the option that has the most matches # Unfortunately this approach is influenced by the number of keywords per option diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb index d0b1e76aa3..eecd91db14 100644 --- a/app/models/bot/alegre.rb +++ b/app/models/bot/alegre.rb @@ -7,6 +7,7 @@ class Error < ::StandardError include AlegreSimilarity include AlegreWebhooks + include AlegreV2 # Text similarity models MEAN_TOKENS_MODEL = 'xlm-r-bert-base-nli-stsb-mean-tokens' @@ -32,7 +33,7 @@ def similar_items_ids_and_scores(team_ids, thresholds = {}) 'UploadedImage' => 'image', }[self.media.type].to_s threshold = [{value: thresholds.dig(media_type.to_sym, :value)}] || Bot::Alegre.get_threshold_for_query(media_type, self, true) - ids_and_scores = Bot::Alegre.get_items_with_similar_media(Bot::Alegre.media_file_url(self), threshold, team_ids, "/#{media_type}/similarity/").to_h + ids_and_scores = Bot::Alegre.get_items_with_similar_media(Bot::Alegre.media_file_url(self), threshold, team_ids, "/#{media_type}/similarity/search/").to_h elsif self.is_text? ids_and_scores = {} threads = [] @@ -150,10 +151,14 @@ def self.run(body) if body.dig(:event) == 'create_project_media' && !pm.nil? Rails.logger.info("[Alegre Bot] [ProjectMedia ##{pm.id}] This item was just created, processing...") self.get_language(pm) - self.send_to_media_similarity_index(pm) - self.send_field_to_similarity_index(pm, 'original_title') - self.send_field_to_similarity_index(pm, 'original_description') - self.relate_project_media_to_similar_items(pm) + if self.get_pm_type(pm) == "audio" + self.relate_project_media(pm) + else + self.send_to_media_similarity_index(pm) + self.send_field_to_similarity_index(pm, 'original_title') + self.send_field_to_similarity_index(pm, 'original_description') + self.relate_project_media_to_similar_items(pm) + end self.get_extracted_text(pm) self.get_flags(pm) self.auto_transcription(pm) @@ -224,7 +229,7 @@ def self.get_items_from_similar_text(team_id, text, fields = nil, threshold = ni threshold ||= self.get_threshold_for_query('text', nil, true) models ||= [self.matching_model_to_use(team_ids)].flatten Hash[self.get_similar_items_from_api( - '/text/similarity/', + '/text/similarity/search/', self.similar_texts_from_api_conditions(text, models, fuzzy, team_ids, fields, threshold), threshold ).collect{|k,v| [k, v.merge(model: v[:model]||Bot::Alegre.default_matching_model)]}] @@ -348,7 +353,7 @@ def self.auto_transcription(pm) def self.get_language_from_alegre(text) lang = 'und' begin - response = self.request_api('post', '/text/langid/', { text: text }) + response = self.request('post', '/text/langid/', { text: text }) lang = response['result']['language'] || lang rescue nil @@ -370,21 +375,21 @@ def self.save_annotation(pm, type, fields) def self.get_flags(pm) if pm.report_type == 'uploadedimage' - result = self.request_api('get', '/image/classification/', { uri: self.media_file_url(pm) }, 'query') + result = self.request('post', '/image/classification/', { uri: self.media_file_url(pm) }) self.save_annotation(pm, 'flag', result['result']) end end def self.get_extracted_text(pm) if pm.report_type == 'uploadedimage' - result = self.request_api('get', '/image/ocr/', { url: self.media_file_url(pm) }, 'query') + result = self.request('post', '/image/ocr/', { url: self.media_file_url(pm) }) self.save_annotation(pm, 'extracted_text', result) if result end end def self.update_audio_transcription(transcription_annotation_id, attempts) annotation = Dynamic.find(transcription_annotation_id) - result = self.request_api('get', '/audio/transcription/', { job_name: annotation.get_field_value('job_name') }) + result = self.request('post', '/audio/transcription/result/', { job_name: annotation.get_field_value('job_name') }) completed = false if result['job_status'] == 'COMPLETED' annotation.disable_es_callbacks = Rails.env.to_s == 'test' @@ -395,7 +400,7 @@ def self.update_audio_transcription(transcription_annotation_id, attempts) elsif result['job_status'] == 'DONE' completed = true end - self.delay_for(10.seconds, retry: 5).update_audio_transcription(annotation.id, attempts + 1) if !completed && attempts < 2000 # Maximum: ~5h of transcription + self.delay_for(10.seconds, retry: 5).update_audio_transcription(annotation.id, attempts + 1) if !completed && attempts < 200 # Maximum: ~5h of transcription end def self.transcribe_audio(pm) @@ -404,7 +409,7 @@ def self.transcribe_audio(pm) url = self.media_file_url(pm) job_name = Digest::MD5.hexdigest(URI(url).open.read) s3_url = url.gsub(/^https?:\/\/[^\/]+/, "s3://#{CheckConfig.get('storage_bucket')}") - result = self.request_api('post', '/audio/transcription/', { url: s3_url, job_name: job_name }) + result = self.request('post', '/audio/transcription/', { url: s3_url, job_name: job_name }) annotation = self.save_annotation(pm, 'transcription', { text: '', job_name: job_name, last_response: result }) if result # FIXME: Calculate schedule interval based on audio duration self.delay_for(10.seconds, retry: 5).update_audio_transcription(annotation.id, 1) @@ -420,7 +425,7 @@ def self.media_file_url(pm) url end - def self.item_doc_id(object, field_name) + def self.item_doc_id(object, field_name=nil) Base64.encode64(["check", object.class.to_s.underscore, object&.id, field_name].join("-")).strip.delete("\n").delete("=") end @@ -469,46 +474,6 @@ def self.get_alegre_tbi(team_id) tbi end - def self.request_api(method, path, params = {}, query_or_body = 'body', retries = 3) - # Release database connection while Alegre API is being called - if RequestStore.store[:pause_database_connection] - ActiveRecord::Base.clear_active_connections! - ActiveRecord::Base.connection.close - end - uri = URI(CheckConfig.get('alegre_host') + path) - klass = 'Net::HTTP::' + method.capitalize - request = klass.constantize.new(uri.path, 'Content-Type' => 'application/json') - if query_or_body == 'query' - request.set_form_data(params) - request = Net::HTTP::Get.new(uri.path+ '?' + request.body) - else - request.body = params.to_json - end - http = Net::HTTP.new(uri.hostname, uri.port) - http.use_ssl = uri.scheme == 'https' - begin - response = http.request(request) - Rails.logger.info("[Alegre Bot] Alegre Bot request: (#{method}, #{path}, #{params.inspect}, #{query_or_body}, #{retries})") - response_body = response.body - Rails.logger.info("[Alegre Bot] Alegre response: #{response_body.inspect}") - ActiveRecord::Base.connection.reconnect! if RequestStore.store[:pause_database_connection] - parsed_response = JSON.parse(response_body) - if parsed_response.dig("queue") == 'audio__Model' && parsed_response.dig("body", "callback_url") != nil - redis = Redis.new(REDIS_CONFIG) - redis_response = redis.blpop("alegre:webhook:#{parsed_response.dig("body", "id")}", 120) - return JSON.parse(redis_response[1]) - end - parsed_response - rescue StandardError => e - if retries > 0 - sleep 1 - self.request_api(method, path, params, query_or_body , retries - 1) - end - Rails.logger.error("[Alegre Bot] Alegre error: #{e.message}") - { 'type' => 'error', 'data' => { 'message' => e.message } } - end - end - def self.extract_project_medias_from_context(search_result) # We currently have two cases of context: # - a straight hash with project_media_id diff --git a/app/models/concerns/alegre_similarity.rb b/app/models/concerns/alegre_similarity.rb index b5187cafbd..ec7ce94f9d 100644 --- a/app/models/concerns/alegre_similarity.rb +++ b/app/models/concerns/alegre_similarity.rb @@ -35,11 +35,11 @@ def get_similar_items(pm) end end - def get_items_with_similarity(type, pm, threshold, query_or_body = 'body') + def get_items_with_similarity(type, pm, threshold) if type == 'text' self.get_merged_items_with_similar_text(pm, threshold) else - results = self.get_items_with_similar_media(self.media_file_url(pm), threshold, pm.team_id, "/#{type}/similarity/", query_or_body).reject{ |id, _score_with_context| pm.id == id } + results = self.get_items_with_similar_media(self.media_file_url(pm), threshold, pm.team_id, "/#{type}/similarity/search/").reject{ |id, _score_with_context| pm.id == id } self.merge_response_with_source_and_target_fields(results, type) end end @@ -110,23 +110,13 @@ def delete_field_from_text_similarity_index(pm, field, quiet=false) end def delete_from_text_similarity_index(doc_id, context, quiet=false) - self.request_api('delete', '/text/similarity/', { + self.request('delete', '/text/similarity/', { doc_id: doc_id, context: context, quiet: quiet }) end - def get_context(pm, field=nil) - context = { - team_id: pm.team_id, - project_media_id: pm.id, - has_custom_id: true - } - context[:field] = field if field - context - end - def send_to_text_similarity_index_package(pm, field, text, doc_id) models ||= self.indexing_models_to_use(pm) language = self.language_for_similarity(pm&.team_id) @@ -141,7 +131,7 @@ def send_to_text_similarity_index_package(pm, field, text, doc_id) end def send_to_text_similarity_index(pm, field, text, doc_id) - self.request_api( + self.request( 'post', '/text/similarity/', self.send_to_text_similarity_index_package(pm, field, text, doc_id) @@ -168,7 +158,7 @@ def delete_from_media_similarity_index(pm, quiet=false) quiet: quiet, context: self.get_context(pm), } - self.request_api( + self.request( 'delete', "/#{type}/similarity/", params @@ -186,7 +176,7 @@ def send_to_media_similarity_index(pm) match_across_content_types: true, requires_callback: true } - self.request_api( + self.request( 'post', "/#{type}/similarity/", params @@ -214,10 +204,10 @@ def get_merged_similar_items(pm, threshold, fields, value, team_ids = [pm&.team_ es_matches end - def get_similar_items_from_api(path, conditions, _threshold = {}, query_or_body = 'body') + def get_similar_items_from_api(path, conditions, _threshold = {}) Rails.logger.error("[Alegre Bot] Sending request to alegre : #{path} , #{conditions.to_json}") response = {} - result = self.request_api('get', path, conditions, query_or_body)&.dig('result') + result = self.request('post', path, conditions)&.dig('result') project_medias = result.collect{ |r| self.extract_project_medias_from_context(r) } if !result.nil? && result.is_a?(Array) project_medias.each do |request_response| request_response.each do |pmid, score_with_context| @@ -293,7 +283,7 @@ def similar_texts_from_api_conditions(text, models, fuzzy, team_id, fields, thre params end - def get_items_with_similar_media(media_url, threshold, team_id, path, query_or_body = 'body') + def get_items_with_similar_media(media_url, threshold, team_id, path) self.get_similar_items_from_api( path, self.similar_media_content_from_api_conditions( @@ -301,8 +291,7 @@ def get_items_with_similar_media(media_url, threshold, team_id, path, query_or_b media_url, threshold ), - threshold, - query_or_body + threshold ) end diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb new file mode 100644 index 0000000000..74d88ddede --- /dev/null +++ b/app/models/concerns/alegre_v2.rb @@ -0,0 +1,242 @@ +require 'active_support/concern' + +module AlegreV2 + extend ActiveSupport::Concern + + module ClassMethods + def host + CheckConfig.get('alegre_host') + end + + def sync_path + "/similarity/sync/audio" + end + + def async_path + "/similarity/async/audio" + end + + def delete_path(project_media) + type = get_type(project_media) + "/#{type}/similarity/" + end + + def release_db + if RequestStore.store[:pause_database_connection] + ActiveRecord::Base.clear_active_connections! + ActiveRecord::Base.connection.close + end + end + + def reconnect_db + ActiveRecord::Base.connection.reconnect! if RequestStore.store[:pause_database_connection] + end + + def get_request_object(method, _path, uri) + full_path = uri.path + full_path += "?#{uri.query}" if uri.query + headers = ["post", "delete"].include?(method.downcase) ? {'Content-Type' => 'application/json'} : {} + return ('Net::HTTP::' + method.capitalize).constantize.new(full_path, headers) + end + + def generate_request(method, path, params) + uri = URI(host + path) + request = get_request_object(method, path, uri) + if method.downcase == 'post' || method.downcase == 'delete' + request.body = params.to_json + end + http = Net::HTTP.new(uri.hostname, uri.port) + http.use_ssl = uri.scheme == 'https' + return http, request + end + + def run_request(http, request) + http.request(request) + end + + def parse_response(http, request) + release_db + response = run_request(http, request) + reconnect_db + JSON.parse(response.body) + end + + def request(method, path, params, retries=3) + http, request = generate_request(method, path, params) + begin + Rails.logger.info("[Alegre Bot] Alegre Bot request: (#{method}, #{path}, #{params.inspect}, #{retries})") + parsed_response = parse_response(http, request) + Rails.logger.info("[Alegre Bot] Alegre response: #{parsed_response.inspect}") + parsed_response + rescue StandardError => e + if retries > 0 + sleep 1 + self.request(method, path, params, retries - 1) + end + Rails.logger.error("[Alegre Bot] Alegre error: #{e.message}") + { 'type' => 'error', 'data' => { 'message' => e.message } } + end + end + + def request_delete(data, project_media) + request("delete", delete_path(project_media), data) + end + + def request_sync(data) + request("post", sync_path, data) + end + + def request_async(data) + request("post", async_path, data) + end + + def get_type(project_media) + type = nil + if project_media.is_text? + type = 'text' + elsif project_media.is_image? + type = 'image' + elsif project_media.is_video? + type = 'video' + elsif project_media.is_audio? + type = 'audio' + end + return type + end + + def generic_package(project_media, field) + { + doc_id: item_doc_id(project_media, field), + context: get_context(project_media, field) + } + end + + def delete_package(project_media, field, params={}, quiet=false) + generic_package(project_media, field).merge( + self.send("delete_package_#{get_type(project_media)}", project_media, field, params) + ).merge( + quiet: quiet + ).merge(params) + end + + def generic_package_audio(project_media, params) + generic_package(project_media, nil).merge( + url: media_file_url(project_media), + ).merge(params) + end + + def delete_package_audio(project_media, _field, params) + generic_package_audio(project_media, params) + end + + def store_package(project_media, field, params={}) + generic_package(project_media, field).merge( + self.send("store_package_#{get_type(project_media)}", project_media, field, params) + ) + end + + def is_not_generic_field(field) + !["audio", "video", "image"].include?(field) + end + + def get_context(project_media, field=nil) + context = { + team_id: project_media.team_id, + project_media_id: project_media.id, + has_custom_id: true + } + context[:field] = field if field && is_not_generic_field(field) + context + end + + def store_package_audio(project_media, _field, params) + generic_package_audio(project_media, params) + end + + def get_sync(project_media, field=nil, params={}) + request_sync( + store_package(project_media, field, params) + ) + end + + def get_async(project_media, field=nil, params={}) + request_async( + store_package(project_media, field, params) + ) + end + + def delete(project_media, field=nil, params={}) + request_delete( + delete_package(project_media, field, params), + project_media + ) + end + + def get_per_model_threshold(project_media, threshold) + type = get_type(project_media) + if type == "text" + {per_model_threshold: threshold.collect{|x| {model: x[:model], value: x[:value]}}} + else + {threshold: threshold[0][:value]} + end + end + + def isolate_relevant_context(project_media, result) + result["context"].select{|x| x["team_id"] == project_media.team_id}.first + end + + def get_target_field(project_media, field) + type = get_type(project_media) + return field if type == "text" + return type if !type.nil? + field || type + end + + def parse_similarity_results(project_media, field, results, relationship_type) + Hash[results.collect{|result| + result["context"] = isolate_relevant_context(project_media, result) + [ + result["context"] && result["context"]["project_media_id"], + { + score: result["score"], + context: result["context"], + model: result["model"], + source_field: get_target_field(project_media, field), + target_field: get_target_field(project_media, result["field"]), + relationship_type: relationship_type + } + ] + }.reject{|k,_| k == project_media.id}] + end + + def get_items(project_media, field, confirmed=false) + relationship_type = confirmed ? Relationship.confirmed_type : Relationship.suggested_type + type = get_type(project_media) + threshold = get_per_model_threshold(project_media, Bot::Alegre.get_threshold_for_query(type, project_media, confirmed)) + parse_similarity_results( + project_media, + field, + get_sync(project_media, field, threshold)["result"], + relationship_type + ) + end + + def get_suggested_items(project_media, field) + get_items(project_media, field) + end + + def get_confirmed_items(project_media, field) + get_items(project_media, field, true) + end + + def get_similar_items_v2(project_media, field) + suggested_or_confirmed = get_suggested_items(project_media, field) + confirmed = get_confirmed_items(project_media, field) + Bot::Alegre.merge_suggested_and_confirmed(suggested_or_confirmed, confirmed, project_media) + end + + def relate_project_media(project_media, field=nil) + self.add_relationships(project_media, self.get_similar_items_v2(project_media, field)) unless project_media.is_blank? + end + end +end diff --git a/app/models/request.rb b/app/models/request.rb index 7ba630dfc5..058d93755c 100644 --- a/app/models/request.rb +++ b/app/models/request.rb @@ -53,13 +53,13 @@ def attach_to_similar_request!(alegre_limit = 20) models_thresholds = self.text_similarity_settings.reject{ |_k, v| v['min_words'] > words } if models_thresholds.count > 0 params = { text: media.quote, models: models_thresholds.keys, per_model_threshold: models_thresholds.transform_values{ |v| v['threshold'] }, limit: alegre_limit, context: context } - similar_request_id = ::Bot::Alegre.request_api('get', '/text/similarity/', params)&.dig('result').to_a.collect{ |result| result&.dig('_source', 'context', 'request_id').to_i }.find{ |id| id != 0 && id < self.id } + similar_request_id = ::Bot::Alegre.request('post', '/text/similarity/search/', params)&.dig('result').to_a.collect{ |result| result&.dig('_source', 'context', 'request_id').to_i }.find{ |id| id != 0 && id < self.id } end elsif ['UploadedImage', 'UploadedAudio', 'UploadedVideo'].include?(media.type) threshold = 0.85 #FIXME: Should be feed setting type = media.type.gsub(/^Uploaded/, '').downcase params = { url: media.file.file.public_url, threshold: threshold, limit: alegre_limit, context: context } - similar_request_id = ::Bot::Alegre.request_api('get', "/#{type}/similarity/", params)&.dig('result').to_a.collect{ |result| result&.dig('context').to_a.collect{ |c| c['request_id'].to_i } }.flatten.find{ |id| id != 0 && id < self.id } + similar_request_id = ::Bot::Alegre.request('post', "/#{type}/similarity/search/", params)&.dig('result').to_a.collect{ |result| result&.dig('context').to_a.collect{ |c| c['request_id'].to_i } }.flatten.find{ |id| id != 0 && id < self.id } end end unless similar_request_id.blank? @@ -194,7 +194,7 @@ def self.send_to_alegre(id) models: request.text_similarity_settings.keys(), context: context } - ::Bot::Alegre.request_api('post', '/text/similarity/', params) + ::Bot::Alegre.request('post', '/text/similarity/', params) elsif ['UploadedImage', 'UploadedAudio', 'UploadedVideo'].include?(media.type) type = media.type.gsub(/^Uploaded/, '').downcase url = media.file&.file&.public_url @@ -204,7 +204,7 @@ def self.send_to_alegre(id) context: context, match_across_content_types: true, } - ::Bot::Alegre.request_api('post', "/#{type}/similarity/", params) + ::Bot::Alegre.request('post', "/#{type}/similarity/", params) end end diff --git a/app/workers/reindex_alegre_workspace.rb b/app/workers/reindex_alegre_workspace.rb index 7a4783ed69..093f8e56af 100644 --- a/app/workers/reindex_alegre_workspace.rb +++ b/app/workers/reindex_alegre_workspace.rb @@ -68,7 +68,7 @@ def check_for_write(running_bucket, event_id, team_id, write_remains=false, in_p # manage dispatch of documents to bulk similarity api call in parallel if running_bucket.length > 500 || write_remains log(event_id, 'Writing to Alegre...') - Parallel.map(running_bucket.each_slice(30).to_a, in_processes: in_processes) { |bucket_slice| Bot::Alegre.request_api('post', '/text/bulk_similarity/', { documents: bucket_slice }) } + Parallel.map(running_bucket.each_slice(30).to_a, in_processes: in_processes) { |bucket_slice| Bot::Alegre.request('post', '/text/bulk_similarity/', { documents: bucket_slice }) } log(event_id, 'Wrote to Alegre.') # track state in case job needs to restart write_last_id(event_id, team_id, running_bucket.last[:context][:project_media_id]) if running_bucket.length > 0 && running_bucket.last[:context] diff --git a/spec/pacts/check_api-alegre.json b/spec/pacts/check_api-alegre.json index efd3351514..71d96bc073 100644 --- a/spec/pacts/check_api-alegre.json +++ b/spec/pacts/check_api-alegre.json @@ -10,9 +10,11 @@ "description": "a request to extract text", "providerState": "an image URL", "request": { - "method": "get", + "method": "post", "path": "/image/ocr/", - "query": "url=https%3A%2F%2Fi.imgur.com%2FewGClFQ.png" + "body": { + "url": "https://i.imgur.com/ewGClFQ.png" + } }, "response": { "status": 200, @@ -24,6 +26,37 @@ } } }, + { + "description": "a request to link similar images", + "providerState": "an image URL", + "request": { + "method": "post", + "path": "/image/similarity/search/", + "body": { + "url": "https://i.imgur.com/ewGClFQ.png", + "threshold": 0.89 + } + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "body": { + "result": [ + { + "id": 1, + "sha256": "9bb1b8da9eec7caaea50099ba0488a1bdd221305a327097057fb8f626b6ba39b", + "phash": 26558343354958209, + "url": "https://i.imgur.com/ewGClFQ.png", + "context": { + }, + "score": 0 + } + ] + } + } + }, { "description": "a request to identify its language", "providerState": "a text exists", @@ -62,9 +95,11 @@ "description": "a request to get image flags", "providerState": "an image URL", "request": { - "method": "get", + "method": "post", "path": "/image/classification/", - "query": "uri=https%3A%2F%2Fi.imgur.com%2FewGClFQ.png" + "body": { + "uri": "https://i.imgur.com/ewGClFQ.png" + } }, "response": { "status": 200, @@ -84,34 +119,6 @@ } } } - }, - { - "description": "a request to link similar images", - "providerState": "an image URL", - "request": { - "method": "get", - "path": "/image/similarity/", - "query": "url=https%3A%2F%2Fi.imgur.com%2FewGClFQ.png&threshold=0.89" - }, - "response": { - "status": 200, - "headers": { - "Content-Type": "application/json" - }, - "body": { - "result": [ - { - "id": 1, - "sha256": "9bb1b8da9eec7caaea50099ba0488a1bdd221305a327097057fb8f626b6ba39b", - "phash": 26558343354958209, - "url": "https://i.imgur.com/ewGClFQ.png", - "context": { - }, - "score": 0 - } - ] - } - } } ], "metadata": { diff --git a/test/contract/alegre_contract_test.rb b/test/contract/alegre_contract_test.rb index 2d83ca96be..4d491c6f14 100644 --- a/test/contract/alegre_contract_test.rb +++ b/test/contract/alegre_contract_test.rb @@ -20,12 +20,13 @@ def setup @flags = {:flags=>{"adult"=>1, "spoof"=>1, "medical"=>2, "violence"=>1, "racy"=>1, "spam"=>0}} end - def stub_similarity_requests(url) + def stub_similarity_requests(url, pm) WebMock.stub_request(:post, 'http://localhost:3100/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://localhost:3100/text/similarity/').to_return(body: { success: true }.to_json) WebMock.stub_request(:post, 'http://localhost:3100/image/similarity/').to_return(body: { 'success': true }.to_json) - WebMock.stub_request(:get, 'http://localhost:3100/image/classification/').with({ query: { uri: url} }).to_return(body:{ result: @flags }.to_json) - WebMock.stub_request(:get, 'http://localhost:3100/image/similarity/').to_return(body: { "result": [] }.to_json) + WebMock.stub_request(:post, 'http://localhost:3100/image/classification/').with(body: { uri: url}).to_return(body:{ result: @flags }.to_json) + WebMock.stub_request(:post, 'http://localhost:3100/image/similarity/search/').with(body: {:url=>"https://i.imgur.com/ewGClFQ.png", :context=>{:has_custom_id=>true, :team_id=>pm.team_id}, :match_across_content_types=>true, :threshold=>0.89}).to_return(body: { "result": [] }.to_json) + WebMock.stub_request(:post, 'http://localhost:3100/image/similarity/search/').with(body: {:url=>"https://i.imgur.com/ewGClFQ.png", :context=>{:has_custom_id=>true, :team_id=>pm.team_id}, :match_across_content_types=>true, :threshold=>0.95}).to_return(body: { "result": [] }.to_json) end # def teardown @@ -57,15 +58,14 @@ def stub_similarity_requests(url) test "should get image flags" do stub_configs({ 'alegre_host' => 'http://localhost:3100' }) do - stub_similarity_requests(@url2) - WebMock.stub_request(:get, 'http://localhost:3100/image/ocr/').with({ query: { url: @url } }).to_return(body: { "text": @extracted_text }.to_json) + WebMock.stub_request(:post, 'http://localhost:3100/image/ocr/').with({ body: { url: @url } }).to_return(body: { "text": @extracted_text }.to_json) Bot::Alegre.unstub(:media_file_url) alegre.given('an image URL'). upon_receiving('a request to get image flags'). with( - method: :get, + method: :post, path: '/image/classification/', - query: { uri: @url }, + body: { uri: @url }, ). will_respond_with( status: 200, @@ -75,6 +75,7 @@ def stub_similarity_requests(url) body: { result: @flags } ) pm1 = create_project_media team: @pm.team, media: create_uploaded_image + stub_similarity_requests(@url2, pm1) Bot::Alegre.stubs(:media_file_url).with(pm1).returns(@url) assert Bot::Alegre.run({ data: { dbid: pm1.id }, event: 'create_project_media' }) assert_not_nil pm1.get_annotations('flag').last @@ -84,15 +85,14 @@ def stub_similarity_requests(url) test "should extract text" do stub_configs({ 'alegre_host' => 'http://localhost:3100' }) do - stub_similarity_requests(@url) - WebMock.stub_request(:get, 'http://localhost:3100/text/similarity/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://localhost:3100/text/similarity/search/').to_return(body: {success: true}.to_json) Bot::Alegre.unstub(:media_file_url) alegre.given('an image URL'). upon_receiving('a request to extract text'). with( - method: :get, + method: :post, path: '/image/ocr/', - query: { url: @url }, + body: { url: @url } ). will_respond_with( status: 200, @@ -103,6 +103,7 @@ def stub_similarity_requests(url) ) pm2 = create_project_media team: @pm.team, media: create_uploaded_image + stub_similarity_requests(@url, pm2) Bot::Alegre.stubs(:media_file_url).with(pm2).returns(@url) assert Bot::Alegre.run({ data: { dbid: pm2.id }, event: 'create_project_media' }) extracted_text_annotation = pm2.get_annotations('extracted_text').last @@ -113,22 +114,19 @@ def stub_similarity_requests(url) test "should link similar images" do stub_configs({ 'alegre_host' => 'http://localhost:3100' }) do - stub_similarity_requests(@url) - WebMock.stub_request(:get, 'http://localhost:3100/image/ocr/').with({ query: { url: @url } }).to_return(body: { "text": @extracted_text }.to_json) + WebMock.stub_request(:post, 'http://localhost:3100/image/classification/').with(body: { uri: @url2}).to_return(body:{ result: @flags }.to_json) + WebMock.stub_request(:post, 'http://localhost:3100/image/ocr/').with({ body: { url: @url } }).to_return(body: { "text": @extracted_text }.to_json) pm1 = create_project_media team: @pm.team, media: create_uploaded_image + stub_similarity_requests(@url, pm1) Bot::Alegre.stubs(:media_file_url).with(pm1).returns(@url) assert Bot::Alegre.run({ data: { dbid: pm1.id }, event: 'create_project_media' }) Bot::Alegre.unstub(:media_file_url) alegre.given('an image URL'). upon_receiving('a request to link similar images'). with( - method: :get, - path: '/image/similarity/', - query: { - url: @url, - threshold: "0.89", - context: {} - } + method: :post, + path: '/image/similarity/search/', + body: {url: @url,threshold: 0.89} ). will_respond_with( status: 200, @@ -147,10 +145,9 @@ def stub_similarity_requests(url) } ] } - ) conditions = {url: @url, threshold: 0.89} - Bot::Alegre.get_similar_items_from_api('/image/similarity/', conditions, 0.89, 'query') + Bot::Alegre.get_similar_items_from_api('/image/similarity/search/', conditions, 0.89) end end end diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb index 3c9c87f229..95a63d299b 100644 --- a/test/controllers/elastic_search_9_test.rb +++ b/test/controllers/elastic_search_9_test.rb @@ -117,27 +117,29 @@ def setup bot.install_to!(team) create_flag_annotation_type create_extracted_text_annotation_type - Bot::Alegre.unstub(:request_api) Rails.stubs(:env).returns('development'.inquiry) stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'result' => { 'language' => 'es' }}.to_json) WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:get, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:get, 'http://alegre/image/similarity/').to_return(body: { - "result": [] - }.to_json) - WebMock.stub_request(:get, 'http://alegre/image/classification/').with({ query: { uri: 'some/path' } }).to_return(body: { + WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/image/classification/').with({ body: { uri: 'some/path' } }).to_return(body: { "result": valid_flags_data }.to_json) - WebMock.stub_request(:get, 'http://alegre/image/ocr/').with({ query: { url: 'some/path' } }).to_return(body: { + WebMock.stub_request(:post, 'http://alegre/image/ocr/').with({ body: { url: 'some/path' } }).to_return(body: { "text": "ocr_text" }.to_json) WebMock.stub_request(:post, 'http://alegre/image/similarity/').to_return(body: 'success') # Text extraction Bot::Alegre.unstub(:media_file_url) pm = create_project_media team: team, media: create_uploaded_image, disable_es_callbacks: false + WebMock.stub_request(:post, 'http://alegre/image/similarity/search/').with(body: {context: {:has_custom_id=>true, :team_id=>pm.team_id}, match_across_content_types: true, threshold: 0.89, url: "some/path"}).to_return(body: { + "result": [] + }.to_json) + WebMock.stub_request(:post, 'http://alegre/image/similarity/search/').with(body: {context: {:has_custom_id=>true, :team_id=>pm.team_id}, match_across_content_types: true, threshold: 0.95, url: "some/path"}).to_return(body: { + "result": [] + }.to_json) Bot::Alegre.stubs(:media_file_url).with(pm).returns("some/path") assert Bot::Alegre.run({ data: { dbid: pm.id }, event: 'create_project_media' }) sleep 2 diff --git a/test/controllers/feeds_controller_test.rb b/test/controllers/feeds_controller_test.rb index c82cbc187c..cd7b60666a 100644 --- a/test/controllers/feeds_controller_test.rb +++ b/test/controllers/feeds_controller_test.rb @@ -105,7 +105,7 @@ def teardown end test "should save request query" do - Bot::Alegre.stubs(:request_api).returns({}) + Bot::Alegre.stubs(:request).returns({}) Sidekiq::Testing.inline! authenticate_with_token @a assert_difference 'Request.count' do @@ -116,28 +116,28 @@ def teardown assert_response :success assert_equal 2, json_response['data'].size assert_equal 2, json_response['meta']['record-count'] - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should save relationship between request and results" do - Bot::Alegre.stubs(:request_api).returns({}) + Bot::Alegre.stubs(:request).returns({}) Sidekiq::Testing.inline! authenticate_with_token @a assert_difference 'Request.count' do get :index, params: { filter: { type: 'text', query: 'Foo', feed_id: @f.id } } end assert_response :success - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should parse the full query" do Bot::Smooch.unstub(:search_for_similar_published_fact_checks) - Bot::Alegre.stubs(:request_api).returns({}) + Bot::Alegre.stubs(:request).returns({}) Sidekiq::Testing.inline! authenticate_with_token @a get :index, params: { filter: { type: 'text', query: 'Foo, bar and test', feed_id: @f.id } } assert_response :success assert_equal 'Foo, bar and test', Media.last.quote - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end end diff --git a/test/controllers/graphql_controller_8_test.rb b/test/controllers/graphql_controller_8_test.rb index 837207a20c..e3220e1b1b 100644 --- a/test/controllers/graphql_controller_8_test.rb +++ b/test/controllers/graphql_controller_8_test.rb @@ -390,12 +390,11 @@ def setup test "should get OCR" do b = create_alegre_bot(name: 'alegre', login: 'alegre') b.approve! - Bot::Alegre.unstub(:request_api) stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do Sidekiq::Testing.fake! do WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ - WebMock.stub_request(:get, 'http://alegre/image/ocr/').with({ query: { url: "some/path" } }).to_return(body: { text: 'Foo bar' }.to_json) - WebMock.stub_request(:get, 'http://alegre/text/similarity/') + WebMock.stub_request(:post, 'http://alegre/image/ocr/').with({ body: { url: "some/path" } }).to_return(body: { text: 'Foo bar' }.to_json) + WebMock.stub_request(:post, 'http://alegre/text/similarity/') u = create_user t = create_team @@ -803,10 +802,9 @@ def setup url = Bot::Alegre.media_file_url(pm) s3_url = url.gsub(/^https?:\/\/[^\/]+/, "s3://#{CheckConfig.get('storage_bucket')}") - Bot::Alegre.unstub(:request_api) - Bot::Alegre.stubs(:request_api).returns({ success: true }) - Bot::Alegre.stubs(:request_api).with('post', '/audio/transcription/', { url: s3_url, job_name: '0c481e87f2774b1bd41a0a70d9b70d11' }).returns({ 'job_status' => 'IN_PROGRESS' }) - Bot::Alegre.stubs(:request_api).with('get', '/audio/transcription/', { job_name: '0c481e87f2774b1bd41a0a70d9b70d11' }).returns({ 'job_status' => 'COMPLETED', 'transcription' => 'Foo bar' }) + Bot::Alegre.stubs(:request).returns({ success: true }) + Bot::Alegre.stubs(:request).with('post', '/audio/transcription/', { url: s3_url, job_name: '0c481e87f2774b1bd41a0a70d9b70d11' }).returns({ 'job_status' => 'IN_PROGRESS' }) + Bot::Alegre.stubs(:request).with('post', '/audio/transcription/result/', { job_name: '0c481e87f2774b1bd41a0a70d9b70d11' }).returns({ 'job_status' => 'COMPLETED', 'transcription' => 'Foo bar' }) WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'result' => { 'language' => 'es' }}.to_json) b = create_bot_user login: 'alegre', name: 'Alegre', approved: true @@ -820,7 +818,7 @@ def setup assert_response :success assert_equal 'Foo bar', JSON.parse(@response.body)['data']['transcribeAudio']['annotation']['data']['text'] - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end end diff --git a/test/controllers/reports_controller_test.rb b/test/controllers/reports_controller_test.rb index 5551ba9049..32e197a704 100644 --- a/test/controllers/reports_controller_test.rb +++ b/test/controllers/reports_controller_test.rb @@ -40,7 +40,7 @@ def from_alegre(pm) pm5 = create_project_media team: @t, media: create_uploaded_video create_project_media team: @t - Bot::Alegre.stubs(:request_api).returns({ 'result' => [from_alegre(@pm), from_alegre(pm), from_alegre(pm2), from_alegre(pm3), from_alegre(pm4), from_alegre(pm5)] }) + Bot::Alegre.stubs(:request).returns({ 'result' => [from_alegre(@pm), from_alegre(pm), from_alegre(pm2), from_alegre(pm3), from_alegre(pm4), from_alegre(pm5)] }) post :index, params: {} assert_response :success @@ -57,7 +57,7 @@ def from_alegre(pm) assert_equal 1, json_response['data'].size assert_equal 1, json_response['meta']['record-count'] - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should return empty set if Alegre doesn't return anything" do @@ -65,14 +65,14 @@ def from_alegre(pm) authenticate_with_token @a 3.times { create_project_media(team: @t) } - Bot::Alegre.stubs(:request_api).returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).returns({ 'result' => [] }) get :index, params: { filter: { similar_to_text: 'Test' } } assert_response :success assert_equal 0, json_response['data'].size assert_equal 0, json_response['meta']['record-count'] - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should return empty set if Alegre Bot is not installed" do @@ -81,13 +81,13 @@ def from_alegre(pm) 3.times { create_project_media } TeamBotInstallation.delete_all BotUser.delete_all - Bot::Alegre.stubs(:request_api).returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).returns({ 'result' => [] }) get :index, params: { filter: { similar_to_text: 'Test' } } assert_response :success assert_equal 0, json_response['data'].size assert_equal 0, json_response['meta']['record-count'] - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end end diff --git a/test/lib/smooch_nlu_test.rb b/test/lib/smooch_nlu_test.rb index 3f1b4cbaf4..dddcdec083 100644 --- a/test/lib/smooch_nlu_test.rb +++ b/test/lib/smooch_nlu_test.rb @@ -64,7 +64,7 @@ def create_team_with_smooch_bot_installed team = create_team_with_smooch_bot_installed nlu = SmoochNlu.new(team.slug) nlu.enable! - Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe') expected_output = { 'en' => { @@ -85,7 +85,7 @@ def create_team_with_smooch_bot_installed end test 'should add keyword if it does not exist' do - Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') end @@ -93,20 +93,20 @@ def create_team_with_smooch_bot_installed test 'should not add keyword if it exists' do team = create_team_with_smooch_bot_installed nlu = SmoochNlu.new(team.slug) - Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') - Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.never + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.never nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') end test 'should delete keyword' do - Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'delete' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'delete' && y == '/text/similarity/' }.once team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).remove_keyword_from_menu_option('en', 'main', 0, 'subscribe to the newsletter') end test 'should not return a menu option if NLU is not enabled' do - Bot::Alegre.stubs(:request_api).never + Bot::Alegre.stubs(:request).never team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).disable! Bot::Smooch.get_installation('smooch_id', 'test') @@ -114,7 +114,7 @@ def create_team_with_smooch_bot_installed end test 'should return a menu option if NLU is enabled' do - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => 'test' } } }, ]}) team = create_team_with_smooch_bot_installed diff --git a/test/models/bot/alegre_2_test.rb b/test/models/bot/alegre_2_test.rb index 120b3c25fc..1e26a8045f 100644 --- a/test/models/bot/alegre_2_test.rb +++ b/test/models/bot/alegre_2_test.rb @@ -16,7 +16,6 @@ def setup create_extracted_text_annotation_type Sidekiq::Testing.inline! Bot::Alegre.stubs(:should_get_similar_items_of_type?).returns(true) - Bot::Alegre.unstub(:request_api) Bot::Alegre.unstub(:media_file_url) @media_path = random_url @params = { url: @media_path, context: { has_custom_id: true, team_id: @team.id }, threshold: 0.9, match_across_content_types: true } @@ -31,7 +30,7 @@ def teardown pm1 = create_project_media team: @team, media: create_uploaded_video pm2 = create_project_media team: @team, media: create_uploaded_video pm3 = create_project_media team: @team, media: create_uploaded_video - Bot::Alegre.stubs(:request_api).with('get', '/video/similarity/', @params, 'body').returns({ + Bot::Alegre.stubs(:request).with('post', '/video/similarity/search/', @params).returns({ result: [ { context: [ @@ -53,7 +52,7 @@ def teardown assert_difference 'Relationship.count' do Bot::Alegre.relate_project_media_to_similar_items(pm3) end - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) Bot::Alegre.unstub(:media_file_url) r = Relationship.last assert_equal pm3, r.target @@ -65,7 +64,7 @@ def teardown pm1 = create_project_media team: @team, media: create_uploaded_audio pm2 = create_project_media team: @team, media: create_uploaded_audio pm3 = create_project_media team: @team, media: create_uploaded_audio - Bot::Alegre.stubs(:request_api).with('get', '/audio/similarity/', @params, 'body').returns({ + Bot::Alegre.stubs(:request).with('post', '/audio/similarity/search/', @params).returns({ result: [ { id: 1, @@ -98,6 +97,7 @@ def teardown assert_equal pm3, r.target assert_equal pm2, r.source assert_equal r.weight, 0.983167 + Bot::Alegre.unstub(:request) end test "should match audio with similar audio from video" do @@ -105,7 +105,7 @@ def teardown pm1 = create_project_media team: @team, media: create_uploaded_video pm2 = create_project_media team: @team, media: create_uploaded_audio pm3 = create_project_media team: @team, media: create_uploaded_audio - Bot::Alegre.stubs(:request_api).with('get', '/audio/similarity/', @params, 'body').returns({ + Bot::Alegre.stubs(:request).with('post', '/audio/similarity/search/', @params).returns({ result: [ { id: 2, @@ -133,7 +133,7 @@ def teardown assert_difference 'Relationship.count' do Bot::Alegre.relate_project_media_to_similar_items(pm3) end - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) Bot::Alegre.unstub(:media_file_url) r = Relationship.last assert_equal pm3, r.target @@ -171,13 +171,13 @@ def teardown } ] }.with_indifferent_access - Bot::Alegre.stubs(:request_api).with('get', '/image/similarity/', @params.merge({ threshold: 0.89 }), 'body').returns(result) - Bot::Alegre.stubs(:request_api).with('get', '/image/similarity/', @params.merge({ threshold: 0.95 }), 'body').returns(result) + Bot::Alegre.stubs(:request).with('post', '/image/similarity/search/', @params.merge({ threshold: 0.89 })).returns(result) + Bot::Alegre.stubs(:request).with('post', '/image/similarity/search/', @params.merge({ threshold: 0.95 })).returns(result) Bot::Alegre.stubs(:media_file_url).with(pm3).returns(@media_path) assert_difference 'Relationship.count' do Bot::Alegre.relate_project_media_to_similar_items(pm3) end - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) Bot::Alegre.unstub(:media_file_url) r = Relationship.last assert_equal pm3, r.target @@ -222,12 +222,12 @@ def teardown ] }.with_indifferent_access Bot::Alegre.stubs(:media_file_url).with(pm1a).returns(@media_path) - Bot::Alegre.stubs(:request_api).with('get', '/image/similarity/', @params.merge({ threshold: 0.89 }), 'body').returns(response) - Bot::Alegre.stubs(:request_api).with('get', '/image/similarity/', @params.merge({ threshold: 0.95 }), 'body').returns(response) + Bot::Alegre.stubs(:request).with('post', '/image/similarity/search/', @params.merge({ threshold: 0.89 })).returns(response) + Bot::Alegre.stubs(:request).with('post', '/image/similarity/search/', @params.merge({ threshold: 0.95 })).returns(response) assert_difference 'Relationship.count' do Bot::Alegre.relate_project_media_to_similar_items(pm1a) end - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) assert_equal pm1b, Relationship.last.source assert_equal pm1a, Relationship.last.target end @@ -237,23 +237,19 @@ def teardown ft = create_field_type field_type: 'image_path', label: 'Image Path' at = create_annotation_type annotation_type: 'reverse_image', label: 'Reverse Image' create_field_instance annotation_type_object: at, name: 'reverse_image_path', label: 'Reverse Image', field_type_object: ft, optional: false - Bot::Alegre.unstub(:request_api) stub_configs({ 'alegre_host' => 'http://alegre.test', 'alegre_token' => 'test' }) do WebMock.stub_request(:post, 'http://alegre.test/text/langid/').to_return(body: { 'result' => { 'language' => 'es' }}.to_json) WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ WebMock.stub_request(:post, 'http://alegre.test/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre.test/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:get, 'http://alegre.test/text/similarity/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre.test/text/similarity/search/').to_return(body: {success: true}.to_json) WebMock.stub_request(:post, 'http://alegre.test/image/similarity/').to_return(body: { "success": true }.to_json) - WebMock.stub_request(:get, 'http://alegre.test/image/similarity/').to_return(body: { - "result": [] - }.to_json) - WebMock.stub_request(:get, 'http://alegre.test/image/classification/').with({ query: { uri: image_path } }).to_return(body: { + WebMock.stub_request(:post, 'http://alegre.test/image/classification/').with({ body: { uri: image_path } }).to_return(body: { "result": valid_flags_data }.to_json) - WebMock.stub_request(:get, 'http://alegre.test/image/ocr/').with({ query: { url: image_path } }).to_return(body: { + WebMock.stub_request(:post, 'http://alegre.test/image/ocr/').with({ body: { url: image_path } }).to_return(body: { "text": "Foo bar" }.to_json) WebMock.stub_request(:post, 'http://alegre.test/image/similarity/').to_return(body: 'success') @@ -261,14 +257,38 @@ def teardown # Similarity t = create_team pm1 = create_project_media team: t, media: create_uploaded_image - Bot::Alegre.stubs(:media_file_url).returns(image_path) - assert Bot::Alegre.run({ data: { dbid: pm1.id }, event: 'create_project_media' }) - Bot::Alegre.unstub(:media_file_url) context = [{ "team_id" => pm1.team.id.to_s, "project_media_id" => pm1.id.to_s }] - WebMock.stub_request(:get, 'http://alegre.test/image/similarity/').with(body: /"url":"#{image_path}"/).to_return(body: { + WebMock.stub_request(:post, "http://alegre.test/image/similarity/search/").with(body: {:url=>image_path, :context=>{:has_custom_id=>true, :team_id=>pm1.team_id}, :match_across_content_types=>true, :threshold=>0.89}).to_return(body: { + "result": [ + { + "id": pm1.id, + "sha256": "1782b1d1993fcd9f6fd8155adc6009a9693a8da7bb96d20270c4bc8a30c97570", + "phash": 17399941807326929, + "url": "https:\/\/www.gstatic.com\/webp\/gallery3\/1.png", + "context": context, + "score": 0 + } + ] + }.to_json) + WebMock.stub_request(:post, "http://alegre.test/image/similarity/search/").with(body: {:url=>image_path, :context=>{:has_custom_id=>true, :team_id=>pm1.team_id}, :match_across_content_types=>true, :threshold=>0.95}).to_return(body: { + "result": [ + { + "id": pm1.id, + "sha256": "1782b1d1993fcd9f6fd8155adc6009a9693a8da7bb96d20270c4bc8a30c97570", + "phash": 17399941807326929, + "url": "https:\/\/www.gstatic.com\/webp\/gallery3\/1.png", + "context": context, + "score": 0 + } + ] + }.to_json) + Bot::Alegre.stubs(:media_file_url).returns(image_path) + assert Bot::Alegre.run({ data: { dbid: pm1.id }, event: 'create_project_media' }) + Bot::Alegre.unstub(:media_file_url) + WebMock.stub_request(:post, 'http://alegre.test/image/similarity/search/').with(body: {url: image_path}).to_return(body: { "result": [ { "id": 1, @@ -287,7 +307,7 @@ def teardown # Flags Bot::Alegre.unstub(:media_file_url) - WebMock.stub_request(:get, 'http://alegre.test/image/classification/').to_return(body: { + WebMock.stub_request(:post, 'http://alegre.test/image/classification/').to_return(body: { "result": valid_flags_data }.to_json) pm3 = create_project_media team: t, media: create_uploaded_image @@ -310,21 +330,11 @@ def teardown test "should pause database connection when calling Alegre" do RequestStore.store[:pause_database_connection] = true assert_nothing_raised do - Bot::Alegre.request_api('post', '/text/langid/') + Bot::Alegre.request('post', '/text/langid/', {}) end RequestStore.store[:pause_database_connection] = false end - test "should block calls on redis blpop for audio request" do - stubbed_response = Net::HTTPSuccess.new(1.0, '200', 'OK') - stubbed_response.stubs(:body).returns({"queue" => "audio__Model", "body" => {"id" => "123", "callback_url" => "http://example.com/callback"}}.to_json) - Net::HTTP.any_instance.stubs(:request).returns(stubbed_response) - Redis.any_instance.stubs(:blpop).with("alegre:webhook:123", 120).returns(["alegre:webhook:123", {"tested" => true}.to_json]) - assert_equal Bot::Alegre.request_api('get', '/audio/similarity/', @params, 'body'), {"tested" => true} - Net::HTTP.any_instance.unstub(:request) - Redis.any_instance.unstub(:blpop) - end - test "should get items with similar title" do create_verification_status_stuff RequestStore.store[:skip_cached_field_update] = false @@ -334,7 +344,7 @@ def teardown pm2 = create_project_media quote: "Blah2", team: @team pm2.analysis = { title: 'Title 1' } pm2.save! - Bot::Alegre.stubs(:request_api).returns({"result" => [{ + Bot::Alegre.stubs(:request).returns({"result" => [{ "_index" => "alegre_similarity", "_type" => "_doc", "_id" => "tMXj53UB36CYclMPXp14", @@ -352,7 +362,7 @@ def teardown }) response = Bot::Alegre.get_items_with_similar_title(pm, Bot::Alegre.get_threshold_for_query('text', pm)) assert_equal response.class, Hash - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should respond to a media_file_url request" do @@ -588,9 +598,9 @@ def teardown pm = create_project_media quote: "Blah", team: @team pm.analysis = { content: 'Description 1' } pm.save! - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) assert Bot::Alegre.delete_from_index(pm) - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should get items with similar description" do @@ -602,7 +612,7 @@ def teardown pm2 = create_project_media quote: "Blah2", team: @team pm2.analysis = { content: 'Description 1' } pm2.save! - Bot::Alegre.stubs(:request_api).returns({ + Bot::Alegre.stubs(:request).returns({ "result" => [ { "_source" => { @@ -621,7 +631,7 @@ def teardown }) response = Bot::Alegre.get_items_with_similar_description(pm, Bot::Alegre.get_threshold_for_query('text', pm)) assert_equal response.class, Hash - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should get items with similar title when using non-elasticsearch matching model" do @@ -633,7 +643,7 @@ def teardown pm2 = create_project_media quote: "Blah2", team: @team pm2.analysis = { title: 'Title 1' } pm2.save! - Bot::Alegre.stubs(:request_api).returns({"result" => [{ + Bot::Alegre.stubs(:request).returns({"result" => [{ "_index" => "alegre_similarity", "_type" => "_doc", "_id" => "tMXj53UB36CYclMPXp14", @@ -652,7 +662,7 @@ def teardown Bot::Alegre.stubs(:matching_model_to_use).with([pm.team_id]).returns(Bot::Alegre::MEAN_TOKENS_MODEL) response = Bot::Alegre.get_items_with_similar_title(pm, [{ key: 'text_elasticsearch_suggestion_threshold', model: 'elasticsearch', value: 0.1, automatic: false }]) assert_equal response.class, Hash - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) Bot::Alegre.unstub(:matching_model_to_use) end @@ -729,9 +739,9 @@ def teardown pm.media.type = "UploadedVideo" pm.media.save! pm.save! - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) assert Bot::Alegre.delete_from_index(pm) - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should pass through the send audio to similarity index call" do @@ -742,9 +752,9 @@ def teardown pm.media.type = "UploadedAudio" pm.media.save! pm.save! - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) assert Bot::Alegre.send_to_media_similarity_index(pm) - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should pass through the send to description similarity index call" do @@ -753,9 +763,9 @@ def teardown pm = create_project_media quote: "Blah", team: @team pm.analysis = { content: 'Description 1' } pm.save! - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) assert Bot::Alegre.send_field_to_similarity_index(pm, 'description') - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should be able to request deletion from index for a text given specific field" do @@ -764,9 +774,9 @@ def teardown pm = create_project_media quote: "Blah", team: @team pm.analysis = { content: 'Description 1' } pm.save! - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) assert Bot::Alegre.delete_from_index(pm, ['description']) - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end end diff --git a/test/models/bot/alegre_3_test.rb b/test/models/bot/alegre_3_test.rb index 3684e72b51..778fc47fb8 100644 --- a/test/models/bot/alegre_3_test.rb +++ b/test/models/bot/alegre_3_test.rb @@ -27,13 +27,13 @@ def teardown test "should return language" do stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do - WebMock.stub_request(:get, 'http://alegre/text/langid/').to_return(body: { + WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'result': { 'language': 'en', 'confidence': 1.0 } }.to_json) - Bot::Alegre.stubs(:request_api).returns({ + Bot::Alegre.stubs(:request).returns({ 'result' => { 'language' => 'en', 'confidence' => 1.0 @@ -43,21 +43,21 @@ def teardown assert_difference 'Annotation.count' do assert_equal 'en', Bot::Alegre.get_language(@pm) end - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end end test "should return language und if there is an error" do stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do - WebMock.stub_request(:get, 'http://alegre/text/langid/').to_return(body: { + WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'foo': 'bar' }.to_json) - Bot::Alegre.stubs(:request_api).raises(RuntimeError) + Bot::Alegre.stubs(:request).raises(RuntimeError) WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ assert_difference 'Annotation.count' do assert_equal 'und', Bot::Alegre.get_language(@pm) end - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end end @@ -73,7 +73,6 @@ def teardown } create_annotation_type_and_fields('Transcription', {}, json_schema) create_annotation_type_and_fields('Smooch', { 'Data' => ['JSON', true] }) - Bot::Alegre.unstub(:request_api) tbi = Bot::Alegre.get_alegre_tbi(@team.id) tbi.set_transcription_similarity_enabled = false tbi.save! @@ -82,11 +81,8 @@ def teardown WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:get, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:post, 'http://alegre/audio/similarity/').to_return(body: { - "success": true - }.to_json) - WebMock.stub_request(:get, 'http://alegre/audio/similarity/').to_return(body: { + WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/audio/similarity/search/').to_return(body: { "result": [] }.to_json) @@ -96,12 +92,13 @@ def teardown Bot::Alegre.stubs(:media_file_url).returns(media_file_url) pm1 = create_project_media team: @pm.team, media: create_uploaded_audio(file: 'rails.mp3') + WebMock.stub_request(:post, "http://alegre/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>media_file_url, :threshold=>0.9}).to_return(body: { + "result": [] + }.to_json) + WebMock.stub_request(:post, 'http://alegre/audio/transcription/result/').with(body: {job_name: "0c481e87f2774b1bd41a0a70d9b70d11"}).to_return(body: { 'job_status' => 'DONE' }.to_json) WebMock.stub_request(:post, 'http://alegre/audio/transcription/').with({ body: { url: s3_file_url, job_name: '0c481e87f2774b1bd41a0a70d9b70d11' }.to_json }).to_return(body: { 'job_status' => 'IN_PROGRESS' }.to_json) - WebMock.stub_request(:get, 'http://alegre/audio/transcription/').with( - body: { job_name: '0c481e87f2774b1bd41a0a70d9b70d11' } - ).to_return(body: { 'job_status' => 'DONE' }.to_json) # Verify with transcription_similarity_enabled = false assert Bot::Alegre.run({ data: { dbid: pm1.id }, event: 'create_project_media' }) a = pm1.annotations('transcription').last @@ -143,7 +140,6 @@ def teardown } create_annotation_type_and_fields('Transcription', {}, json_schema) create_annotation_type_and_fields('Smooch', { 'Data' => ['JSON', true] }) - Bot::Alegre.unstub(:request_api) tbi = Bot::Alegre.get_alegre_tbi(@team.id) tbi.set_transcription_similarity_enabled = false tbi.save! @@ -152,13 +148,10 @@ def teardown WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:get, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json) WebMock.stub_request(:post, 'http://alegre/audio/similarity/').to_return(body: { "success": true }.to_json) - WebMock.stub_request(:get, 'http://alegre/audio/similarity/').to_return(body: { - "result": [] - }.to_json) media_file_url = 'https://example.com/test/data/rails.mp3' s3_file_url = "s3://check-api-test/test/data/rails.mp3" @@ -166,12 +159,13 @@ def teardown Bot::Alegre.stubs(:media_file_url).returns(media_file_url) pm1 = create_project_media team: @pm.team, media: create_uploaded_audio(file: 'rails.mp3') + WebMock.stub_request(:post, "http://alegre/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>media_file_url, :threshold=>0.9}).to_return(body: { + "result": [] + }.to_json) WebMock.stub_request(:post, 'http://alegre/audio/transcription/').with({ body: { url: s3_file_url, job_name: '0c481e87f2774b1bd41a0a70d9b70d11' }.to_json }).to_return(body: { 'job_status' => 'IN_PROGRESS' }.to_json) - WebMock.stub_request(:get, 'http://alegre/audio/transcription/').with( - body: { job_name: '0c481e87f2774b1bd41a0a70d9b70d11' } - ).to_return(body: { 'job_status' => 'COMPLETED', 'transcription' => 'Foo bar' }.to_json) + WebMock.stub_request(:post, 'http://alegre/audio/transcription/result/').with(body: {job_name: "0c481e87f2774b1bd41a0a70d9b70d11"}).to_return(body: { 'job_status' => 'COMPLETED', 'transcription' => 'Foo bar' }.to_json) # Verify with transcription_similarity_enabled = false assert Bot::Alegre.run({ data: { dbid: pm1.id }, event: 'create_project_media' }) a = pm1.annotations('transcription').last @@ -206,13 +200,13 @@ def teardown test "should return true when bot is called successfully" do stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') - WebMock.stub_request(:get, 'http://alegre/text/langid/').to_return(body: { + WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'result': { 'language': 'en', 'confidence': 1.0 } }.to_json) - Bot::Alegre.stubs(:request_api).returns({ + Bot::Alegre.stubs(:request).returns({ 'result' => { 'language' => 'en', 'confidence' => 1.0 @@ -220,7 +214,7 @@ def teardown }) WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ assert Bot::Alegre.run({ data: { dbid: @pm.id }, event: 'create_project_media' }) - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end end @@ -420,7 +414,7 @@ def self.extract_project_medias_from_context(search_result) end test "should not return a malformed hash" do - Bot::Alegre.stubs(:request_api).returns({"result"=> [{ + Bot::Alegre.stubs(:request).returns({"result"=> [{ "_index"=>"alegre_similarity", "_type"=>"_doc", "_id"=>"i8XY53UB36CYclMPF5wC", @@ -444,7 +438,7 @@ def self.extract_project_medias_from_context(search_result) response = Bot::Alegre.get_similar_items_from_api("blah", {}) assert_equal response.class, Hash assert_equal response, {1932=>{:score=>200, :context=>{"team_id"=>1692, "field"=>"title|description", "project_media_id"=>1932, "contexts_count"=>2}, :model=>nil}} - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should generate correct text conditions for api request" do @@ -468,7 +462,7 @@ def self.extract_project_medias_from_context(search_result) pm2.save! Bot::Alegre.stubs(:matching_model_to_use).with([pm.team_id]).returns(Bot::Alegre::ELASTICSEARCH_MODEL) Bot::Alegre.stubs(:matching_model_to_use).with(pm2.team_id).returns(Bot::Alegre::ELASTICSEARCH_MODEL) - Bot::Alegre.stubs(:request_api).returns({"result" => [{ + Bot::Alegre.stubs(:request).returns({"result" => [{ "_index" => "alegre_similarity", "_type" => "_doc", "_id" => "tMXj53UB36CYclMPXp14", @@ -486,7 +480,7 @@ def self.extract_project_medias_from_context(search_result) }) response = Bot::Alegre.get_similar_items(pm) assert_equal response.class, Hash - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) Bot::Alegre.unstub(:matching_model_to_use) end @@ -499,7 +493,7 @@ def self.extract_project_medias_from_context(search_result) pm2 = create_project_media quote: "Blah2", team: @team pm2.analysis = { title: 'This is also a long enough Title so as to allow an actual check of other titles' } pm2.save! - Bot::Alegre.stubs(:request_api).returns({"result" => [{ + Bot::Alegre.stubs(:request).returns({"result" => [{ "_index" => "alegre_similarity", "_type" => "_doc", "_id" => "tMXj53UB36CYclMPXp14", @@ -517,7 +511,7 @@ def self.extract_project_medias_from_context(search_result) }) response = Bot::Alegre.get_items_with_similar_text(pm, ['title'], [{key: 'text_elasticsearch_suggestion_threshold', model: 'elasticsearch', value: 0.7, automatic: false}], 'blah') assert_equal response.class, Hash - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should not get items with similar short text when they are text-based" do @@ -529,7 +523,7 @@ def self.extract_project_medias_from_context(search_result) pm2 = create_project_media quote: "Blah2", team: @team pm2.analysis = { title: 'This is also a long enough Title so as to allow an actual check of other titles' } pm2.save! - Bot::Alegre.stubs(:request_api).returns({"result" => [{ + Bot::Alegre.stubs(:request).returns({"result" => [{ "_index" => "alegre_similarity", "_type" => "_doc", "_id" => "tMXj53UB36CYclMPXp14", @@ -548,7 +542,7 @@ def self.extract_project_medias_from_context(search_result) response = Bot::Alegre.get_items_with_similar_text(pm, ['title'], [{key: 'text_elasticsearch_matching_threshold', model: 'elasticsearch', value: 0.7, automatic: true}], 'blah foo bar') assert_equal response.class, Hash assert_not_empty response - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end @@ -572,9 +566,9 @@ def self.extract_project_medias_from_context(search_result) pm.media.type = "UploadedVideo" pm.media.save! pm.save! - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) assert Bot::Alegre.send_to_media_similarity_index(pm) - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should not resort matches if format is unknown" do diff --git a/test/models/bot/alegre_test.rb b/test/models/bot/alegre_test.rb index d2e890ef41..3db5f0dd5b 100644 --- a/test/models/bot/alegre_test.rb +++ b/test/models/bot/alegre_test.rb @@ -40,10 +40,10 @@ def teardown test "should capture error when failing to call service" do stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do - WebMock.stub_request(:get, 'http://alegre/text/langid/').to_return(body: 'bad JSON response') + WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response') WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response') WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') - WebMock.stub_request(:get, 'http://alegre/text/similarity/').to_return(body: 'success') + WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: 'success') WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ Bot::Alegre.any_instance.stubs(:get_language).raises(RuntimeError) assert_nothing_raised do @@ -129,7 +129,7 @@ def teardown pm1 = create_project_media project: p, quote: "for testing short text", team: @team pm2 = create_project_media project: p, quote: "testing short text", team: @team pm2.analysis = { content: 'short text' } - Bot::Alegre.stubs(:request_api).returns({ + Bot::Alegre.stubs(:request).returns({ "result" => [ { "_score" => 26.493948, @@ -147,7 +147,7 @@ def teardown # Relation should be confirmed if at least one field size > threshold pm3 = create_project_media project: p, quote: 'This is also a long enough title', team: @team pm4 = create_project_media project: p, quote: 'This is also a long enough title so as to allow an actual check of other titles', team: @team - Bot::Alegre.stubs(:request_api).returns({ + Bot::Alegre.stubs(:request).returns({ "result" => [ { "_score" => 26.493948, @@ -162,7 +162,7 @@ def teardown end r = Relationship.last assert_equal Relationship.confirmed_type, r.relationship_type - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should set similarity relationship based on date threshold" do @@ -170,7 +170,7 @@ def teardown p = create_project team: @team pm1 = create_project_media project: p, quote: "This is also a long enough Title so as to allow an actual check of other titles", team: @team pm2 = create_project_media project: p, quote: "This is also a long enough Title so as to allow an actual check of other titles 2", team: @team - Bot::Alegre.stubs(:request_api).returns({ + Bot::Alegre.stubs(:request).returns({ "result" => [ { "_score" => 26.493948, @@ -197,7 +197,7 @@ def teardown end r = Relationship.last assert_equal Relationship.suggested_type, r.relationship_type - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should index report data" do @@ -222,7 +222,7 @@ def teardown test "should use OCR data for similarity matching 2" do pm = create_project_media team: @team, media: create_uploaded_image pm2 = create_project_media team: @team, media: create_uploaded_image - Bot::Alegre.stubs(:request_api).returns({"result"=> [{ + Bot::Alegre.stubs(:request).returns({"result"=> [{ "_index"=>"alegre_similarity", "_type"=>"_doc", "_id"=>"i8XY53UB36CYclMPF5wC", @@ -250,7 +250,7 @@ def teardown assert_equal r.model, "elasticsearch" assert_equal Bot::Alegre.get_pm_type(r.source), "image" assert_equal Bot::Alegre.get_pm_type(r.target), "image" - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end diff --git a/test/models/bot/alegre_v2_test.rb b/test/models/bot/alegre_v2_test.rb new file mode 100644 index 0000000000..6f9af1d175 --- /dev/null +++ b/test/models/bot/alegre_v2_test.rb @@ -0,0 +1,582 @@ +require_relative '../../test_helper' + +class Bot::AlegreTest < ActiveSupport::TestCase + def setup + super + ft = DynamicAnnotation::FieldType.where(field_type: 'language').last || create_field_type(field_type: 'language', label: 'Language') + at = create_annotation_type annotation_type: 'language', label: 'Language' + create_field_instance annotation_type_object: at, name: 'language', label: 'Language', field_type_object: ft, optional: false + @bot = create_alegre_bot(name: "alegre", login: "alegre") + @bot.approve! + p = create_project + p.team.set_languages = ['en','pt','es'] + p.team.save! + @bot.install_to!(p.team) + @team = p.team + m = create_claim_media quote: 'I like apples' + @pm = create_project_media project: p, media: m + create_flag_annotation_type + create_extracted_text_annotation_type + Sidekiq::Testing.inline! + WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ + end + + def teardown + super + end + + test "should generate media file url" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.media_file_url(pm1).class, String + end + + test "should generate item_doc_id" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.item_doc_id(pm1).class, String + end + + test "should return proper types per object" do + p = create_project team: @team + pm1 = create_project_media project: p, team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.get_type(pm1), "audio" + pm2 = create_project_media project: p, team: @team, media: create_uploaded_video + assert_equal Bot::Alegre.get_type(pm2), "video" + pm3 = create_project_media project: p, team: @team, media: create_uploaded_image + assert_equal Bot::Alegre.get_type(pm3), "image" + pm4 = create_project_media project: p, quote: "testing short text", team: @team + assert_equal Bot::Alegre.get_type(pm4), "text" + end + + + test "should have host and paths" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.host, CheckConfig.get('alegre_host') + assert_equal Bot::Alegre.sync_path, "/similarity/sync/audio" + assert_equal Bot::Alegre.async_path, "/similarity/async/audio" + assert_equal Bot::Alegre.delete_path(pm1), "/audio/similarity/" + end + + test "should release and reconnect db" do + RequestStore.store[:pause_database_connection] = true + assert_equal Bot::Alegre.release_db.class, Thread::ConditionVariable + assert_equal Bot::Alegre.reconnect_db[0].class, PG::Result + RequestStore.store[:pause_database_connection] = false + end + + test "should create a generic_package" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.generic_package(pm1, "audio"), {:doc_id=>Bot::Alegre.item_doc_id(pm1, "audio"), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}} + end + + test "should create a generic_package_audio" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.generic_package_audio(pm1, {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1)} + assert_equal Bot::Alegre.store_package_audio(pm1, "audio", {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1)} + assert_equal Bot::Alegre.store_package(pm1, "audio", {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1)} + end + + test "should create a context for audio" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.get_context(pm1, "audio"), {:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true} + end + + test "should create a delete_package" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + package = Bot::Alegre.delete_package(pm1, "audio") + assert_equal package[:doc_id], Bot::Alegre.item_doc_id(pm1, nil) + assert_equal package[:context], {:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true} + assert_equal package[:url].class, String + assert_equal package[:quiet], false + end + + test "should run audio async request" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + response = { + "message": "Message pushed successfully", + "queue": "audio__Model", + "body": { + "callback_url": "http:\/\/alegre:3100\/presto\/receive\/add_item\/audio", + "id": "f0d43d29-853d-4099-9e92-073203afa75b", + "url": Bot::Alegre.media_file_url(pm1), + "text": nil, + "raw": { + "limit": 200, + "url": Bot::Alegre.media_file_url(pm1), + "callback_url": "http:\/\/example.com\/search_results", + "doc_id": Bot::Alegre.item_doc_id(pm1, "audio"), + "context": Bot::Alegre.get_context(pm1, "audio"), + "created_at": "2023-10-27T22:40:14.205586", + "command": "search", + "threshold": 0.0, + "per_model_threshold": {}, + "match_across_content_types": false, + "requires_callback": true, + "final_task": "search" + } + } + } + WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/async/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1)}).to_return(body: response.to_json) + assert_equal JSON.parse(Bot::Alegre.get_async(pm1).to_json), JSON.parse(response.to_json) + end + + test "should isolate relevant_context" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.isolate_relevant_context(pm1, {"context"=>[{"team_id"=>pm1.team_id}]}), {"team_id"=>pm1.team_id} + end + + test "should return field or type on get_target_field" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + Bot::Alegre.stubs(:get_type).returns(nil) + assert_equal Bot::Alegre.get_target_field(pm1, "blah"), "blah" + Bot::Alegre.unstub(:get_type) + end + + test "should generate per model threshold for text" do + p = create_project team: @team + pm1 = create_project_media project: p, quote: "testing short text", team: @team + sample = [{:value=>0.9, :key=>"vector_hash_suggestion_threshold", :automatic=>false, :model=>"vector"}] + assert_equal Bot::Alegre.get_per_model_threshold(pm1, sample), {:per_model_threshold=>[{:model=>"vector", :value=>0.9}]} + end + + test "should generate per model threshold" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + sample = [{:value=>0.9, :key=>"audio_hash_suggestion_threshold", :automatic=>false, :model=>"hash"}] + assert_equal Bot::Alegre.get_per_model_threshold(pm1, sample), {:threshold=>0.9} + end + + test "should get target field" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + assert_equal Bot::Alegre.get_target_field(pm1, nil), "audio" + end + + test "should parse similarity results" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + results = [ + { + "id"=>15346, + "doc_id"=>"Y2hlY2stcHJvamVjdF9tZWRpYS0yMzE4MC1hdWRpbw", + "chromaprint_fingerprint"=>[ + -714426431, + -731146431, + -731138797, + -597050061 + ], + "url"=>"https://qa-assets.checkmedia.org/uploads/uploaded_audio/47237/51845f9bbf47bcfc47e90ab2083f94c1.mp3", + "context"=>[{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>pm1.id}], + "score"=>1.0, + "model"=>"audio"}, + { + "id"=>15347, + "doc_id"=>"Y2hlY2stcHJvamVjdF9tZWRpYS0yMzE4MS1hdWRpbw", + "chromaprint_fingerprint"=>[ + -546788830, + -566629838, + -29630958, + -29638141 + ], + "url"=>"https://qa-assets.checkmedia.org/uploads/uploaded_audio/47238/e6cd55fd06742929124cdeaeebfa58d6.mp3", + "context"=>[{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>23181}], + "score"=>0.915364583333333, + "model"=>"audio" + } + ] + assert_equal Bot::Alegre.parse_similarity_results(pm1, nil, results, Relationship.suggested_type), {23181=>{:score=>0.915364583333333, :context=>{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>23181}, :model=>"audio", :source_field=>"audio", :target_field=>"audio", :relationship_type=>Relationship.suggested_type}} + end + + test "should run audio sync request" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + response = { + "result": [ + { + "id": 1, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75b", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.ogg", + "context": [ + { + "team_id": 1 + } + ], + "score": 1.0, + "model": "audio" + } + ] + } + WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1)}).to_return(body: response.to_json) + assert_equal JSON.parse(Bot::Alegre.get_sync(pm1).to_json), JSON.parse(response.to_json) + end + + test "should run delete request" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + response = {"requested"=> + {"limit"=>200, + "url"=>"https://qa-assets.checkmedia.org/uploads/uploaded_audio/47237/51845f9bbf47bcfc47e90ab2083f94c1.mp3", + "callback_url"=>nil, + "doc_id"=>"Y2hlY2stcHJvamVjdF9tZWRpYS0yMzE4MC1hdWRpbw", + "context"=>{"team_id"=>183, "project_media_id"=>23180, "has_custom_id"=>true}, + "created_at"=>nil, + "command"=>"delete", + "threshold"=>0.0, + "per_model_threshold"=>{}, + "match_across_content_types"=>false, + "requires_callback"=>false}, + "result"=>{"url"=>"https://qa-assets.checkmedia.org/uploads/uploaded_audio/47237/51845f9bbf47bcfc47e90ab2083f94c1.mp3", "deleted"=>1} + } + WebMock.stub_request(:delete, Bot::Alegre.host+Bot::Alegre.delete_path(pm1)).to_return(body: response.to_json) + assert_equal JSON.parse(Bot::Alegre.delete(pm1).to_json), JSON.parse(response.to_json) + end + + test "should get_items" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + response = { + "result": [ + { + "id": 1, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75b", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.ogg", + "context": [ + { + "team_id": pm1.team_id, + "project_media_id": pm1.id, + "has_custom_id": true, + } + ], + "score": 1.0, + "model": "audio" + }, + { + "id": 2, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75c", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.mp3", + "context": [ + { + "team_id": pm1.team_id, + "project_media_id": pm1.id+1, + "has_custom_id": true, + } + ], + "score": 1.0, + "model": "audio" + } + ] + } + WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json) + assert_equal Bot::Alegre.get_items(pm1, nil), {(pm1.id+1)=>{:score=>1.0, :context=>{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>(pm1.id+1)}, :model=>"audio", :source_field=>"audio", :target_field=>"audio", :relationship_type=>Relationship.suggested_type}} + end + + test "should get_suggested_items" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + response = { + "result": [ + { + "id": 1, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75b", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.ogg", + "context": [ + { + "team_id": pm1.team_id, + "project_media_id": pm1.id, + "has_custom_id": true, + } + ], + "score": 1.0, + "model": "audio" + }, + { + "id": 2, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75c", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.mp3", + "context": [ + { + "team_id": pm1.team_id, + "project_media_id": pm1.id+1, + "has_custom_id": true, + } + ], + "score": 0.91, + "model": "audio" + } + ] + } + WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json) + assert_equal Bot::Alegre.get_items(pm1, nil), {(pm1.id+1)=>{:score=>0.91, :context=>{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>(pm1.id+1)}, :model=>"audio", :source_field=>"audio", :target_field=>"audio", :relationship_type=>Relationship.suggested_type}} + end + + test "should get_confirmed_items zzz" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + response = { + "result": [ + { + "id": 1, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75b", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.ogg", + "context": [ + { + "team_id": pm1.team_id, + "project_media_id": pm1.id, + "has_custom_id": true, + } + ], + "score": 1.0, + "model": "audio" + }, + { + "id": 2, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75c", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.mp3", + "context": [ + { + "team_id": pm1.team_id, + "project_media_id": pm1.id+1, + "has_custom_id": true, + } + ], + "score": 0.91, + "model": "audio" + } + ] + } + WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json) + assert_equal Bot::Alegre.get_confirmed_items(pm1, nil), {(pm1.id+1)=>{:score=>0.91, :context=>{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>(pm1.id+1)}, :model=>"audio", :source_field=>"audio", :target_field=>"audio", :relationship_type=>Relationship.confirmed_type}} + end + + test "should get_similar_items_v2" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + response = { + "result": [ + { + "id": 1, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75b", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.ogg", + "context": [ + { + "team_id": pm1.team_id, + "project_media_id": pm1.id, + "has_custom_id": true, + } + ], + "score": 1.0, + "model": "audio" + }, + { + "id": 2, + "doc_id": "f0d43d29-853d-4099-9e92-073203afa75c", + "chromaprint_fingerprint": [ + 377259661, + 376226445, + 305001149, + 306181093, + 1379918309, + 1383899364, + 1995219172, + 1974379732, + 1957603396, + 1961789696, + 1416464641, + 1429048627, + 1999429922, + 1999380774, + 2100043878, + 2083467494, + -59895634, + -118617698, + -122811506 + ], + "url": "http:\/\/devingaffney.com\/files\/audio.mp3", + "context": [ + { + "team_id": pm1.team_id, + "project_media_id": pm1.id+1, + "has_custom_id": true, + } + ], + "score": 0.91, + "model": "audio" + } + ] + } + WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json) + assert_equal Bot::Alegre.get_similar_items_v2(pm1, nil), {} + end + + test "should relate project media for audio" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + pm2 = create_project_media team: @team, media: create_uploaded_audio + Bot::Alegre.stubs(:get_similar_items_v2).returns({pm2.id=>{:score=>0.91, :context=>{"team_id"=>pm2.team_id, "has_custom_id"=>true, "project_media_id"=>pm2.id}, :model=>"audio", :source_field=>"audio", :target_field=>"audio", :relationship_type=>Relationship.suggested_type}}) + relationship = nil + assert_difference 'Relationship.count' do + relationship = Bot::Alegre.relate_project_media(pm1) + end + assert_equal relationship.source, pm2 + assert_equal relationship.target, pm1 + assert_equal relationship.relationship_type, Relationship.suggested_type + Bot::Alegre.unstub(:get_similar_items_v2) + end +end \ No newline at end of file diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index 2ad590b041..3d7021bb24 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -138,7 +138,7 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query without details on tipline bot v2" do - WebMock.stub_request(:get, /\/text\/similarity\//).to_return(body: {}.to_json) + WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, random_string, claim, random_string, random_string, '1' assert_saved_query_type 'default_requests' @@ -208,7 +208,7 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query with details on tipline bot v2" do - WebMock.stub_request(:get, /\/text\/similarity\//).to_return(body: {}.to_json) + WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, '2', random_string, claim, '1' assert_saved_query_type 'default_requests' @@ -753,9 +753,9 @@ def send_message_outside_24_hours_window(template, pm = nil) test 'should process menu option using NLU' do # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns(true) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns(true) # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "want" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords for the newsletter menu option nlu = SmoochNlu.new(@team.slug) @@ -768,7 +768,7 @@ def send_message_outside_24_hours_window(template, pm = nil) subscription_option_id = @installation.get_smooch_workflows[0]['smooch_state_main']['smooch_menu_options'][2]['smooch_menu_option_id'] # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns({ 'result' => [ + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /want/ }.returns({ 'result' => [ { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, { '_score' => 0.2, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } ]}) @@ -782,7 +782,7 @@ def send_message_outside_24_hours_window(template, pm = nil) assert_state 'main' # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns({ 'result' => [ + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /want/ }.returns({ 'result' => [ { '_score' => 0.96, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, { '_score' => 0.91, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } ]}) @@ -798,7 +798,7 @@ def send_message_outside_24_hours_window(template, pm = nil) assert_state 'main' # Delete two keywords, so expect two calls to Alegre - Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'delete' && y == '/text/similarity/' }.twice + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'delete' && y == '/text/similarity/' }.twice nlu.remove_keyword_from_menu_option('en', 'main', 2, 'I want to subscribe to the newsletter') nlu.remove_keyword_from_menu_option('en', 'main', 2, 'I want to unsubscribe from the newsletter') end @@ -821,9 +821,9 @@ def send_message_outside_24_hours_window(template, pm = nil) Sidekiq::Testing.fake! do WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "who are you" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /who are you/ }.returns(true) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /who are you/ }.returns(true) # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "who are you" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && (z[:text] =~ /who are you/).nil? }.returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && (z[:text] =~ /who are you/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords to a new "About Us" resource nlu = SmoochNlu.new(@team.slug) @@ -833,7 +833,7 @@ def send_message_outside_24_hours_window(template, pm = nil) r.add_keyword('who are you') # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "who are you" - Bot::Alegre.stubs(:request_api).with{ |x, y, z| x == 'get' && y == '/text/similarity/' && z[:text] =~ /who are you/ }.returns({ 'result' => [ + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /who are you/ }.returns({ 'result' => [ { '_score' => 0.9, '_source' => { 'context' => { 'resource_id' => 0 } } }, { '_score' => 0.8, '_source' => { 'context' => { 'resource_id' => r.id } } } ]}) @@ -851,8 +851,9 @@ def send_message_outside_24_hours_window(template, pm = nil) assert_no_saved_query # Delete one keyword, so expect one call to Alegre - Bot::Alegre.expects(:request_api).with{ |x, y, _z| x == 'delete' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'delete' && y == '/text/similarity/' }.once r.remove_keyword('who are you') + Bot::Alegre.unstub(:request) end end diff --git a/test/models/request_test.rb b/test/models/request_test.rb index 9b0a079352..89d0c27871 100644 --- a/test/models/request_test.rb +++ b/test/models/request_test.rb @@ -116,55 +116,55 @@ def setup end test "should send text request to Alegre" do - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) assert_nothing_raised do create_request(media: create_claim_media) end - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should send media request to Alegre" do - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) assert_nothing_raised do create_request(media: create_uploaded_image) end - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should attach to similar text long" do - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) f = create_feed m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f m2 = Media.create! type: 'Claim', quote: 'Foo bar foo bar 2' r2 = create_request media: m2, feed: f response = { 'result' => [{ '_source' => { 'context' => { 'request_id' => r1.id } } }] } - Bot::Alegre.stubs(:request_api).with('get', '/text/similarity/', { text: 'Foo bar foo bar 2', models: [::Bot::Alegre::ELASTICSEARCH_MODEL, ::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::ELASTICSEARCH_MODEL => 0.85, ::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) + Bot::Alegre.stubs(:request).with('post', '/text/similarity/search/', { text: 'Foo bar foo bar 2', models: [::Bot::Alegre::ELASTICSEARCH_MODEL, ::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::ELASTICSEARCH_MODEL => 0.85, ::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) r2.attach_to_similar_request! #Alegre should be called with ES and vector model for request with 4 or more words assert_equal r1, r2.reload.similar_to_request assert_equal [r2], r1.reload.similar_requests - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should attach to similar text short" do - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) f = create_feed m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f m2 = Media.create! type: 'Claim', quote: 'Foo bar 2' r2 = create_request media: m2, feed: f response = { 'result' => [{ '_source' => { 'context' => { 'request_id' => r1.id } } }] } - Bot::Alegre.stubs(:request_api).with('get', '/text/similarity/', { text: 'Foo bar 2', models: [::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) + Bot::Alegre.stubs(:request).with('post', '/text/similarity/search/', { text: 'Foo bar 2', models: [::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) r2.attach_to_similar_request! #Alegre should only be called with vector models for 2 or 3 word request assert_equal r1, r2.reload.similar_to_request assert_equal [r2], r1.reload.similar_requests - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should not attach to similar text short" do - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) f = create_feed m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f @@ -174,26 +174,26 @@ def setup # Alegre should not be called for a one word request assert_not_equal r1, r2.reload.similar_to_request assert_not_equal [r2], r1.reload.similar_requests - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should attach to similar media" do - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) f = create_feed m1 = create_uploaded_image r1 = create_request request_type: 'image', media: m1, feed: f m2 = create_uploaded_image r2 = create_request request_type: 'image', media: m2, feed: f response = { 'result' => [{ 'context' => [{ 'request_id' => r1.id }] }] } - Bot::Alegre.stubs(:request_api).with('get', '/image/similarity/', { url: m2.file.file.public_url, threshold: 0.85, limit: 20, context: { feed_id: f.id } }).returns(response) + Bot::Alegre.stubs(:request).with('post', '/image/similarity/search/', { url: m2.file.file.public_url, threshold: 0.85, limit: 20, context: { feed_id: f.id } }).returns(response) r2.attach_to_similar_request! assert_equal r1, r2.reload.similar_to_request assert_equal [r2], r1.reload.similar_requests - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should attach to similar link" do - Bot::Alegre.stubs(:request_api).returns(true) + Bot::Alegre.stubs(:request).returns(true) f = create_feed m = create_valid_media create_request request_type: 'text', media: m @@ -203,6 +203,7 @@ def setup r2.attach_to_similar_request! assert_equal r1, r2.reload.similar_to_request assert_equal [r2], r1.reload.similar_requests + Bot::Alegre.unstub(:request) end test "should set fields" do @@ -213,7 +214,7 @@ def setup end test "should update fields" do - Bot::Alegre.stubs(:request_api).returns({}) + Bot::Alegre.stubs(:request).returns({}) m1 = create_uploaded_image m2 = create_uploaded_image r1 = create_request media: m1 @@ -226,11 +227,11 @@ def setup assert_equal r4.created_at.to_s, r1.reload.last_submitted_at.to_s assert_equal 2, r1.reload.medias_count assert_equal 4, r1.reload.requests_count - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should return medias" do - Bot::Alegre.stubs(:request_api).returns({}) + Bot::Alegre.stubs(:request).returns({}) create_request create_uploaded_image m1 = create_uploaded_image @@ -241,11 +242,11 @@ def setup r3 = create_request media: m2 r3.similar_to_request = r1 ; r3.save! assert_equal [m1, m2].map(&:id).sort, r1.reload.medias.map(&:id).sort - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should cache team names that fact-checked a request" do - Bot::Alegre.stubs(:request_api).returns({}) + Bot::Alegre.stubs(:request).returns({}) RequestStore.store[:skip_cached_field_update] = false u = create_user is_admin: true f = create_feed @@ -279,7 +280,7 @@ def setup ProjectMediaRequest.create!(project_media: create_project_media(team: t4), request: r) assert_equal 'Bar, Foo, Test', r.reload.fact_checked_by assert_equal 3, r.reload.fact_checked_by_count - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) User.unstub(:current) end @@ -326,12 +327,12 @@ def setup end test "should cache media type" do - Bot::Alegre.stubs(:request_api).returns({}) + Bot::Alegre.stubs(:request).returns({}) RequestStore.store[:skip_cached_field_update] = false m = create_uploaded_image r = create_request media: m assert_equal 'UploadedImage', r.media_type(true) - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should not have a circular dependency" do diff --git a/test/workers/reindex_alegre_workspace_test.rb b/test/workers/reindex_alegre_workspace_test.rb index c4c1b1c6a3..bba484f1e7 100644 --- a/test/workers/reindex_alegre_workspace_test.rb +++ b/test/workers/reindex_alegre_workspace_test.rb @@ -25,14 +25,14 @@ def setup @tbi.save Bot::Alegre.stubs(:get_alegre_tbi).returns(TeamBotInstallation.new) Sidekiq::Testing.inline! - Bot::Alegre.stubs(:request_api).with('post', '/text/bulk_similarity/', anything).returns("done") + Bot::Alegre.stubs(:request).with('post', '/text/bulk_similarity/', anything).returns("done") end def teardown super [@tbi, @pm, @m, @p, @team, @bot].collect(&:destroy) Bot::Alegre.unstub(:get_alegre_tbi) - Bot::Alegre.unstub(:request_api) + Bot::Alegre.unstub(:request) end test "should trigger reindex" do @@ -90,7 +90,7 @@ def teardown assert_equal Array, response.class end - test "tests the parallel request_api" do + test "tests the parallel request" do package = { :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), :text=>"Some text",