Merge branch 'develop' into epic/CV2-4441-articles

Dockerfile

@@ -12,6 +12,11 @@ ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8
 ENV LANGUAGE C.UTF-8
 
+# Setup a user account
+ENV DEPLOYUSER=checkdeploy
+RUN useradd ${DEPLOYUSER} -s /bin/bash -m
+
+
 RUN apt-get update -qq && apt-get install -y --no-install-recommends curl
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
@@ -26,6 +31,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     libtag1-dev \
     lsof
 
+# CMD and helper scripts
+COPY --chown=root:root production/bin /opt/bin
+
 # tx client
 RUN curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
 
@@ -47,4 +55,4 @@ RUN echo 'require "irb/ext/save-history"' > ~/.irbrc && \
 RUN chmod +x /app/docker-entrypoint.sh
 RUN chmod +x /app/docker-background.sh
 EXPOSE 3000
-CMD ["/app/docker-entrypoint.sh"]
+CMD ["/app/docker-entrypoint.sh"]

app/graph/mutations/project_media_mutations.rb

@@ -45,6 +45,7 @@ class Create < Mutations::CreateMutation
     argument :set_tags, JsonStringType, required: false, camelize: false
     argument :set_title, GraphQL::Types::String, required: false, camelize: false
     argument :set_status, GraphQL::Types::String, required: false, camelize: false # Status identifier (for example, "in_progress")
+    argument :set_original_claim, GraphQL::Types::String, required: false, camelize: false
   end
 
   class Update < Mutations::UpdateMutation

app/graph/mutations/team_mutations.rb

@@ -47,7 +47,6 @@ class Update < Mutations::UpdateMutation
     argument :language, GraphQL::Types::String, required: false
     argument :languages, JsonStringType, required: false
     argument :language_detection, GraphQL::Types::Boolean, required: false, camelize: false
-    argument :list_columns, JsonStringType, required: false, camelize: false
     argument :tipline_inbox_filters, GraphQL::Types::String, required: false, camelize: false
     argument :suggested_matches_filters, GraphQL::Types::String, required: false, camelize: false
     argument :outgoing_urls_utm_code, GraphQL::Types::String, required: false, camelize: false

app/graph/types/cluster_type.rb

@@ -18,12 +18,6 @@ def fact_checks_count
 
   field :center, ProjectMediaType, null: true
 
-  def center
-    RecordLoader
-      .for(ProjectMedia)
-      .load(object.project_media_id)
-  end
-
   field :first_item_at, GraphQL::Types::Int, null: true
 
   def first_item_at

app/graph/types/project_media_type.rb

@@ -26,7 +26,6 @@ class ProjectMediaType < DefaultObject
   field :last_seen, GraphQL::Types::String, null: true
   field :status, GraphQL::Types::String, null: true
   field :share_count, GraphQL::Types::Int, null: true
-  field :list_columns_values, JsonStringType, null: true
   field :feed_columns_values, JsonStringType, null: true
   field :report_status, GraphQL::Types::String, null: true
   field :confirmed_as_similar_by_name, GraphQL::Types::String, null: true

app/graph/types/team_type.rb

@@ -39,7 +39,6 @@ class TeamType < DefaultObject
   field :get_language_detection, GraphQL::Types::Boolean, null: true
   field :get_report, JsonStringType, null: true
   field :get_fieldsets, JsonStringType, null: true
-  field :list_columns, JsonStringType, null: true
   field :url, GraphQL::Types::String, null: true
   field :data_report, JsonStringType, null: true
   field :available_newsletter_header_types, JsonStringType, null: true # List of header type strings
@@ -112,7 +111,6 @@ def get_fieldsets
     object.get_fieldsets
   end
 
-  field :list_columns, JsonStringType, null: true
   field :get_data_report_url, GraphQL::Types::String, null: true
 
   def get_data_report_url

app/helpers/validations_helper.rb

@@ -49,15 +49,4 @@ def fieldsets_format
     errors.add(:settings, JSON::Validator.fully_validate(schema, fieldsets)) if !JSON::Validator.validate(schema, fieldsets)
   end
 
-  def list_columns_format
-    return if self.get_list_columns.blank?
-    schema = {
-      type: 'array',
-      items: {
-        type: 'string',
-      }
-    }
-    columns = self.get_list_columns
-    errors.add(:settings, JSON::Validator.fully_validate(schema, columns)) if !JSON::Validator.validate(schema, columns)
-  end
 end

app/models/concerns/project_media_creators.rb

@@ -1,4 +1,6 @@
 require 'active_support/concern'
+require 'open-uri'
+require 'uri'
 
 module ProjectMediaCreators
   extend ActiveSupport::Concern
@@ -28,6 +30,62 @@ def create_annotation
     end
   end
 
+  def create_original_claim
+    claim = self.set_original_claim.strip
+    if claim.match?(/\A#{URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/)
+      uri = URI.parse(claim)
+      content_type = fetch_content_type(uri)
+
+      case content_type
+      when /^image\//
+        self.media = create_media_from_url('UploadedImage', claim)
+      when /^video\//
+        self.media = create_media_from_url('UploadedVideo', claim)
+      when /^audio\//
+        self.media = create_media_from_url('UploadedAudio', claim)
+      else
+        self.media = create_link_media(claim)
+      end
+    else
+      self.media = create_claim_media(claim)
+    end
+  end
+
+  def fetch_content_type(uri)
+    response = Net::HTTP.get_response(uri)
+    response['content-type']
+  end
+
+  def create_media_from_url(type, url)
+    klass = type.constantize
+    file = download_file(url)
+    m = klass.new
+    m.file = file
+    m.save!
+    m
+  end
+
+  def download_file(url)
+    raise "Invalid URL when creating media from original claim attribute" unless url =~ /\A#{URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/
+
+    file = Tempfile.new(['download', File.extname(url)])
+    file.binmode
+    file.write(URI(url).open.read)
+    file.rewind
+    file
+  end
+
+  def create_claim_media(text)
+    Claim.create!(quote: text)
+  end
+
+  def create_link_media(url)
+    team = self.team || Team.current
+    pender_key = team.get_pender_key if team
+    url_from_pender = Link.normalized(url, pender_key)
+    Link.find_by(url: url_from_pender) || Link.create!(url: url, pender_key: pender_key)
+  end
+
   def set_quote_metadata
     media = self.media
     case media.type

app/models/concerns/project_media_getters.rb

@@ -72,10 +72,6 @@ def full_url
     "#{CheckConfig.get('checkdesk_client')}/#{self.team.slug}#{project_prefix}/media/#{self.id}"
   end
 
-  def created_at_timestamp
-    self.created_at.to_i
-  end
-
   def updated_at_timestamp
     self.updated_at.to_i
   end

app/models/concerns/smooch_menus.rb

@@ -67,6 +67,13 @@ def send_message_to_user_with_main_menu_appended(uid, text, workflow, language,
         rows: rows
       }
 
+      # Set extra and fallback
+      extra, fallback = self.smooch_menus_set_extra_and_fallback(main, text, language)
+
+      self.send_message_to_user(uid, fallback.join("\n"), extra, false, true, event)
+    end
+
+    def smooch_menus_set_extra_and_fallback(main, text, language)
       extra = {
         override: {
           whatsapp: {
@@ -114,7 +121,7 @@ def send_message_to_user_with_main_menu_appended(uid, text, workflow, language,
         fallback = [text]
       end
 
-      self.send_message_to_user(uid, fallback.join("\n"), extra, false, true, event)
+      return extra, fallback
     end
 
     def adjust_language_options(rows, language, number_of_options)

app/models/concerns/smooch_messages.rb

@@ -278,15 +278,58 @@ def bundle_list_of_messages(list, last, reject_payload = false)
       self.adjust_media_type(bundle)
     end
 
-    def handle_bundle_messages(type, list, last, app_id, annotated, send_message = true)
-      bundle = self.bundle_list_of_messages(list, last)
-      if ['default_requests', 'irrelevant_search_result_requests'].include?(type)
-        self.process_message(bundle, app_id, send_message, type)
+    def bundle_list_of_messages_to_items(list, last)
+      # Collect messages from list based on media files, long text and short text
+      # so we have three types of messages
+      # Long text (text with number of words > min_number_of_words_for_tipline_submit_shortcut)
+      # Short text (text with number of words <= min_number_of_words_for_tipline_submit_shortcut)
+      # Media (image, audio, video, etc)
+      messages = []
+      # Define a text variable to hold short text
+      text = []
+      list.collect{ |m| JSON.parse(m) }.sort_by{ |m| m['received'].to_f }.each do |message|
+        if message['type'] == 'text'
+          # Get an item for long text (message that match number of words condition)
+          if message['payload'].nil?
+            messages << message if ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer)
+            text << message['text']
+          end
+        else
+          # Get an item for each media file
+          message['text'] = [message['text'], message['mediaUrl'].to_s].compact.join("\n#{Bot::Smooch::MESSAGE_BOUNDARY}")
+          text << message['text']
+          messages << self.adjust_media_type(message)
+        end
+      end
+      # collect all text in right order and add a boundary so we can easily split messages if needed
+      all_text = text.reject{ |t| t.blank? }.join("\n#{Bot::Smooch::MESSAGE_BOUNDARY}")
+      if messages.blank?
+        # No messages exist (this happens when all messages are short text)
+        # So will create a new message of type text and assign short text to it
+        message = last.clone
+        message['text'] = all_text
+        messages << message
+      else
+        # Attach all existing text (media text, long text and short text) to each item
+        messages.each do |raw|
+          # Define a new key `request_body` so we can append all text to request body
+          raw['request_body'] = all_text
+        end
       end
-      if ['timeout_requests', 'menu_options_requests', 'resource_requests', 'relevant_search_result_requests', 'timeout_search_requests'].include?(type)
-        key = "smooch:banned:#{bundle['authorId']}"
-        if Rails.cache.read(key).nil?
-          [annotated].flatten.uniq.each_with_index { |a, i| self.save_message_later(bundle, app_id, type, a, i * 60) }
+      messages
+    end
+
+    def handle_bundle_messages(type, list, last, app_id, annotated, send_message = true)
+      messages = self.bundle_list_of_messages_to_items(list, last)
+      messages.each do |message|
+        if ['default_requests', 'irrelevant_search_result_requests'].include?(type)
+          self.process_message(message, app_id, send_message, type)
+        end
+        if ['timeout_requests', 'menu_options_requests', 'resource_requests', 'relevant_search_result_requests', 'timeout_search_requests'].include?(type)
+          key = "smooch:banned:#{message['authorId']}"
+          if Rails.cache.read(key).nil?
+            [annotated].flatten.uniq.each_with_index { |a, i| self.save_message_later(message, app_id, type, a, i * 60) }
+          end
         end
       end
     end
@@ -364,6 +407,8 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req
     end
 
     def smooch_save_tipline_request(message, associated, app_id, author, request_type, associated_obj)
+      message['text'] = message['request_body'] unless message['request_body'].blank?
+      message.delete('request_body')
       fields = { smooch_data: message.merge({ app_id: app_id }) }
       result = self.smooch_api_get_messages(app_id, message['authorId'])
       fields[:smooch_conversation_id] = result.conversation.id unless result.nil? || result.conversation.nil?

app/models/concerns/smooch_search.rb

@@ -256,7 +256,7 @@ def send_search_results_to_user(uid, results, team_id, platform)
       end
       reports.reject{ |r| r.blank? }.each do |report|
         response = nil
-        no_body = (platform == 'Facebook Messenger')
+        no_body = (platform == 'Facebook Messenger' && !report.report_design_field_value('published_article_url').blank?)
         response = self.send_message_to_user(uid, report.report_design_text(nil, no_body), {}, false, true, 'search_result') if report.report_design_field_value('use_text_message')
         response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }, false, true, 'search_result') if !report.report_design_field_value('use_text_message') && report.report_design_field_value('use_visual_card')
         id = self.get_id_from_send_response(response)

app/models/concerns/team_duplication.rb

@@ -41,9 +41,6 @@ def self.duplicate(t, custom_slug = nil, custom_name = nil)
     end
 
     def self.modify_settings(old_team, new_team)
-      team_task_map = self.team_task_map
-      new_list_columns = old_team.get_list_columns.to_a.collect{|lc| lc.include?("task_value_") ? "task_value_#{team_task_map[lc.split("_").last.to_i]}" : lc}
-      new_team.set_list_columns = new_list_columns unless new_list_columns.blank?
       new_team.set_languages = old_team.get_languages
       new_team.set_language = old_team.get_language
       new_team

app/models/concerns/team_validations.rb

@@ -15,6 +15,5 @@ module TeamValidations
     validate :language_format
     validate :languages_format
     validate :fieldsets_format
-    validate :list_columns_format
   end
 end

app/models/feed.rb

@@ -140,7 +140,7 @@ def clusters_count(args = {})
   def filtered_clusters(args = {})
     team_ids = args[:team_ids]
     channels = args[:channels]
-    query = self.clusters
+    query = self.clusters.joins(:project_media)
 
     # Filter by workspace
     query = query.where.not("ARRAY[?] && team_ids", self.team_ids - team_ids.to_a.map(&:to_i)) if !team_ids.blank? && team_ids != self.team_ids

app/models/project_media.rb

@@ -1,5 +1,5 @@
 class ProjectMedia < ApplicationRecord
-  attr_accessor :quote, :quote_attributions, :file, :media_type, :set_annotation, :set_tasks_responses, :previous_project_id, :cached_permissions, :is_being_created, :related_to_id, :skip_rules, :set_claim_description, :set_claim_context, :set_fact_check, :set_tags, :set_title, :set_status
+  attr_accessor :quote, :quote_attributions, :file, :media_type, :set_annotation, :set_tasks_responses, :previous_project_id, :cached_permissions, :is_being_created, :related_to_id, :skip_rules, :set_claim_description, :set_claim_context, :set_fact_check, :set_tags, :set_title, :set_status, :set_original_claim
 
   belongs_to :media
   has_one :claim_description
@@ -32,6 +32,7 @@ class ProjectMedia < ApplicationRecord
   validates_presence_of :custom_title, if: proc { |pm| pm.title_field == 'custom_title' }
 
   before_validation :set_team_id, :set_channel, :set_project_id, on: :create
+  before_validation :create_original_claim, if: proc { |pm| pm.set_original_claim.present? }, on: :create
   after_create :create_annotation, :create_metrics_annotation, :send_slack_notification, :create_relationship, :create_team_tasks, :create_claim_description_and_fact_check, :create_tags
   after_create :add_source_creation_log, unless: proc { |pm| pm.source_id.blank? }
   after_commit :apply_rules_and_actions_on_create, :set_quote_metadata, :notify_team_bots_create, on: [:create]
@@ -330,19 +331,6 @@ def method_missing(method, *args, &block)
     end
   end
 
-  def list_columns_values
-    values = {}
-    columns = self.team.list_columns || Team.default_list_columns
-    columns.each do |column|
-      c = column.with_indifferent_access
-      if c[:show]
-        key = c[:key]
-        values[key] = self.send(key)
-      end
-    end
-    values
-  end
-
   def feed_columns_values
     values = {}
     columns = [