Merge branch 'develop'

Dockerfile

@@ -12,6 +12,11 @@ ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8
 ENV LANGUAGE C.UTF-8
 
+# Setup a user account
+ENV DEPLOYUSER=checkdeploy
+RUN useradd ${DEPLOYUSER} -s /bin/bash -m
+
+
 RUN apt-get update -qq && apt-get install -y --no-install-recommends curl
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
@@ -26,6 +31,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     libtag1-dev \
     lsof
 
+# CMD and helper scripts
+COPY --chown=root:root production/bin /opt/bin
+
 # tx client
 RUN curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
 
@@ -47,4 +55,4 @@ RUN echo 'require "irb/ext/save-history"' > ~/.irbrc && \
 RUN chmod +x /app/docker-entrypoint.sh
 RUN chmod +x /app/docker-background.sh
 EXPOSE 3000
-CMD ["/app/docker-entrypoint.sh"]
+CMD ["/app/docker-entrypoint.sh"]

app/graph/mutations/project_media_mutations.rb

@@ -45,6 +45,7 @@ class Create < Mutations::CreateMutation
     argument :set_tags, JsonStringType, required: false, camelize: false
     argument :set_title, GraphQL::Types::String, required: false, camelize: false
     argument :set_status, GraphQL::Types::String, required: false, camelize: false # Status identifier (for example, "in_progress")
+    argument :set_original_claim, GraphQL::Types::String, required: false, camelize: false
   end
 
   class Update < Mutations::UpdateMutation

app/graph/mutations/team_mutations.rb

@@ -47,7 +47,6 @@ class Update < Mutations::UpdateMutation
     argument :language, GraphQL::Types::String, required: false
     argument :languages, JsonStringType, required: false
     argument :language_detection, GraphQL::Types::Boolean, required: false, camelize: false
-    argument :list_columns, JsonStringType, required: false, camelize: false
     argument :tipline_inbox_filters, GraphQL::Types::String, required: false, camelize: false
     argument :suggested_matches_filters, GraphQL::Types::String, required: false, camelize: false
     argument :outgoing_urls_utm_code, GraphQL::Types::String, required: false, camelize: false

app/graph/types/project_media_type.rb

@@ -19,13 +19,13 @@ class ProjectMediaType < DefaultObject
   field :description, GraphQL::Types::String, null: true
   field :picture, GraphQL::Types::String, null: true
   field :virality, GraphQL::Types::Int, null: true
+  field :fact_check_published_on, GraphQL::Types::Int, null: true
   field :requests_count, GraphQL::Types::Int, null: true
   field :demand, GraphQL::Types::Int, null: true
   field :linked_items_count, GraphQL::Types::Int, null: true
   field :last_seen, GraphQL::Types::String, null: true
   field :status, GraphQL::Types::String, null: true
   field :share_count, GraphQL::Types::Int, null: true
-  field :list_columns_values, JsonStringType, null: true
   field :feed_columns_values, JsonStringType, null: true
   field :report_status, GraphQL::Types::String, null: true
   field :confirmed_as_similar_by_name, GraphQL::Types::String, null: true

app/graph/types/team_type.rb

@@ -39,7 +39,6 @@ class TeamType < DefaultObject
   field :get_language_detection, GraphQL::Types::Boolean, null: true
   field :get_report, JsonStringType, null: true
   field :get_fieldsets, JsonStringType, null: true
-  field :list_columns, JsonStringType, null: true
   field :url, GraphQL::Types::String, null: true
   field :data_report, JsonStringType, null: true
   field :available_newsletter_header_types, JsonStringType, null: true # List of header type strings
@@ -112,7 +111,6 @@ def get_fieldsets
     object.get_fieldsets
   end
 
-  field :list_columns, JsonStringType, null: true
   field :get_data_report_url, GraphQL::Types::String, null: true
 
   def get_data_report_url

app/helpers/validations_helper.rb

@@ -49,15 +49,4 @@ def fieldsets_format
     errors.add(:settings, JSON::Validator.fully_validate(schema, fieldsets)) if !JSON::Validator.validate(schema, fieldsets)
   end
 
-  def list_columns_format
-    return if self.get_list_columns.blank?
-    schema = {
-      type: 'array',
-      items: {
-        type: 'string',
-      }
-    }
-    columns = self.get_list_columns
-    errors.add(:settings, JSON::Validator.fully_validate(schema, columns)) if !JSON::Validator.validate(schema, columns)
-  end
 end

app/mailers/feed_invitation_mailer.rb

@@ -3,6 +3,7 @@ class FeedInvitationMailer < ApplicationMailer
 
   def notify(record_id)
     record = FeedInvitation.find_by_id record_id
+    return if record.nil?
     @recipient = record.email
     @user = record.user
     @feed = record.feed

app/models/concerns/alegre_v2.rb

@@ -148,8 +148,23 @@ def get_type(project_media)
       type
     end
 
+    def content_hash(project_media, field)
+      if Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS.include?(field)
+        Digest::MD5.hexdigest(project_media.send(field))
+      else
+        if project_media.is_link?
+          return Digest::MD5.hexdigest(project_media.media.url)
+        elsif project_media.is_a?(TemporaryProjectMedia)
+          return Rails.cache.read("url_sha:#{project_media.url}")
+        elsif !project_media.is_text?
+          return project_media.media.file.filename.split(".").first
+        end
+      end
+    end
+
     def generic_package(project_media, field)
       {
+        content_hash: content_hash(project_media, field),
         doc_id: item_doc_id(project_media, field),
         context: get_context(project_media, field)
       }

app/models/concerns/project_media_cached_fields.rb

@@ -135,7 +135,7 @@ def title_or_description_update
         {
           model: TiplineRequest,
           if: proc { |tr| tr.associated_type == 'ProjectMedia' },
-          affected_ids: proc { |tr| tr.associated.related_items_ids },
+          affected_ids: proc { |tr| tr.associated&.related_items_ids },
           events: {
             create: :recalculate,
           }
@@ -247,7 +247,7 @@ def title_or_description_update
         {
           model: Dynamic,
           if: proc { |d| d.annotation_type == 'report_design' },
-          affected_ids: proc { |d| d.annotated.related_items_ids },
+          affected_ids: proc { |d| d.annotated&.related_items_ids },
           events: {
             save: :cached_field_project_media_report_status_save,
           }

app/models/concerns/project_media_creators.rb

@@ -1,4 +1,6 @@
 require 'active_support/concern'
+require 'open-uri'
+require 'uri'
 
 module ProjectMediaCreators
   extend ActiveSupport::Concern
@@ -28,6 +30,62 @@ def create_annotation
     end
   end
 
+  def create_original_claim
+    claim = self.set_original_claim.strip
+    if claim.match?(/\A#{URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/)
+      uri = URI.parse(claim)
+      content_type = fetch_content_type(uri)
+
+      case content_type
+      when /^image\//
+        self.media = create_media_from_url('UploadedImage', claim)
+      when /^video\//
+        self.media = create_media_from_url('UploadedVideo', claim)
+      when /^audio\//
+        self.media = create_media_from_url('UploadedAudio', claim)
+      else
+        self.media = create_link_media(claim)
+      end
+    else
+      self.media = create_claim_media(claim)
+    end
+  end
+
+  def fetch_content_type(uri)
+    response = Net::HTTP.get_response(uri)
+    response['content-type']
+  end
+
+  def create_media_from_url(type, url)
+    klass = type.constantize
+    file = download_file(url)
+    m = klass.new
+    m.file = file
+    m.save!
+    m
+  end
+
+  def download_file(url)
+    raise "Invalid URL when creating media from original claim attribute" unless url =~ /\A#{URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/
+
+    file = Tempfile.new(['download', File.extname(url)])
+    file.binmode
+    file.write(URI(url).open.read)
+    file.rewind
+    file
+  end
+
+  def create_claim_media(text)
+    Claim.create!(quote: text)
+  end
+
+  def create_link_media(url)
+    team = self.team || Team.current
+    pender_key = team.get_pender_key if team
+    url_from_pender = Link.normalized(url, pender_key)
+    Link.find_by(url: url_from_pender) || Link.create!(url: url, pender_key: pender_key)
+  end
+
   def set_quote_metadata
     media = self.media
     case media.type

app/models/concerns/project_media_getters.rb

@@ -72,10 +72,6 @@ def full_url
     "#{CheckConfig.get('checkdesk_client')}/#{self.team.slug}#{project_prefix}/media/#{self.id}"
   end
 
-  def created_at_timestamp
-    self.created_at.to_i
-  end
-
   def updated_at_timestamp
     self.updated_at.to_i
   end

app/models/concerns/smooch_capi.rb

@@ -57,15 +57,13 @@ def store_capi_media(media_id, mime_type)
       req = Net::HTTP::Get.new(uri.request_uri, 'Content-Type' => 'application/json', 'Authorization' => "Bearer #{self.config['capi_permanent_token']}")
       response = http.request(req)
       media_url = JSON.parse(response.body)['url']
-
-      uri = URI(media_url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true
-      req = Net::HTTP::Get.new(uri.request_uri, 'Authorization' => "Bearer #{self.config['capi_permanent_token']}")
-      response = http.request(req)
       path = "capi/#{media_id}"
-      CheckS3.write(path, mime_type, response.body)
-      CheckS3.public_url(path)
+      self.write_file_to_s3(
+        media_url,
+        path,
+        mime_type,
+        {'Authorization' => "Bearer #{self.config['capi_permanent_token']}"}
+      )
     end
 
     def handle_capi_system_message(message)

app/models/concerns/smooch_menus.rb

@@ -19,7 +19,7 @@ def send_message_to_user_with_main_menu_appended(uid, text, workflow, language,
       allowed_types = ['query_state', 'subscription_state', 'custom_resource']
       ['smooch_state_main', 'smooch_state_secondary'].each_with_index do |state, i|
         rows = []
-        options = workflow[state].to_h['smooch_menu_options'].to_a
+        options = begin workflow[state].to_h['smooch_menu_options'].to_a rescue [] end
         next if options.empty?
         options.select{ |o| allowed_types.include?(o['smooch_menu_option_value']) }.each do |option|
           title = option['smooch_menu_option_label']
@@ -67,6 +67,13 @@ def send_message_to_user_with_main_menu_appended(uid, text, workflow, language,
         rows: rows
       }
 
+      # Set extra and fallback
+      extra, fallback = self.smooch_menus_set_extra_and_fallback(main, text, language)
+
+      self.send_message_to_user(uid, fallback.join("\n"), extra, false, true, event)
+    end
+
+    def smooch_menus_set_extra_and_fallback(main, text, language)
       extra = {
         override: {
           whatsapp: {
@@ -114,7 +121,7 @@ def send_message_to_user_with_main_menu_appended(uid, text, workflow, language,
         fallback = [text]
       end
 
-      self.send_message_to_user(uid, fallback.join("\n"), extra, false, true, event)
+      return extra, fallback
     end
 
     def adjust_language_options(rows, language, number_of_options)

app/models/concerns/smooch_messages.rb

@@ -278,15 +278,59 @@ def bundle_list_of_messages(list, last, reject_payload = false)
       self.adjust_media_type(bundle)
     end
 
-    def handle_bundle_messages(type, list, last, app_id, annotated, send_message = true)
-      bundle = self.bundle_list_of_messages(list, last)
-      if ['default_requests', 'irrelevant_search_result_requests'].include?(type)
-        self.process_message(bundle, app_id, send_message, type)
+    def bundle_list_of_messages_to_items(list, last)
+      # Collect messages from list based on media files, long text and short text
+      # so we have three types of messages
+      # Long text (text with number of words > min_number_of_words_for_tipline_submit_shortcut)
+      # Short text (text with number of words <= min_number_of_words_for_tipline_submit_shortcut)
+      # Media (image, audio, video, etc)
+      messages = []
+      # Define a text variable to hold short text
+      text = []
+      list.collect{ |m| JSON.parse(m) }.sort_by{ |m| m['received'].to_f }.each do |message|
+        if message['type'] == 'text'
+          # Get an item for long text (message that match number of words condition)
+          if message['payload'].nil?
+            messages << message if ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer)
+            text << message['text']
+          end
+        elsif !message['mediaUrl'].blank?
+          # Get an item for each media file
+          message['text'] = [message['text'], message['mediaUrl'].to_s].compact.join("\n#{Bot::Smooch::MESSAGE_BOUNDARY}")
+          text << message['text']
+          messages << self.adjust_media_type(message)
+        end
       end
-      if ['timeout_requests', 'menu_options_requests', 'resource_requests', 'relevant_search_result_requests', 'timeout_search_requests'].include?(type)
-        key = "smooch:banned:#{bundle['authorId']}"
-        if Rails.cache.read(key).nil?
-          [annotated].flatten.uniq.each_with_index { |a, i| self.save_message_later(bundle, app_id, type, a, i * 60) }
+      # collect all text in right order and add a boundary so we can easily split messages if needed
+      all_text = text.reject{ |t| t.blank? }.join("\n#{Bot::Smooch::MESSAGE_BOUNDARY}")
+      if messages.blank?
+        # No messages exist (this happens when all messages are short text)
+        # So will create a new message of type text and assign short text to it
+        message = last.clone
+        message['type'] = 'text'
+        message['text'] = all_text
+        messages << message
+      else
+        # Attach all existing text (media text, long text and short text) to each item
+        messages.each do |raw|
+          # Define a new key `request_body` so we can append all text to request body
+          raw['request_body'] = all_text
+        end
+      end
+      messages
+    end
+
+    def handle_bundle_messages(type, list, last, app_id, annotated, send_message = true)
+      messages = self.bundle_list_of_messages_to_items(list, last)
+      messages.each do |message|
+        if ['default_requests', 'irrelevant_search_result_requests'].include?(type)
+          self.process_message(message, app_id, send_message, type)
+        end
+        if ['timeout_requests', 'menu_options_requests', 'resource_requests', 'relevant_search_result_requests', 'timeout_search_requests'].include?(type)
+          key = "smooch:banned:#{message['authorId']}"
+          if Rails.cache.read(key).nil?
+            [annotated].flatten.uniq.each_with_index { |a, i| self.save_message_later(message, app_id, type, a, i * 60) }
+          end
         end
       end
     end
@@ -364,6 +408,8 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req
     end
 
     def smooch_save_tipline_request(message, associated, app_id, author, request_type, associated_obj)
+      message['text'] = message['request_body'] unless message['request_body'].blank?
+      message.delete('request_body')
       fields = { smooch_data: message.merge({ app_id: app_id }) }
       result = self.smooch_api_get_messages(app_id, message['authorId'])
       fields[:smooch_conversation_id] = result.conversation.id unless result.nil? || result.conversation.nil?

app/models/concerns/smooch_search.rb

@@ -198,13 +198,24 @@ def save_locally_and_return_url(media_url, type, feed_id)
         audio: 'audio/ogg',
         video: 'video/mp4'
       }[type.to_sym]
+      path = "feed/#{feed.id}/#{SecureRandom.hex}"
+      self.write_file_to_s3(
+        media_url,
+        path,
+        mime,
+        headers
+      )
+    end
+
+    def write_file_to_s3(media_url, path, mime, headers)
       uri = URI(media_url)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = uri.scheme == 'https'
       req = Net::HTTP::Get.new(uri.request_uri, headers)
       response = http.request(req)
-      path = "feed/#{feed.id}/#{SecureRandom.hex}"
-      CheckS3.write(path, mime, response.body)
+      body = response.body
+      CheckS3.write(path, mime, body)
+      Rails.cache.write("url_sha:#{media_url}", Digest::MD5.hexdigest(body), expires_in: 60*3)
       CheckS3.public_url(path)
     end
 
@@ -245,7 +256,7 @@ def send_search_results_to_user(uid, results, team_id, platform)
       end
       reports.reject{ |r| r.blank? }.each do |report|
         response = nil
-        no_body = (platform == 'Facebook Messenger')
+        no_body = (platform == 'Facebook Messenger' && !report.report_design_field_value('published_article_url').blank?)
         response = self.send_message_to_user(uid, report.report_design_text(nil, no_body), {}, false, true, 'search_result') if report.report_design_field_value('use_text_message')
         response = self.send_message_to_user(uid, '', { 'type' => 'image', 'mediaUrl' => report.report_design_image_url }, false, true, 'search_result') if !report.report_design_field_value('use_text_message') && report.report_design_field_value('use_visual_card')
         id = self.get_id_from_send_response(response)

app/models/concerns/team_duplication.rb

@@ -41,9 +41,6 @@ def self.duplicate(t, custom_slug = nil, custom_name = nil)
     end
 
     def self.modify_settings(old_team, new_team)
-      team_task_map = self.team_task_map
-      new_list_columns = old_team.get_list_columns.to_a.collect{|lc| lc.include?("task_value_") ? "task_value_#{team_task_map[lc.split("_").last.to_i]}" : lc}
-      new_team.set_list_columns = new_list_columns unless new_list_columns.blank?
       new_team.set_languages = old_team.get_languages
       new_team.set_language = old_team.get_language
       new_team