Cv2 3435 alegre with presto for audio (#1647)

* CV2-3435 add alegre webhook

* update fixtures to match actual alegre resposne

* remove unnecessary copypasting

* fix test call

* small fix

* try different setup

* readd request

* downgrade to type checking for the moment

* replace test with known direct relationship count changing test

* update stub

* finally fix test

* add newline per codeclimate

* CV2-3435 updates to alegre to work locally

* update test

* remove token passing

* resolve error in returned value of request_api

* add prefix, add blank safety check

* CV2-3435 update tests as per @caiosba commentary

* Clean-up and test simplification

---------

Co-authored-by: Caio <[email protected]>

app/controllers/api/v1/webhooks_controller.rb

@@ -7,7 +7,8 @@ def index
         bot_name_to_class = {
           smooch: Bot::Smooch,
           keep: Bot::Keep,
-          fetch: Bot::Fetch
+          fetch: Bot::Fetch,
+          alegre: Bot::Alegre,
         }
         unless bot_name_to_class.has_key?(params[:name].to_sym)
           render_error('Bot not found', 'ID_NOT_FOUND', 404) and return

app/models/bot/alegre.rb

@@ -6,6 +6,7 @@ class Error < ::StandardError
   end
 
   include AlegreSimilarity
+  include AlegreWebhooks
 
   # Text similarity models
   MEAN_TOKENS_MODEL = 'xlm-r-bert-base-nli-stsb-mean-tokens'
@@ -491,7 +492,13 @@ def self.request_api(method, path, params = {}, query_or_body = 'body', retries
       response_body = response.body
       Rails.logger.info("[Alegre Bot] Alegre response: #{response_body.inspect}")
       ActiveRecord::Base.connection.reconnect! if RequestStore.store[:pause_database_connection]
-      JSON.parse(response_body)
+      parsed_response = JSON.parse(response_body)
+      if parsed_response.dig("queue") == 'audio__Model' && parsed_response.dig("body", "callback_url") != nil
+        redis = Redis.new(REDIS_CONFIG)
+        redis_response = redis.blpop("alegre:webhook:#{parsed_response.dig("body", "id")}", 120)
+        return JSON.parse(redis_response[1])
+      end
+      parsed_response
     rescue StandardError => e
       if retries > 0
         sleep 1

app/models/concerns/alegre_similarity.rb

@@ -19,7 +19,7 @@ def get_similar_items(pm)
       type = Bot::Alegre.get_pm_type(pm)
       Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 2/5] Type is #{type.blank? ? "blank" : type}"
       unless type.blank?
-        if !self.should_get_similar_items_of_type?('master', pm.team_id) || !self.should_get_similar_items_of_type?(type, pm.team_id)
+        if !Bot::Alegre.should_get_similar_items_of_type?('master', pm.team_id) || !Bot::Alegre.should_get_similar_items_of_type?(type, pm.team_id)
           Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 3/5] ProjectMedia cannot be checked for similar items"
           return {}
         else

app/models/concerns/alegre_webhooks.rb

@@ -0,0 +1,26 @@
+class AlegreCallbackError < StandardError
+end
+
+module AlegreWebhooks
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def valid_request?(request)
+      token = request.params['token'] || request.query_parameters['token']
+      !token.blank? && token == CheckConfig.get('alegre_token')
+    end
+
+    def webhook(request)
+      begin
+        doc_id = request.params.dig('data', 'requested', 'body', 'id')
+        raise 'Unexpected params format' if doc_id.blank?
+        redis = Redis.new(REDIS_CONFIG)
+        key = "alegre:webhook:#{doc_id}"
+        redis.lpush(key, request.params.to_json)
+        redis.expire(key, 1.day.to_i)
+      rescue StandardError => e
+        CheckSentry.notify(AlegreCallbackError.new(e.message), { alegre_response: request.params })
+      end
+    end
+  end
+end

config/config.yml.example

@@ -11,7 +11,6 @@ development: &default
   elasticsearch_index:
   elasticsearch_log: true
   elasticsearch_sync: false
-
   # WARNING For production, don't use a wildcard: set the allowed domains explicitly as a regular expression, e.g.
   # '(https?://.*\.?(meedan.com|meedan.org))'
   allowed_origins: '.*'

test/controllers/webhooks_controller_test.rb

@@ -232,4 +232,24 @@ def setup
     assert_equal '200', response.code
     assert_match /ignored/, response.body
   end
+
+  test "should process Alegre webhook" do
+    CheckSentry.expects(:notify).never
+    redis = Redis.new(REDIS_CONFIG)
+    redis.del('foo')
+    payload = { 'action' => 'audio', 'data' => { 'requested' => { 'body' => { 'id' => 'foo', 'context' => { 'project_media_id' => random_number } } } } }
+    assert_nil redis.lpop('alegre:webhook:foo')
+
+    post :index, params: { name: :alegre, token: CheckConfig.get('alegre_token') }.merge(payload)
+    response = JSON.parse(redis.lpop('alegre:webhook:foo'))
+    assert_equal 'foo', response.dig('data', 'requested', 'body', 'id')
+
+    travel_to Time.now.since(2.days)
+    assert_nil redis.lpop('alegre:webhook:foo')
+  end
+
+  test "should report error if can't process Alegre webhook" do
+    CheckSentry.expects(:notify).once
+    post :index, params: { name: :alegre, token: CheckConfig.get('alegre_token') }.merge({ foo: 'bar' })
+  end
 end

test/models/bot/alegre_2_test.rb

@@ -315,6 +315,16 @@ def teardown
     RequestStore.store[:pause_database_connection] = false
   end
 
+  test "should block calls on redis blpop for audio request" do
+    stubbed_response = Net::HTTPSuccess.new(1.0, '200', 'OK')
+    stubbed_response.stubs(:body).returns({"queue" => "audio__Model", "body" => {"id" => "123", "callback_url" => "http://example.com/callback"}}.to_json)
+    Net::HTTP.any_instance.stubs(:request).returns(stubbed_response)
+    Redis.any_instance.stubs(:blpop).with("alegre:webhook:123", 120).returns(["alegre:webhook:123", {"tested" => true}.to_json])
+    assert_equal Bot::Alegre.request_api('get', '/audio/similarity/', @params, 'body'), {"tested" => true}
+    Net::HTTP.any_instance.unstub(:request)
+    Redis.any_instance.unstub(:blpop)
+  end
+
   test "should get items with similar title" do
     create_verification_status_stuff
     RequestStore.store[:skip_cached_field_update] = false