From 8122caed1763381f0cf39b345ad14daca86f7a2d Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Mon, 17 Jun 2024 12:06:53 -0700
Subject: [PATCH 01/12] CV2-4719 add unique identifier as doc_id at check-api
 level

---
 app/models/concerns/alegre_v2.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb
index 247dbd7bc6..9187b6340c 100644
--- a/app/models/concerns/alegre_v2.rb
+++ b/app/models/concerns/alegre_v2.rb
@@ -148,8 +148,14 @@ def get_type(project_media)
       type
     end
 
+    def content_hash(project_media, field)
+      #Adding specific method so that we can easily extend and mutate how we count unique items as necessary...
+      item_doc_id(project_media, field)
+    end
+
     def generic_package(project_media, field)
       {
+        content_hash: content_hash(project_media, field),
         doc_id: item_doc_id(project_media, field),
         context: get_context(project_media, field)
       }

From 806c9dbb0a7427a411bcbe3f02d619e187ec731d Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Tue, 18 Jun 2024 09:40:33 -0700
Subject: [PATCH 02/12] more tweaking for initial caching strategy

---
 app/models/concerns/alegre_v2.rb | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb
index 9187b6340c..28d93812e8 100644
--- a/app/models/concerns/alegre_v2.rb
+++ b/app/models/concerns/alegre_v2.rb
@@ -149,8 +149,17 @@ def get_type(project_media)
     end
 
     def content_hash(project_media, field)
-      #Adding specific method so that we can easily extend and mutate how we count unique items as necessary...
-      item_doc_id(project_media, field)
+      if Bot::Alegre.ALL_TEXT_SIMILARITY_FIELDS.include?(field)
+        Digest::MD5.hexdigest(project_media.send(field))
+      else
+        if project_media.is_link?
+          return Digest::MD5.hexdigest(project_media.media.url)
+        if !project_media.is_text?
+          return project_media.media.file.split(".").first
+        elsif project_media.is_a?(TemporaryProjectMedia)
+          return Digest::MD5.hexdigest(project_media.url)
+        end
+      end
     end
 
     def generic_package(project_media, field)

From 8fb3a0c45af4866461bf5298f8f5876757b604ce Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Tue, 18 Jun 2024 10:45:45 -0700
Subject: [PATCH 03/12] fix typo

---
 app/models/concerns/alegre_v2.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb
index 28d93812e8..330e6b6c6a 100644
--- a/app/models/concerns/alegre_v2.rb
+++ b/app/models/concerns/alegre_v2.rb
@@ -154,7 +154,7 @@ def content_hash(project_media, field)
       else
         if project_media.is_link?
           return Digest::MD5.hexdigest(project_media.media.url)
-        if !project_media.is_text?
+        elsif !project_media.is_text?
           return project_media.media.file.split(".").first
         elsif project_media.is_a?(TemporaryProjectMedia)
           return Digest::MD5.hexdigest(project_media.url)

From 12d8daa118621b9be78fc530f8c484358194d7a5 Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Tue, 18 Jun 2024 14:44:39 -0700
Subject: [PATCH 04/12] add in shim for caching the sha on capi messages

---
 app/models/concerns/alegre_v2.rb   | 2 +-
 app/models/concerns/smooch_capi.rb | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb
index 330e6b6c6a..749627977c 100644
--- a/app/models/concerns/alegre_v2.rb
+++ b/app/models/concerns/alegre_v2.rb
@@ -157,7 +157,7 @@ def content_hash(project_media, field)
         elsif !project_media.is_text?
           return project_media.media.file.split(".").first
         elsif project_media.is_a?(TemporaryProjectMedia)
-          return Digest::MD5.hexdigest(project_media.url)
+          return Rails.cache.read("url_sha:#{project_media.url}")
         end
       end
     end
diff --git a/app/models/concerns/smooch_capi.rb b/app/models/concerns/smooch_capi.rb
index cc50bd13fe..05dab644d4 100644
--- a/app/models/concerns/smooch_capi.rb
+++ b/app/models/concerns/smooch_capi.rb
@@ -64,7 +64,9 @@ def store_capi_media(media_id, mime_type)
       req = Net::HTTP::Get.new(uri.request_uri, 'Authorization' => "Bearer #{self.config['capi_permanent_token']}")
       response = http.request(req)
       path = "capi/#{media_id}"
-      CheckS3.write(path, mime_type, response.body)
+      body = response.body
+      CheckS3.write(path, mime_type, body)
+      Rails.cache.write("url_sha:#{media_url}", Digest::MD5.hexdigest(body), expires_in: 60*3)
       CheckS3.public_url(path)
     end
 

From 25a59dfcedba1c10d65795dfdecf87083f630a87 Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Tue, 18 Jun 2024 16:50:13 -0700
Subject: [PATCH 05/12] updates

---
 app/models/concerns/alegre_v2.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb
index 749627977c..bfa61c6812 100644
--- a/app/models/concerns/alegre_v2.rb
+++ b/app/models/concerns/alegre_v2.rb
@@ -149,7 +149,7 @@ def get_type(project_media)
     end
 
     def content_hash(project_media, field)
-      if Bot::Alegre.ALL_TEXT_SIMILARITY_FIELDS.include?(field)
+      if Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS.include?(field)
         Digest::MD5.hexdigest(project_media.send(field))
       else
         if project_media.is_link?

From bc0df1a8a210ba39976146782d3dab83306c06e8 Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Wed, 19 Jun 2024 11:14:52 -0700
Subject: [PATCH 06/12] resolve testing errors

---
 app/models/concerns/alegre_v2.rb  |  2 +-
 test/models/bot/alegre_2_test.rb  | 12 +++++-----
 test/models/bot/alegre_3_test.rb  |  8 +++----
 test/models/bot/alegre_v2_test.rb | 40 +++++++++++++++----------------
 4 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb
index bfa61c6812..43c446fc54 100644
--- a/app/models/concerns/alegre_v2.rb
+++ b/app/models/concerns/alegre_v2.rb
@@ -155,7 +155,7 @@ def content_hash(project_media, field)
         if project_media.is_link?
           return Digest::MD5.hexdigest(project_media.media.url)
         elsif !project_media.is_text?
-          return project_media.media.file.split(".").first
+          return project_media.media.file.filename.split(".").first
         elsif project_media.is_a?(TemporaryProjectMedia)
           return Rails.cache.read("url_sha:#{project_media.url}")
         end
diff --git a/test/models/bot/alegre_2_test.rb b/test/models/bot/alegre_2_test.rb
index d54ce886b4..29e1056640 100644
--- a/test/models/bot/alegre_2_test.rb
+++ b/test/models/bot/alegre_2_test.rb
@@ -32,7 +32,7 @@ def teardown
     pm1 = create_project_media team: @team, media: create_uploaded_video
     pm2 = create_project_media team: @team, media: create_uploaded_video
     pm3 = create_project_media team: @team, media: create_uploaded_video
-    params = {:doc_id => Bot::Alegre.item_doc_id(pm3), :context => {:team_id => pm3.team_id, :project_media_id => pm3.id, :has_custom_id => true, :temporary_media => false}, :url => @media_path}
+    params = {:content_hash=>Bot::Alegre.content_hash(pm3, nil), :doc_id => Bot::Alegre.item_doc_id(pm3), :context => {:team_id => pm3.team_id, :project_media_id => pm3.id, :has_custom_id => true, :temporary_media => false}, :url => @media_path}
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/video', params.merge({ threshold: 0.9, confirmed: false })).returns(true)
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/video', params.merge({ threshold: 0.9, confirmed: true })).returns(true)
     Redis.any_instance.stubs(:get).returns({
@@ -70,7 +70,7 @@ def teardown
     pm1 = create_project_media team: @team, media: create_uploaded_audio
     pm2 = create_project_media team: @team, media: create_uploaded_audio
     pm3 = create_project_media team: @team, media: create_uploaded_audio
-    params = {:doc_id => Bot::Alegre.item_doc_id(pm3), :context => {:team_id => pm3.team_id, :project_media_id => pm3.id, :has_custom_id => true, :temporary_media => false}, :url => @media_path}
+    params = {:content_hash=>Bot::Alegre.content_hash(pm3, nil), :doc_id => Bot::Alegre.item_doc_id(pm3), :context => {:team_id => pm3.team_id, :project_media_id => pm3.id, :has_custom_id => true, :temporary_media => false}, :url => @media_path}
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/audio', params.merge({ threshold: 0.9, confirmed: false })).returns(true)
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/audio', params.merge({ threshold: 0.9, confirmed: true })).returns(true)
     
@@ -110,8 +110,8 @@ def teardown
     pm1 = create_project_media team: @team, media: create_uploaded_video
     pm2 = create_project_media team: @team, media: create_uploaded_audio
     pm3 = create_project_media team: @team, media: create_uploaded_audio
-    request_params = {:doc_id=>Bot::Alegre.item_doc_id(pm3), :context=>{:team_id=>pm3.team_id, :project_media_id=>pm3.id, :has_custom_id=>true, :temporary_media=>false}, :url=>@media_path, :threshold=>0.9, :confirmed=>true}
-    request_params_unconfirmed = {:doc_id=>Bot::Alegre.item_doc_id(pm3), :context=>{:team_id=>pm3.team_id, :project_media_id=>pm3.id, :has_custom_id=>true, :temporary_media=>false}, :url=>@media_path, :threshold=>0.9, :confirmed=>false}
+    request_params = {:content_hash=>Bot::Alegre.content_hash(pm3, nil), :doc_id=>Bot::Alegre.item_doc_id(pm3), :context=>{:team_id=>pm3.team_id, :project_media_id=>pm3.id, :has_custom_id=>true, :temporary_media=>false}, :url=>@media_path, :threshold=>0.9, :confirmed=>true}
+    request_params_unconfirmed = {:content_hash=>Bot::Alegre.content_hash(pm3, nil), :doc_id=>Bot::Alegre.item_doc_id(pm3), :context=>{:team_id=>pm3.team_id, :project_media_id=>pm3.id, :has_custom_id=>true, :temporary_media=>false}, :url=>@media_path, :threshold=>0.9, :confirmed=>false}
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/audio', request_params).returns({
       result: [
         {
@@ -221,7 +221,7 @@ def teardown
         }
       ]
     }.with_indifferent_access
-    params = {:doc_id => Bot::Alegre.item_doc_id(pm3), :context => {:team_id => pm3.team_id, :project_media_id => pm3.id, :has_custom_id => true, :temporary_media => false}, :url => @media_path}
+    params = {:content_hash=>Bot::Alegre.content_hash(pm3, nil), :doc_id => Bot::Alegre.item_doc_id(pm3), :context => {:team_id => pm3.team_id, :project_media_id => pm3.id, :has_custom_id => true, :temporary_media => false}, :url => @media_path}
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/image', params.merge({ threshold: 0.89, confirmed: false })).returns(result)
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/image', params.merge({ threshold: 0.95, confirmed: true })).returns(result)
     Bot::Alegre.stubs(:media_file_url).with(pm3).returns(@media_path)
@@ -289,7 +289,7 @@ def teardown
         relationship_type: Relationship.suggested_type
       }
     }.to_yaml)
-    params = {:doc_id => Bot::Alegre.item_doc_id(pm1a), :context => {:team_id => pm1a.team_id, :project_media_id => pm1a.id, :has_custom_id => true, :temporary_media => false}, :url => @media_path}
+    params = {:content_hash=>Bot::Alegre.content_hash(pm1a, nil), :doc_id => Bot::Alegre.item_doc_id(pm1a), :context => {:team_id => pm1a.team_id, :project_media_id => pm1a.id, :has_custom_id => true, :temporary_media => false}, :url => @media_path}
     Bot::Alegre.stubs(:media_file_url).with(pm1a).returns(@media_path)
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/image', params.merge({ threshold: 0.89, confirmed: false })).returns(true)
     Bot::Alegre.stubs(:request).with('post', '/similarity/async/image', params.merge({ threshold: 0.95, confirmed: true })).returns(true)
diff --git a/test/models/bot/alegre_3_test.rb b/test/models/bot/alegre_3_test.rb
index f4304d1ddc..5a7a90cfe5 100644
--- a/test/models/bot/alegre_3_test.rb
+++ b/test/models/bot/alegre_3_test.rb
@@ -91,10 +91,10 @@ def teardown
       Bot::Alegre.stubs(:media_file_url).returns(media_file_url)
 
       pm1 = create_project_media team: @pm.team, media: create_uploaded_audio(file: 'rails.mp3')
-      WebMock.stub_request(:post, "http://alegre/similarity/async/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>media_file_url, :threshold=>0.9, :confirmed=> false}).to_return(body: {
+      WebMock.stub_request(:post, "http://alegre/similarity/async/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>media_file_url, :threshold=>0.9, :confirmed=> false}).to_return(body: {
         "result": []
       }.to_json)
-      WebMock.stub_request(:post, "http://alegre/similarity/async/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>media_file_url, :threshold=>0.9, :confirmed=> true}).to_return(body: {
+      WebMock.stub_request(:post, "http://alegre/similarity/async/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>media_file_url, :threshold=>0.9, :confirmed=> true}).to_return(body: {
         "result": []
       }.to_json)
       WebMock.stub_request(:post, 'http://alegre/audio/transcription/result/').with(body: {job_name: "0c481e87f2774b1bd41a0a70d9b70d11"}).to_return(body: { 'job_status' => 'DONE' }.to_json)
@@ -160,10 +160,10 @@ def teardown
       Bot::Alegre.stubs(:media_file_url).returns(media_file_url)
 
       pm1 = create_project_media team: @pm.team, media: create_uploaded_audio(file: 'rails.mp3')
-      WebMock.stub_request(:post, "http://alegre/similarity/async/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>media_file_url, :threshold=>0.9, :confirmed=>true}).to_return(body: {
+      WebMock.stub_request(:post, "http://alegre/similarity/async/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>media_file_url, :threshold=>0.9, :confirmed=>true}).to_return(body: {
         "result": []
       }.to_json)
-      WebMock.stub_request(:post, "http://alegre/similarity/async/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>media_file_url, :threshold=>0.9, :confirmed=>false}).to_return(body: {
+      WebMock.stub_request(:post, "http://alegre/similarity/async/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>media_file_url, :threshold=>0.9, :confirmed=>false}).to_return(body: {
         "result": []
       }.to_json)
       WebMock.stub_request(:post, 'http://alegre/audio/transcription/').with({
diff --git a/test/models/bot/alegre_v2_test.rb b/test/models/bot/alegre_v2_test.rb
index 2ed345990c..bcc453066c 100644
--- a/test/models/bot/alegre_v2_test.rb
+++ b/test/models/bot/alegre_v2_test.rb
@@ -108,38 +108,38 @@ def teardown
 
   test "should create a generic_package for audio" do
     pm1 = create_project_media team: @team, media: create_uploaded_audio
-    assert_equal Bot::Alegre.generic_package(pm1, "audio"), {:doc_id=>Bot::Alegre.item_doc_id(pm1, "audio"), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}}
+    assert_equal Bot::Alegre.generic_package(pm1, "audio"), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, "audio"), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}}
   end
 
   test "should create a generic_package for image" do
     pm1 = create_project_media team: @team, media: create_uploaded_image
-    assert_equal Bot::Alegre.generic_package(pm1, "image"), {:doc_id=>Bot::Alegre.item_doc_id(pm1, "image"), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}}
+    assert_equal Bot::Alegre.generic_package(pm1, "image"), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, "image"), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}}
   end
 
   test "should create a generic_package for video" do
     pm1 = create_project_media team: @team, media: create_uploaded_video
-    assert_equal Bot::Alegre.generic_package(pm1, "video"), {:doc_id=>Bot::Alegre.item_doc_id(pm1, "video"), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}}
+    assert_equal Bot::Alegre.generic_package(pm1, "video"), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, "video"), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}}
   end
 
   test "should create a generic_package_audio" do
     pm1 = create_project_media team: @team, media: create_uploaded_audio
-    assert_equal Bot::Alegre.generic_package_audio(pm1, {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
-    assert_equal Bot::Alegre.store_package_audio(pm1, "audio", {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
-    assert_equal Bot::Alegre.store_package(pm1, "audio", {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.generic_package_audio(pm1, {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.store_package_audio(pm1, "audio", {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.store_package(pm1, "audio", {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
   end
 
   test "should create a generic_package_image" do
     pm1 = create_project_media team: @team, media: create_uploaded_image
-    assert_equal Bot::Alegre.generic_package_image(pm1, {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
-    assert_equal Bot::Alegre.store_package_image(pm1, "image", {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
-    assert_equal Bot::Alegre.store_package(pm1, "image", {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.generic_package_image(pm1, {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.store_package_image(pm1, "image", {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.store_package(pm1, "image", {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
   end
 
   test "should create a generic_package_video" do
     pm1 = create_project_media team: @team, media: create_uploaded_video
-    assert_equal Bot::Alegre.generic_package_image(pm1, {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
-    assert_equal Bot::Alegre.store_package_image(pm1, "video", {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
-    assert_equal Bot::Alegre.store_package(pm1, "video", {}), {:doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.generic_package_image(pm1, {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.store_package_image(pm1, "video", {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
+    assert_equal Bot::Alegre.store_package(pm1, "video", {}), {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1, nil), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}
   end
 
   test "should create a context for audio" do
@@ -210,7 +210,7 @@ def teardown
         }
       }
     }
-    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/async/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}).to_return(body: response.to_json)
+    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/async/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}).to_return(body: response.to_json)
     assert_equal JSON.parse(Bot::Alegre.get_async(pm1).to_json), JSON.parse(response.to_json)
   end
 
@@ -363,7 +363,7 @@ def teardown
         }
       ]
     }
-    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}).to_return(body: response.to_json)
+    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1)}).to_return(body: response.to_json)
     assert_equal JSON.parse(Bot::Alegre.get_sync(pm1).to_json), JSON.parse(response.to_json)
   end
 
@@ -479,7 +479,7 @@ def teardown
         }
       ]
     }
-    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json)
+    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json)
     assert_equal Bot::Alegre.get_items(pm1, nil), {(pm1.id+1)=>{:score=>1.0, :context=>{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>(pm1.id+1)}, :model=>"audio", :source_field=>"audio", :target_field=>"audio", :relationship_type=>Relationship.suggested_type}}
   end
 
@@ -561,7 +561,7 @@ def teardown
         }
       ]
     }
-    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json)
+    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json)
     assert_equal Bot::Alegre.get_items(pm1, nil), {(pm1.id+1)=>{:score=>0.91, :context=>{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>(pm1.id+1), "temporary_media"=>false}, :model=>"audio", :source_field=>"audio", :target_field=>"audio", :relationship_type=>Relationship.suggested_type}}
   end
 
@@ -643,7 +643,7 @@ def teardown
         }
       ]
     }
-    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json)
+    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json)
     assert_equal Bot::Alegre.get_confirmed_items(pm1, nil), {(pm1.id+1)=>{:score=>0.91, :context=>{"team_id"=>pm1.team_id, "has_custom_id"=>true, "project_media_id"=>(pm1.id+1), "temporary_media"=>false}, :model=>"audio", :source_field=>"audio", :target_field=>"audio", :relationship_type=>Relationship.confirmed_type}}
   end
 
@@ -685,8 +685,8 @@ def teardown
         }
       }
     }
-    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/async/audio").with(body: {:doc_id => Bot::Alegre.item_doc_id(pm1),:context => {:team_id => pm1.team_id, :project_media_id => pm1.id, :has_custom_id => true, :temporary_media => false}, :url => Bot::Alegre.media_file_url(pm1), :threshold => 0.9, :confirmed => false}).to_return(body: response.to_json)
-    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/async/audio").with(body: {:doc_id => Bot::Alegre.item_doc_id(pm1),:context => {:team_id => pm1.team_id, :project_media_id => pm1.id, :has_custom_id => true, :temporary_media => false}, :url => Bot::Alegre.media_file_url(pm1), :threshold => 0.9, :confirmed => true}).to_return(body: response.to_json)
+    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/async/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id => Bot::Alegre.item_doc_id(pm1),:context => {:team_id => pm1.team_id, :project_media_id => pm1.id, :has_custom_id => true, :temporary_media => false}, :url => Bot::Alegre.media_file_url(pm1), :threshold => 0.9, :confirmed => false}).to_return(body: response.to_json)
+    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/async/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id => Bot::Alegre.item_doc_id(pm1),:context => {:team_id => pm1.team_id, :project_media_id => pm1.id, :has_custom_id => true, :temporary_media => false}, :url => Bot::Alegre.media_file_url(pm1), :threshold => 0.9, :confirmed => true}).to_return(body: response.to_json)
     assert_equal Bot::Alegre.get_similar_items_v2_async(pm1, nil), true
   end
 
@@ -768,7 +768,7 @@ def teardown
         }
       ]
     }
-    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json)
+    WebMock.stub_request(:post, "#{CheckConfig.get('alegre_host')}/similarity/sync/audio").with(body: {:content_hash=>Bot::Alegre.content_hash(pm1, nil), :doc_id=>Bot::Alegre.item_doc_id(pm1), :context=>{:team_id=>pm1.team_id, :project_media_id=>pm1.id, :has_custom_id=>true, :temporary_media=>false}, :url=>Bot::Alegre.media_file_url(pm1), :threshold=>0.9}).to_return(body: response.to_json)
     assert_equal Bot::Alegre.get_similar_items_v2(pm1, nil), {}
   end
 

From 736d0f966b3638bb0637525e4b59e3becb46229a Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Wed, 19 Jun 2024 15:45:44 -0700
Subject: [PATCH 07/12] fix last broken test param

---
 test/controllers/elastic_search_9_test.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb
index 4f6d12dcf9..a015628a52 100644
--- a/test/controllers/elastic_search_9_test.rb
+++ b/test/controllers/elastic_search_9_test.rb
@@ -71,10 +71,10 @@ def setup
       # Text extraction
       Bot::Alegre.unstub(:media_file_url)
       pm = create_project_media team: team, media: create_uploaded_image, disable_es_callbacks: false
-      WebMock.stub_request(:post, 'http://alegre/similarity/async/image').with(body: {doc_id: Bot::Alegre.item_doc_id(pm), context: {:has_custom_id=>true, :project_media_id=>pm.id, :team_id=>pm.team_id, :temporary_media=>false}, threshold: 0.89, url: "some/path", confirmed: false}).to_return(body: {
+      WebMock.stub_request(:post, 'http://alegre/similarity/async/image').with(body: {content_hash: Bot::Alegre.content_hash(pm, nil), doc_id: Bot::Alegre.item_doc_id(pm), context: {:has_custom_id=>true, :project_media_id=>pm.id, :team_id=>pm.team_id, :temporary_media=>false}, threshold: 0.89, url: "some/path", confirmed: false}).to_return(body: {
         "result": []
       }.to_json)
-      WebMock.stub_request(:post, 'http://alegre/similarity/async/image').with(body: {doc_id: Bot::Alegre.item_doc_id(pm), context: {:has_custom_id=>true, :project_media_id=>pm.id, :team_id=>pm.team_id, :temporary_media=>false}, threshold: 0.95, url: "some/path", confirmed: true}).to_return(body: {
+      WebMock.stub_request(:post, 'http://alegre/similarity/async/image').with(body: {content_hash: Bot::Alegre.content_hash(pm, nil), doc_id: Bot::Alegre.item_doc_id(pm), context: {:has_custom_id=>true, :project_media_id=>pm.id, :team_id=>pm.team_id, :temporary_media=>false}, threshold: 0.95, url: "some/path", confirmed: true}).to_return(body: {
         "result": []
       }.to_json)
       Bot::Alegre.stubs(:media_file_url).with(pm).returns("some/path")

From 02b8ccf581daab226f98f6bfc5d9dbf2842d7c9a Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Thu, 20 Jun 2024 07:20:36 -0700
Subject: [PATCH 08/12] add tests for each branch of content_hash function

---
 app/models/concerns/alegre_v2.rb  |  4 ++--
 test/models/bot/alegre_v2_test.rb | 40 +++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb
index 43c446fc54..de219a6db4 100644
--- a/app/models/concerns/alegre_v2.rb
+++ b/app/models/concerns/alegre_v2.rb
@@ -154,10 +154,10 @@ def content_hash(project_media, field)
       else
         if project_media.is_link?
           return Digest::MD5.hexdigest(project_media.media.url)
-        elsif !project_media.is_text?
-          return project_media.media.file.filename.split(".").first
         elsif project_media.is_a?(TemporaryProjectMedia)
           return Rails.cache.read("url_sha:#{project_media.url}")
+        elsif !project_media.is_text?
+          return project_media.media.file.filename.split(".").first
         end
       end
     end
diff --git a/test/models/bot/alegre_v2_test.rb b/test/models/bot/alegre_v2_test.rb
index bcc453066c..4a513ab5a5 100644
--- a/test/models/bot/alegre_v2_test.rb
+++ b/test/models/bot/alegre_v2_test.rb
@@ -1172,4 +1172,44 @@ def teardown
     pm = create_project_media team: @team, media: create_uploaded_video
     assert_equal({}, Bot::Alegre.get_similar_items_v2(pm, nil))
   end
+
+  test "should generate content_hash for text types" do
+    pm = create_project_media
+    data = {
+      title: 'Report text title',
+      text: 'Report text content',
+      headline: 'Visual card title',
+      description: 'Visual card content'
+    }
+    publish_report(pm, {}, nil, data)
+    pm = ProjectMedia.find(pm.id).reload
+    assert_equal("eb02b714673c8af17b108836ce750070", Bot::Alegre.content_hash(pm, "report_text_title")
+    assert_equal("b476da9a44932178529f6896e0346af7", Bot::Alegre.content_hash(pm, nil))
+  end
+
+  test "should generate content_hash for link types" do
+    url = "http://example.com/newslink"
+    pender_url = CheckConfig.get('pender_url_private') + '/api/medias'
+    raw = {"json+ld": {}}
+    response = {'type':'media','data': {'url': url, 'type': 'item', 'raw': raw}}.to_json
+    WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response)
+    pm = create_project_media url: url
+    assert_equal("7621482de494568a442bfac13b1ceeb2", Bot::Alegre.content_hash(pm, nil))
+  end
+
+  test "should generate content_hash for media types" do
+    pm = create_project_media media: create_uploaded_image
+    assert_equal("110c700b3c4b02286cbfa3b700af8a57", Bot::Alegre.content_hash(pm, nil))
+  end
+
+  test "should generate content_hash for temporary types" do
+    pm = TemporaryProjectMedia.new
+    pm.url = "http://example.com/asset.mp3"
+    pm.id = Digest::MD5.hexdigest(pm.url).to_i(16)
+    pm.team_id = [1]
+    pm.type = "audio"
+    assert_equal(nil, Bot::Alegre.content_hash(pm, nil))
+    Rails.cache.write("url_sha:#{project_media.url}", Digest::MD5.hexdigest("blah"), expires_in: 60*3)
+    assert_equal("6f1ed002ab5595859014ebf0951522d9", Bot::Alegre.content_hash(pm, nil))
+  end
 end

From 00beb028d17738e2a90072c6fc718abb463d8f42 Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Thu, 20 Jun 2024 10:06:08 -0700
Subject: [PATCH 09/12] fix typo

---
 test/models/bot/alegre_v2_test.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/models/bot/alegre_v2_test.rb b/test/models/bot/alegre_v2_test.rb
index 4a513ab5a5..ba3b40f8a5 100644
--- a/test/models/bot/alegre_v2_test.rb
+++ b/test/models/bot/alegre_v2_test.rb
@@ -1183,7 +1183,7 @@ def teardown
     }
     publish_report(pm, {}, nil, data)
     pm = ProjectMedia.find(pm.id).reload
-    assert_equal("eb02b714673c8af17b108836ce750070", Bot::Alegre.content_hash(pm, "report_text_title")
+    assert_equal("eb02b714673c8af17b108836ce750070", Bot::Alegre.content_hash(pm, "report_text_title"))
     assert_equal("b476da9a44932178529f6896e0346af7", Bot::Alegre.content_hash(pm, nil))
   end
 

From 3523b8af71280e74a8ceee6e1b07dfc143479271 Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Thu, 20 Jun 2024 16:59:29 -0700
Subject: [PATCH 10/12] updates

---
 test/models/bot/alegre_v2_test.rb | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/test/models/bot/alegre_v2_test.rb b/test/models/bot/alegre_v2_test.rb
index ba3b40f8a5..ce44280f2b 100644
--- a/test/models/bot/alegre_v2_test.rb
+++ b/test/models/bot/alegre_v2_test.rb
@@ -1173,7 +1173,7 @@ def teardown
     assert_equal({}, Bot::Alegre.get_similar_items_v2(pm, nil))
   end
 
-  test "should generate content_hash for text types" do
+  test "should generate content_hash for named field on text types" do
     pm = create_project_media
     data = {
       title: 'Report text title',
@@ -1184,7 +1184,6 @@ def teardown
     publish_report(pm, {}, nil, data)
     pm = ProjectMedia.find(pm.id).reload
     assert_equal("eb02b714673c8af17b108836ce750070", Bot::Alegre.content_hash(pm, "report_text_title"))
-    assert_equal("b476da9a44932178529f6896e0346af7", Bot::Alegre.content_hash(pm, nil))
   end
 
   test "should generate content_hash for link types" do
@@ -1209,7 +1208,7 @@ def teardown
     pm.team_id = [1]
     pm.type = "audio"
     assert_equal(nil, Bot::Alegre.content_hash(pm, nil))
-    Rails.cache.write("url_sha:#{project_media.url}", Digest::MD5.hexdigest("blah"), expires_in: 60*3)
+    Rails.cache.write("url_sha:#{pm.url}", Digest::MD5.hexdigest("blah"), expires_in: 60*3)
     assert_equal("6f1ed002ab5595859014ebf0951522d9", Bot::Alegre.content_hash(pm, nil))
   end
 end

From ebb1d20dfd9c46699c21ef6d6627c59d07a2a3d7 Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Fri, 21 Jun 2024 11:02:18 -0700
Subject: [PATCH 11/12] refactor to DRY up file writing on messages

---
 app/models/concerns/smooch_capi.rb   | 17 +++++++----------
 app/models/concerns/smooch_search.rb | 22 +++++++++++++++++++---
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/app/models/concerns/smooch_capi.rb b/app/models/concerns/smooch_capi.rb
index 05dab644d4..910e55b75f 100644
--- a/app/models/concerns/smooch_capi.rb
+++ b/app/models/concerns/smooch_capi.rb
@@ -57,17 +57,14 @@ def store_capi_media(media_id, mime_type)
       req = Net::HTTP::Get.new(uri.request_uri, 'Content-Type' => 'application/json', 'Authorization' => "Bearer #{self.config['capi_permanent_token']}")
       response = http.request(req)
       media_url = JSON.parse(response.body)['url']
-
-      uri = URI(media_url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true
-      req = Net::HTTP::Get.new(uri.request_uri, 'Authorization' => "Bearer #{self.config['capi_permanent_token']}")
-      response = http.request(req)
       path = "capi/#{media_id}"
-      body = response.body
-      CheckS3.write(path, mime_type, body)
-      Rails.cache.write("url_sha:#{media_url}", Digest::MD5.hexdigest(body), expires_in: 60*3)
-      CheckS3.public_url(path)
+      self.write_file_to_s3(
+        media_url,
+        path,
+        mime_type,
+        true,
+        {'Authorization' => "Bearer #{self.config['capi_permanent_token']}"}
+      )
     end
 
     def handle_capi_system_message(message)
diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb
index 788d212b45..066f404004 100644
--- a/app/models/concerns/smooch_search.rb
+++ b/app/models/concerns/smooch_search.rb
@@ -198,13 +198,29 @@ def save_locally_and_return_url(media_url, type, feed_id)
         audio: 'audio/ogg',
         video: 'video/mp4'
       }[type.to_sym]
+      path = "feed/#{feed.id}/#{SecureRandom.hex}"
+      self.write_file_to_s3(
+        media_url,
+        path,
+        mime,
+        false,
+        headers
+      )
+    end
+    
+    def write_file_to_s3(media_url, path, mime, force_ssl, headers)
       uri = URI(media_url)
       http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = uri.scheme == 'https'
+      if force_ssl
+        http.use_ssl = true
+      else
+        http.use_ssl = uri.scheme == 'https'
+      end
       req = Net::HTTP::Get.new(uri.request_uri, headers)
       response = http.request(req)
-      path = "feed/#{feed.id}/#{SecureRandom.hex}"
-      CheckS3.write(path, mime, response.body)
+      body = response.body
+      CheckS3.write(path, mime, body)
+      Rails.cache.write("url_sha:#{media_url}", Digest::MD5.hexdigest(body), expires_in: 60*3)
       CheckS3.public_url(path)
     end
 

From e2950999c1d7252fd95fcf754e20db941b1f84a5 Mon Sep 17 00:00:00 2001
From: Devin Gaffney <itsme@devingaffney.com>
Date: Sat, 22 Jun 2024 12:35:06 -0700
Subject: [PATCH 12/12] Refactor as per review

---
 app/models/concerns/smooch_capi.rb   |  1 -
 app/models/concerns/smooch_search.rb | 11 +++--------
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/app/models/concerns/smooch_capi.rb b/app/models/concerns/smooch_capi.rb
index 910e55b75f..5d3be805ad 100644
--- a/app/models/concerns/smooch_capi.rb
+++ b/app/models/concerns/smooch_capi.rb
@@ -62,7 +62,6 @@ def store_capi_media(media_id, mime_type)
         media_url,
         path,
         mime_type,
-        true,
         {'Authorization' => "Bearer #{self.config['capi_permanent_token']}"}
       )
     end
diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb
index 066f404004..f998e97843 100644
--- a/app/models/concerns/smooch_search.rb
+++ b/app/models/concerns/smooch_search.rb
@@ -203,19 +203,14 @@ def save_locally_and_return_url(media_url, type, feed_id)
         media_url,
         path,
         mime,
-        false,
         headers
       )
     end
-    
-    def write_file_to_s3(media_url, path, mime, force_ssl, headers)
+
+    def write_file_to_s3(media_url, path, mime, headers)
       uri = URI(media_url)
       http = Net::HTTP.new(uri.host, uri.port)
-      if force_ssl
-        http.use_ssl = true
-      else
-        http.use_ssl = uri.scheme == 'https'
-      end
+      http.use_ssl = uri.scheme == 'https'
       req = Net::HTTP::Get.new(uri.request_uri, headers)
       response = http.request(req)
       body = response.body