cx collection uploads

Gemfile

@@ -49,6 +49,7 @@ gem "rubocop-rspec"
 gem 'active_model_serializers', '>= 0.10.13'
 gem 'acts-as-list'
 gem 'aws-sdk-rails', '>= 3.8.0'
+gem 'aws-sdk-s3'
 gem 'caracal', '>= 1.4.1'
 gem 'carrierwave', '>= 2.2.1'
 gem 'devise', '>= 4.8.1'

Gemfile.lock

@@ -18,50 +18,51 @@ GEM
     aasm-diagram (0.1.3)
       aasm (~> 5.0, >= 4.12)
       ruby-graphviz (~> 1.2)
-    actioncable (7.1.1)
-      actionpack (= 7.1.1)
-      activesupport (= 7.1.1)
+    actioncable (7.1.2)
+      actionpack (= 7.1.2)
+      activesupport (= 7.1.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
       zeitwerk (~> 2.6)
-    actionmailbox (7.1.1)
-      actionpack (= 7.1.1)
-      activejob (= 7.1.1)
-      activerecord (= 7.1.1)
-      activestorage (= 7.1.1)
-      activesupport (= 7.1.1)
+    actionmailbox (7.1.2)
+      actionpack (= 7.1.2)
+      activejob (= 7.1.2)
+      activerecord (= 7.1.2)
+      activestorage (= 7.1.2)
+      activesupport (= 7.1.2)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.1.1)
-      actionpack (= 7.1.1)
-      actionview (= 7.1.1)
-      activejob (= 7.1.1)
-      activesupport (= 7.1.1)
+    actionmailer (7.1.2)
+      actionpack (= 7.1.2)
+      actionview (= 7.1.2)
+      activejob (= 7.1.2)
+      activesupport (= 7.1.2)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.2)
-    actionpack (7.1.1)
-      actionview (= 7.1.1)
-      activesupport (= 7.1.1)
+    actionpack (7.1.2)
+      actionview (= 7.1.2)
+      activesupport (= 7.1.2)
       nokogiri (>= 1.8.5)
+      racc
       rack (>= 2.2.4)
       rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    actiontext (7.1.1)
-      actionpack (= 7.1.1)
-      activerecord (= 7.1.1)
-      activestorage (= 7.1.1)
-      activesupport (= 7.1.1)
+    actiontext (7.1.2)
+      actionpack (= 7.1.2)
+      activerecord (= 7.1.2)
+      activestorage (= 7.1.2)
+      activesupport (= 7.1.2)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.1.1)
-      activesupport (= 7.1.1)
+    actionview (7.1.2)
+      activesupport (= 7.1.2)
       builder (~> 3.1)
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
@@ -71,22 +72,22 @@ GEM
       activemodel (>= 4.1)
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
-    activejob (7.1.1)
-      activesupport (= 7.1.1)
+    activejob (7.1.2)
+      activesupport (= 7.1.2)
       globalid (>= 0.3.6)
-    activemodel (7.1.1)
-      activesupport (= 7.1.1)
-    activerecord (7.1.1)
-      activemodel (= 7.1.1)
-      activesupport (= 7.1.1)
+    activemodel (7.1.2)
+      activesupport (= 7.1.2)
+    activerecord (7.1.2)
+      activemodel (= 7.1.2)
+      activesupport (= 7.1.2)
       timeout (>= 0.4.0)
-    activestorage (7.1.1)
-      actionpack (= 7.1.1)
-      activejob (= 7.1.1)
-      activerecord (= 7.1.1)
-      activesupport (= 7.1.1)
+    activestorage (7.1.2)
+      actionpack (= 7.1.2)
+      activejob (= 7.1.2)
+      activerecord (= 7.1.2)
+      activesupport (= 7.1.2)
       marcel (~> 1.0)
-    activesupport (7.1.1)
+    activesupport (7.1.2)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -104,7 +105,7 @@ GEM
     aes_key_wrap (1.1.0)
     ast (2.4.2)
     aws-eventstream (1.2.0)
-    aws-partitions (1.848.0)
+    aws-partitions (1.849.0)
     aws-record (2.13.0)
       aws-sdk-dynamodb (~> 1, >= 1.85.0)
     aws-sdk-core (3.186.0)
@@ -115,6 +116,9 @@ GEM
     aws-sdk-dynamodb (1.96.0)
       aws-sdk-core (~> 3, >= 3.184.0)
       aws-sigv4 (~> 1.1)
+    aws-sdk-kms (1.72.0)
+      aws-sdk-core (~> 3, >= 3.184.0)
+      aws-sigv4 (~> 1.1)
     aws-sdk-rails (3.9.0)
       aws-record (~> 2)
       aws-sdk-ses (~> 1, >= 1.50.0)
@@ -123,13 +127,17 @@ GEM
       aws-sessionstore-dynamodb (~> 2)
       concurrent-ruby (~> 1)
       railties (>= 5.2.0)
+    aws-sdk-s3 (1.136.0)
+      aws-sdk-core (~> 3, >= 3.181.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.6)
     aws-sdk-ses (1.56.0)
       aws-sdk-core (~> 3, >= 3.184.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-sesv2 (1.41.0)
       aws-sdk-core (~> 3, >= 3.184.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.65.0)
+    aws-sdk-sqs (1.67.0)
       aws-sdk-core (~> 3, >= 3.184.0)
       aws-sigv4 (~> 1.1)
     aws-sessionstore-dynamodb (2.1.0)
@@ -241,7 +249,7 @@ GEM
       activesupport (>= 6.0.0)
       railties (>= 6.0.0)
     io-console (0.6.0)
-    irb (1.8.3)
+    irb (1.9.0)
       rdoc
       reline (>= 0.3.8)
     jbuilder (2.11.5)
@@ -279,7 +287,7 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     logstop (0.3.1)
-    loofah (2.21.4)
+    loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
@@ -302,7 +310,7 @@ GEM
     multi_xml (0.6.0)
     multipart-post (2.3.0)
     mutex_m (0.2.0)
-    net-imap (0.4.4)
+    net-imap (0.4.5)
       date
       net-protocol
     net-pop (0.1.2)
@@ -356,7 +364,7 @@ GEM
     public_suffix (5.0.3)
     puma (6.4.0)
       nio4r (~> 2.0)
-    racc (1.7.2)
+    racc (1.7.3)
     rack (2.2.8)
     rack-cors (2.0.1)
       rack (>= 2.0.0)
@@ -369,20 +377,20 @@ GEM
     rackup (1.0.0)
       rack (< 3)
       webrick
-    rails (7.1.1)
-      actioncable (= 7.1.1)
-      actionmailbox (= 7.1.1)
-      actionmailer (= 7.1.1)
-      actionpack (= 7.1.1)
-      actiontext (= 7.1.1)
-      actionview (= 7.1.1)
-      activejob (= 7.1.1)
-      activemodel (= 7.1.1)
-      activerecord (= 7.1.1)
-      activestorage (= 7.1.1)
-      activesupport (= 7.1.1)
+    rails (7.1.2)
+      actioncable (= 7.1.2)
+      actionmailbox (= 7.1.2)
+      actionmailer (= 7.1.2)
+      actionpack (= 7.1.2)
+      actiontext (= 7.1.2)
+      actionview (= 7.1.2)
+      activejob (= 7.1.2)
+      activemodel (= 7.1.2)
+      activerecord (= 7.1.2)
+      activestorage (= 7.1.2)
+      activesupport (= 7.1.2)
       bundler (>= 1.15.0)
-      railties (= 7.1.1)
+      railties (= 7.1.2)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
       actionview (>= 5.0.1.rc1)
@@ -399,9 +407,9 @@ GEM
     rails-html-sanitizer (1.6.0)
       loofah (~> 2.21)
       nokogiri (~> 1.14)
-    railties (7.1.1)
-      actionpack (= 7.1.1)
-      activesupport (= 7.1.1)
+    railties (7.1.2)
+      actionpack (= 7.1.2)
+      activesupport (= 7.1.2)
       irb
       rackup (>= 1.0.0)
       rake (>= 12.2)
@@ -421,7 +429,7 @@ GEM
     redis-namespace (1.11.0)
       redis (>= 4)
     regexp_parser (2.8.2)
-    reline (0.3.9)
+    reline (0.4.0)
       io-console (~> 0.5)
     request_store (1.5.1)
       rack (>= 1.4)
@@ -566,6 +574,7 @@ DEPENDENCIES
   acts-as-list
   acts-as-taggable-on
   aws-sdk-rails (>= 3.8.0)
+  aws-sdk-s3
   bootsnap
   brakeman
   bullet

app/assets/stylesheets/site.scss

@@ -147,7 +147,7 @@ footer {
 .well {
   border: 1px solid #d6d7d9;
   padding: 20px;
-  margin: 10px 0;
+  margin-bottom: 10px;
   overflow-wrap: break-word;
 }
 

app/controllers/admin/cx_collection_details_controller.rb

@@ -1,5 +1,5 @@
 class Admin::CxCollectionDetailsController < AdminController
-  before_action :set_cx_collection_detail, only: %i[ show edit upload update destroy ]
+  before_action :set_cx_collection_detail, only: %i[ show edit upload upload_csv update destroy ]
   before_action :set_cx_collections, only: %i[ new edit upload ]
 
   def index
@@ -26,8 +26,8 @@ def create
 
     respond_to do |format|
       if @cx_collection_detail.save
-        format.html { redirect_to admin_cx_collection_detail_url(@cx_collection_detail), notice: "Cx collection detail was successfully created." }
-        format.json { render :show, status: :created, location: @cx_collection_detail }
+        format.html { redirect_to upload_admin_cx_collection_detail_url(@cx_collection_detail), notice: "CX Collection Detail was successfully created." }
+        format.json { render :upload, status: :created, location: @cx_collection_detail }
       else
         format.html { render :new, status: :unprocessable_entity }
         format.json { render json: @cx_collection_detail.errors, status: :unprocessable_entity }
@@ -38,7 +38,7 @@ def create
   def update
     respond_to do |format|
       if @cx_collection_detail.update(cx_collection_detail_params)
-        format.html { redirect_to admin_cx_collection_detail_url(@cx_collection_detail), notice: "Cx collection detail was successfully updated." }
+        format.html { redirect_to admin_cx_collection_detail_url(@cx_collection_detail), notice: "CX Collection Detail was successfully updated." }
         format.json { render :show, status: :ok, location: @cx_collection_detail }
       else
         format.html { render :edit, status: :unprocessable_entity }
@@ -51,19 +51,40 @@ def destroy
     @cx_collection_detail.destroy
 
     respond_to do |format|
-      format.html { redirect_to cx_collection_details_url, notice: "Cx collection detail was successfully destroyed." }
+      format.html { redirect_to cx_collection_details_url, notice: "CX Collection Detail was successfully destroyed." }
       format.json { head :no_content }
     end
   end
 
+  # Handle a large-ish csv upload (5+ MB) to S3
+  def upload_csv
+    file = params[:file] # Assuming the file comes from a form field named 'file'
+    bucket = ENV.fetch("S3_UPLOADS_AWS_BUCKET_NAME")
+
+    s3 = Aws::S3::Resource.new
+    key = "cx_data_collections/cx-upload-#{Time.now.to_i}-#{file.original_filename}}"
+    obj = s3.bucket(bucket).object(file.original_filename)
+    # Upload the file
+    response = obj.upload_file(file.path)
+
+    CxCollectionDetailUpload.create!({
+      user_id: current_user.id,
+      cx_collection_detail_id: @cx_collection_detail.id,
+      size: obj.size,
+      key: obj.key,
+    })
+
+    render :upload, message: 'File successfully uploaded'
+  end
+
   private
     def set_cx_collection_detail
       @cx_collection_detail = CxCollectionDetail.find(params[:id])
     end
 
     def set_cx_collections
       if service_manager_permissions?
-        @cx_collections = CxCollection.all.includes(:organization)
+        @cx_collections = CxCollection.all.includes(:organization).order('organizations.abbreviation')
       else
         @cx_collections = current_user.organization.cx_collections
       end

app/controllers/admin/service_providers_controller.rb

@@ -111,9 +111,6 @@ def service_provider_params
         :description,
         :notes,
         :slug,
-        :department,
-        :department_abbreviation,
-        :bureau,
         :inactive,
         :new,
         :tag_list,

app/models/cx_collection_detail_upload.rb

@@ -0,0 +1,4 @@
+class CxCollectionDetailUpload < ApplicationRecord
+    belongs_to :user
+    belongs_to :cx_collection_detail
+end

app/models/website.rb

@@ -191,7 +191,7 @@ def validate_domain_format
   end
 
   def validate_domain_suffix
-    errors.add(:domain, 'domain must have a valid suffix, like .gov or .mil') unless domain.present? && APPROVED_DOMAINS.any? { |word| domain.end_with?(word) }
+    errors.add(:domain, 'domain must have a valid suffix, like .gov or .mil') unless domain.present? && APPROVED_WEBSITE_DOMAINS.any? { |word| domain.end_with?(word) }
   end
 
   def site_scanner_json_request

app/serializers/service_serializer.rb

@@ -19,8 +19,6 @@ class ServiceSerializer < ActiveModel::Serializer
              :transactional,
              :notes,
              :hisp,
-             :department,
-             :bureau,
              :service_abbreviation,
              :service_slug,
              :service_owner_email,