Human friendly report id; limit workers for frontend test; add queue components; change data types to TEXT

.env.example

@@ -46,4 +46,5 @@ MINIO_ROOT_USER=EXAMPLEID
 # if using docker
 CLAMAV_ENDPOINT=http://clamav-rest:8080
 # if running locally
-# CLAMAV_ENDPOINT=http://localhost:8081
+# CLAMAV_ENDPOINT=http://localhost:8081
+REDIS_PASS=SUPERSECUREPASSWORD

docs/adr/0013-add-job-queue-and-worker.md

@@ -0,0 +1,26 @@
+# 13. Add job queue and worker
+
+# Date
+20201-02-13
+
+## Status
+
+Accepted
+
+## Context
+
+In order to satisfy the [RA-5](https://nvd.nist.gov/800-53/Rev4/control/RA-5)
+control around vulnerability scanning, we wish to scan all user-uploaded files
+with a malware detection service. We want to satisfy the following requirements.
+1. Scanning can be done asyncronously so as not to negatively impact the user experience.
+2. Scanning should be loosely coupled to main application to allow for more resiliance and fault tolerance.
+3. Scanning should be retried if malware detection service is unavailable.
+4. Scanning should run on a seperate instance to prevent a negative impact to the user experience.
+
+## Decision
+
+We will use redis as a queue and build a worker node which will take jobs from the queue, send them to the malware detection service and then update the database with the scan results.
+
+## Consequences
+
+All of the above requirements are filled. This does introduce some additional complexity in the form of a redis instance and worker instance. However, cloud.gov supplies managed redis instances and the worker will be built on top of an official node.js buildpack, so any additional maintenance from running these instances is negligable.

docs/boundary_diagram.md

@@ -1,7 +1,7 @@
 System Boundary Diagram
 =======================
 
-![rendered boundary diagram](http://www.plantuml.com/plantuml/png/dLP1Rniv3xtdL_3BeKY19kwNz50i5eiwiPjcaMpTZDjU2WIoXXvhev5KaR9hBFhVTwYC9nuxNHGz9DP8ukEZI3xf3R6Ad6fxy3_Zj4qbme8vq6-3GLIhayhm8iqIOTJUCJe-qRuUg38QvPOsXJIuU7KyiShnnvlhWLMCnCtKtOLth8p3U184cmrRXEbq24Mj8iExD8EPJwvKSGrBWwk32KRoxYZa7uVs1Hm-Rlr860-RcQ43TEMnq04_1rxiO1u-t-2Kk4Jh6xGUpcWpUzYRNZQWFa9dGw1S2PVJcmRll6Lvf5Oa33uNSEcN5zQdyl0v_7C0qEtGCbdvPRCj3q-QSRi7e5WJOtqt_iQ2TAViR7iRMA_AuvcookdCBZiQNfyt3FhMdPSx5KmFOZYv-e_TqIVEf_7xCwlM6EaFU3HzM2JncoE3jn9JN0D7DPyR3OHnQJIs-8zdmducNBMEM1fMRBphLhlG0RfgE5gjLWtnqEFHBulFE8DX2DRe32US_l8b_bcyWF6t496QDDoYqYmh8qEXWNb8cFRMecRAWALYTG8toR895i4uOZbRWfYSCwwIK7Qv-NYFiQMtGoel_OZNUR9MlYSOvpxMC5Fwlefo322OGYxpWH2zHgAdtMkhQYN5yzPO1D8gyynfpNbHzV2JZAuSOtJAWdfa0utVHvFS7ob8JT6-ez4iPw6S2MDFN4KiFbp3IB6QAS9DyTsE2XcsPWHj_xSdxUQcIh7ptS_X3ttIMGjUdC4iwNlamsuYjoJxY0GabQHXBkUWY91_78NKdVJbhVpknJ2sYXeafHTvwaF2XE3dsmdwsyy8s3yaohlMqVS3-Vj-yD19H--AST7Ooq-cQLnb4GfJENZllhYRjqDO6WLJK-FnkI8ifLB0HvXSVGAjOlcbsLNu57LsVncLOyHCTrjYAb9tf1H93M1vExoP7AiGebysW8LIHUweNdixN4MqHainuWfkPRp9cd799u8Rxmpxc6kkYQZPiYsEMm0togaA8JmKiJGBF0Fhgwen5SnyBS3bU3gOl2-QVn-dWz7uUZmTmz6hLsVF6tJ1RXkyXnmn-CW4Z1PhvBWMEDQXPxYb_8y2MMTENKF_70td6I6QQi7W7LZZxW-6re8MT2qLIr1YC9gn11KHqEcu3lArCho0wNK1_pyvXUGi4e7d1SQL8GJ1BKyEq9LvAAoxgjw-jqg_Sj9xRG4pAxseqJ7rwFU2xoBUr-iG_JSZkeLV4nBJpwTY3wlBpQ-3zaIoKonH7ihXTaWW873q-kNhbq-PjXqLaQF19NOjiPVbtUBgcQl42tJIuOoKPKzitm-RwTFTikgCbXZDV0sc54jUxuVUkJmtFQUTQaPK9GrMqOXNOKp8457xMERu0tsrdNaNzIyZz0ZGsLFc6tKMssNGWAuCtZZUO_7BCBj4EkL5zrLabKH98QYeQjoy9VAM7Ghx02K-2G3F88M6L7iVf1vY9vcjb4bxRQJpucfmCOA92MYDmttDp4zQISAbORciHDZV5SSIUlJMebnl9OA85Xdsyf3f0sWJjKLjn5SUNbpjQAvSqLLxEnRTqsbFV3icapIpXXRPiK0r2q3Dzz7fQLlw7Ujjvxwk3SF2-tiH93ENZjum0lOGauE7qmZTQMImHv2Tr-G8W_Nhtil2Ew2aDHBDastX6k06wLgjxqOxklqmUzlUc7dY3ReorVPV)
+![rendered boundary diagram](http://www.plantuml.com/plantuml/png/dLPHRnit37xFh-3oKAH04sDfNpOCGvtOJJCagvjpsvS2WQoZRIqwIHKfkyRG_pvHELzySJCKVKYiayIF7qNoaruJgyYfiGU_6ATjgX6Mp85-7moYMfyi32_JB156xHsZunFjcu6ger5kPLD8W_DNnnEh77-ulXfOnKYSfsxFlMDb7CQJ8DXis29CfqEe6XKPtgOPp7nojOeRM1bS7qmmadT7eVmuj2_Wy67h1y9uc6U63j3Lnq87_1nu-GZpyFaMfyOLMf_HUZYZxUnXRtgLGNs4peP0kHekfjSPxwnbUQJM90m-LN3XL-VMf_hmEVnx0D3jq3AvyAkyhGnFyhZT0r1jYB6v7NzbGRfLThQx3QnNzV5CMULqOLSS3Q_ECeE-TUVbLWNJWnXEBlzdT_I9osdyySog3KRw4nvCxnT9_9R8u8t453V0KStdHWDXN1cDBVx3cR2V2LUj8zQ65HllkjKkT82k6exMwtKc7dey-cNn2MSm3C4QNU24qz--nh-g5p3-6Y9IJiCDAisoCZ8KCinzmhIt5ZLJ0QmLgnEuJfPDi0Z64SlP4iJad76B7CeUDn_lCBRq7f4gIq_nEalMolS4uzp71cPAtosY70C8PY4dV214hv7e--wrLOsIudbZB09fLNYMj9Qyg6RuKKQNZZ4wPK4zi87izy6az4SAKbEqxsWqArcePyBO4oyYLk-lOAHOpHJXjlXkHfKCMpE2jl_RatPpiqYnyztFuPC-wMKBNhs4MT9to8VTH6b9zX49I2f9mrpEGH4XFns5r9tqvOtyxiKmjeeoajBBClK-OKBmyzq4_UDd1Epl4-LTw-ZRWVnzExZlfAEtrRZAzl9JQ1gti0YLMJXuvpwxwpT3M1a5KzFWyPaYhALIm4UON7u4hMBzEU-gV8gwkB-8oZ5YeRihCGjIjwGKIGrWVJEydXoh4A9VPS1AgQfqLA-z7QuZMgEb674DDpBUPisuv1F1jNU6VImrboFAMtR5SGVWMZcrG0ZtHIpD0i_0-iKYcmfckHRWOZmTJDvL-T-7wM0qlXfFnt3qwjNBvnbTiDi6xo5733uo0QF51Zbk18vrw1dkAF-3G3Q5KzVGFqR3UKQ8Ph5ay0wiSNS7Gsj12xgMYYMeCHZDM8EA2EXqt0JvMXjUmlIgWbzFJY4vYqJWUObnRGX1SEkJ0tHr6GhhhgftxwtQhvtqNbl0p4el6tHCFVgzuBk8z_KwHF_LY6xXbuJ4zEEfsCFgSlFpe3sHx1HBb8TAk3j8WW97lprXsxPKH8u6LzZrn5wQzwjhnrmbNgAJ5cUaedjY-swOfq_twwerMc4qymsOMYnvinzxv_BCz9nsiXbHrJHOHoDUXJ4XGqJjOqq25-YBxSoxg7yMeKz0gctVV2lFOhiA6j3LmHl7-on-5cQFo5PwqNqHNIMHKWWggWQtZucovIaA7K2DtmM0Pv129jLU2Ac7s6bcgsNINrljFBySd8zWOW9Qut1VDCkRLf9mOLXkAr7sjzMnX1wzjIZtMuqWgaM6lRna-W3QH6rH6_5LXkUNZqHNxkX5UprMtTjfJtqx9fCqiuQMnR50vGMW_7rqUjgMViTwrxbl6iEmzFvE5CdCfQMtZ80znEJW_ZH2Tne5x07aqNDoXC7wJUzfuHrGqXg9vicsS1au8rsfpUteaN3VpzwqLsQPU8skJetz3m00)
 
 UML Source
 ----------
@@ -40,7 +40,7 @@ end note
 Rel(www_s3, personnel, "download file attachments", "https GET (443)")
 Rel(aws_alb, cloudgov_router, "proxies requests", "https GET/POST/PUT/DELETE (443)")
 Rel(cloudgov_router, www_app, "proxies requests", "https GET/POST/PUT/DELETE (443)")
-Rel(worker_app, clamav, "scans files", "http POST (8080)")
+Rel(worker_app, clamav, "scans files", "https POST (9443)")
 Rel(worker_app, HSES, "retrieve Grantee data", "https GET (443)")
 Rel(www_app, HSES, "authenticates user", "OAuth2")
 Rel(personnel, HSES, "verify identity", "https GET/POST (443)")
@@ -65,7 +65,7 @@ Lay_R(HSES, aws)
 Instructions
 ------------
 
-1. [Edit this diagram with plantuml.com](http://www.plantuml.com/plantuml/uml/dLP1Rniv3xtdL_3BeKY19kwNz50i5eiwiPjcaMpTZDjU2WIoXXvhev5KaR9hBFhVTwYC9nuxNHGz9DP8ukEZI3xf3R6Ad6fxy3_Zj4qbme8vq6-3GLIhayhm8iqIOTJUCJe-qRuUg38QvPOsXJIuU7KyiShnnvlhWLMCnCtKtOLth8p3U184cmrRXEbq24Mj8iExD8EPJwvKSGrBWwk32KRoxYZa7uVs1Hm-Rlr860-RcQ43TEMnq04_1rxiO1u-t-2Kk4Jh6xGUpcWpUzYRNZQWFa9dGw1S2PVJcmRll6Lvf5Oa33uNSEcN5zQdyl0v_7C0qEtGCbdvPRCj3q-QSRi7e5WJOtqt_iQ2TAViR7iRMA_AuvcookdCBZiQNfyt3FhMdPSx5KmFOZYv-e_TqIVEf_7xCwlM6EaFU3HzM2JncoE3jn9JN0D7DPyR3OHnQJIs-8zdmducNBMEM1fMRBphLhlG0RfgE5gjLWtnqEFHBulFE8DX2DRe32US_l8b_bcyWF6t496QDDoYqYmh8qEXWNb8cFRMecRAWALYTG8toR895i4uOZbRWfYSCwwIK7Qv-NYFiQMtGoel_OZNUR9MlYSOvpxMC5Fwlefo322OGYxpWH2zHgAdtMkhQYN5yzPO1D8gyynfpNbHzV2JZAuSOtJAWdfa0utVHvFS7ob8JT6-ez4iPw6S2MDFN4KiFbp3IB6QAS9DyTsE2XcsPWHj_xSdxUQcIh7ptS_X3ttIMGjUdC4iwNlamsuYjoJxY0GabQHXBkUWY91_78NKdVJbhVpknJ2sYXeafHTvwaF2XE3dsmdwsyy8s3yaohlMqVS3-Vj-yD19H--AST7Ooq-cQLnb4GfJENZllhYRjqDO6WLJK-FnkI8ifLB0HvXSVGAjOlcbsLNu57LsVncLOyHCTrjYAb9tf1H93M1vExoP7AiGebysW8LIHUweNdixN4MqHainuWfkPRp9cd799u8Rxmpxc6kkYQZPiYsEMm0togaA8JmKiJGBF0Fhgwen5SnyBS3bU3gOl2-QVn-dWz7uUZmTmz6hLsVF6tJ1RXkyXnmn-CW4Z1PhvBWMEDQXPxYb_8y2MMTENKF_70td6I6QQi7W7LZZxW-6re8MT2qLIr1YC9gn11KHqEcu3lArCho0wNK1_pyvXUGi4e7d1SQL8GJ1BKyEq9LvAAoxgjw-jqg_Sj9xRG4pAxseqJ7rwFU2xoBUr-iG_JSZkeLV4nBJpwTY3wlBpQ-3zaIoKonH7ihXTaWW873q-kNhbq-PjXqLaQF19NOjiPVbtUBgcQl42tJIuOoKPKzitm-RwTFTikgCbXZDV0sc54jUxuVUkJmtFQUTQaPK9GrMqOXNOKp8457xMERu0tsrdNaNzIyZz0ZGsLFc6tKMssNGWAuCtZZUO_7BCBj4EkL5zrLabKH98QYeQjoy9VAM7Ghx02K-2G3F88M6L7iVf1vY9vcjb4bxRQJpucfmCOA92MYDmttDp4zQISAbORciHDZV5SSIUlJMebnl9OA85Xdsyf3f0sWJjKLjn5SUNbpjQAvSqLLxEnRTqsbFV3icapIpXXRPiK0r2q3Dzz7fQLlw7Ujjvxwk3SF2-tiH93ENZjum0lOGauE7qmZTQMImHv2Tr-G8W_Nhtil2Ew2aDHBDastX6k06wLgjxqOxklqmUzlUc7dY3ReorVPV)
+1. [Edit this diagram with plantuml.com](http://www.plantuml.com/plantuml/uml/dLPHRnit37xFh-3oKAH04sDfNpOCGvtOJJCagvjpsvS2WQoZRIqwIHKfkyRG_pvHELzySJCKVKYiayIF7qNoaruJgyYfiGU_6ATjgX6Mp85-7moYMfyi32_JB156xHsZunFjcu6ger5kPLD8W_DNnnEh77-ulXfOnKYSfsxFlMDb7CQJ8DXis29CfqEe6XKPtgOPp7nojOeRM1bS7qmmadT7eVmuj2_Wy67h1y9uc6U63j3Lnq87_1nu-GZpyFaMfyOLMf_HUZYZxUnXRtgLGNs4peP0kHekfjSPxwnbUQJM90m-LN3XL-VMf_hmEVnx0D3jq3AvyAkyhGnFyhZT0r1jYB6v7NzbGRfLThQx3QnNzV5CMULqOLSS3Q_ECeE-TUVbLWNJWnXEBlzdT_I9osdyySog3KRw4nvCxnT9_9R8u8t453V0KStdHWDXN1cDBVx3cR2V2LUj8zQ65HllkjKkT82k6exMwtKc7dey-cNn2MSm3C4QNU24qz--nh-g5p3-6Y9IJiCDAisoCZ8KCinzmhIt5ZLJ0QmLgnEuJfPDi0Z64SlP4iJad76B7CeUDn_lCBRq7f4gIq_nEalMolS4uzp71cPAtosY70C8PY4dV214hv7e--wrLOsIudbZB09fLNYMj9Qyg6RuKKQNZZ4wPK4zi87izy6az4SAKbEqxsWqArcePyBO4oyYLk-lOAHOpHJXjlXkHfKCMpE2jl_RatPpiqYnyztFuPC-wMKBNhs4MT9to8VTH6b9zX49I2f9mrpEGH4XFns5r9tqvOtyxiKmjeeoajBBClK-OKBmyzq4_UDd1Epl4-LTw-ZRWVnzExZlfAEtrRZAzl9JQ1gti0YLMJXuvpwxwpT3M1a5KzFWyPaYhALIm4UON7u4hMBzEU-gV8gwkB-8oZ5YeRihCGjIjwGKIGrWVJEydXoh4A9VPS1AgQfqLA-z7QuZMgEb674DDpBUPisuv1F1jNU6VImrboFAMtR5SGVWMZcrG0ZtHIpD0i_0-iKYcmfckHRWOZmTJDvL-T-7wM0qlXfFnt3qwjNBvnbTiDi6xo5733uo0QF51Zbk18vrw1dkAF-3G3Q5KzVGFqR3UKQ8Ph5ay0wiSNS7Gsj12xgMYYMeCHZDM8EA2EXqt0JvMXjUmlIgWbzFJY4vYqJWUObnRGX1SEkJ0tHr6GhhhgftxwtQhvtqNbl0p4el6tHCFVgzuBk8z_KwHF_LY6xXbuJ4zEEfsCFgSlFpe3sHx1HBb8TAk3j8WW97lprXsxPKH8u6LzZrn5wQzwjhnrmbNgAJ5cUaedjY-swOfq_twwerMc4qymsOMYnvinzxv_BCz9nsiXbHrJHOHoDUXJ4XGqJjOqq25-YBxSoxg7yMeKz0gctVV2lFOhiA6j3LmHl7-on-5cQFo5PwqNqHNIMHKWWggWQtZucovIaA7K2DtmM0Pv129jLU2Ac7s6bcgsNINrljFBySd8zWOW9Qut1VDCkRLf9mOLXkAr7sjzMnX1wzjIZtMuqWgaM6lRna-W3QH6rH6_5LXkUNZqHNxkX5UprMtTjfJtqx9fCqiuQMnR50vGMW_7rqUjgMViTwrxbl6iEmzFvE5CdCfQMtZ80znEJW_ZH2Tne5x07aqNDoXC7wJUzfuHrGqXg9vicsS1au8rsfpUteaN3VpzwqLsQPU8skJetz3m00)
 1. Copy and paste the final UML into the UML Source section
 1. Update the img src and edit link target to the current values
 

docs/openapi/index.yaml

@@ -24,6 +24,11 @@ components:
       properties:
         id:
           type: number
+        displayId:
+          type: string
+          description: >
+            A human readable report ID, of the form `R[Region]-AR-[ID]`. A report with
+            the ID of 20 in region 8 would have a displayId of `R08-AR-20`
         userId:
           type: number
           description: The id of the author of the report

package.json

@@ -142,6 +142,7 @@
     "adm-zip": "^0.5.1",
     "aws-sdk": "^2.826.0",
     "axios": "^0.21.1",
+    "bull": "^3.20.1",
     "chromedriver": "^87.0.0",
     "client-oauth2": "^4.3.3",
     "cookie-session": "^1.4.0",

src/migrations/20210215161922-add-statuses-to-file-model.js

@@ -0,0 +1,12 @@
+module.exports = {
+  up: async (queryInterface) => {
+    await queryInterface.sequelize.query('ALTER TYPE "enum_Files_status" ADD VALUE \'QUEUEING_FAILED\';');
+    await queryInterface.sequelize.query('ALTER TYPE "enum_Files_status" ADD VALUE \'SCANNING_QUEUED\';');
+  },
+  down: async (queryInterface) => {
+    let query = 'DELETE FROM pg_enum WHERE enumlabel = \'QUEUEING_FAILED\' AND enumtypid = ( SELECT oid FROM pg_type WHERE typname = \'enum_Files_status\')';
+    await queryInterface.sequelize.query(query);
+    query = 'DELETE FROM pg_enum WHERE enumlabel = \'SCANNING_QUEUED\' AND enumtypid = ( SELECT oid FROM pg_type WHERE typname = \'enum_Files_status\')';
+    await queryInterface.sequelize.query(query);
+  },
+};

src/migrations/20210215181557-convert-string-types-to-text.js

@@ -0,0 +1,17 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    Promise.all([
+      queryInterface.changeColumn('ActivityReports', 'additionalNotes', { type: Sequelize.TEXT }),
+      queryInterface.changeColumn('ActivityReports', 'context', { type: Sequelize.TEXT }),
+      queryInterface.changeColumn('ActivityReports', 'resourcesUsed', { type: Sequelize.TEXT }),
+    ]);
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    Promise.all([
+      queryInterface.changeColumn('ActivityReports', 'additionalNotes', { type: Sequelize.STRING }),
+      queryInterface.changeColumn('ActivityReports', 'context', { type: Sequelize.STRING }),
+      queryInterface.changeColumn('ActivityReports', 'resourcesUsed', { type: Sequelize.STRING }),
+    ]);
+  },
+};

src/models/activityReport.js

@@ -38,6 +38,12 @@ export default (sequelize, DataTypes) => {
     }
   }
   ActivityReport.init({
+    displayId: {
+      type: DataTypes.VIRTUAL,
+      get() {
+        return `R${this.regionId.toString().padStart(2, '0')}-AR-${this.id}`;
+      },
+    },
     userId: {
       type: DataTypes.INTEGER,
     },
@@ -49,10 +55,10 @@ export default (sequelize, DataTypes) => {
       allowNull: true,
     },
     resourcesUsed: {
-      type: DataTypes.STRING,
+      type: DataTypes.TEXT,
     },
     additionalNotes: {
-      type: DataTypes.STRING,
+      type: DataTypes.TEXT,
     },
     numberOfParticipants: {
       type: DataTypes.INTEGER,
@@ -94,7 +100,7 @@ export default (sequelize, DataTypes) => {
       type: DataTypes.ARRAY(DataTypes.STRING),
     },
     context: {
-      type: DataTypes.STRING,
+      type: DataTypes.TEXT,
     },
     pageState: {
       type: DataTypes.JSON,

src/models/file.js

@@ -26,7 +26,16 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
     },
     status: {
-      type: DataTypes.ENUM('UPLOADING', 'UPLOADED', 'UPLOAD_FAILED', 'SCANNING', 'APPROVED', 'REJECTED'),
+      type: DataTypes.ENUM(
+        'UPLOADING',
+        'UPLOADED',
+        'UPLOAD_FAILED',
+        'QUEUEING_FAILED',
+        'SCANNING_QUEUED',
+        'SCANNING',
+        'APPROVED',
+        'REJECTED',
+      ),
       allowNull: false,
     },
     attachmentType: {

src/routes/files/handlers.js

@@ -3,10 +3,12 @@ import * as fs from 'fs';
 import handleErrors from '../../lib/apiErrorHandler';
 import { File } from '../../models';
 import s3Uploader from '../../lib/s3Uploader';
+import addToScanQueue from '../../services/queue';
 
 import ActivityReportPolicy from '../../policies/activityReport';
 import { activityReportById } from '../../services/activityReports';
 import { userById } from '../../services/users';
+import { auditLogger } from '../../logger';
 
 const fileType = require('file-type');
 const multiparty = require('multiparty');
@@ -20,14 +22,25 @@ const logContext = {
   namespace,
 };
 
+const fileStatuses = {
+  uploading: 'UPLOADING',
+  uploaded: 'UPLOADED',
+  uploadFailed: 'UPLOAD_FAILED',
+  queued: 'SCANNING_QUEUED',
+  queueingFailed: 'QUEUEING_FAILED',
+  scanning: 'SCANNING',
+  approved: 'APPROVED',
+  rejected: 'REJECTED',
+};
+
 export const createFileMetaData = async (
   originalFileName, s3FileName, reportId, attachmentType, fileSize) => {
   const newFile = {
     activityReportId: reportId,
     originalFileName,
     attachmentType,
     key: s3FileName,
-    status: 'UPLOADING',
+    status: fileStatuses.uploading,
     fileSize,
   };
   let file;
@@ -111,13 +124,23 @@ export default async function uploadHandler(req, res) {
     }
     try {
       await s3Uploader(buffer, fileName, type);
-      await updateStatus(metadata.id, 'UPLOADED');
+      await updateStatus(metadata.id, fileStatuses.uploaded);
       res.status(200).send({ id: metadata.id });
     } catch (err) {
       if (metadata) {
-        await updateStatus(metadata.id, 'UPLOAD_FAILED');
+        await updateStatus(metadata.id, fileStatuses.uploadFailed);
       }
       await handleErrors(req, res, err, logContext);
+      return;
+    }
+    try {
+      await addToScanQueue({ key: metadata.key });
+      await updateStatus(metadata.id, fileStatuses.queued);
+    } catch (err) {
+      if (metadata) {
+        await updateStatus(metadata.id, fileStatuses.queueingFailed);
+        auditLogger.error(`${logContext} Failed to queue ${metadata.originalFileName}. Error: ${err}`);
+      }
     }
   });
 }

src/routes/files/handlers.test.js

@@ -7,6 +7,7 @@ import db, {
 } from '../../models';
 import app from '../../app';
 import s3Uploader from '../../lib/s3Uploader';
+import * as queue from '../../services/queue';
 import SCOPES from '../../middleware/scopeConstants';
 import { REPORT_STATUSES } from '../../constants';
 import ActivityReportPolicy from '../../policies/activityReport';
@@ -43,6 +44,7 @@ const mockUser = {
 
 const mockSession = jest.fn();
 mockSession.userId = mockUser.id;
+const mockAddToScanQueue = jest.spyOn(queue, 'default').mockImplementation(() => jest.fn());
 
 const reportObject = {
   activityRecipientType: 'grantee',
@@ -93,6 +95,7 @@ describe('File Upload', () => {
           fileId = res.body.id;
           expect(s3Uploader).toHaveBeenCalled();
         });
+      expect(mockAddToScanQueue).toHaveBeenCalled();
     });
     it('checks the metadata was uploaded to the database', async () => {
       ActivityReportPolicy.mockImplementation(() => ({

src/services/queue.js

@@ -0,0 +1,10 @@
+import Queue from 'bull';
+
+const REDIS_PORT = process.env.REDIS_PORT || 6379;
+const { REDIS_HOST, REDIS_PASS } = process.env;
+
+const scanQueue = new Queue('scan', `redis://${REDIS_HOST}:${REDIS_PORT}`, { redis: { password: REDIS_PASS } });
+
+export default async function addToScanQueue(fileKey) {
+  await scanQueue.add(fileKey);
+}

terraform/dev/main.tf

@@ -74,13 +74,8 @@ resource "cloudfoundry_service_instance" "document_upload_bucket" {
 # ClamAV networking
 ###
 
-data "cloudfoundry_app" "clamav_server" {
-  name_or_id = var.clamav_server_app_name
-  space      = data.cloudfoundry_space.space.id
-}
-
-data "cloudfoundry_app" "clamav_rest" {
-  name_or_id = var.clamav_rest_app_name
+data "cloudfoundry_app" "clamav_api" {
+  name_or_id = var.clamav_api_app_name
   space      = data.cloudfoundry_space.space.id
 }
 
@@ -90,14 +85,9 @@ data "cloudfoundry_app" "ttahub" {
 }
 
 resource "cloudfoundry_network_policy" "clamav_routing" {
-  policy {
-    source_app      = data.cloudfoundry_app.clamav_rest.id
-    destination_app = data.cloudfoundry_app.clamav_server.id
-    port            = "3310"
-  }
   policy {
     source_app      = data.cloudfoundry_app.ttahub.id
-    destination_app = data.cloudfoundry_app.clamav_rest.id
-    port            = "8080"
+    destination_app = data.cloudfoundry_app.clamav_api.id
+    port            = "9443"
   }
 }

terraform/dev/variables.tf

@@ -45,14 +45,8 @@ variable "env" {
   default     = "dev"
 }
 
-variable "clamav_server_app_name" {
-  type        = string
-  description = "app name for clamd server"
-  default     = "clamav-server-ttahub-dev"
-}
-
-variable "clamav_rest_app_name" {
+variable "clamav_api_app_name" {
   type        = string
   description = "app name for clamav api server"
-  default     = "clamav-rest-ttahub-dev"
+  default     = "clamav-api-ttahub-dev"
 }

terraform/prod/main.tf

@@ -77,13 +77,8 @@ resource "cloudfoundry_service_instance" "document_upload_bucket" {
 # Run them after the ttahub and ClamAV apps have all been deployed
 ###
 
-data "cloudfoundry_app" "clamav_server" {
-  name_or_id = var.clamav_server_app_name
-  space      = data.cloudfoundry_space.space.id
-}
-
-data "cloudfoundry_app" "clamav_rest" {
-  name_or_id = var.clamav_rest_app_name
+data "cloudfoundry_app" "clamav_api" {
+  name_or_id = var.clamav_api_app_name
   space      = data.cloudfoundry_space.space.id
 }
 
@@ -93,14 +88,9 @@ data "cloudfoundry_app" "ttahub" {
 }
 
 resource "cloudfoundry_network_policy" "clamav_routing" {
-  policy {
-    source_app      = data.cloudfoundry_app.clamav_rest.id
-    destination_app = data.cloudfoundry_app.clamav_server.id
-    port            = "3310"
-  }
   policy {
     source_app      = data.cloudfoundry_app.ttahub.id
-    destination_app = data.cloudfoundry_app.clamav_rest.id
-    port            = "8080"
+    destination_app = data.cloudfoundry_app.clamav_api.id
+    port            = "9443"
   }
 }

terraform/prod/variables.tf

@@ -45,14 +45,8 @@ variable "env" {
   default     = "prod"
 }
 
-variable "clamav_server_app_name" {
-  type        = string
-  description = "app name for clamd server"
-  default     = "clamav-server-ttahub-prod"
-}
-
-variable "clamav_rest_app_name" {
+variable "clamav_api_app_name" {
   type        = string
   description = "app name for clamav api server"
-  default     = "clamav-rest-ttahub-prod"
+  default     = "clamav-api-ttahub-prod"
 }

terraform/sandbox/main.tf

@@ -80,8 +80,8 @@ data "cloudfoundry_space" "dev_space" {
   name     = var.cf_dev_space_name
 }
 
-data "cloudfoundry_app" "clamav_rest" {
-  name_or_id = var.clamav_rest_app_name
+data "cloudfoundry_app" "clamav_api" {
+  name_or_id = var.clamav_api_app_name
   space      = data.cloudfoundry_space.dev_space.id
 }
 
@@ -93,7 +93,7 @@ data "cloudfoundry_app" "ttahub" {
 resource "cloudfoundry_network_policy" "clamav_routing" {
   policy {
     source_app      = data.cloudfoundry_app.ttahub.id
-    destination_app = data.cloudfoundry_app.clamav_rest.id
-    port            = "8080"
+    destination_app = data.cloudfoundry_app.clamav_api.id
+    port            = "9443"
   }
 }