Merge pull request #4387 from guardian/file-size-limit

file size limit for upload/ingest (optional and configurable)
guardian · Dec 23, 2024 · 97764d8 · 97764d8
2 parents 0b56a49 + 8ab87d9
commit 97764d8
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 2 deletions.
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala
@@ -54,6 +54,8 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B
   val maybeIngestBucket: Option[String] = stringOpt("s3.ingest.bucket")
   val maybeFailBucket: Option[String] = stringOpt("s3.fail.bucket")
 
+  val maybeUploadLimitInBytes: Option[Int] = intOpt("upload.limit.mb").map(_ * 1_000_000)
+
   // Note: had to make these lazy to avoid init order problems ;_;
   val domainRoot: String = string("domain.root")
   val domainRootOverride: Option[String] = stringOpt("domain.root-override")

diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala
@@ -141,7 +141,19 @@ class ImageLoaderController(auth: Authentication,
 
         val approximateReceiveCount = getApproximateReceiveCount(sqsMessage)
 
-        if (approximateReceiveCount > 2) {
+        if(config.maybeUploadLimitInBytes.exists(_ < s3IngestObject.contentLength)){
+          val errorMessage = s"File size exceeds the maximum allowed size (${config.maybeUploadLimitInBytes.get / 1_000_000}MB). Moving to fail bucket."
+          logger.warn(logMarker, errorMessage)
+          store.moveObjectToFailedBucket(s3IngestObject.key)
+          s3IngestObject.maybeMediaIdFromUiUpload foreach { imageId =>
+            uploadStatusTable.updateStatus( // fire & forget, since there's nothing else we can do
+              imageId, UploadStatus(StatusType.Failed, Some(errorMessage))
+            )
+          }
+          metrics.failedIngestsFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
+          Future.unit
+        }
+        else if (approximateReceiveCount > 2) {
           metrics.abandonedMessagesFromQueue.incrementBothWithAndWithoutDimensions(metricDimensions)
           val errorMessage = s"File processing has been attempted $approximateReceiveCount times. Moving to fail bucket."
           logger.warn(logMarker, errorMessage)

diff --git a/kahuna/app/views/main.scala.html b/kahuna/app/views/main.scala.html
@@ -85,6 +85,7 @@
           permissionsDefault: "@kahunaConfig.permissionsDefault",
           defaultShouldBlurGraphicImages: @kahunaConfig.defaultShouldBlurGraphicImages,
           shouldUploadStraightToBucket: @kahunaConfig.shouldUploadStraightToBucket,
+          maybeUploadLimitInBytes: @kahunaConfig.maybeUploadLimitInBytes,
           announcements: @Html(announcements),
           imageTypes: @Html(imageTypes),
         }

diff --git a/kahuna/public/js/upload/manager.js b/kahuna/public/js/upload/manager.js
@@ -31,7 +31,21 @@ upload.factory('uploadManager',
         };
     }
 
-    async function createJobItems(files){
+    async function createJobItems(_files){
+
+      const maybeUploadLimitInBytes = window._clientConfig.maybeUploadLimitInBytes;
+      const maybeFilesAboveSizeLimit = !!maybeUploadLimitInBytes && _files.filter(file => file.size > maybeUploadLimitInBytes);
+
+      if (maybeFilesAboveSizeLimit && maybeFilesAboveSizeLimit.length > 0){
+        alert(`The following files will be skipped as they are above the size limit of ${maybeUploadLimitInBytes / 1_000_000}MB:\n${
+          maybeFilesAboveSizeLimit.map(file => file.name).join("\n")
+        }`);
+      }
+
+      const files = maybeFilesAboveSizeLimit && maybeFilesAboveSizeLimit.length > 0
+        ? _files.filter(file => !maybeFilesAboveSizeLimit.includes(file))
+        : _files;
+
       if (window._clientConfig.shouldUploadStraightToBucket) {
         const mediaIdToFileMap = Object.fromEntries(
           await Promise.all(