3855 - Move files to s3 bucket

.env.template

@@ -52,3 +52,9 @@ SEPAL_PASSWORD="password"
 GEE_PRIVATE_KEY='private key json content'
 
 FRA_MAIL_ENABLED=false
+
+# File S3
+AWS_ACCESS_KEY_ID=your-access-key-id
+AWS_SECRET_ACCESS_KEY=your-secret-access-key
+AWS_REGION=eu-west-1
+S3_BUCKET_NAME=fra-platform-s3

package.json

@@ -158,6 +158,7 @@
     "webpack-dev-server": "^4.7.4"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.637.0",
     "@google/earthengine": "^0.1.316",
     "@googlemaps/js-api-loader": "^1.13.10",
     "@googlemaps/react-wrapper": "^1.1.29",

src/meta/file/file.ts

@@ -1,16 +1,18 @@
+import { Readable } from 'stream'
+
 import { Label } from 'meta/assessment'
 
 export type FileSummary = {
   readonly createdAt: string
   readonly id: number
   readonly name: string
-  readonly size: number
+  size: number
   readonly uuid: string
   readonly repositoryItemUuid: string
 }
 
 export type File = FileSummary & {
-  readonly file: Buffer
+  file: Readable
 }
 
 export type FileUsage = {

src/meta/user/userProfilePicture.ts

@@ -1,4 +1,7 @@
+import { Readable } from 'stream'
+
 export interface UserProfilePicture {
   name: string
-  data: string
+  uuid: string
+  data?: Readable
 }

src/server/api/cycleData/print/report.ts

@@ -75,7 +75,14 @@ const getPdf = async (req: Request, fileName: string): Promise<Buffer> => {
     return pdfBuffer
   }
 
-  return cachedPdfInfo.file.file
+  // Convert Readable to Buffer
+  const chunks = []
+  // eslint-disable-next-line no-restricted-syntax
+  for await (const chunk of cachedPdfInfo.file.file) {
+    chunks.push(chunk)
+  }
+
+  return Buffer.concat(chunks)
 }
 
 export const report = async (req: Request, res: Response) => {

src/server/api/cycleData/repository/getRepositoryFile.ts

@@ -6,6 +6,7 @@ import { Translations } from 'meta/translation'
 
 import { AssessmentController } from 'server/controller/assessment'
 import { CycleDataController } from 'server/controller/cycleData'
+import { FileStorage, FileStorageUtils } from 'server/service/fileStorage'
 import Requests from 'server/utils/requests'
 import { Responses } from 'server/utils/responses'
 
@@ -26,7 +27,10 @@ export const getRepositoryFile = async (req: Request, res: Response) => {
     const extension = file.name.split('.').pop()
     const fileName = `${label}.${extension}`
 
-    Responses.sendFile(res, fileName, file.file)
+    const key = repositoryItem.fileUuid
+    const fileStream = await FileStorage.getFile({ key })
+
+    Responses.sendFileStream(res, fileName, fileStream, FileStorageUtils.getContentType(extension))
   } catch (e) {
     Requests.sendErr(res, e)
   }

src/server/api/file/get.ts

@@ -1,6 +1,6 @@
 import { Request, Response } from 'express'
 
-import { FileRepository, fileTypes } from 'server/service/file'
+import { FileRepository, fileTypes } from 'server/service/file_deprecated'
 
 type Query = {
   countryIso: string

src/server/api/file/getBiomassStockFile.ts

@@ -2,7 +2,7 @@ import { Response } from 'express'
 
 import { BiomassStockFileRequest } from 'meta/api/request'
 
-import { FileRepository, fileTypes } from 'server/service/file'
+import { FileRepository, fileTypes } from 'server/service/file_deprecated'
 import { Requests } from 'server/utils'
 
 export const getBiomassStockFile = async (req: BiomassStockFileRequest, res: Response) => {

src/server/api/file/getDataDownloadFile.ts

@@ -2,7 +2,7 @@ import { Response } from 'express'
 
 import { CycleRequest } from 'meta/api/request'
 
-import { FileRepository, fileTypes } from 'server/service/file'
+import { FileRepository, fileTypes } from 'server/service/file_deprecated'
 
 type Request = CycleRequest<{
   fileName: string

src/server/api/file/getSdgMetadata.ts

@@ -3,7 +3,7 @@ import { Response } from 'express'
 import { CycleDataRequest } from 'meta/api/request'
 import { Lang } from 'meta/lang'
 
-import { FileRepository, fileTypes } from 'server/service/file'
+import { FileRepository, fileTypes } from 'server/service/file_deprecated'
 import { Requests } from 'server/utils'
 
 type Request = CycleDataRequest<{ key: 'Metadata-15-01-01' | 'Metadata-15-02-01' }>

src/server/api/file/getUserGuide.ts

@@ -1,6 +1,6 @@
 import { Request, Response } from 'express'
 
-import { FileRepository, fileTypes } from 'server/service/file'
+import { FileRepository, fileTypes } from 'server/service/file_deprecated'
 
 type Query = {
   language: string

src/server/api/user/getProfilePicture.ts

@@ -1,14 +1,15 @@
-import { Request, Response } from 'express'
 import * as path from 'path'
-import Requests from 'server/utils/requests'
+import { Request, Response } from 'express'
+
 import { UserController } from 'server/controller/user'
+import Requests from 'server/utils/requests'
 
 export const getProfilePicture = async (req: Request, res: Response) => {
   const { id } = req.params
   try {
     const profilePicture = await UserController.getProfilePicture({ id: Number(id) })
     if (profilePicture && profilePicture.data) {
-      res.end(profilePicture.data, 'binary')
+      profilePicture.data.pipe(res)
     } else {
       res.sendFile(path.resolve(__dirname, '..', '..', 'static', 'avatar.png'))
     }

src/server/controller/cycleData/report/create.ts

@@ -6,6 +6,7 @@ import { FileSummary } from 'meta/file'
 import { BaseProtocol, DB } from 'server/db'
 import { RepositoryRepository } from 'server/repository/assessmentCycle/repository'
 import { FileRepository } from 'server/repository/public/file'
+import { FileStorage } from 'server/service/fileStorage'
 
 import { bufferToPdfMulterFile } from './utils'
 
@@ -30,7 +31,10 @@ export const create = async (props: Props): Promise<Returned> => {
   const pdfMulterFile = bufferToPdfMulterFile({ buffer, fileName })
 
   return DB.tx(async (t: BaseProtocol) => {
-    const file = await FileRepository.create({ file: pdfMulterFile }, t)
+    const file = await FileRepository.create({ fileName: pdfMulterFile.originalname }, t)
+    const { uuid: key } = file
+    const body = pdfMulterFile.buffer
+    await FileStorage.uploadFile({ key, body })
 
     const repositoryItemProps: Partial<RepositoryItem> = {
       countryIso,

src/server/controller/cycleData/report/getOne.ts

@@ -6,6 +6,7 @@ import { File } from 'meta/file'
 
 import { RepositoryRepository } from 'server/repository/assessmentCycle/repository'
 import { FileRepository } from 'server/repository/public/file'
+import { FileStorage } from 'server/service/fileStorage'
 
 type Props = {
   assessment: Assessment
@@ -25,7 +26,10 @@ export const getOne = async (props: Props): Promise<Returned> => {
     const repositoryItem = await RepositoryRepository.getOne(props)
 
     const fileRepositoryProps = { fileUuid: repositoryItem.fileUuid }
-    const file = await FileRepository.getOne(fileRepositoryProps)
+    const fileSummary = await FileRepository.getOne(fileRepositoryProps)
+    const { uuid: key } = fileSummary
+    const fileData = await FileStorage.getFile({ key })
+    const file = { ...fileSummary, file: fileData }
 
     return { file, repositoryItem }
   } catch (error) {

src/server/controller/cycleData/report/updateFile.ts

@@ -6,6 +6,7 @@ import { FileSummary } from 'meta/file'
 import { BaseProtocol, DB } from 'server/db'
 import { RepositoryRepository } from 'server/repository/assessmentCycle/repository'
 import { FileRepository } from 'server/repository/public/file'
+import { FileStorage } from 'server/service/fileStorage'
 
 import { bufferToPdfMulterFile } from './utils'
 
@@ -30,7 +31,10 @@ export const updateFile = async (props: Props): Promise<Returned> => {
     const repositoryItem = await RepositoryRepository.getOne(getRepositoryItemProps, t)
 
     const pdfMulterFile = bufferToPdfMulterFile({ buffer, fileName })
-    const newFile = await FileRepository.create({ file: pdfMulterFile }, t)
+    const newFile = await FileRepository.create({ fileName: pdfMulterFile.originalname }, t)
+    const { uuid: key } = newFile
+    const body = pdfMulterFile.buffer
+    await FileStorage.uploadFile({ key, body })
 
     const updateRepositoryItemProps: RepositoryItem = { ...repositoryItem, fileUuid: newFile.uuid }
     const updateRepositoryProps = { assessment, cycle, repositoryItem: updateRepositoryItemProps }

src/server/controller/cycleData/repository/getFileMeta.ts

@@ -3,6 +3,7 @@ import { FileMeta } from 'meta/file'
 
 import { RepositoryRepository } from 'server/repository/assessmentCycle/repository'
 import { FileRepository } from 'server/repository/public/file'
+import { FileStorage } from 'server/service/fileStorage'
 
 type Props = {
   assessment: Assessment
@@ -24,6 +25,8 @@ export const getFileMeta = async (props: Props): Promise<Returned> => {
     FileRepository.getSummary({ fileUuid: repositoryItem.fileUuid }),
   ])
 
+  summary.size = await FileStorage.getFileSize({ key: repositoryItem.fileUuid })
+
   return {
     usages,
     summary: { ...summary, repositoryItemUuid: repositoryItem.uuid },

src/server/controller/cycleData/repository/getManyFiles.ts

@@ -1,10 +1,13 @@
+import { Readable } from 'stream'
+
 import { CountryIso } from 'meta/area'
 import { Assessment, Cycle } from 'meta/assessment'
 import { Lang } from 'meta/lang'
 import { Translations } from 'meta/translation'
 
 import { RepositoryRepository } from 'server/repository/assessmentCycle/repository'
 import { FileRepository } from 'server/repository/public/file'
+import { FileStorage } from 'server/service/fileStorage'
 
 type Props = {
   assessment: Assessment
@@ -15,7 +18,7 @@ type Props = {
 
 type Returned = Array<{
   fileName: string
-  file: Buffer
+  file: Readable
 }>
 
 export const getManyFiles = async (props: Props): Promise<Returned> => {
@@ -27,6 +30,12 @@ export const getManyFiles = async (props: Props): Promise<Returned> => {
   const repositoryProps = { fileUuids: repositoryItems.map((item) => item.fileUuid) }
   const files = await FileRepository.getMany(repositoryProps)
 
+  // eslint-disable-next-line no-restricted-syntax
+  for await (const file of files) {
+    const { uuid: key } = file
+    file.file = await FileStorage.getFile({ key })
+  }
+
   return files.map((file) => {
     const repositoryItem = repositoryItems.find((item) => item.fileUuid === file.uuid)
     const label = Translations.getLabel({ translation: repositoryItem.props.translation, language: Lang.en })

src/server/controller/file/createMany.ts

@@ -5,6 +5,7 @@ import { User } from 'meta/user'
 import { BaseProtocol, DB } from 'server/db'
 import { ActivityLogRepository } from 'server/repository/public/activityLog'
 import { FileRepository } from 'server/repository/public/file'
+import { FileStorage } from 'server/service/fileStorage'
 
 type Props = {
   assessment: Assessment
@@ -19,9 +20,14 @@ export const createMany = async (props: Props, client: BaseProtocol = DB): Promi
   return client.tx(async (t) => {
     return Promise.all(
       files.map(async (multerFile) => {
-        const file = await FileRepository.create({ ...props, file: multerFile }, t)
+        const fileName = multerFile.originalname
+        const file = await FileRepository.create({ ...props, fileName }, t)
+        const { uuid } = file
+        const key = uuid
+        const body = multerFile.buffer
+        await FileStorage.uploadFile({ key, body })
 
-        const target = { fileName: file.name, uuid: file.uuid }
+        const target = { fileName, uuid }
         const message = ActivityLogMessage.fileCreate
         const activityLog = { target, section: 'assessment', message, user }
         const activityLogParams = { activityLog, assessment, cycle }

src/server/controller/user/getProfilePicture.ts

@@ -0,0 +1,21 @@
+import { UserProfilePicture } from 'meta/user/userProfilePicture'
+
+import { BaseProtocol, DB } from 'server/db'
+import { UserRepository } from 'server/repository/public/user'
+import { FileStorage } from 'server/service/fileStorage'
+
+export const getProfilePicture = async (
+  props: {
+    id: number
+  },
+  client: BaseProtocol = DB
+): Promise<UserProfilePicture> => {
+  const { id } = props
+  const userProfilePicture = await UserRepository.getProfilePicture({ id }, client)
+  const key = userProfilePicture.uuid
+  const data = await FileStorage.getFile({ key })
+  return {
+    ...userProfilePicture,
+    data,
+  }
+}

src/server/controller/user/index.ts

@@ -8,6 +8,7 @@ import { create } from './create'
 import { createResetPassword } from './createResetPassword'
 import { findByInvitation } from './findByInvitation'
 import { findByResetPassword } from './findByResetPassword'
+import { getProfilePicture } from './getProfilePicture'
 import { invite } from './invite'
 import { remove } from './remove'
 import { removeInvitation } from './removeInvitation'
@@ -24,7 +25,7 @@ export const UserController = {
   getManyInvitations: UserInvitationRepository.getMany,
   getCountInvitations: UserInvitationRepository.getCount,
   getOne: UserRepository.getOne,
-  getProfilePicture: UserRepository.getProfilePicture,
+  getProfilePicture,
   remove,
   invite,
   acceptInvitation,

src/server/controller/user/update.ts

@@ -5,6 +5,7 @@ import { BaseProtocol, DB } from 'server/db'
 import { ActivityLogRepository } from 'server/repository/public/activityLog'
 import { FileRepository } from 'server/repository/public/file'
 import { UserRepository } from 'server/repository/public/user'
+import { FileStorage } from 'server/service/fileStorage'
 
 export const update = async (
   props: {
@@ -18,7 +19,10 @@ export const update = async (
 
   return client.tx(async (t) => {
     if (profilePicture) {
-      const createdFile = await FileRepository.create({ file: profilePicture }, client)
+      const createdFile = await FileRepository.create({ fileName: profilePicture.originalname }, client)
+      const { uuid: key } = createdFile
+
+      await FileStorage.uploadFile({ key, body: profilePicture.buffer })
       userToUpdate.profilePictureFileUuid = createdFile.uuid
     }
 

src/server/repository/adapter/file.ts

@@ -4,18 +4,12 @@ import { File } from 'meta/file'
 
 export type FileDB = {
   created_at: string
-  file?: Buffer
   id: number
   name: string
   size: number
   uuid: string
 }
 
 export const FileAdapter = (fileDB: FileDB): File => {
-  const { file, ...rest } = fileDB
-
-  return {
-    ...Objects.camelize(rest),
-    file,
-  }
+  return Objects.camelize(fileDB)
 }

src/server/repository/public/file/create.ts

@@ -6,18 +6,18 @@ import { FileAdapter } from 'server/repository/adapter'
 import { fieldsFileSummary } from './fields'
 
 type Props = {
-  file: Express.Multer.File
+  fileName: string
 }
 
 export const create = async (props: Props, client: BaseProtocol = DB): Promise<FileSummary> => {
-  const { file } = props
+  const { fileName } = props
 
   return client.one(
     `
-        insert into public.file (name, file)
-        values ($1, $2)
+        insert into public.file (name)
+        values ($1)
         returning ${fieldsFileSummary.join(', ')}`,
-    [file.originalname, file.buffer],
+    [fileName],
     FileAdapter
   )
 }

src/server/repository/public/file/fields.ts

@@ -1,2 +1 @@
-export const fieldsFileSummary = ['id', 'name', 'uuid', 'created_at', 'length(file) as size']
-export const fieldsFile = [...fieldsFileSummary, 'file']
+export const fieldsFileSummary = ['id', 'name', 'uuid', 'created_at']