diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index e7e892226..bb777384d 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -29,7 +29,7 @@ jobs: context: ./ file: ./Dockerfile push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip:${{ github.sha }} + tags: ${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip:${{ github.sha }},${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip:latest deploy: needs: build runs-on: ubuntu-latest diff --git a/.github/workflows/build-and-publish-dev-preview.yml b/.github/workflows/build-and-publish-dev-preview.yml new file mode 100644 index 000000000..db0ed2c13 --- /dev/null +++ b/.github/workflows/build-and-publish-dev-preview.yml @@ -0,0 +1,48 @@ +name: Build & Publish Stable Preview +on: + push: + branches: ["master"] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check Out Repo + uses: actions/checkout@v2 + with: + registry-url: 'https://npm.pkg.github.com' + - run: echo "registry=https://registry.yarnpkg.com/" > .npmrc + - run: echo "@internxt:registry=https://npm.pkg.github.com" >> .npmrc + # You cannot read packages from other private repos with GITHUB_TOKEN + # You have to use a PAT instead https://github.com/actions/setup-node/issues/49 + - run: echo //npm.pkg.github.com/:_authToken=${{ secrets.PERSONAL_ACCESS_TOKEN }} >> .npmrc + - run: echo "always-auth=true" >> .npmrc + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build and push to drive-server-wip + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./development.Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip-dev:${{ github.sha }} + dispatch_update_preview_image: + needs: build + runs-on: ubuntu-latest + steps: + - name: Dispatch Update Preview Image Command + uses: myrotvorets/trigger-repository-dispatch-action@1.0.0 + with: + token: ${{ secrets.PAT }} + repo: internxt/environments + type: update-preview-image-command + payload: | + { + "image": "${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip", + "newImage": "${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip-dev", + "newTag": "${{ github.sha }}" + } diff --git a/.github/workflows/deploy-feature.yml b/.github/workflows/deploy-feature.yml deleted file mode 100644 index 9870cd9da..000000000 --- a/.github/workflows/deploy-feature.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Deploy feature -on: - pull_request: - workflow_dispatch: - inputs: - pr-number: - type: number -jobs: - retrive-pr: - runs-on: ubuntu-latest - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - PR=$(gh pr view ${{ github.event.inputs.pr-number }} --json headRefName,state,headRepository,number,title) - echo 'PR_BRANCH=$(jq -r '.headRefName' <<< "$PR")' >> $GITHUB_ENV - build: - needs: retrive-pr - runs-on: ubuntu-latest - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v2 - with: - ref: $PR_BRANCH - registry-url: 'https://npm.pkg.github.com' - - name: Create .npmrc file - run: | - echo "registry=https://registry.yarnpkg.com/" > .npmrc - echo "@internxt:registry=https://npm.pkg.github.com" >> .npmrc - # You cannot read packages from other private repos with GITHUB_TOKEN - # You have to use a PAT instead https://github.com/actions/setup-node/issues/49 - echo //npm.pkg.github.com/:_authToken=${{ secrets.PERSONAL_ACCESS_TOKEN }} >> .npmrc - echo "always-auth=true" >> .npmrc - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Build and push to drive-server-wip - run: echo "Would build the image with tag drive-server-wip:${{ github.sha }}-pr${{ github.event.inputs.pr-number }}" - # uses: docker/build-push-action@v2 - # with: - # context: ./ - # file: ./development.Dockerfile - # push: true - # tags: ${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip-dev:${{ github.sha }}-pr${{ github.event.inputs.pr-number }} - deploy: - needs: build - runs-on: ubuntu-latest - environment: - name: develop - steps: - - uses: actions/checkout@v2 - - name: Update drive-server-wip image - run: echo "Would set the image drive-server-wip-dev:${{ github.sha }}-pr${{ github.event.inputs.pr-number }}" - # uses: steebchen/kubectl@v2.0.0 - # with: # defaults to latest kubectl binary version - # config: ${{ secrets.KUBE_CONFIG_DATA_DEVELOPMENT }} - # version: v1.22.2 - # command: set image --record deployment/drive-server-wip-dev drive-server-wip-dev=${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip-dev:${{ github.sha }}-pr${{ github.event.inputs.pr-number }} - # - name: Verify drive-server deployment - # uses: steebchen/kubectl@v2.0.0 - # with: - # config: ${{ secrets.KUBE_CONFIG_DATA_DEVELOPMENT }} - # version: v1.22.2 # specify kubectl binary version explicitly - # command: rollout status deployment/drive-server-wip-dev \ No newline at end of file diff --git a/.github/workflows/deploy-preview-feature.yml b/.github/workflows/deploy-preview-feature.yml new file mode 100644 index 000000000..2f4570aa1 --- /dev/null +++ b/.github/workflows/deploy-preview-feature.yml @@ -0,0 +1,111 @@ +name: Deploy PR Preview +on: + pull_request: + types: [opened, reopened, synchronize] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check Out Repo + uses: actions/checkout@v2 + with: + registry-url: 'https://npm.pkg.github.com' + - run: echo "registry=https://registry.yarnpkg.com/" > .npmrc + - run: echo "@internxt:registry=https://npm.pkg.github.com" >> .npmrc + # You cannot read packages from other private repos with GITHUB_TOKEN + # You have to use a PAT instead https://github.com/actions/setup-node/issues/49 + - run: echo //npm.pkg.github.com/:_authToken=${{ secrets.PERSONAL_ACCESS_TOKEN }} >> .npmrc + - run: echo "always-auth=true" >> .npmrc + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build and push to drive-server-wip + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./development.Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_USERNAME }}/drive-server-wip-dev:preview-${{ github.event.number }}-${{ github.event.pull_request.head.sha }} + add_preview_label: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions-ecosystem/action-add-labels@v1 + with: + labels: | + preview + dispatch_update_deployment: + needs: add_preview_label + runs-on: ubuntu-latest + if: ${{ contains(github.event.pull_request.labels.*.name, 'deployed') }} + steps: + - name: Dispatch Update Preview Repository Command + uses: myrotvorets/trigger-repository-dispatch-action@1.0.0 + with: + token: ${{ secrets.PAT }} + repo: internxt/environments + type: update-preview-command + payload: | + { + "github": { + "payload": { + "repository": { + "name": "${{ github.event.repository.name }}", + "full_name": "${{ github.event.repository.full_name }}" + }, + "issue": { + "number": ${{ github.event.number }}, + "labels": ${{ toJSON(github.event.pull_request.labels) }} + } + } + }, + "slash_command": { + "args": { + "named": { + "deployment": "${{ github.event.repository.name }}", + "tag": "preview-${{ github.event.number }}-${{ github.event.pull_request.head.sha }}", + "imageSuffix": "-dev" + } + } + } + } + dispatch_check_deployment: + needs: add_preview_label + runs-on: ubuntu-latest + steps: + - name: Dispatch Check Preview Repository Command + uses: myrotvorets/trigger-repository-dispatch-action@1.0.0 + with: + token: ${{ secrets.PAT }} + repo: internxt/environments + type: check-preview-command + payload: | + { + "github": { + "payload": { + "repository": { + "name": "${{ github.event.repository.name }}", + "full_name": "${{ github.event.repository.full_name }}", + "html_url": "${{ github.event.repository.html_url }}" + }, + "issue": { + "number": ${{ github.event.number }}, + "labels": ${{ toJSON(github.event.pull_request.labels) }}, + "pull_request": { + "html_url": "${{ github.event.pull_request.html_url }}" + } + } + } + }, + "slash_command": { + "args": { + "named": { + "notify": "true" + } + } + } + } \ No newline at end of file diff --git a/.github/workflows/dispatch-cleanup-preview.yml b/.github/workflows/dispatch-cleanup-preview.yml new file mode 100644 index 000000000..815cbf1ba --- /dev/null +++ b/.github/workflows/dispatch-cleanup-preview.yml @@ -0,0 +1,27 @@ +name: Clean Up PR Preview +on: + pull_request: + types: [closed] +jobs: + dispatch_cleanup_deployment: + runs-on: ubuntu-latest + steps: + - name: Dispatch Cleanup Preview Repository Command + uses: myrotvorets/trigger-repository-dispatch-action@1.0.0 + with: + token: ${{ secrets.PAT }} + repo: internxt/environments + type: cleanup-preview-command + payload: | + { + "github": { + "payload": { + "repository": { + "name": "${{ github.event.repository.name }}" + }, + "issue": { + "number": ${{ github.event.number }} + } + } + } + } \ No newline at end of file diff --git a/migrations/20240306175726-convert-gb-to-bytes-limits.js b/migrations/20240306175726-convert-gb-to-bytes-limits.js new file mode 100644 index 000000000..a04a3423e --- /dev/null +++ b/migrations/20240306175726-convert-gb-to-bytes-limits.js @@ -0,0 +1,45 @@ +'use strict'; + +module.exports = { + async up(queryInterface, Sequelize) { + const convertGbToBytes = (gbValue) => gbValue * 1024 * 1024 * 1024; + + const maxFileUploadSizeLimits = await queryInterface.sequelize.query( + "SELECT id, value FROM limits WHERE label = 'max-file-upload-size'", + { type: Sequelize.QueryTypes.SELECT }, + ); + + for (const limit of maxFileUploadSizeLimits) { + const bytesValue = convertGbToBytes(parseInt(limit.value)).toString(); + + await queryInterface.sequelize.query( + 'UPDATE limits SET value = :bytesValue WHERE id = :id', + { + replacements: { bytesValue, id: limit.id }, + type: Sequelize.QueryTypes.UPDATE, + }, + ); + } + }, + + async down(queryInterface, Sequelize) { + const convertBytesToGB = (gbValue) => gbValue / 1024 / 1024 / 1024; + + const maxFileUploadSizeLimits = await queryInterface.sequelize.query( + "SELECT id, value FROM limits WHERE label = 'max-file-upload-size'", + { type: Sequelize.QueryTypes.SELECT }, + ); + + for (const limit of maxFileUploadSizeLimits) { + const bytesValue = convertBytesToGB(parseInt(limit.value)).toString(); + + await queryInterface.sequelize.query( + 'UPDATE limits SET value = :bytesValue WHERE id = :id', + { + replacements: { bytesValue, id: limit.id }, + type: Sequelize.QueryTypes.UPDATE, + }, + ); + } + }, +}; diff --git a/package.json b/package.json index 37c99ead5..72fa25a1c 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@nestjs/jwt": "^10.2.0", "@nestjs/passport": "^10.0.2", "@nestjs/platform-express": "^10.2.8", + "@nestjs/schedule": "^4.0.1", "@nestjs/sequelize": "^10.0.0", "@nestjs/swagger": "^7.1.16", "@nestjs/throttler": "^5.0.1", diff --git a/seeders/20240227151157-add-paid-plans-dev.js b/seeders/20240227151157-add-paid-plans-dev.js index 38d17a8b8..96aef8539 100644 --- a/seeders/20240227151157-add-paid-plans-dev.js +++ b/seeders/20240227151157-add-paid-plans-dev.js @@ -11,9 +11,28 @@ module.exports = { ]; for (let plan of plans) { + const planAlreadyMapped = await queryInterface.sequelize.query( + `SELECT id FROM paid_plans WHERE plan_id = :planId LIMIT 1`, + { + replacements: { + planId: plan.planId, + }, + type: Sequelize.QueryTypes.SELECT, + }, + ); + + if (planAlreadyMapped.length > 0) { + continue; + } + const tiers = await queryInterface.sequelize.query( - `SELECT id FROM tiers WHERE label = '${plan.name}' LIMIT 1`, - { type: Sequelize.QueryTypes.SELECT }, + `SELECT id FROM tiers WHERE label = :label LIMIT 1`, + { + replacements: { + label: plan.name, + }, + type: Sequelize.QueryTypes.SELECT, + }, ); if (tiers.length > 0) { diff --git a/seeders/20240227151200-public-sharings-user.js b/seeders/20240227151200-public-sharings-user.js new file mode 100644 index 000000000..da403d3d7 --- /dev/null +++ b/seeders/20240227151200-public-sharings-user.js @@ -0,0 +1,54 @@ +'use strict'; +const { v4 } = require('uuid'); +const { Op } = require('sequelize'); + +const referralCode = v4(); + +const PublicSharingUser = { + user_id: 'Public Shared Items User', + name: 'Internxt', + lastname: 'Internxt', + email: 'public-sharings@internxt.com', + username: 'public-sharings@internxt.com', + bridge_user: 'public-sharings@internxt.com', + password: 'publicSharingUser', + mnemonic: 'internxt public sharings mnemonic', + h_key: 'internxt salt', + referrer: null, + referral_code: referralCode, + uuid: '00000000-0000-0000-0000-000000000000', + credit: 0, + welcome_pack: true, + register_completed: true, +}; + +module.exports = { + async up(queryInterface) { + const existingUsers = await queryInterface.sequelize.query( + 'SELECT email FROM users WHERE email = :email OR uuid = :uuid', + { + replacements: { + email: PublicSharingUser.email, + uuid: PublicSharingUser.uuid, + }, + type: queryInterface.sequelize.QueryTypes.SELECT, + }, + ); + + if (existingUsers.length === 0) { + await queryInterface.bulkInsert('users', [PublicSharingUser]); + } + }, + + async down(queryInterface) { + await queryInterface.bulkDelete( + 'users', + { + email: { [Op.in]: [PublicSharingUser.email] }, + }, + {}, + ); + }, +}; + +module.exports.users = { PublicSharingUser }; diff --git a/src/app.module.ts b/src/app.module.ts index dcc761009..1623026b1 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -23,6 +23,7 @@ import { FuzzySearchModule } from './modules/fuzzy-search/fuzzy-search.module'; import { SharingModule } from './modules/sharing/sharing.module'; import { AppSumoModule } from './modules/app-sumo/app-sumo.module'; import { PlanModule } from './modules/plan/plan.module'; +import { ScheduleModule } from '@nestjs/schedule'; @Module({ imports: [ @@ -97,6 +98,7 @@ import { PlanModule } from './modules/plan/plan.module'; limit: 5, }, ]), + ScheduleModule.forRoot(), NotificationModule, FileModule, FolderModule, diff --git a/src/modules/feature-limit/feature-limit.usecase.ts b/src/modules/feature-limit/feature-limit.usecase.ts index 76a178b9c..3f2d7ce27 100644 --- a/src/modules/feature-limit/feature-limit.usecase.ts +++ b/src/modules/feature-limit/feature-limit.usecase.ts @@ -40,6 +40,10 @@ export class FeatureLimitUsecases { user: User, data: LimitTypeMapping[T], ): Promise { + if (!user.tierId) { + return false; + } + const limit = await this.limitsRepository.findLimitByLabelAndTier( user.tierId, limitLabel, diff --git a/src/modules/feature-limit/limit.domain.ts b/src/modules/feature-limit/limit.domain.ts index daf98ea70..cce13f70a 100644 --- a/src/modules/feature-limit/limit.domain.ts +++ b/src/modules/feature-limit/limit.domain.ts @@ -26,6 +26,14 @@ export class Limit { return this.type === LimitTypes.Boolean; } + getLimitValue() { + if (this.isBooleanLimit()) { + return this.value === 'true'; + } + + return Number(this.value); + } + private isFeatureEnabled() { return this.isBooleanLimit() && this.value === 'true'; } diff --git a/src/modules/feature-limit/limits.enum.ts b/src/modules/feature-limit/limits.enum.ts index 7b9ac8625..e250e3d18 100644 --- a/src/modules/feature-limit/limits.enum.ts +++ b/src/modules/feature-limit/limits.enum.ts @@ -1,6 +1,7 @@ export enum LimitLabels { MaxSharedItems = 'max-shared-items', MaxSharedItemInvites = 'max-shared-invites', + MaxTrashStorageDays = 'max-trash-storage-days', } export enum LimitTypes { diff --git a/src/modules/feature-limit/models/tier.model.ts b/src/modules/feature-limit/models/tier.model.ts index 8d8a73d61..e6960d467 100644 --- a/src/modules/feature-limit/models/tier.model.ts +++ b/src/modules/feature-limit/models/tier.model.ts @@ -5,7 +5,12 @@ import { PrimaryKey, DataType, AllowNull, + HasMany, + BelongsToMany, } from 'sequelize-typescript'; +import { Limitmodel } from './limit.model'; +import { TierLimitsModel } from './tier-limits.model'; +import { UserModel } from '../../../modules/user/user.model'; export interface TierAttributes { id: string; @@ -37,4 +42,12 @@ export class TierModel extends Model implements TierAttributes { @Column updatedAt: Date; + + @BelongsToMany(() => Limitmodel, { + through: () => TierLimitsModel, + }) + limits: Limitmodel[]; + + @HasMany(() => UserModel, 'tierId') + users?: UserModel[]; } diff --git a/src/modules/file/file.repository.ts b/src/modules/file/file.repository.ts index 6ed859445..b2b8cf95a 100644 --- a/src/modules/file/file.repository.ts +++ b/src/modules/file/file.repository.ts @@ -1,7 +1,7 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { InjectModel } from '@nestjs/sequelize'; import { File, FileAttributes, FileOptions, FileStatus } from './file.domain'; -import { FindOptions, Op } from 'sequelize'; +import { FindOptions, Op, Sequelize } from 'sequelize'; import { User } from '../user/user.domain'; import { Folder } from '../folder/folder.domain'; @@ -10,6 +10,10 @@ import { ShareModel } from '../share/share.repository'; import { ThumbnailModel } from '../thumbnail/thumbnail.model'; import { FileModel } from './file.model'; import { SharingModel } from '../sharing/models'; +import { UserModel } from '../user/user.model'; +import { TierModel } from '../feature-limit/models/tier.model'; +import { Limitmodel } from '../feature-limit/models/limit.model'; +import { LimitLabels } from '../feature-limit/limits.enum'; export interface FileRepository { deleteByFileId(fileId: any): Promise; @@ -361,6 +365,52 @@ export class SequelizeFileRepository implements FileRepository { return count; } + async getTrashedExpiredFiles(limit?: number) { + const files = await this.fileModel.findAll({ + attributes: ['status', 'plainName', 'updatedAt', 'fileId', 'uuid'], + replacements: { limitLabel: LimitLabels.MaxTrashStorageDays }, + where: { + status: 'TRASHED', + updatedAt: { + [Op.lt]: Sequelize.literal(`now() - (SELECT value :: integer + FROM + limits + JOIN tiers_limits ON limits.id = tiers_limits.limit_id + JOIN tiers ON tiers_limits.tier_id = tiers.id + WHERE + tiers.id = "user"."tier_id" AND limits.label = :limitLabel) * INTERVAL '1 day'`), + }, + }, + include: { + model: UserModel, + attributes: ['uuid', 'id', 'tierId'], + }, + limit, + }); + return files; + } + + async deleteTrashedFilesById(fileIds: File['fileId'][]): Promise { + await this.fileModel.update( + { + removed: true, + removedAt: new Date(), + status: FileStatus.DELETED, + updatedAt: new Date(), + }, + { + where: { + fileId: { + [Op.in]: fileIds, + }, + status: { + [Op.eq]: FileStatus.TRASHED, + }, + }, + }, + ); + } + async updateFilesStatusToTrashed( user: User, fileIds: File['fileId'][], diff --git a/src/modules/file/file.usecase.ts b/src/modules/file/file.usecase.ts index 63c0b4d79..34afa9470 100644 --- a/src/modules/file/file.usecase.ts +++ b/src/modules/file/file.usecase.ts @@ -25,6 +25,7 @@ import { SequelizeFileRepository } from './file.repository'; import { FolderUseCases } from '../folder/folder.usecase'; import { ReplaceFileDto } from './dto/replace-file.dto'; import { FileDto } from './dto/file.dto'; +import { Op } from 'sequelize'; type SortParams = Array<[SortableFileAttributes, 'ASC' | 'DESC']>; @@ -177,15 +178,20 @@ export class FileUseCases { offset: number; sort?: SortParams; withoutThumbnails?: boolean; + updatedAfter?: Date; } = { limit: 20, offset: 0, }, ): Promise { let filesWithMaybePlainName; + const updatedAfterCondition = options.updatedAfter + ? { updatedAt: { [Op.gte]: options.updatedAfter } } + : null; + if (options?.withoutThumbnails) filesWithMaybePlainName = await this.fileRepository.findAllCursor( - { ...where, userId }, + { ...where, userId, ...updatedAfterCondition }, options.limit, options.offset, options.sort, @@ -193,7 +199,7 @@ export class FileUseCases { else filesWithMaybePlainName = await this.fileRepository.findAllCursorWithThumbnails( - { ...where, userId }, + { ...where, userId, ...updatedAfterCondition }, options.limit, options.offset, options.sort, diff --git a/src/modules/folder/folder.repository.ts b/src/modules/folder/folder.repository.ts index 29d3fc01a..f1c6e5dee 100644 --- a/src/modules/folder/folder.repository.ts +++ b/src/modules/folder/folder.repository.ts @@ -1,6 +1,6 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { InjectModel } from '@nestjs/sequelize'; -import { FindOptions, Op } from 'sequelize'; +import { FindOptions, Op, Sequelize } from 'sequelize'; import { v4 } from 'uuid'; import { Folder } from './folder.domain'; @@ -12,6 +12,8 @@ import { Pagination } from '../../lib/pagination'; import { FolderModel } from './folder.model'; import { SharingModel } from '../sharing/models'; import { CalculateFolderSizeTimeoutException } from './exception/calculate-folder-size-timeout.exception'; +import { UserModel } from '../user/user.model'; +import { LimitLabels } from '../feature-limit/limits.enum'; function mapSnakeCaseToCamelCase(data) { const camelCasedObject = {}; @@ -421,6 +423,24 @@ export class SequelizeFolderRepository implements FolderRepository { ); } + async deleteByIds(folderIds: Folder['id'][]): Promise { + await this.folderModel.update( + { + removed: true, + removedAt: new Date(), + deleted: true, + deletedAt: new Date(), + }, + { + where: { + id: { + [Op.in]: folderIds, + }, + }, + }, + ); + } + async findAllCursorWhereUpdatedAfter( where: Partial, updatedAfter: Date, @@ -446,6 +466,39 @@ export class SequelizeFolderRepository implements FolderRepository { return folders.map((folder) => this.toDomain(folder)); } + async getTrashedExpiredFolders(limit?: number) { + const folders = await this.folderModel.findAll({ + replacements: { limitLabel: LimitLabels.MaxTrashStorageDays }, + attributes: [ + 'deleted', + 'plainName', + 'updatedAt', + 'id', + 'uuid', + 'deletedAt', + ], + where: { + deleted: true, + removed: false, + deletedAt: { + [Op.lt]: Sequelize.literal(`now() - (SELECT value :: integer + FROM + limits + JOIN tiers_limits ON limits.id = tiers_limits.limit_id + JOIN tiers ON tiers_limits.tier_id = tiers.id + WHERE + tiers.id = "user"."tier_id" AND limits.label = :limitLabel) * INTERVAL '1 day'`), + }, + }, + include: { + model: UserModel, + attributes: ['uuid', 'id', 'tierId'], + }, + limit, + }); + return folders; + } + async calculateFolderSize(folderUuid: string): Promise { try { const calculateSizeQuery = ` diff --git a/src/modules/folder/folder.usecase.ts b/src/modules/folder/folder.usecase.ts index f34fad779..b8e245bad 100644 --- a/src/modules/folder/folder.usecase.ts +++ b/src/modules/folder/folder.usecase.ts @@ -17,6 +17,7 @@ import { import { FolderAttributes } from './folder.attributes'; import { SequelizeFolderRepository } from './folder.repository'; import { SequelizeFileRepository } from '../file/file.repository'; +import { Op } from 'sequelize'; const invalidName = /[\\/]|^\s*$/; @@ -410,13 +411,22 @@ export class FolderUseCases { async getFolders( userId: FolderAttributes['userId'], where: Partial, - options: { limit: number; offset: number; sort?: SortParams } = { + options: { + limit: number; + offset: number; + sort?: SortParams; + updatedAfter?: Date; + } = { limit: 20, offset: 0, }, ): Promise { + const updatedAfterCondition = options.updatedAfter + ? { updatedAt: { [Op.gte]: options.updatedAfter } } + : null; + const foldersWithMaybePlainName = await this.folderRepository.findAllCursor( - { ...where, userId }, + { ...where, userId, ...updatedAfterCondition }, options.limit, options.offset, options.sort, diff --git a/src/modules/sharing/sharing.controller.ts b/src/modules/sharing/sharing.controller.ts index dda83021d..b848128ee 100644 --- a/src/modules/sharing/sharing.controller.ts +++ b/src/modules/sharing/sharing.controller.ts @@ -58,6 +58,9 @@ import { ThrottlerGuard } from '../../guards/throttler.guard'; import { SetSharingPasswordDto } from './dto/set-sharing-password.dto'; import { UuidDto } from '../../common/uuid.dto'; import { HttpExceptionFilter } from '../../lib/http/http-exception.filter'; +import { ApplyLimit } from '../feature-limit/decorators/apply-limit.decorator'; +import { LimitLabels } from '../feature-limit/limits.enum'; +import { FeatureLimit } from '../feature-limit/feature-limits.guard'; @ApiTags('Sharing') @Controller('sharings') @@ -248,14 +251,14 @@ export class SharingController { } @Post('/invites/send') - /* @ApplyLimit({ + @ApplyLimit({ limitLabels: [LimitLabels.MaxSharedItemInvites, LimitLabels.MaxSharedItems], dataSources: [ { sourceKey: 'body', fieldName: 'itemId' }, { sourceKey: 'body', fieldName: 'itemType' }, ], }) - @UseGuards(FeatureLimit) */ + @UseGuards(FeatureLimit) createInvite( @UserDecorator() user: User, @Body() createInviteDto: CreateInviteDto, @@ -599,11 +602,11 @@ export class SharingController { } @Post('/') - /* @ApplyLimit({ + @ApplyLimit({ limitLabels: [LimitLabels.MaxSharedItems], dataSources: [{ sourceKey: 'body', fieldName: 'itemId' }], }) - @UseGuards(FeatureLimit) */ + @UseGuards(FeatureLimit) createSharing( @UserDecorator() user, @Body() acceptInviteDto: CreateSharingDto, diff --git a/src/modules/trash/trash.controller.ts b/src/modules/trash/trash.controller.ts index 484e9feab..1419253e5 100644 --- a/src/modules/trash/trash.controller.ts +++ b/src/modules/trash/trash.controller.ts @@ -41,6 +41,8 @@ import logger from '../../externals/logger'; import { v4 } from 'uuid'; import { Response } from 'express'; import { HttpExceptionFilter } from '../../lib/http/http-exception.filter'; +import { FeatureLimitUsecases } from '../feature-limit/feature-limit.usecase'; +import { Cron } from '@nestjs/schedule'; @ApiTags('Trash') @Controller('storage/trash') @@ -51,6 +53,7 @@ export class TrashController { private userUseCases: UserUseCases, private notificationService: NotificationService, private trashUseCases: TrashUseCases, + private featureLimitsUseCases: FeatureLimitUsecases, ) {} @Get('/paginated') @@ -78,20 +81,39 @@ export class TrashController { throw new BadRequestException('Limit should be between 1 and 50'); } + let userTierId = user.tierId; + if (!userTierId) { + const freeTier = await this.featureLimitsUseCases.getFreeTier(); + userTierId = freeTier.id; + } + + const storageDaysLimit = + await this.featureLimitsUseCases.getTierMaxTrashStorageDays(userTierId); + const storageDays = storageDaysLimit?.getLimitValue(); + const maxStorageDate = new Date(); + if (typeof storageDays === 'number') { + maxStorageDate.setDate(maxStorageDate.getDate() - storageDays); + } + try { let result: File[] | Folder[]; if (type === 'files') { result = await this.fileUseCases.getFiles( user.id, - { status: FileStatus.TRASHED }, - { limit, offset }, + { + status: FileStatus.TRASHED, + }, + { limit, offset, updatedAfter: maxStorageDate }, ); } else { result = await this.folderUseCases.getFolders( user.id, - { deleted: true, removed: false }, - { limit, offset }, + { + deleted: true, + removed: false, + }, + { limit, offset, updatedAfter: maxStorageDate }, ); } @@ -294,4 +316,9 @@ export class TrashController { await this.trashUseCases.deleteItems(user, [], [folders[0]]); } + + @Cron('*/5 * * * *', { name: 'deleteExpiredItems' }) + async removeExpiredItems() { + await this.trashUseCases.removeExpiredItems(); + } } diff --git a/src/modules/trash/trash.module.ts b/src/modules/trash/trash.module.ts index 74da57473..00fcb3971 100644 --- a/src/modules/trash/trash.module.ts +++ b/src/modules/trash/trash.module.ts @@ -9,6 +9,7 @@ import { TrashUseCases } from './trash.usecase'; import { ShareModule } from '../share/share.module'; import { ShareModel } from '../share/share.repository'; import { FileModel } from '../file/file.model'; +import { FeatureLimitModule } from '../feature-limit/feature-limit.module'; @Module({ imports: [ @@ -19,6 +20,7 @@ import { FileModel } from '../file/file.model'; NotificationModule, UserModule, ShareModule, + FeatureLimitModule, ], controllers: [TrashController], providers: [Logger, TrashUseCases], diff --git a/src/modules/trash/trash.usecase.ts b/src/modules/trash/trash.usecase.ts index 52b3f4a46..d7d081e3a 100644 --- a/src/modules/trash/trash.usecase.ts +++ b/src/modules/trash/trash.usecase.ts @@ -1,9 +1,11 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { Folder } from '../folder/folder.domain'; import { User } from '../user/user.domain'; import { File, FileStatus } from '../file/file.domain'; import { FolderUseCases } from '../folder/folder.usecase'; import { FileUseCases } from '../file/file.usecase'; +import { SequelizeFileRepository } from '../file/file.repository'; +import { SequelizeFolderRepository } from '../folder/folder.repository'; @Injectable() export class TrashUseCases { @@ -13,6 +15,8 @@ export class TrashUseCases { constructor( private fileUseCases: FileUseCases, private folderUseCases: FolderUseCases, + private filesRepository: SequelizeFileRepository, + private foldersRepository: SequelizeFolderRepository, ) {} /** @@ -75,4 +79,55 @@ export class TrashUseCases { ); } } + + async deteleTrashedExpiredFiles() { + const limit = 1000; + let hasMore = true; + + while (hasMore) { + const files = await this.filesRepository.getTrashedExpiredFiles(limit); + await this.filesRepository.deleteTrashedFilesById( + files.map((file) => file.fileId), + ); + hasMore = files.length === limit; + } + } + + async deleteTrashedExpiredFolders() { + const limit = 1000; + let hasMore = true; + + while (hasMore) { + const folders = + await this.foldersRepository.getTrashedExpiredFolders(limit); + await this.foldersRepository.deleteByIds(folders.map((f) => f.id)); + hasMore = folders.length === limit; + } + } + + async removeExpiredItems() { + const startTime = new Date(); + try { + Logger.log( + `[TRASH/REMOVE-EXPIRED-ITEMS]: Cron job started at ${startTime.toISOString()}.`, + ); + + await this.deleteTrashedExpiredFolders(); + await this.deteleTrashedExpiredFiles(); + + const endTime = new Date(); + const duration = endTime.getTime() - startTime.getTime(); + Logger.log( + `[TRASH/REMOVE-EXPIRED-ITEMS]: Cron job completed successfully. Started at: ${startTime.toISOString()}, finished at: ${endTime.toISOString()}, total duration: ${duration} ms.`, + ); + } catch (err) { + const errorTime = new Date(); + Logger.error( + `[TRASH/REMOVE-EXPIRED-ITEMS]: Error encountered. Started at: ${startTime.toISOString()}, error occurred at: ${errorTime.toISOString()}, error details: ${ + err.message + }, stack trace: ${err.stack}`, + ); + throw err; + } + } } diff --git a/src/modules/user/user.model.ts b/src/modules/user/user.model.ts index d3c61e5a0..3dee95313 100644 --- a/src/modules/user/user.model.ts +++ b/src/modules/user/user.model.ts @@ -14,6 +14,7 @@ import { import { FolderModel } from '../folder/folder.model'; import { UserAttributes } from './user.attributes'; +import { TierModel } from '../feature-limit/models/tier.model'; @Table({ underscored: true, @@ -125,6 +126,10 @@ export class UserModel extends Model implements UserAttributes { lastPasswordChangedAt: Date; @AllowNull + @ForeignKey(() => TierModel) @Column tierId: string; + + @BelongsTo(() => TierModel, 'tierId') + tier: TierModel; } diff --git a/yarn.lock b/yarn.lock index a22cd232d..245330d98 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1609,6 +1609,14 @@ multer "1.4.4-lts.1" tslib "2.6.2" +"@nestjs/schedule@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@nestjs/schedule/-/schedule-4.0.1.tgz#beb69f974e7adc96fd799f555ba0121c10d2d61b" + integrity sha512-cz2FNjsuoma+aGsG0cMmG6Dqg/BezbBWet1UTHtAuu6d2mXNTVcmoEQM2DIVG5Lfwb2hfSE2yZt8Moww+7y+mA== + dependencies: + cron "3.1.6" + uuid "9.0.1" + "@nestjs/schematics@^10.0.1", "@nestjs/schematics@^10.0.3": version "10.0.3" resolved "https://registry.yarnpkg.com/@nestjs/schematics/-/schematics-10.0.3.tgz#0f48af0a20983ffecabcd8763213a3e53d43f270" @@ -2689,6 +2697,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.182.tgz#05301a4d5e62963227eaafe0ce04dd77c54ea5c2" integrity sha512-/THyiqyQAP9AfARo4pF+aCGcyiQ94tX/Is2I7HofNRqoYLgN1PBoOWu2/zTA5zMxzP5EFutMtWtGAFRKUe961Q== +"@types/luxon@~3.3.0": + version "3.3.8" + resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-3.3.8.tgz#84dbf2d020a9209a272058725e168f21d331a67e" + integrity sha512-jYvz8UMLDgy3a5SkGJne8H7VA7zPV2Lwohjx0V8V31+SqAjNmurWMkk9cQhfvlcnXWudBpK9xPM1n4rljOcHYQ== + "@types/mime@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" @@ -4199,6 +4212,14 @@ cron-parser@^4.2.1: dependencies: luxon "^3.0.1" +cron@3.1.6: + version "3.1.6" + resolved "https://registry.yarnpkg.com/cron/-/cron-3.1.6.tgz#e7e1798a468e017c8d31459ecd7c2d088f97346c" + integrity sha512-cvFiQCeVzsA+QPM6fhjBtlKGij7tLLISnTSvFxVdnFGLdz+ZdXN37kNe0i2gefmdD17XuZA6n2uPVwzl4FxW/w== + dependencies: + "@types/luxon" "~3.3.0" + luxon "~3.4.0" + cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -6475,6 +6496,11 @@ luxon@^3.0.1: resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929" integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw== +luxon@~3.4.0: + version "3.4.4" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.4.tgz#cf20dc27dc532ba41a169c43fdcc0063601577af" + integrity sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA== + macos-release@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/macos-release/-/macos-release-2.5.0.tgz#067c2c88b5f3fb3c56a375b2ec93826220fa1ff2" @@ -8479,7 +8505,7 @@ uuid@9.0.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== -uuid@^9.0.1: +uuid@9.0.1, uuid@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==