From dcdd75e28c2d0466e1ffab226e62dbc41d5157ca Mon Sep 17 00:00:00 2001 From: ArceDanielShok Date: Wed, 16 Oct 2024 17:10:20 -0300 Subject: [PATCH 1/5] fix: resolved multiple bugs reported by QA and updated the endpoint call --- src/apps/sync-engine/BindingManager.ts | 11 ++++++++++- .../application/FileSyncOrchestrator.ts | 2 -- .../files/infrastructure/NodeWinLocalFileSystem.ts | 1 - .../files/infrastructure/SDKRemoteFileSystem.ts | 2 +- .../folders/application/FolderCreator.ts | 5 ++++- src/context/virtual-drive/folders/domain/Folder.ts | 2 +- .../folders/infrastructure/HttpRemoteFolderSystem.ts | 7 +++++-- 7 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index e6abe9067..9ee142bf2 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -38,6 +38,7 @@ export class BindingsManager { private queueManager: QueueManager | null = null; private lastHydrated = ''; + private lastMoved = ''; constructor( private readonly container: DependencyContainer, @@ -108,11 +109,18 @@ export class BindingsManager { try { Logger.debug('Path received from rename callback', absolutePath); + if (this.lastMoved === absolutePath) { + Logger.debug('Same file moved'); + this.lastMoved = ''; + callback(true); + return; + } + const isTempFile = await isTemporaryFile(absolutePath); Logger.debug('[isTemporaryFile]', isTempFile); - if (isTempFile) { + if (isTempFile && !contentsId.startsWith('FOLDER')) { Logger.debug('File is temporary, skipping'); callback(true); return; @@ -128,6 +136,7 @@ export class BindingsManager { }); fn(absolutePath, contentsId, callback); Logger.debug('Finish Rename', absolutePath); + this.lastMoved = absolutePath; } catch (error) { Logger.error('Error during rename or move operation', error); } diff --git a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts index 5d13b31cc..36550f899 100644 --- a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts +++ b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts @@ -13,8 +13,6 @@ export class FileSyncOrchestrator { for (const absolutePath of absolutePaths) { const tempFile = await isTemporaryFile(absolutePath); - Logger.debug('[isTemporaryFile]', tempFile); - if (tempFile) { Logger.debug(`Skipping temporary file: ${absolutePath}`); continue; diff --git a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts index 6a3d56951..ba4af7364 100644 --- a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts @@ -89,7 +89,6 @@ export class NodeWinLocalFileSystem implements LocalFileSystem { path: string, newIdentity: `${string}-${string}` ): Promise { - Logger.info('[updateFileIdentity]: ', path, newIdentity); const isNotDirectory = true; return this.virtualDrive.updateFileIdentity( path, diff --git a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts index ec0ca91d4..5304a6014 100644 --- a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts @@ -233,7 +233,7 @@ export class SDKRemoteFileSystem implements RemoteFileSystem { async move(file: File): Promise { await this.sdk.moveFile({ fileId: file.contentsId, - destination: Number(file.folderId), + destination: Number(file.folderId.value), destinationPath: uuidv4.v4(), bucketId: this.bucket, }); diff --git a/src/context/virtual-drive/folders/application/FolderCreator.ts b/src/context/virtual-drive/folders/application/FolderCreator.ts index 202a170a4..5250667ea 100644 --- a/src/context/virtual-drive/folders/application/FolderCreator.ts +++ b/src/context/virtual-drive/folders/application/FolderCreator.ts @@ -1,7 +1,10 @@ import { SyncEngineIpc } from '../../../../apps/sync-engine/ipcRendererSyncEngine'; import { EventBus } from '../../shared/domain/EventBus'; import { Folder } from '../domain/Folder'; +import { FolderId } from '../domain/FolderId'; +import { FolderPath } from '../domain/FolderPath'; import { FolderRepository } from '../domain/FolderRepository'; +import { FolderUuid } from '../domain/FolderUuid'; import { OfflineFolder } from '../domain/OfflineFolder'; import { RemoteFolderSystem } from '../domain/file-systems/RemoteFolderSystem'; import { FolderPlaceholderConverter } from './FolderPlaceholderConverter'; @@ -22,7 +25,7 @@ export class FolderCreator { const attributes = await this.remote.persist(offlineFolder); - const folder = Folder.create(attributes); + const folder = Folder.from(attributes); await this.repository.add(folder); diff --git a/src/context/virtual-drive/folders/domain/Folder.ts b/src/context/virtual-drive/folders/domain/Folder.ts index 4c4384f02..045254664 100644 --- a/src/context/virtual-drive/folders/domain/Folder.ts +++ b/src/context/virtual-drive/folders/domain/Folder.ts @@ -35,7 +35,7 @@ export class Folder extends AggregateRoot { } public get id(): number { - return this._id.value; + return this._id?.value; } public get uuid(): string { diff --git a/src/context/virtual-drive/folders/infrastructure/HttpRemoteFolderSystem.ts b/src/context/virtual-drive/folders/infrastructure/HttpRemoteFolderSystem.ts index a3f4b37dc..555f479cd 100644 --- a/src/context/virtual-drive/folders/infrastructure/HttpRemoteFolderSystem.ts +++ b/src/context/virtual-drive/folders/infrastructure/HttpRemoteFolderSystem.ts @@ -178,8 +178,11 @@ export class HttpRemoteFolderSystem implements RemoteFolderSystem { private async existFolder(offline: OfflineFolder): Promise { try { - const response = await this.trashClient.get( - `${process.env.NEW_DRIVE_URL}/drive/folders/content/${offline.parentUuid}/folders/existence?plainName=${offline.basename}` + const response = await this.trashClient.post( + `${process.env.NEW_DRIVE_URL}/drive/folders/content/${offline.parentUuid}/folders/existence`, + { + plainNames: [offline.basename], + } ); Logger.debug('[FOLDER FILE SYSTEM] Folder already exists', response.data); From 3016e541d4a17c0de120b1e773843c9ff59ff2b3 Mon Sep 17 00:00:00 2001 From: ArceDanielShok Date: Thu, 17 Oct 2024 15:34:10 -0300 Subject: [PATCH 2/5] fix resolve issues with backup --- src/apps/backups/Backups.ts | 26 +++++++++---------- src/apps/backups/batches/GroupFilesBySize.ts | 2 +- src/apps/backups/diff/DiffFilesCalculator.ts | 4 +-- .../application/update/FileBatchUpdater.ts | 4 +-- .../localTree/application/LocalTreeBuilder.ts | 3 +++ 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/apps/backups/Backups.ts b/src/apps/backups/Backups.ts index 84cb695dc..29aba6fd2 100644 --- a/src/apps/backups/Backups.ts +++ b/src/apps/backups/Backups.ts @@ -46,14 +46,11 @@ export class Backup { info: BackupInfo, abortController: AbortController ): Promise { - Logger.info('[BACKUPS] Backing:', info); - const localTreeEither = await this.localTreeBuilder.run( info.pathname as AbsolutePath ); if (localTreeEither.isLeft()) { - Logger.error('[BACKUPS] local tree is left', localTreeEither); return localTreeEither.getLeft(); } @@ -175,17 +172,20 @@ export class Backup { const batches = AddedFilesBatchCreator.run(added); for (const batch of batches) { - if (abortController.signal.aborted) { - return; + try { + if (abortController.signal.aborted) { + return; + } + // eslint-disable-next-line no-await-in-loop + await this.fileBatchUploader.run( + localRootPath, + tree, + batch, + abortController.signal + ); + } catch (error) { + Logger.error('Error uploading files', error); } - // eslint-disable-next-line no-await-in-loop - await this.fileBatchUploader.run( - localRootPath, - tree, - batch, - abortController.signal - ); - this.backed += batch.length; Logger.debug('[Backed]', this.backed); diff --git a/src/apps/backups/batches/GroupFilesBySize.ts b/src/apps/backups/batches/GroupFilesBySize.ts index 92e35f244..82877f5ab 100644 --- a/src/apps/backups/batches/GroupFilesBySize.ts +++ b/src/apps/backups/batches/GroupFilesBySize.ts @@ -2,7 +2,7 @@ import { LocalFile } from '../../../context/local/localFile/domain/LocalFile'; export class GroupFilesBySize { static small(files: Array) { - return files.filter((file) => file.isSmall()); + return files.filter((file) => file.isSmall() && file); } static medium(files: Array) { diff --git a/src/apps/backups/diff/DiffFilesCalculator.ts b/src/apps/backups/diff/DiffFilesCalculator.ts index 8e02c07fb..820dc0ff0 100644 --- a/src/apps/backups/diff/DiffFilesCalculator.ts +++ b/src/apps/backups/diff/DiffFilesCalculator.ts @@ -4,7 +4,7 @@ import { AbsolutePath } from '../../../context/local/localFile/infrastructure/Ab import { LocalTree } from '../../../context/local/localTree/domain/LocalTree'; import { File } from '../../../context/virtual-drive/files/domain/File'; import { RemoteTree } from '../../../context/virtual-drive/remoteTree/domain/RemoteTree'; -import { relative } from '../utils/relative'; +import { relativeV2 } from '../utils/relative'; import Logger from 'electron-log'; export type FilesDiff = { @@ -24,7 +24,7 @@ export class DiffFilesCalculator { const rootPath = local.root.path; local.files.forEach((local) => { - const remotePath = relative(rootPath, local.path); + const remotePath = relativeV2(rootPath, local.path); const remoteExists = remote.has(remotePath); diff --git a/src/context/local/localFile/application/update/FileBatchUpdater.ts b/src/context/local/localFile/application/update/FileBatchUpdater.ts index 07334da7a..83de69e52 100644 --- a/src/context/local/localFile/application/update/FileBatchUpdater.ts +++ b/src/context/local/localFile/application/update/FileBatchUpdater.ts @@ -4,7 +4,7 @@ import { LocalFileHandler } from '../../domain/LocalFileUploader'; import { RemoteTree } from '../../../../virtual-drive/remoteTree/domain/RemoteTree'; import { SimpleFileOverrider } from '../../../../virtual-drive/files/application/override/SimpleFileOverrider'; import { LocalFolder } from '../../../localFolder/domain/LocalFolder'; -import { relative } from '../../../../../apps/backups/utils/relative'; +import { relativeV2 } from '../../../../../apps/backups/utils/relative'; @Service() export class FileBatchUpdater { @@ -33,7 +33,7 @@ export class FileBatchUpdater { const contentsId = upload.getRight(); - const remotePath = relative(localRoot.path, localFile.path); + const remotePath = relativeV2(localRoot.path, localFile.path); const file = remoteTree.get(remotePath); diff --git a/src/context/local/localTree/application/LocalTreeBuilder.ts b/src/context/local/localTree/application/LocalTreeBuilder.ts index 3f38df047..13a455446 100644 --- a/src/context/local/localTree/application/LocalTreeBuilder.ts +++ b/src/context/local/localTree/application/LocalTreeBuilder.ts @@ -19,6 +19,9 @@ export default class LocalTreeBuilder { const { files, folders } = await this.generator.getAll(currentFolder.path); files.forEach((fileAttributes) => { + if (fileAttributes.size === 0) { + return; + } const file = LocalFile.from(fileAttributes); tree.addFile(currentFolder, file); }); From 13c2412ce8fa105f35de8bf2d1a26bea758b5ff7 Mon Sep 17 00:00:00 2001 From: ArceDanielShok Date: Mon, 21 Oct 2024 16:23:38 -0300 Subject: [PATCH 3/5] some fix --- src/apps/main/remote-sync/handlers.ts | 16 +----- src/apps/renderer/localize/locales/en.json | 6 ++- src/apps/renderer/localize/locales/es.json | 6 ++- src/apps/renderer/localize/locales/fr.json | 6 ++- src/apps/shared/IPC/events/sync-engine.ts | 2 +- .../UserAvaliableSpaceValidator.ts | 5 ++ .../application/RemoteTreeBuilder.ts | 2 +- .../remoteTree/domain/RemoteItemsGenerator.ts | 2 +- .../infrastructure/IpcRemoteItemsGenerator.ts | 4 +- .../SQLiteRemoteItemsGenerator.ts | 49 ++++++++++++++++++- 10 files changed, 71 insertions(+), 27 deletions(-) diff --git a/src/apps/main/remote-sync/handlers.ts b/src/apps/main/remote-sync/handlers.ts index 04fbda465..f727d25a3 100644 --- a/src/apps/main/remote-sync/handlers.ts +++ b/src/apps/main/remote-sync/handlers.ts @@ -198,20 +198,8 @@ ipcMain.handle('START_REMOTE_SYNC', async () => { setIsProcessing(false); }); -ipcMain.handle('FORCE_REFRESH_BACKUPS', async () => { - Logger.info('Received start remote sync event'); - const deviceUuid = configStore.get('deviceUuid'); - const backupsFolder: RemoteSyncedFolder[] = - await remoteSyncManager.getFolderChildren(deviceUuid); - setIsProcessing(true); - await Promise.all( - backupsFolder.map(async (folder) => { - if (!folder.id) return; - await sleep(200); - await startRemoteSync(folder.id); - }) - ); - setIsProcessing(false); +ipcMain.handle('FORCE_REFRESH_BACKUPS', async (_, folderId: number) => { + await startRemoteSync(folderId); }); remoteSyncManager.onStatusChange((newStatus) => { diff --git a/src/apps/renderer/localize/locales/en.json b/src/apps/renderer/localize/locales/en.json index 0d5ba3340..59b58f7db 100644 --- a/src/apps/renderer/localize/locales/en.json +++ b/src/apps/renderer/localize/locales/en.json @@ -363,7 +363,8 @@ "file-does-not-exist": "File doesn't exist", "file-too-big": "Max upload size is 20GB. Please try smaller files.", "file-non-extension": "Files without extensions are not supported.", - "duplicated-node": "It appears that there are duplicate file or folder names in this directory. Please rename one of them to ensure synchronization" + "duplicated-node": "It appears that there are duplicate file or folder names in this directory. Please rename one of them to ensure synchronization", + "not-enough-space": "You have not enough space to complete the operation" }, "error-messages": { "no-internet": "Looks like you are not connected to the internet, we'll try again later", @@ -379,7 +380,8 @@ "file-does-not-exist": "This file was present when we compared your local folder with your Internxt drive but disappeared when we tried to access it. If you deleted this file, don't worry, this error should dissapear the next time the sync process starts.", "file-too-big": "Max upload size is 20GB. Please try smaller files.", "file-non-extension": "Files without extensions are not supported. Not synchronized.", - "duplicated-node": "There are two elements (file or folder) with the same name on a folder. Rename one of them to sync them both" + "duplicated-node": "There are two elements (file or folder) with the same name on a folder. Rename one of them to sync them both", + "not-enough-space": "You have not enough space to complete the operation" }, "report-modal": { "actions": { diff --git a/src/apps/renderer/localize/locales/es.json b/src/apps/renderer/localize/locales/es.json index 685f0892d..dc9497ee9 100644 --- a/src/apps/renderer/localize/locales/es.json +++ b/src/apps/renderer/localize/locales/es.json @@ -362,7 +362,8 @@ "file-does-not-exist": "El archivo no existe", "file-too-big": "El tamaño máximo de carga es de 20GB. Por favor, intenta con archivos más pequeños.", "file-non-extension": "Archivos sin extensión no son soportados", - "duplicated-node": "Hay un conflicto con el elemento" + "duplicated-node": "Hay un conflicto con el elemento", + "not-enough-space": "No tienes suficiente espacio para completar la operación." }, "error-messages": { "no-internet": "No estás conectado a Internet, lo intentaremos más tarde", @@ -378,7 +379,8 @@ "file-does-not-exist": "Este archivo estaba presente cuando comparamos su carpeta local con su unidad Internxt, pero desapareció cuando intentamos acceder a él. Si has eliminado este archivo, no te preocupes, este error debería desaparecer la próxima vez que se inicie el proceso de sincronización", "file-too-big": "El tamaño máximo de carga es de 20GB. Por favor, intenta con archivos más pequeños.", "file-non-extension": "Los archivos sin extensiones no son soportados. No sincronizado", - "duplicated-node": "Hay dos elementos (archivo o carpeta) con el mismo nombre en una carpeta. Cambia el nombre de uno de ellos para sincronizar ambos." + "duplicated-node": "Hay dos elementos (archivo o carpeta) con el mismo nombre en una carpeta. Cambia el nombre de uno de ellos para sincronizar ambos.", + "not-enough-space": "No tienes suficiente espacio para completar la operación." }, "report-modal": { "actions": { diff --git a/src/apps/renderer/localize/locales/fr.json b/src/apps/renderer/localize/locales/fr.json index eddf95f88..7d0cd4e24 100644 --- a/src/apps/renderer/localize/locales/fr.json +++ b/src/apps/renderer/localize/locales/fr.json @@ -357,7 +357,8 @@ "no-permission": "Permissions insuffisantes", "file-does-not-exist": "Le fichier n'existe pas", "file-too-big": "La taille maximale de téléchargement est de 20 GB. Veuillez essayer des fichiers plus petits.", - "duplicated-node": "Il y a un conflit avec l'élément." + "duplicated-node": "Il y a un conflit avec l'élément.", + "not-enough-space": "Vous n'avez pas assez d'espace pour compléter l'opération." }, "error-messages": { "no-internet": "Il semble que vous ne soyez pas connecté à l'internet, nous réessayerons plus tard", @@ -373,7 +374,8 @@ "file-does-not-exist": "Ce fichier était présent lorsque nous avons comparé votre dossier local avec votre disque interne, mais il a disparu lorsque nous avons essayé d'y accéder. Si vous avez supprimé ce fichier, ne vous inquiétez pas, cette erreur devrait disparaître au prochain démarrage du processus de synchronisation.", "file-too-big": "La taille maximale de téléchargement est de 20 GB. Veuillez essayer des fichiers plus petits.", "file-non-extension": "Les archives sans extensions ne sont pas supportées. Non synchronisées", - "duplicated-node": "Il y a deux éléments (fichier ou dossier) avec le même nom dans un dossier. Renommez l'un d'eux pour les synchroniser tous les deux." + "duplicated-node": "Il y a deux éléments (fichier ou dossier) avec le même nom dans un dossier. Renommez l'un d'eux pour les synchroniser tous les deux.", + "not-enough-space": "Vous n'avez pas assez d'espace pour compléter l'opération." }, "report-modal": { "actions": { diff --git a/src/apps/shared/IPC/events/sync-engine.ts b/src/apps/shared/IPC/events/sync-engine.ts index 7e17d6eb2..9b35f7e1e 100644 --- a/src/apps/shared/IPC/events/sync-engine.ts +++ b/src/apps/shared/IPC/events/sync-engine.ts @@ -129,7 +129,7 @@ export type SyncEngineInvocableFunctions = { folders: DriveFolder[]; }>; START_REMOTE_SYNC: () => Promise; - FORCE_REFRESH_BACKUPS: () => Promise; + FORCE_REFRESH_BACKUPS: (folderId: number) => Promise; }; // TODO: change how errors are reported to the ui diff --git a/src/context/user/usage/application/UserAvaliableSpaceValidator.ts b/src/context/user/usage/application/UserAvaliableSpaceValidator.ts index 4dc8b2731..27d0e76fb 100644 --- a/src/context/user/usage/application/UserAvaliableSpaceValidator.ts +++ b/src/context/user/usage/application/UserAvaliableSpaceValidator.ts @@ -1,3 +1,4 @@ +import Logger from 'electron-log'; import { Service } from 'diod'; import { UserUsageRepository } from '../domain/UserUsageRepository'; @@ -8,6 +9,10 @@ export class UserAvaliableSpaceValidator { async run(desiredSpaceToUse: number): Promise { const usage = await this.repository.getUsage(); + Logger.info( + `Checking if user has enough space to use ${desiredSpaceToUse} bytes. User has ${usage.free()} bytes available.` + ); + return desiredSpaceToUse < usage.free(); } } diff --git a/src/context/virtual-drive/remoteTree/application/RemoteTreeBuilder.ts b/src/context/virtual-drive/remoteTree/application/RemoteTreeBuilder.ts index 9df069492..ce1b060d2 100644 --- a/src/context/virtual-drive/remoteTree/application/RemoteTreeBuilder.ts +++ b/src/context/virtual-drive/remoteTree/application/RemoteTreeBuilder.ts @@ -14,7 +14,7 @@ export class RemoteTreeBuilder { async run(rootFolderId: number, refresh = false): Promise { if (refresh) { Logger.debug('[REMOTE TREE BUILDER] Force refresh'); - await this.remoteItemsGenerator.forceRefresh(); + await this.remoteItemsGenerator.forceRefresh(rootFolderId); } const items = await this.remoteItemsGenerator.getAllItemsByFolderId( diff --git a/src/context/virtual-drive/remoteTree/domain/RemoteItemsGenerator.ts b/src/context/virtual-drive/remoteTree/domain/RemoteItemsGenerator.ts index 579d8cb66..88c483b95 100644 --- a/src/context/virtual-drive/remoteTree/domain/RemoteItemsGenerator.ts +++ b/src/context/virtual-drive/remoteTree/domain/RemoteItemsGenerator.ts @@ -8,5 +8,5 @@ export abstract class RemoteItemsGenerator { folderId: number ): Promise<{ files: ServerFile[]; folders: ServerFolder[] }>; - abstract forceRefresh(): Promise; + abstract forceRefresh(folderId: number): Promise; } diff --git a/src/context/virtual-drive/remoteTree/infrastructure/IpcRemoteItemsGenerator.ts b/src/context/virtual-drive/remoteTree/infrastructure/IpcRemoteItemsGenerator.ts index 9580d49d1..633a8826b 100644 --- a/src/context/virtual-drive/remoteTree/infrastructure/IpcRemoteItemsGenerator.ts +++ b/src/context/virtual-drive/remoteTree/infrastructure/IpcRemoteItemsGenerator.ts @@ -104,7 +104,7 @@ export class IpcRemoteItemsGenerator implements RemoteItemsGenerator { return { files, folders }; } - async forceRefresh(): Promise { - await this.ipc.invoke('FORCE_REFRESH_BACKUPS'); + async forceRefresh(folderId: number): Promise { + await this.ipc.invoke('FORCE_REFRESH_BACKUPS', folderId); } } diff --git a/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts b/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts index de80c2538..68b696959 100644 --- a/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts +++ b/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts @@ -11,6 +11,7 @@ import { import { RemoteItemsGenerator } from '../domain/RemoteItemsGenerator'; import { getUpdatedRemoteItems, + getUpdatedRemoteItemsByFolder, startRemoteSync, } from '../../../../apps/main/remote-sync/handlers'; @@ -56,7 +57,51 @@ export class SQLiteRemoteItemsGenerator implements RemoteItemsGenerator { return { files, folders }; } - async forceRefresh(): Promise { - await startRemoteSync(); + async getAllItemsByFolderId( + folderId: number + ): Promise<{ files: ServerFile[]; folders: ServerFolder[] }> { + const updatedRemoteItems = await getUpdatedRemoteItemsByFolder(folderId); + + const files = updatedRemoteItems.files.map((updatedFile) => { + return { + bucket: updatedFile.bucket, + createdAt: updatedFile.createdAt, + encrypt_version: '03-aes', + fileId: updatedFile.fileId, + folderId: updatedFile.folderId, + id: updatedFile.id, + modificationTime: updatedFile.modificationTime, + name: updatedFile.name, + plainName: updatedFile.plainName, + size: updatedFile.size, + type: updatedFile.type ?? null, + updatedAt: updatedFile.updatedAt, + userId: updatedFile.userId, + status: updatedFile.status as ServerFileStatus, + uuid: updatedFile.uuid, + }; + }); + + const folders = updatedRemoteItems.folders.map( + (updatedFolder) => { + return { + bucket: updatedFolder.bucket ?? null, + createdAt: updatedFolder.createdAt, + id: updatedFolder.id, + name: updatedFolder.name, + parentId: updatedFolder.parentId ?? null, + updatedAt: updatedFolder.updatedAt, + plain_name: updatedFolder.plainName ?? null, + status: updatedFolder.status as ServerFolderStatus, + uuid: updatedFolder.uuid, + }; + } + ); + + return { files, folders }; + } + + async forceRefresh(folderId: number): Promise { + await startRemoteSync(folderId); } } From 4c24b73a1f05e83611649ea7ab8fe9cd343af131 Mon Sep 17 00:00:00 2001 From: ArceDanielShok Date: Fri, 1 Nov 2024 09:58:06 -0300 Subject: [PATCH 4/5] optimized upload flow --- package.json | 2 +- release/app/package.json | 2 +- .../main/remote-sync/RemoteSyncManager.ts | 1 - src/apps/main/remote-sync/handlers.ts | 5 +- .../common/QueueManager.ts | 86 ++++++++++++++----- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index 7065af44f..d291745f5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "internxt-drive", - "version": "2.2.0", + "version": "2.2.1", "author": "Internxt ", "description": "Internxt Drive client UI", "license": "AGPL-3.0", diff --git a/release/app/package.json b/release/app/package.json index fb77bc359..2d4fb1bfb 100644 --- a/release/app/package.json +++ b/release/app/package.json @@ -1,6 +1,6 @@ { "name": "internxt-drive", - "version": "2.2.0", + "version": "2.2.1", "description": "Internxt Drive client UI", "main": "./dist/main/main.js", "author": "Internxt ", diff --git a/src/apps/main/remote-sync/RemoteSyncManager.ts b/src/apps/main/remote-sync/RemoteSyncManager.ts index 4bbba0940..1ea40c7e7 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.ts @@ -230,7 +230,6 @@ export class RemoteSyncManager { } private addLastSyncingFinishedTimestamp() { - if (this.status !== 'SYNCING') return; Logger.info('Adding last syncing finished timestamp'); this.lastSyncingFinishedTimestamp = new Date(); } diff --git a/src/apps/main/remote-sync/handlers.ts b/src/apps/main/remote-sync/handlers.ts index f727d25a3..ede381f95 100644 --- a/src/apps/main/remote-sync/handlers.ts +++ b/src/apps/main/remote-sync/handlers.ts @@ -235,7 +235,8 @@ export async function updateRemoteSync(): Promise { Logger.info('Last files sync at', lastFilesSyncAt); const folderId = lastFilesSyncAt ? undefined : userData?.root_folder_id; await startRemoteSync(folderId); - const isSyncing = await checkSyncEngineInProcess(2_000); + const isSyncing = await checkSyncEngineInProcess(5000); + Logger.info('Is syncing', isSyncing); if (isSyncing) { Logger.info('Remote sync is already running'); return; @@ -249,7 +250,7 @@ export async function fallbackRemoteSync(): Promise { ipcMain.handle('SYNC_MANUALLY', async () => { Logger.info('[Manual Sync] Received manual sync event'); - const isSyncing = await checkSyncEngineInProcess(5_000); + const isSyncing = await checkSyncEngineInProcess(5000); if (isSyncing) return; await updateRemoteSync(); await fallbackRemoteSync(); diff --git a/src/apps/sync-engine/dependency-injection/common/QueueManager.ts b/src/apps/sync-engine/dependency-injection/common/QueueManager.ts index 9c2a531e1..d8261ff87 100644 --- a/src/apps/sync-engine/dependency-injection/common/QueueManager.ts +++ b/src/apps/sync-engine/dependency-injection/common/QueueManager.ts @@ -1,11 +1,14 @@ +/* eslint-disable no-await-in-loop */ import { HandleAction, IQueueManager, QueueItem, HandleActions, + typeQueue, } from 'virtual-drive/dist'; import Logger from 'electron-log'; import fs from 'fs'; +import _ from 'lodash'; export type QueueHandler = { handleAdd: HandleAction; @@ -37,8 +40,10 @@ export class QueueManager implements IQueueManager { changeSize: false, }; - private readonly notify: QueueManagerCallback; + private enqueueTimeout: NodeJS.Timeout | null = null; + private enqueueDelay = 2000; + private readonly notify: QueueManagerCallback; private readonly persistPath: string; actions: HandleActions; @@ -96,6 +101,7 @@ export class QueueManager implements IQueueManager { this.queues = JSON.parse(data); } } + public clearQueue(): void { this.queues = { add: [], @@ -116,15 +122,26 @@ export class QueueManager implements IQueueManager { Logger.debug('Task already exists in queue. Skipping.'); return; } + this.queues[task.type].push(task); this.sortQueue(task.type); this.saveQueueStateToFile(); - if (!this.isProcessing[task.type]) { - this.processQueue(task.type); + this.resetEnqueueTimeout(); + } + + private resetEnqueueTimeout(): void { + if (this.enqueueTimeout) { + clearTimeout(this.enqueueTimeout); } + + // Inicia el temporizador de espera + this.enqueueTimeout = setTimeout(() => { + Logger.debug('Processing all tasks'); + this.processAll(); + }, this.enqueueDelay); } - private sortQueue(type: string): void { + private sortQueue(type: typeQueue): void { this.queues[type].sort((a, b) => { if (a.isFolder && b.isFolder) { return 0; @@ -139,32 +156,61 @@ export class QueueManager implements IQueueManager { }); } - private async processQueue(type: string): Promise { - if (this.isProcessing[type]) { - return; - } + private async processQueue(type: typeQueue): Promise { + if (this.isProcessing[type]) return; this.isProcessing[type] = true; + const start = Date.now(); + Logger.debug(`[TIME] Processing ${type} tasks started at ${start}`); + + if (type === typeQueue.add) { + await this.processInChunks(type, 5); + } else { + await this.processSequentially(type); + } + + this.isProcessing[type] = false; + const end = Date.now(); + Logger.debug(`[TIME] Processing ${type} tasks ended at ${end}`); + Logger.debug(`[TIME] Processing ${type} tasks took ${end - start}ms`); + } + + private async processInChunks( + type: typeQueue, + chunkSize: number + ): Promise { + const chunks = _.chunk(this.queues[type], chunkSize); + + for (const chunk of chunks) { + await Promise.all(chunk.map((task) => this.processTask(type, task))); + this.queues[type] = this.queues[type].slice(chunk.length); + } + } + + private async processSequentially(type: typeQueue): Promise { while (this.queues[type].length > 0) { const task = this.queues[type].shift(); this.saveQueueStateToFile(); - if (task) { - Logger.debug(`Processing ${type} task: ${JSON.stringify(task)}`); - Logger.debug(`Tasks length: ${this.queues[type].length}`); - try { - await this.actions[task.type](task); - } catch (error) { - Logger.error(`Failed to process ${type} task:`, task, error); - } - } + + if (task) await this.processTask(type, task); + } + } + + private async processTask(type: typeQueue, task: QueueItem): Promise { + Logger.debug(`Processing ${type} task: ${JSON.stringify(task)}`); + try { + await this.actions[task.type](task); + } catch (error) { + Logger.error(`Failed to process ${type} task:`, task, error); } - this.isProcessing[type] = false; } public async processAll(): Promise { - const taskTypes = Object.keys(this.queues); + const taskTypes = Object.keys(this.queues) as typeQueue[]; await this.notify.onTaskProcessing(); - await Promise.all(taskTypes.map((type) => this.processQueue(type))); + await Promise.all( + taskTypes.map((type: typeQueue) => this.processQueue(type)) + ); await this.notify.onTaskSuccess(); } } From 7c80b126359468d2c786e880288a46ede7ff73ac Mon Sep 17 00:00:00 2001 From: ArceDanielShok Date: Tue, 5 Nov 2024 12:26:00 -0300 Subject: [PATCH 5/5] remove some log --- .../folders/infrastructure/HttpRemoteFolderSystem.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/context/virtual-drive/folders/infrastructure/HttpRemoteFolderSystem.ts b/src/context/virtual-drive/folders/infrastructure/HttpRemoteFolderSystem.ts index 555f479cd..db064b1d7 100644 --- a/src/context/virtual-drive/folders/infrastructure/HttpRemoteFolderSystem.ts +++ b/src/context/virtual-drive/folders/infrastructure/HttpRemoteFolderSystem.ts @@ -163,9 +163,9 @@ export class HttpRemoteFolderSystem implements RemoteFolderSystem { status: FolderStatuses.EXISTS, }; } catch (error: unknown) { - Logger.error('[FOLDER FILE SYSTEM] Error creating folder', error); + Logger.error('[FOLDER FILE SYSTEM] Error creating folder'); if (axios.isAxiosError(error)) { - Logger.error('[Is Axios Error]', error.response?.data); + Logger.error('[Is Axios Error]', error.response); const existing = await this.existFolder(offline); return existing.status !== FolderStatuses.EXISTS ? Promise.reject(error)