From 01eb6249b233401af272fb26698337cf8cd6488e Mon Sep 17 00:00:00 2001 From: joan vicens Date: Wed, 10 Jan 2024 10:15:23 +0100 Subject: [PATCH 01/32] fix: ignore duplicated nodes --- .../items/application/Traverser.ts | 70 +++++++++---------- .../virtual-drive/items/domain/FolderNode.ts | 3 +- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/context/virtual-drive/items/application/Traverser.ts b/src/context/virtual-drive/items/application/Traverser.ts index 53c964a71..681bf8a02 100644 --- a/src/context/virtual-drive/items/application/Traverser.ts +++ b/src/context/virtual-drive/items/application/Traverser.ts @@ -1,14 +1,5 @@ +import * as Sentry from '@sentry/electron/renderer'; import Logger from 'electron-log'; -import { - FolderStatus, - FolderStatuses, -} from '../../folders/domain/FolderStatus'; -import { Folder } from '../../folders/domain/Folder'; -import { EitherTransformer } from '../../shared/application/EitherTransformer'; -import { createFileFromServerFile } from '../../files/application/FileCreatorFromServerFile'; -import { createFolderFromServerFolder } from '../../folders/application/FolderCreatorFromServerFolder'; -import { NameDecrypt } from '../domain/NameDecrypt'; -import { Tree } from '../domain/Tree'; import { ServerFile, ServerFileStatus, @@ -17,6 +8,16 @@ import { ServerFolder, ServerFolderStatus, } from '../../../shared/domain/ServerFolder'; +import { createFileFromServerFile } from '../../files/application/FileCreatorFromServerFile'; +import { createFolderFromServerFolder } from '../../folders/application/FolderCreatorFromServerFolder'; +import { Folder } from '../../folders/domain/Folder'; +import { + FolderStatus, + FolderStatuses, +} from '../../folders/domain/FolderStatus'; +import { EitherTransformer } from '../../shared/application/EitherTransformer'; +import { NameDecrypt } from '../domain/NameDecrypt'; +import { Tree } from '../domain/Tree'; type Items = { files: Array; @@ -69,17 +70,17 @@ export class Traverser { return folder.parentId === currentFolder.id; }); - filesInThisFolder.forEach((file) => { - if (!this.fileStatusesToFilter.includes(file.status)) { + filesInThisFolder.forEach((serverFile) => { + if (!this.fileStatusesToFilter.includes(serverFile.status)) { return; } const decryptedName = this.decrypt.decryptName( - file.name, - file.folderId.toString(), - file.encrypt_version + serverFile.name, + serverFile.folderId.toString(), + serverFile.encrypt_version ); - const extensionToAdd = file.type ? `.${file.type}` : ''; + const extensionToAdd = serverFile.type ? `.${serverFile.type}` : ''; const relativeFilePath = `${currentFolder.path}/${decryptedName}${extensionToAdd}`.replaceAll( @@ -87,17 +88,16 @@ export class Traverser { '/' ); - EitherTransformer.handleWithEither(() => - createFileFromServerFile(file, relativeFilePath) - ).fold( - (error) => { - Logger.warn( - `[Traverser] File with path ${relativeFilePath} could not be created: `, - error - ); + EitherTransformer.handleWithEither(() => { + const file = createFileFromServerFile(serverFile, relativeFilePath); + tree.addFile(currentFolder, file); + }).fold( + (error): void => { + Logger.warn('[Traverser] Error adding file:', error); + Sentry.captureException(error); }, - (file) => { - tree.addFile(currentFolder, file); + () => { + // no-op } ); }); @@ -118,18 +118,18 @@ export class Traverser { return; } - EitherTransformer.handleWithEither(() => - createFolderFromServerFolder(serverFolder, name) - ).fold( + EitherTransformer.handleWithEither(() => { + const folder = createFolderFromServerFolder(serverFolder, name); + + tree.addFolder(currentFolder, folder); + + return folder; + }).fold( (error) => { - Logger.warn( - `[Traverser] Folder with path ${name} could not be created: `, - error - ); + Logger.warn(`[Traverser] Error adding folder: ${error} `); + Sentry.captureException(error); }, (folder) => { - tree.addFolder(currentFolder, folder); - if (folder.hasStatus(FolderStatuses.EXISTS)) { // The folders and the files inside trashed or deleted folders // will have the status "EXISTS", to avoid filtering witch folders and files diff --git a/src/context/virtual-drive/items/domain/FolderNode.ts b/src/context/virtual-drive/items/domain/FolderNode.ts index f14c98718..3a2565cfd 100644 --- a/src/context/virtual-drive/items/domain/FolderNode.ts +++ b/src/context/virtual-drive/items/domain/FolderNode.ts @@ -1,6 +1,7 @@ import { Folder } from '../../folders/domain/Folder'; import { FileNode } from './FileNode'; import { Node } from './Node'; + export class FolderNode { private constructor( public readonly folder: Folder, @@ -17,7 +18,7 @@ export class FolderNode { addChild(node: Node): void { if (this.children.has(node.id)) { - throw new Error('Child already exists'); + throw new Error(`Duplicated node detected: ${node.id}`); } this.children.set(node.id, node); From e674d84f00d7a322d21739a9e95d51282c04530f Mon Sep 17 00:00:00 2001 From: joan vicens Date: Wed, 10 Jan 2024 12:07:48 +0100 Subject: [PATCH 02/32] feat: comunicate the traverser error to user --- .../background-processes/process-issues.ts | 10 +++++++ src/apps/main/fordwardToWindows.ts | 8 +++++ src/apps/renderer/localize/locales/en.json | 8 +++-- src/apps/renderer/localize/locales/es.json | 6 ++-- src/apps/renderer/localize/locales/fr.json | 6 ++-- src/apps/renderer/messages/process-error.ts | 2 ++ src/apps/renderer/pages/Widget/Item.tsx | 9 ++++-- src/apps/shared/IPC/events/sync-engine.ts | 4 +++ src/apps/shared/types.ts | 8 +++-- .../dependency-injection/common/traverser.ts | 7 ++++- .../dependency-injection/items/builder.ts | 2 ++ .../items/application/Traverser.ts | 29 +++++++++++++++++-- 12 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/apps/main/background-processes/process-issues.ts b/src/apps/main/background-processes/process-issues.ts index f62c58376..fbbcd4570 100644 --- a/src/apps/main/background-processes/process-issues.ts +++ b/src/apps/main/background-processes/process-issues.ts @@ -72,6 +72,16 @@ ipcMain.on('SYNC_INFO_UPDATE', (_, payload: ProcessInfoUpdatePayload) => { } }); +ipcMain.on('SYNC_PROBLEM', (_, payload) => { + addProcessIssue({ + action: 'GENERATE_TREE', + process: 'SYNC', + errorName: 'DUPLICATED_NODE', + kind: 'LOCAL', + name: payload.additionalData.name, + }); +}); + ipcMain.on('BACKUP_ISSUE', (_, issue: ProcessIssue) => { addProcessIssue(issue); }); diff --git a/src/apps/main/fordwardToWindows.ts b/src/apps/main/fordwardToWindows.ts index b0cc9f986..a592cef9a 100644 --- a/src/apps/main/fordwardToWindows.ts +++ b/src/apps/main/fordwardToWindows.ts @@ -1,5 +1,6 @@ import { broadcastToWindows } from './windows'; import { ipcMainDrive } from './ipcs/mainDrive'; +import { ipcMainSyncEngine } from './ipcs/ipcMainSyncEngine'; import { FileErrorInfo } from '../shared/IPC/events/drive'; ipcMainDrive.on('FILE_DELETED', (_, payload) => { @@ -130,3 +131,10 @@ ipcMainDrive.on('FILE_DELETION_ERROR', (_, payload: FileErrorInfo) => { name: nameWithExtension, }); }); + +ipcMainSyncEngine.on('SYNC_PROBLEM', (_, payload) => { + broadcastToWindows('sync-info-update', { + action: 'SYNC_PROBLEM', + name: payload.additionalData.name, + }); +}); diff --git a/src/apps/renderer/localize/locales/en.json b/src/apps/renderer/localize/locales/en.json index 95b1a0527..228ebbdbe 100644 --- a/src/apps/renderer/localize/locales/en.json +++ b/src/apps/renderer/localize/locales/en.json @@ -321,7 +321,8 @@ "no-permission": "Insufficient permissions", "file-does-not-exist": "File doesn't exist", "file-too-big": "Max upload size is 20GB. Please try smaller files.", - "file-non-extension": "Files without extensions are not supported." + "file-non-extension": "Files without extensions are not supported.", + "duplicated-node": "There is a conflict with the element" }, "error-messages": { "no-internet": "Looks like you are not connected to the internet, we'll try again later", @@ -334,9 +335,10 @@ "unknown": "An unknown error ocurred while trying to sync your files", "empty-file": "We don't support files with a size of 0 bytes because of our processes of sharding and encryption", "bad-response": "We got a bad response from our servers while processing this file. Please, try starting the sync process again.", - "file-does-not-exist": "This file was present when we compared your local folder with your Internxt drive but dissapeared when we tried to access it. If you deleted this file, don't worry, this error should dissapear the next time the sync process starts.", + "file-does-not-exist": "This file was present when we compared your local folder with your Internxt drive but disappeared when we tried to access it. If you deleted this file, don't worry, this error should dissapear the next time the sync process starts.", "file-too-big": "Max upload size is 20GB. Please try smaller files.", - "file-non-extension": "Files without extensions are not supported. Not synchronized." + "file-non-extension": "Files without extensions are not supported. Not synchronized.", + "duplicated-node": "There are two elements (file or folder) with the same name on a folder. Rename one of them to sync them both" }, "report-modal": { "actions": { diff --git a/src/apps/renderer/localize/locales/es.json b/src/apps/renderer/localize/locales/es.json index 99c0e4fde..e66dd663f 100644 --- a/src/apps/renderer/localize/locales/es.json +++ b/src/apps/renderer/localize/locales/es.json @@ -320,7 +320,8 @@ "no-permission": "Permisos insuficientes", "file-does-not-exist": "El archivo no existe", "file-too-big": "El tamaño máximo de carga es de 20GB. Por favor, intenta con archivos más pequeños.", - "file-non-extension": "Archivos sin extensión no son soportados" + "file-non-extension": "Archivos sin extensión no son soportados", + "duplicated-node": "Hay un conflicto con el elemento" }, "error-messages": { "no-internet": "No estás conectado a Internet, lo intentaremos más tarde", @@ -335,7 +336,8 @@ "bad-response": "Error de servidor al procesar este archivo. Por favor, intente iniciar de nuevo el proceso de sincronización", "file-does-not-exist": "Este archivo estaba presente cuando comparamos su carpeta local con su unidad Internxt, pero desapareció cuando intentamos acceder a él. Si has eliminado este archivo, no te preocupes, este error debería desaparecer la próxima vez que se inicie el proceso de sincronización", "file-too-big": "El tamaño máximo de carga es de 20GB. Por favor, intenta con archivos más pequeños.", - "file-non-extension": "Los archivos sin extensiones no son soportados. No sincronizado" + "file-non-extension": "Los archivos sin extensiones no son soportados. No sincronizado", + "duplicated-node": "Hay dos elementos (archivo o carpeta) con el mismo nombre en una carpeta. Cambia el nombre de uno de ellos para sincronizar ambos." }, "report-modal": { "actions": { diff --git a/src/apps/renderer/localize/locales/fr.json b/src/apps/renderer/localize/locales/fr.json index d34615e7b..b57c8821c 100644 --- a/src/apps/renderer/localize/locales/fr.json +++ b/src/apps/renderer/localize/locales/fr.json @@ -314,7 +314,8 @@ "no-internet-connection": "Pas de connexion internet", "no-permission": "Permissions insuffisantes", "file-does-not-exist": "Le fichier n'existe pas", - "file-too-big": "La taille maximale de téléchargement est de 20 GB. Veuillez essayer des fichiers plus petits." + "file-too-big": "La taille maximale de téléchargement est de 20 GB. Veuillez essayer des fichiers plus petits.", + "duplicated-node": "Il y a un conflit avec l'élément." }, "error-messages": { "no-internet": "Il semble que vous ne soyez pas connecté à l'internet, nous réessayerons plus tard", @@ -329,7 +330,8 @@ "bad-response": "Nous avons reçu une mauvaise réponse de nos serveurs lors du traitement de ce fichier. Veuillez essayer de relancer le processus de synchronisation.", "file-does-not-exist": "Ce fichier était présent lorsque nous avons comparé votre dossier local avec votre disque interne, mais il a disparu lorsque nous avons essayé d'y accéder. Si vous avez supprimé ce fichier, ne vous inquiétez pas, cette erreur devrait disparaître au prochain démarrage du processus de synchronisation.", "file-too-big": "La taille maximale de téléchargement est de 20 GB. Veuillez essayer des fichiers plus petits.", - "file-non-extension": "Les archives sans extensions ne sont pas supportées. Non synchronisées" + "file-non-extension": "Les archives sans extensions ne sont pas supportées. Non synchronisées", + "duplicated-node": "Il y a deux éléments (fichier ou dossier) avec le même nom dans un dossier. Renommez l'un d'eux pour les synchroniser tous les deux." }, "report-modal": { "actions": { diff --git a/src/apps/renderer/messages/process-error.ts b/src/apps/renderer/messages/process-error.ts index 34df86b4a..d381f4d8c 100644 --- a/src/apps/renderer/messages/process-error.ts +++ b/src/apps/renderer/messages/process-error.ts @@ -12,6 +12,7 @@ export const shortMessages: ProcessErrorMessages = { UNKNOWN: 'issues.short-error-messages.unknown', FILE_TOO_BIG: 'issues.short-error-messages.file-too-big', FILE_NON_EXTENSION: 'issues.short-error-messages.file-non-extension', + DUPLICATED_NODE: 'issues.short-error-messages.duplicated-node', }; export const longMessages: ProcessErrorMessages = { @@ -25,4 +26,5 @@ export const longMessages: ProcessErrorMessages = { UNKNOWN: 'issues.error-messages.unknown', FILE_TOO_BIG: 'issues.error-messages.file-too-big', FILE_NON_EXTENSION: 'issues.error-messages.file-non-extension', + DUPLICATED_NODE: 'issues.error-messages.duplicated-node', }; diff --git a/src/apps/renderer/pages/Widget/Item.tsx b/src/apps/renderer/pages/Widget/Item.tsx index 5bd6f9ab8..f1397d12a 100644 --- a/src/apps/renderer/pages/Widget/Item.tsx +++ b/src/apps/renderer/pages/Widget/Item.tsx @@ -68,7 +68,8 @@ export function Item({ action === 'DOWNLOAD_ERROR' || action === 'UPLOAD_ERROR' || action === 'RENAME_ERROR' || - action === 'METADATA_READ_ERROR') + action === 'METADATA_READ_ERROR' || + action === 'GENERATE_TREE') ? 'text-red' : undefined }`} @@ -78,7 +79,8 @@ export function Item({ action === 'DOWNLOAD_ERROR' || action === 'UPLOAD_ERROR' || action === 'RENAME_ERROR' || - action === 'METADATA_READ_ERROR') + action === 'METADATA_READ_ERROR' || + action === 'GENERATE_TREE') ? description : undefined } @@ -123,7 +125,8 @@ export function Item({ action === 'DOWNLOAD_ERROR' || action === 'UPLOAD_ERROR' || action === 'RENAME_ERROR' || - action === 'METADATA_READ_ERROR') && ( + action === 'METADATA_READ_ERROR' || + action === 'GENERATE_TREE') && ( )} diff --git a/src/apps/shared/IPC/events/sync-engine.ts b/src/apps/shared/IPC/events/sync-engine.ts index bf09401ed..2b9494796 100644 --- a/src/apps/shared/IPC/events/sync-engine.ts +++ b/src/apps/shared/IPC/events/sync-engine.ts @@ -128,6 +128,10 @@ export type SyncEngineInvocableFunctions = { // TODO: change how errors are reported to the ui export type ProcessInfoUpdate = { SYNC_INFO_UPDATE: (payload: ProcessInfoUpdatePayload) => void; + SYNC_PROBLEM: (payload: { + key: string; + additionalData: Record; + }) => void; }; export type FromProcess = FilesEvents & diff --git a/src/apps/shared/types.ts b/src/apps/shared/types.ts index 8c204661b..d77562c62 100644 --- a/src/apps/shared/types.ts +++ b/src/apps/shared/types.ts @@ -149,7 +149,10 @@ export type ProcessErrorName = | 'FILE_NON_EXTENSION' // Unknown error - | 'UNKNOWN'; + | 'UNKNOWN' + + // Duplicated node path + | 'DUPLICATED_NODE'; export class ProcessError extends Error { details: ErrorDetails; @@ -213,7 +216,8 @@ export type ProcessIssue = ProcessInfoBase & { | 'DOWNLOAD_ERROR' | 'RENAME_ERROR' | 'DELETE_ERROR' - | 'METADATA_READ_ERROR'; + | 'METADATA_READ_ERROR' + | 'GENERATE_TREE'; errorName: ProcessErrorName; process: 'SYNC' | 'BACKUPS'; diff --git a/src/apps/sync-engine/dependency-injection/common/traverser.ts b/src/apps/sync-engine/dependency-injection/common/traverser.ts index b25de6491..be17a4e0c 100644 --- a/src/apps/sync-engine/dependency-injection/common/traverser.ts +++ b/src/apps/sync-engine/dependency-injection/common/traverser.ts @@ -1,6 +1,7 @@ import crypt from '../../../../context/shared/infrastructure/crypt'; import { Traverser } from '../../../../context/virtual-drive/items/application/Traverser'; import { DependencyInjectionUserProvider } from './user'; +import { ipcRendererSyncEngine } from '../../ipcRendererSyncEngine'; export class DependencyInjectionTraverserProvider { private static traverser: Traverser; @@ -12,7 +13,11 @@ export class DependencyInjectionTraverserProvider { const user = DependencyInjectionUserProvider.get(); - const traverser = Traverser.existingItems(crypt, user.root_folder_id); + const traverser = Traverser.existingItems( + crypt, + ipcRendererSyncEngine, + user.root_folder_id + ); DependencyInjectionTraverserProvider.traverser = traverser; diff --git a/src/apps/sync-engine/dependency-injection/items/builder.ts b/src/apps/sync-engine/dependency-injection/items/builder.ts index c6f66a759..20a7c7ffd 100644 --- a/src/apps/sync-engine/dependency-injection/items/builder.ts +++ b/src/apps/sync-engine/dependency-injection/items/builder.ts @@ -19,10 +19,12 @@ export function buildItemsContainer(): ItemsContainer { const existingItemsTraverser = Traverser.existingItems( nameDecryptor, + ipcRendererSyncEngine, user.root_folder_id ); const allStatusesTraverser = Traverser.allItems( nameDecryptor, + ipcRendererSyncEngine, user.root_folder_id ); const treeBuilder = new TreeBuilder( diff --git a/src/context/virtual-drive/items/application/Traverser.ts b/src/context/virtual-drive/items/application/Traverser.ts index 681bf8a02..f6a312553 100644 --- a/src/context/virtual-drive/items/application/Traverser.ts +++ b/src/context/virtual-drive/items/application/Traverser.ts @@ -1,5 +1,6 @@ import * as Sentry from '@sentry/electron/renderer'; import Logger from 'electron-log'; +import { SyncEngineIpc } from '../../../../apps/sync-engine/ipcRendererSyncEngine'; import { ServerFile, ServerFileStatus, @@ -27,22 +28,32 @@ type Items = { export class Traverser { constructor( private readonly decrypt: NameDecrypt, + private readonly ipc: SyncEngineIpc, private readonly baseFolderId: number, private readonly fileStatusesToFilter: Array, private readonly folderStatusesToFilter: Array ) {} - static existingItems(decrypt: NameDecrypt, baseFolderId: number): Traverser { + static existingItems( + decrypt: NameDecrypt, + ipc: SyncEngineIpc, + baseFolderId: number + ): Traverser { return new Traverser( decrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS], [ServerFolderStatus.EXISTS] ); } - static allItems(decrypt: NameDecrypt, baseFolderId: number): Traverser { - return new Traverser(decrypt, baseFolderId, [], []); + static allItems( + decrypt: NameDecrypt, + ipc: SyncEngineIpc, + baseFolderId: number + ): Traverser { + return new Traverser(decrypt, ipc, baseFolderId, [], []); } private createRootFolder(): Folder { @@ -95,6 +106,12 @@ export class Traverser { (error): void => { Logger.warn('[Traverser] Error adding file:', error); Sentry.captureException(error); + this.ipc.send('SYNC_PROBLEM', { + key: 'node-duplicated', + additionalData: { + name: serverFile.plainName, + }, + }); }, () => { // no-op @@ -128,6 +145,12 @@ export class Traverser { (error) => { Logger.warn(`[Traverser] Error adding folder: ${error} `); Sentry.captureException(error); + this.ipc.send('SYNC_PROBLEM', { + key: 'node-duplicated', + additionalData: { + name, + }, + }); }, (folder) => { if (folder.hasStatus(FolderStatuses.EXISTS)) { From 26760f0a00bc8ed5b14228f1685c5eb2c7e0ff9c Mon Sep 17 00:00:00 2001 From: joan vicens Date: Wed, 10 Jan 2024 12:56:54 +0100 Subject: [PATCH 03/32] test: update traverser test --- .../virtual-drive/items/application/Traverser.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/context/virtual-drive/items/application/Traverser.test.ts b/tests/context/virtual-drive/items/application/Traverser.test.ts index 62adf56d4..7c98bce10 100644 --- a/tests/context/virtual-drive/items/application/Traverser.test.ts +++ b/tests/context/virtual-drive/items/application/Traverser.test.ts @@ -8,10 +8,12 @@ import { } from '../../../../../src/context/shared/domain/ServerFolder'; import { Traverser } from '../../../../../src/context/virtual-drive/items/application/Traverser'; import { ContentsIdMother } from '../../contents/domain/ContentsIdMother'; +import { IpcRendererSyncEngineMock } from '../../shared/__mock__/IpcRendererSyncEngineMock'; import { FakeNameDecrypt } from '../infrastructure/FakeNameDecrypt'; describe('Traverser', () => { const nameDecrypt = new FakeNameDecrypt(); + const ipc = new IpcRendererSyncEngineMock(); it('first level files starts with /', () => { const baseFolderId = 6; @@ -29,6 +31,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] @@ -64,6 +67,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] @@ -91,6 +95,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] @@ -124,6 +129,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] @@ -157,6 +163,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] @@ -197,6 +204,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] @@ -224,6 +232,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] @@ -259,6 +268,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] @@ -294,6 +304,7 @@ describe('Traverser', () => { }; const SUT = new Traverser( nameDecrypt, + ipc, baseFolderId, [ServerFileStatus.EXISTS, ServerFileStatus.TRASHED], [ServerFolderStatus.EXISTS] From 03f9e0b9e9df864a8e23e8dcf84e33849f74f7f0 Mon Sep 17 00:00:00 2001 From: joan vicens Date: Wed, 10 Jan 2024 17:10:13 +0100 Subject: [PATCH 04/32] chore: allow long error messages --- src/apps/renderer/localize/locales/en.json | 2 +- src/apps/renderer/pages/ProcessIssues/List.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/renderer/localize/locales/en.json b/src/apps/renderer/localize/locales/en.json index 228ebbdbe..04b2169f6 100644 --- a/src/apps/renderer/localize/locales/en.json +++ b/src/apps/renderer/localize/locales/en.json @@ -322,7 +322,7 @@ "file-does-not-exist": "File doesn't exist", "file-too-big": "Max upload size is 20GB. Please try smaller files.", "file-non-extension": "Files without extensions are not supported.", - "duplicated-node": "There is a conflict with the element" + "duplicated-node": "It appears that there are duplicate file or folder names in this directory. Please rename one of them to ensure synchronization" }, "error-messages": { "no-internet": "Looks like you are not connected to the internet, we'll try again later", diff --git a/src/apps/renderer/pages/ProcessIssues/List.tsx b/src/apps/renderer/pages/ProcessIssues/List.tsx index a6887aade..eee9a0e82 100644 --- a/src/apps/renderer/pages/ProcessIssues/List.tsx +++ b/src/apps/renderer/pages/ProcessIssues/List.tsx @@ -202,7 +202,7 @@ function Item({

{translate(shortMessages[errorName])} From 6d1cb35d35d578d6d2e24391e36bfbd9dc7cc62c Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 18 Jan 2024 09:51:49 -0500 Subject: [PATCH 05/32] *add retry mechanism to try deleting folder *add logic to delete queue events --- .../callbacks-controllers/buildControllers.ts | 4 +- .../controllers/DeleteController.ts | 51 +++++++++++++++++-- .../files/FilesContainer.ts | 2 + .../dependency-injection/files/builder.ts | 7 +++ .../folders/FoldersContainer.ts | 4 ++ .../dependency-injection/folders/builder.ts | 8 +++ src/apps/sync-engine/index.ts | 3 ++ .../files/application/FileDeleter.ts | 4 +- .../FileFolderContainerDetector.ts | 18 +++++++ .../application/FolderContainerDetector.ts | 27 ++++++++++ .../folders/application/FolderFinder.ts | 8 +++ .../Offline/OfflineFolderCreator.ts | 2 + .../folders/application/RetryFolderDeleter.ts | 50 ++++++++++++++++++ .../errors/MaxRetriesDeletingFolderError.ts | 5 ++ .../virtual-drive/shared/domain/DelayQueue.ts | 31 ++++++++--- 15 files changed, 207 insertions(+), 17 deletions(-) create mode 100644 src/context/virtual-drive/files/application/FileFolderContainerDetector.ts create mode 100644 src/context/virtual-drive/folders/application/FolderContainerDetector.ts create mode 100644 src/context/virtual-drive/folders/application/RetryFolderDeleter.ts create mode 100644 src/context/virtual-drive/folders/domain/errors/MaxRetriesDeletingFolderError.ts diff --git a/src/apps/sync-engine/callbacks-controllers/buildControllers.ts b/src/apps/sync-engine/callbacks-controllers/buildControllers.ts index 1f749a02d..d4ea27b56 100644 --- a/src/apps/sync-engine/callbacks-controllers/buildControllers.ts +++ b/src/apps/sync-engine/callbacks-controllers/buildControllers.ts @@ -16,7 +16,9 @@ export function buildControllers(container: DependencyContainer) { const deleteController = new DeleteController( container.fileDeleter, - container.folderDeleter + container.retryFolderDeleter, + container.fileFolderContainerDetector, + container.folderContainerDetector ); const renameOrMoveController = new RenameOrMoveController( diff --git a/src/apps/sync-engine/callbacks-controllers/controllers/DeleteController.ts b/src/apps/sync-engine/callbacks-controllers/controllers/DeleteController.ts index 65ee7fa18..f6be1e79d 100644 --- a/src/apps/sync-engine/callbacks-controllers/controllers/DeleteController.ts +++ b/src/apps/sync-engine/callbacks-controllers/controllers/DeleteController.ts @@ -1,8 +1,11 @@ import { FileDeleter } from '../../../../context/virtual-drive/files/application/FileDeleter'; -import { FolderDeleter } from '../../../../context/virtual-drive/folders/application/FolderDeleter'; +import { RetryFolderDeleter } from '../../../../context/virtual-drive/folders/application/RetryFolderDeleter'; import { DelayQueue } from '../../../../context/virtual-drive/shared/domain/DelayQueue'; import { CallbackController } from './CallbackController'; import Logger from 'electron-log'; +import { FileFolderContainerDetector } from '../../../../context/virtual-drive/files/application/FileFolderContainerDetector'; +import { Folder } from '../../../../context/virtual-drive/folders/domain/Folder'; +import { FolderContainerDetector } from '../../../../context/virtual-drive/folders/application/FolderContainerDetector'; export class DeleteController extends CallbackController { private readonly filesQueue: DelayQueue; @@ -10,7 +13,9 @@ export class DeleteController extends CallbackController { constructor( private readonly fileDeleter: FileDeleter, - private readonly folderDeleter: FolderDeleter + private readonly retryFolderDeleter: RetryFolderDeleter, + private readonly fileFolderContainerDetector: FileFolderContainerDetector, + private readonly folderContainerDetector: FolderContainerDetector ) { super(); @@ -19,7 +24,12 @@ export class DeleteController extends CallbackController { }; const deleteFolder = async (folder: string) => { - await this.folderDeleter.run(folder); + try { + await this.retryFolderDeleter.run(folder); + } catch (error) { + Logger.error('Error deleting folder: ', error); + // TODO: create tree of placeholders that are not deleted + } }; const canDeleteFolders = () => { @@ -34,8 +44,7 @@ export class DeleteController extends CallbackController { ); const canDeleteFiles = () => { - // Files cannot be deleted if there are folders on the queue - return this.foldersQueue.size === 0; + return this.foldersQueue.isEmpty; }; this.filesQueue = new DelayQueue('files', deleteFile, canDeleteFiles); @@ -54,10 +63,42 @@ export class DeleteController extends CallbackController { if (this.isFolderPlaceholder(trimmedId)) { const [_, folderUuid] = trimmedId.split(':'); Logger.debug(`Adding folder: ${folderUuid} to the trash queue`); + this.CleanQueuesByFolder(folderUuid); this.foldersQueue.push(folderUuid); return; } throw new Error(`Placeholder Id not identified: ${trimmedId}`); } + + private CleanQueuesByFolder(folderUuid: Folder['uuid']) { + // always remove files from the filesQueue if a folder is added + this.CleanQueueFile(folderUuid); + // remove child folders from the queue if a parent folder exists + this.CleanQueueFolder(folderUuid); + } + + private CleanQueueFile(folderUuid: Folder['uuid']) { + const files = this.filesQueue.values; + const filesToDelete = files.filter((file) => + this.fileFolderContainerDetector.run(file, folderUuid) + ); + filesToDelete.forEach((file) => { + this.filesQueue.removeOne(file); + }); + } + + private CleanQueueFolder(folderUuid: Folder['uuid']) { + const folders = this.foldersQueue.values; + const reversedFolders = folders.reverse(); + reversedFolders.forEach((folder) => { + const isParentFolder = this.folderContainerDetector.run( + folder, + folderUuid + ); + if (isParentFolder) { + this.foldersQueue.removeOne(folder); + } + }); + } } diff --git a/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts b/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts index 5ba634247..ea7334e25 100644 --- a/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts +++ b/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts @@ -9,10 +9,12 @@ import { FilesPlaceholderCreator } from '../../../../context/virtual-drive/files import { RepositoryPopulator } from '../../../../context/virtual-drive/files/application/RepositoryPopulator'; import { RetrieveAllFiles } from '../../../../context/virtual-drive/files/application/RetrieveAllFiles'; import { SameFileWasMoved } from '../../../../context/virtual-drive/files/application/SameFileWasMoved'; +import { FileFolderContainerDetector } from '../../../../context/virtual-drive/files/application/FileFolderContainerDetector'; export interface FilesContainer { fileFinderByContentsId: FileFinderByContentsId; fileDeleter: FileDeleter; + fileFolderContainerDetector: FileFolderContainerDetector; filePathUpdater: FilePathUpdater; fileCreator: FileCreator; filePlaceholderCreatorFromContentsId: FilePlaceholderCreatorFromContentsId; diff --git a/src/apps/sync-engine/dependency-injection/files/builder.ts b/src/apps/sync-engine/dependency-injection/files/builder.ts index e9cf5c424..2b1bfb434 100644 --- a/src/apps/sync-engine/dependency-injection/files/builder.ts +++ b/src/apps/sync-engine/dependency-injection/files/builder.ts @@ -24,6 +24,7 @@ import { SDKRemoteFileSystem } from '../../../../context/virtual-drive/files/inf import { NodeWinLocalFileSystem } from '../../../../context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem'; import { LocalFileIdProvider } from '../../../../context/virtual-drive/shared/application/LocalFileIdProvider'; import { DependencyInjectionHttpClientsProvider } from '../common/clients'; +import { FileFolderContainerDetector } from '../../../../context/virtual-drive/files/application/FileFolderContainerDetector'; export async function buildFilesContainer( folderContainer: FoldersContainer, @@ -63,6 +64,11 @@ export async function buildFilesContainer( ipcRendererSyncEngine ); + const fileFolderContainerDetector = new FileFolderContainerDetector( + repository, + folderContainer.folderFinder + ); + const sameFileWasMoved = new SameFileWasMoved( repository, localFileSystem, @@ -120,6 +126,7 @@ export async function buildFilesContainer( fileDeleter, filePathUpdater, fileCreator, + fileFolderContainerDetector, filePlaceholderCreatorFromContentsId: filePlaceholderCreatorFromContentsId, createFilePlaceholderOnDeletionFailed: createFilePlaceholderOnDeletionFailed, diff --git a/src/apps/sync-engine/dependency-injection/folders/FoldersContainer.ts b/src/apps/sync-engine/dependency-injection/folders/FoldersContainer.ts index bcb774e3d..10de3f7ce 100644 --- a/src/apps/sync-engine/dependency-injection/folders/FoldersContainer.ts +++ b/src/apps/sync-engine/dependency-injection/folders/FoldersContainer.ts @@ -1,5 +1,6 @@ import { AllParentFoldersStatusIsExists } from '../../../../context/virtual-drive/folders/application/AllParentFoldersStatusIsExists'; import { FolderByPartialSearcher } from '../../../../context/virtual-drive/folders/application/FolderByPartialSearcher'; +import { FolderContainerDetector } from '../../../../context/virtual-drive/folders/application/FolderContainerDetector'; import { FolderCreator } from '../../../../context/virtual-drive/folders/application/FolderCreator'; import { FolderDeleter } from '../../../../context/virtual-drive/folders/application/FolderDeleter'; import { FolderFinder } from '../../../../context/virtual-drive/folders/application/FolderFinder'; @@ -9,6 +10,7 @@ import { FoldersPlaceholderCreator } from '../../../../context/virtual-drive/fol import { OfflineFolderCreator } from '../../../../context/virtual-drive/folders/application/Offline/OfflineFolderCreator'; import { OfflineFolderPathUpdater } from '../../../../context/virtual-drive/folders/application/Offline/OfflineFolderPathUpdater'; import { RetrieveAllFolders } from '../../../../context/virtual-drive/folders/application/RetrieveAllFolders'; +import { RetryFolderDeleter } from '../../../../context/virtual-drive/folders/application/RetryFolderDeleter'; import { SynchronizeOfflineModifications } from '../../../../context/virtual-drive/folders/application/SynchronizeOfflineModifications'; import { SynchronizeOfflineModificationsOnFolderCreated } from '../../../../context/virtual-drive/folders/application/SynchronizeOfflineModificationsOnFolderCreated'; import { FolderPlaceholderUpdater } from '../../../../context/virtual-drive/folders/application/UpdatePlaceholderFolder'; @@ -16,7 +18,9 @@ import { FolderPlaceholderUpdater } from '../../../../context/virtual-drive/fold export interface FoldersContainer { folderCreator: FolderCreator; folderFinder: FolderFinder; + folderContainerDetector: FolderContainerDetector; folderDeleter: FolderDeleter; + retryFolderDeleter: RetryFolderDeleter; allParentFoldersStatusIsExists: AllParentFoldersStatusIsExists; folderPathUpdater: FolderPathUpdater; folderByPartialSearcher: FolderByPartialSearcher; diff --git a/src/apps/sync-engine/dependency-injection/folders/builder.ts b/src/apps/sync-engine/dependency-injection/folders/builder.ts index b40d16044..f68151366 100644 --- a/src/apps/sync-engine/dependency-injection/folders/builder.ts +++ b/src/apps/sync-engine/dependency-injection/folders/builder.ts @@ -27,6 +27,8 @@ import { DependencyInjectionEventRepository } from '../common/eventRepository'; import { DependencyInjectionVirtualDrive } from '../common/virtualDrive'; import { SharedContainer } from '../shared/SharedContainer'; import { FoldersContainer } from './FoldersContainer'; +import { RetryFolderDeleter } from '../../../../context/virtual-drive/folders/application/RetryFolderDeleter'; +import { FolderContainerDetector } from '../../../../context/virtual-drive/folders/application/FolderContainerDetector'; export async function buildFoldersContainer( shredContainer: SharedContainer @@ -57,6 +59,8 @@ export async function buildFoldersContainer( allParentFoldersStatusIsExists ); + const retryFolderDeleter = new RetryFolderDeleter(folderDeleter); + const folderCreator = new FolderCreator( repository, remoteFileSystem, @@ -124,12 +128,16 @@ export async function buildFoldersContainer( shredContainer.relativePathToAbsoluteConverter ); + const folderContainerDetector = new FolderContainerDetector(repository); + return { folderCreator, folderFinder, folderDeleter, + retryFolderDeleter, allParentFoldersStatusIsExists: allParentFoldersStatusIsExists, folderPathUpdater, + folderContainerDetector, folderByPartialSearcher, synchronizeOfflineModificationsOnFolderCreated, offline: { diff --git a/src/apps/sync-engine/index.ts b/src/apps/sync-engine/index.ts index c5ea70358..61b7814aa 100644 --- a/src/apps/sync-engine/index.ts +++ b/src/apps/sync-engine/index.ts @@ -81,5 +81,8 @@ setUp() }) .catch((error) => { Logger.error('[SYNC ENGINE] Error setting up', error); + if (error.toString().includes('Error: ConnectSyncRoot failed')) { + Logger.info('[SYNC ENGINE] We neeed to restart the app virtual drive'); + } ipcRenderer.send('SYNC_ENGINE_PROCESS_SETUP_FAILED'); }); diff --git a/src/context/virtual-drive/files/application/FileDeleter.ts b/src/context/virtual-drive/files/application/FileDeleter.ts index 70ac81bfd..185232c63 100644 --- a/src/context/virtual-drive/files/application/FileDeleter.ts +++ b/src/context/virtual-drive/files/application/FileDeleter.ts @@ -65,7 +65,7 @@ export class FileDeleter { ); const message = error instanceof Error ? error.message : 'Unknown error'; - + this.local.createPlaceHolder(file); this.ipc.send('FILE_DELETION_ERROR', { name: file.name, extension: file.type, @@ -80,8 +80,6 @@ export class FileDeleter { errorName: 'BAD_RESPONSE', process: 'SYNC', }); - - this.local.createPlaceHolder(file); } } } diff --git a/src/context/virtual-drive/files/application/FileFolderContainerDetector.ts b/src/context/virtual-drive/files/application/FileFolderContainerDetector.ts new file mode 100644 index 000000000..b9b2cd822 --- /dev/null +++ b/src/context/virtual-drive/files/application/FileFolderContainerDetector.ts @@ -0,0 +1,18 @@ +import { FolderFinder } from '../../folders/application/FolderFinder'; +import { File } from '../../files/domain/File'; +import { Folder } from '../../folders/domain/Folder'; +import { FileRepository } from '../domain/FileRepository'; + +export class FileFolderContainerDetector { + constructor( + private readonly repository: FileRepository, + private readonly folderFinder: FolderFinder + ) {} + + run(contentId: File['contentsId'], folderContentId: Folder['uuid']): boolean { + const file = this.repository.searchByPartial({ contentsId: contentId }); + const folder = this.folderFinder.findFromId(file?.folderId); + const [_, folderUuid] = folder.placeholderId.split(':'); + return folderUuid === folderContentId; + } +} diff --git a/src/context/virtual-drive/folders/application/FolderContainerDetector.ts b/src/context/virtual-drive/folders/application/FolderContainerDetector.ts new file mode 100644 index 000000000..14221aa69 --- /dev/null +++ b/src/context/virtual-drive/folders/application/FolderContainerDetector.ts @@ -0,0 +1,27 @@ +import { Folder } from '../domain/Folder'; +import { FolderRepository } from '../domain/FolderRepository'; + +export class FolderContainerDetector { + constructor(private readonly repository: FolderRepository) {} + + run( + fodlerContentId: Folder['uuid'], + parentFolderContentId: Folder['uuid'] + ): boolean { + const folder = this.repository.searchByPartial({ uuid: fodlerContentId }); + + if (!folder) { + throw new Error('Folder not found'); + } + + const parent = this.repository.searchByPartial({ + id: folder.parentId as number, + }); + + if (!parent) { + throw new Error('Parent folder not found'); + } + + return parent.uuid === parentFolderContentId; + } +} diff --git a/src/context/virtual-drive/folders/application/FolderFinder.ts b/src/context/virtual-drive/folders/application/FolderFinder.ts index 8f50d9b46..6db686c50 100644 --- a/src/context/virtual-drive/folders/application/FolderFinder.ts +++ b/src/context/virtual-drive/folders/application/FolderFinder.ts @@ -25,4 +25,12 @@ export class FolderFinder { return folder; } + + findFromId(id: Folder['id'] | undefined): Folder { + const folder = this.repository.searchByPartial({ id }); + if (!folder) { + throw new Error('Folder not found'); + } + return folder; + } } diff --git a/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts b/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts index 30650be19..9bd900a35 100644 --- a/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts +++ b/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts @@ -3,6 +3,7 @@ import { OfflineFolder } from '../../domain/OfflineFolder'; import { OfflineFolderRepository } from '../../domain/OfflineFolderRepository'; import { FolderFinder } from '../FolderFinder'; import { FolderRepository } from '../../domain/FolderRepository'; +import { FolderStatuses } from '../../domain/FolderStatus'; export class OfflineFolderCreator { constructor( @@ -16,6 +17,7 @@ export class OfflineFolderCreator { const onlineFolder = this.repository.searchByPartial({ path: folderPath.value, + status: FolderStatuses.EXISTS, }); if (onlineFolder) { diff --git a/src/context/virtual-drive/folders/application/RetryFolderDeleter.ts b/src/context/virtual-drive/folders/application/RetryFolderDeleter.ts new file mode 100644 index 000000000..e4f6a9dc6 --- /dev/null +++ b/src/context/virtual-drive/folders/application/RetryFolderDeleter.ts @@ -0,0 +1,50 @@ +import Logger from 'electron-log'; +import { FolderDeleter } from './FolderDeleter'; +import { MaxRetriesDeletingFolderError } from '../domain/errors/MaxRetriesDeletingFolderError'; + +export class RetryFolderDeleter { + private static NUMBER_OF_RETRIES = 2; + private static MILLISECOND_BETWEEN_TRIES = 1_000; + private static INITIAL_DELAY = 100; + constructor(private readonly deleter: FolderDeleter) {} + async retryDeleter(asyncFunction: () => Promise) { + let retryCount = 0; + + while (retryCount <= RetryFolderDeleter.NUMBER_OF_RETRIES) { + try { + const result = await asyncFunction(); + return result; + } catch (error: unknown) { + if (error instanceof Error) { + Logger.warn( + `Folder deleter attempt ${retryCount + 1} failed: ${error.message}` + ); + } else { + Logger.warn( + `Folder deleter attempt ${ + retryCount + 1 + } failed with an unknown error.` + ); + } + + await new Promise((resolve) => { + setTimeout(resolve, RetryFolderDeleter.MILLISECOND_BETWEEN_TRIES); + }); + + retryCount++; + } + } + throw new MaxRetriesDeletingFolderError( + RetryFolderDeleter.NUMBER_OF_RETRIES + ); + } + + async run(folder: string): Promise { + await new Promise((resolve) => { + setTimeout(resolve, RetryFolderDeleter.INITIAL_DELAY); + }); + + const deleter = () => this.deleter.run(folder); + return this.retryDeleter(deleter); + } +} diff --git a/src/context/virtual-drive/folders/domain/errors/MaxRetriesDeletingFolderError.ts b/src/context/virtual-drive/folders/domain/errors/MaxRetriesDeletingFolderError.ts new file mode 100644 index 000000000..0d79a399f --- /dev/null +++ b/src/context/virtual-drive/folders/domain/errors/MaxRetriesDeletingFolderError.ts @@ -0,0 +1,5 @@ +export class MaxRetriesDeletingFolderError extends Error { + constructor(retriesNumber: number) { + super(`Max retries (${retriesNumber}) reached. Deleter still failed.`); + } +} diff --git a/src/context/virtual-drive/shared/domain/DelayQueue.ts b/src/context/virtual-drive/shared/domain/DelayQueue.ts index dd7a44c34..19b31f561 100644 --- a/src/context/virtual-drive/shared/domain/DelayQueue.ts +++ b/src/context/virtual-drive/shared/domain/DelayQueue.ts @@ -20,26 +20,29 @@ export class DelayQueue { } } - private setTimeout() { + private setTimeout(delay = DelayQueue.DELAY) { this.clearTimeout(); this.timeout = setTimeout(async () => { Logger.debug('Will try to run delay queue for: ', this.name); if (this.canLoop()) { Logger.debug('Running delay queue for: ', this.name); - const reversedItems = Array.from(this.queue.entries()).reverse(); - - for (const [item] of reversedItems) { - await this.fn(item); - this.queue.delete(item); + await this.executeFn(reversedItems[0][0]); + if (!this.isEmpty) { + Logger.debug('Restarting delay queue is not empty'); + this.setTimeout(500); } - return; } Logger.debug(this.name, 'delay queue blocked'); this.setTimeout(); - }, DelayQueue.DELAY); + }, delay); + } + + private async executeFn(item: string) { + this.queue.delete(item); + await this.fn(item); } push(value: string) { @@ -56,4 +59,16 @@ export class DelayQueue { this.clearTimeout(); this.queue.clear(); } + + removeOne(value: string) { + this.queue.delete(value); + } + + get values(): string[] { + return Array.from(this.queue.keys()); + } + + get isEmpty(): boolean { + return this.queue.size === 0; + } } From c9224c66caddeebfdc739a80eafdc8e17e6e0716 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 25 Jan 2024 12:16:32 -0500 Subject: [PATCH 06/32] Add polling monitor start and stop functionality --- src/apps/sync-engine/BindingManager.ts | 6 ++++ .../shared/SharedContainer.ts | 4 +++ .../dependency-injection/shared/builder.ts | 10 ++++++ .../shared/application/PollingMonitorStart.ts | 8 +++++ .../shared/application/PollingMonitorStop.ts | 8 +++++ .../shared/domain/PollingMonitor.ts | 31 +++++++++++++++++++ 6 files changed, 67 insertions(+) create mode 100644 src/context/virtual-drive/shared/application/PollingMonitorStart.ts create mode 100644 src/context/virtual-drive/shared/application/PollingMonitorStop.ts create mode 100644 src/context/virtual-drive/shared/domain/PollingMonitor.ts diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index 76c4ea410..39aa12c69 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -37,6 +37,7 @@ export class BindingsManager { async start(version: string, providerId: string) { await this.stop(); + this.container.pollingMonitorStart.run(this.monitorSimulateFn); const controllers = buildControllers(this.container); @@ -215,6 +216,7 @@ export class BindingsManager { async stop() { await this.container.virtualDrive.disconnectSyncRoot(); + this.container.pollingMonitorStop.run(); } async cleanUp() { @@ -276,4 +278,8 @@ export class BindingsManager { Logger.error('[SYNC ENGINE] ', error); } } + + private async monitorSimulateFn(): Promise { + Logger.info('[SYNC ENGINE] Starting monitoring polling...'); + } } diff --git a/src/apps/sync-engine/dependency-injection/shared/SharedContainer.ts b/src/apps/sync-engine/dependency-injection/shared/SharedContainer.ts index 1649eceac..1386789f7 100644 --- a/src/apps/sync-engine/dependency-injection/shared/SharedContainer.ts +++ b/src/apps/sync-engine/dependency-injection/shared/SharedContainer.ts @@ -1,7 +1,11 @@ import { AbsolutePathToRelativeConverter } from '../../../../context/virtual-drive/shared/application/AbsolutePathToRelativeConverter'; +import { PollingMonitorStart } from '../../../../context/virtual-drive/shared/application/PollingMonitorStart'; +import { PollingMonitorStop } from '../../../../context/virtual-drive/shared/application/PollingMonitorStop'; import { RelativePathToAbsoluteConverter } from '../../../../context/virtual-drive/shared/application/RelativePathToAbsoluteConverter'; export interface SharedContainer { absolutePathToRelativeConverter: AbsolutePathToRelativeConverter; relativePathToAbsoluteConverter: RelativePathToAbsoluteConverter; + pollingMonitorStart: PollingMonitorStart; + pollingMonitorStop: PollingMonitorStop; } diff --git a/src/apps/sync-engine/dependency-injection/shared/builder.ts b/src/apps/sync-engine/dependency-injection/shared/builder.ts index b7b086734..87923cbac 100644 --- a/src/apps/sync-engine/dependency-injection/shared/builder.ts +++ b/src/apps/sync-engine/dependency-injection/shared/builder.ts @@ -1,9 +1,13 @@ import { AbsolutePathToRelativeConverter } from '../../../../context/virtual-drive/shared/application/AbsolutePathToRelativeConverter'; +import { PollingMonitorStart } from '../../../../context/virtual-drive/shared/application/PollingMonitorStart'; +import { PollingMonitorStop } from '../../../../context/virtual-drive/shared/application/PollingMonitorStop'; import { RelativePathToAbsoluteConverter } from '../../../../context/virtual-drive/shared/application/RelativePathToAbsoluteConverter'; +import { PollingMonitor } from '../../../../context/virtual-drive/shared/domain/PollingMonitor'; import { DependencyInjectionLocalRootFolderPath } from '../common/localRootFolderPath'; import { SharedContainer } from './SharedContainer'; export function buildSharedContainer(): SharedContainer { + const MONITORING_PULLING_INTERVAL = 1_000; // 1 second const localRootFolderPath = DependencyInjectionLocalRootFolderPath.get(); const absolutePathToRelativeConverter = new AbsolutePathToRelativeConverter( localRootFolderPath @@ -13,8 +17,14 @@ export function buildSharedContainer(): SharedContainer { localRootFolderPath ); + const pollingMonitor = new PollingMonitor(MONITORING_PULLING_INTERVAL); + const pollingMonitorStart = new PollingMonitorStart(pollingMonitor); + const pollingMonitorStop = new PollingMonitorStop(pollingMonitor); + return { absolutePathToRelativeConverter, relativePathToAbsoluteConverter, + pollingMonitorStart, + pollingMonitorStop, }; } diff --git a/src/context/virtual-drive/shared/application/PollingMonitorStart.ts b/src/context/virtual-drive/shared/application/PollingMonitorStart.ts new file mode 100644 index 000000000..6c0f2918c --- /dev/null +++ b/src/context/virtual-drive/shared/application/PollingMonitorStart.ts @@ -0,0 +1,8 @@ +import { MonitorFn, PollingMonitor } from '../domain/PollingMonitor'; + +export class PollingMonitorStart { + constructor(private readonly polling: PollingMonitor) {} + run(fn: MonitorFn) { + return this.polling.start(fn); + } +} diff --git a/src/context/virtual-drive/shared/application/PollingMonitorStop.ts b/src/context/virtual-drive/shared/application/PollingMonitorStop.ts new file mode 100644 index 000000000..cdaee08c3 --- /dev/null +++ b/src/context/virtual-drive/shared/application/PollingMonitorStop.ts @@ -0,0 +1,8 @@ +import { PollingMonitor } from '../domain/PollingMonitor'; + +export class PollingMonitorStop { + constructor(private readonly polling: PollingMonitor) {} + run() { + return this.polling.stop(); + } +} diff --git a/src/context/virtual-drive/shared/domain/PollingMonitor.ts b/src/context/virtual-drive/shared/domain/PollingMonitor.ts new file mode 100644 index 000000000..9683e8386 --- /dev/null +++ b/src/context/virtual-drive/shared/domain/PollingMonitor.ts @@ -0,0 +1,31 @@ +export type MonitorFn = () => Promise; +export class PollingMonitor { + constructor( + private readonly delay: number // private readonly fn: () => Promise + ) {} + + private timeout: NodeJS.Timeout | null = null; + + private clearTimeout() { + if (this.timeout) { + clearTimeout(this.timeout); + this.timeout = null; + } + } + + private setTimeout(fn: MonitorFn) { + this.clearTimeout(); + this.timeout = setTimeout(async () => { + await fn(); + this.setTimeout(fn); + }, this.delay); + } + + start(fn: MonitorFn) { + this.setTimeout(fn); + } + + stop() { + this.clearTimeout(); + } +} From 90c3aa54ceaed12719227fc34c4caa48ddb4b783 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Fri, 26 Jan 2024 09:39:15 -0500 Subject: [PATCH 07/32] Update polling interval in BindingManager and Builder --- src/apps/sync-engine/BindingManager.ts | 13 ++++++++++--- .../dependency-injection/shared/builder.ts | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index 39aa12c69..fc8b6d548 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -37,7 +37,7 @@ export class BindingsManager { async start(version: string, providerId: string) { await this.stop(); - this.container.pollingMonitorStart.run(this.monitorSimulateFn); + this.pillingStart(); const controllers = buildControllers(this.container); @@ -279,7 +279,14 @@ export class BindingsManager { } } - private async monitorSimulateFn(): Promise { - Logger.info('[SYNC ENGINE] Starting monitoring polling...'); + private async pillingStart() { + return this.container.pollingMonitorStart.run(this.pilling.bind(this)); + } + + private async pilling(): Promise { + Logger.info('[SYNC ENGINE] Monitoring polling...'); + const result = + await this.container.virtualDrive.getPlaceholderWithStatePending(); + Logger.info('[SYNC ENGINE] result', result); } } diff --git a/src/apps/sync-engine/dependency-injection/shared/builder.ts b/src/apps/sync-engine/dependency-injection/shared/builder.ts index 87923cbac..79046f40b 100644 --- a/src/apps/sync-engine/dependency-injection/shared/builder.ts +++ b/src/apps/sync-engine/dependency-injection/shared/builder.ts @@ -7,7 +7,7 @@ import { DependencyInjectionLocalRootFolderPath } from '../common/localRootFolde import { SharedContainer } from './SharedContainer'; export function buildSharedContainer(): SharedContainer { - const MONITORING_PULLING_INTERVAL = 1_000; // 1 second + const MONITORING_PULLING_INTERVAL = 5_000; // 1 second const localRootFolderPath = DependencyInjectionLocalRootFolderPath.get(); const absolutePathToRelativeConverter = new AbsolutePathToRelativeConverter( localRootFolderPath From 56d5de61da4af28494791e13c62a0a01868700b5 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Fri, 2 Feb 2024 14:54:05 -0500 Subject: [PATCH 08/32] update syncronizer --- src/apps/sync-engine/BindingManager.ts | 14 +- .../boundaryBridge/BoundaryBridgeContainer.ts | 2 + .../boundaryBridge/build.ts | 7 + .../files/FilesContainer.ts | 2 + .../dependency-injection/files/builder.ts | 11 ++ .../dependency-injection/shared/builder.ts | 2 +- .../application/FileSyncOrchestrator.ts | 20 +++ .../files/application/FileContentsUpdater.ts | 13 ++ .../files/application/FileCreator.ts | 20 ++- .../files/application/FileSyncronizer.ts | 120 ++++++++++++++++++ .../domain/file-systems/LocalFileSystem.ts | 2 + .../infrastructure/NodeWinLocalFileSystem.ts | 9 ++ .../infrastructure/dtos/ReplaceFileDTO.ts | 6 + .../shared/domain/PollingMonitor.ts | 4 +- .../files/__mocks__/LocalFileSystemMock.ts | 5 + 15 files changed, 219 insertions(+), 18 deletions(-) create mode 100644 src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts create mode 100644 src/context/virtual-drive/files/application/FileContentsUpdater.ts create mode 100644 src/context/virtual-drive/files/application/FileSyncronizer.ts create mode 100644 src/context/virtual-drive/files/infrastructure/dtos/ReplaceFileDTO.ts diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index fc8b6d548..eb2d9e2a0 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -37,7 +37,7 @@ export class BindingsManager { async start(version: string, providerId: string) { await this.stop(); - this.pillingStart(); + this.pollingStart(); const controllers = buildControllers(this.container); @@ -279,14 +279,14 @@ export class BindingsManager { } } - private async pillingStart() { - return this.container.pollingMonitorStart.run(this.pilling.bind(this)); + private async pollingStart() { + return this.container.pollingMonitorStart.run(this.polling.bind(this)); } - private async pilling(): Promise { + private async polling(): Promise { Logger.info('[SYNC ENGINE] Monitoring polling...'); - const result = - await this.container.virtualDrive.getPlaceholderWithStatePending(); - Logger.info('[SYNC ENGINE] result', result); + const fileInPendingPaths = + (await this.container.virtualDrive.getPlaceholderWithStatePending()) as Array; + await this.container.fileSyncOrchestrator.run(fileInPendingPaths); } } diff --git a/src/apps/sync-engine/dependency-injection/boundaryBridge/BoundaryBridgeContainer.ts b/src/apps/sync-engine/dependency-injection/boundaryBridge/BoundaryBridgeContainer.ts index a885d570f..eb12b6e6c 100644 --- a/src/apps/sync-engine/dependency-injection/boundaryBridge/BoundaryBridgeContainer.ts +++ b/src/apps/sync-engine/dependency-injection/boundaryBridge/BoundaryBridgeContainer.ts @@ -1,5 +1,7 @@ import { FileCreationOrchestrator } from '../../../../context/virtual-drive/boundaryBridge/application/FileCreationOrchestrator'; +import { FileSyncOrchestrator } from '../../../../context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator'; export interface BoundaryBridgeContainer { fileCreationOrchestrator: FileCreationOrchestrator; + fileSyncOrchestrator: FileSyncOrchestrator; } diff --git a/src/apps/sync-engine/dependency-injection/boundaryBridge/build.ts b/src/apps/sync-engine/dependency-injection/boundaryBridge/build.ts index bb065d7e1..7afb7e851 100644 --- a/src/apps/sync-engine/dependency-injection/boundaryBridge/build.ts +++ b/src/apps/sync-engine/dependency-injection/boundaryBridge/build.ts @@ -2,6 +2,7 @@ import { BoundaryBridgeContainer } from './BoundaryBridgeContainer'; import { ContentsContainer } from '../contents/ContentsContainer'; import { FilesContainer } from '../files/FilesContainer'; import { FileCreationOrchestrator } from '../../../../context/virtual-drive/boundaryBridge/application/FileCreationOrchestrator'; +import { FileSyncOrchestrator } from '../../../../context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator'; export function buildBoundaryBridgeContainer( contentsContainer: ContentsContainer, @@ -13,7 +14,13 @@ export function buildBoundaryBridgeContainer( filesContainer.sameFileWasMoved ); + const fileSyncOrchestrator = new FileSyncOrchestrator( + contentsContainer.contentsUploader, + filesContainer.fileSyncronizer + ); + return { fileCreationOrchestrator, + fileSyncOrchestrator, }; } diff --git a/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts b/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts index 5ba634247..0106ae090 100644 --- a/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts +++ b/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts @@ -9,12 +9,14 @@ import { FilesPlaceholderCreator } from '../../../../context/virtual-drive/files import { RepositoryPopulator } from '../../../../context/virtual-drive/files/application/RepositoryPopulator'; import { RetrieveAllFiles } from '../../../../context/virtual-drive/files/application/RetrieveAllFiles'; import { SameFileWasMoved } from '../../../../context/virtual-drive/files/application/SameFileWasMoved'; +import { FileSyncronizer } from '../../../../context/virtual-drive/files/application/FileSyncronizer'; export interface FilesContainer { fileFinderByContentsId: FileFinderByContentsId; fileDeleter: FileDeleter; filePathUpdater: FilePathUpdater; fileCreator: FileCreator; + fileSyncronizer: FileSyncronizer; filePlaceholderCreatorFromContentsId: FilePlaceholderCreatorFromContentsId; createFilePlaceholderOnDeletionFailed: CreateFilePlaceholderOnDeletionFailed; sameFileWasMoved: SameFileWasMoved; diff --git a/src/apps/sync-engine/dependency-injection/files/builder.ts b/src/apps/sync-engine/dependency-injection/files/builder.ts index e9cf5c424..6256d0a7a 100644 --- a/src/apps/sync-engine/dependency-injection/files/builder.ts +++ b/src/apps/sync-engine/dependency-injection/files/builder.ts @@ -24,6 +24,7 @@ import { SDKRemoteFileSystem } from '../../../../context/virtual-drive/files/inf import { NodeWinLocalFileSystem } from '../../../../context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem'; import { LocalFileIdProvider } from '../../../../context/virtual-drive/shared/application/LocalFileIdProvider'; import { DependencyInjectionHttpClientsProvider } from '../common/clients'; +import { FileSyncronizer } from '../../../../context/virtual-drive/files/application/FileSyncronizer'; export async function buildFilesContainer( folderContainer: FoldersContainer, @@ -115,11 +116,21 @@ export async function buildFilesContainer( eventHistory ); + const fileSyncronizer = new FileSyncronizer( + repository, + localFileSystem, + fileCreator, + sharedContainer.absolutePathToRelativeConverter, + folderContainer.folderCreator, + folderContainer.offline.folderCreator + ); + const container: FilesContainer = { fileFinderByContentsId, fileDeleter, filePathUpdater, fileCreator, + fileSyncronizer, filePlaceholderCreatorFromContentsId: filePlaceholderCreatorFromContentsId, createFilePlaceholderOnDeletionFailed: createFilePlaceholderOnDeletionFailed, diff --git a/src/apps/sync-engine/dependency-injection/shared/builder.ts b/src/apps/sync-engine/dependency-injection/shared/builder.ts index 79046f40b..089719499 100644 --- a/src/apps/sync-engine/dependency-injection/shared/builder.ts +++ b/src/apps/sync-engine/dependency-injection/shared/builder.ts @@ -7,7 +7,7 @@ import { DependencyInjectionLocalRootFolderPath } from '../common/localRootFolde import { SharedContainer } from './SharedContainer'; export function buildSharedContainer(): SharedContainer { - const MONITORING_PULLING_INTERVAL = 5_000; // 1 second + const MONITORING_PULLING_INTERVAL = 60 * 60 * 1000; const localRootFolderPath = DependencyInjectionLocalRootFolderPath.get(); const absolutePathToRelativeConverter = new AbsolutePathToRelativeConverter( localRootFolderPath diff --git a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts new file mode 100644 index 000000000..80e9afc13 --- /dev/null +++ b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts @@ -0,0 +1,20 @@ +import { RetryContentsUploader } from '../../contents/application/RetryContentsUploader'; +import { FileSyncronizer } from '../../files/application/FileSyncronizer'; + +export class FileSyncOrchestrator { + constructor( + private readonly contentsUploader: RetryContentsUploader, + private readonly fileSyncronizer: FileSyncronizer + ) {} + + run(absolutePaths: string[]): Promise { + return Promise.all( + absolutePaths.map(async (absolutePath) => { + await this.fileSyncronizer.run( + absolutePath, + this.contentsUploader.run.bind(this.contentsUploader) + ); + }) + ); + } +} diff --git a/src/context/virtual-drive/files/application/FileContentsUpdater.ts b/src/context/virtual-drive/files/application/FileContentsUpdater.ts new file mode 100644 index 000000000..e0a189bc9 --- /dev/null +++ b/src/context/virtual-drive/files/application/FileContentsUpdater.ts @@ -0,0 +1,13 @@ +// import { FileRepository } from '../domain/FileRepository'; +// import { RemoteFileSystem } from '../domain/file-systems/RemoteFileSystem'; + +// export class FileContentsUpdater { +// constructor( +// private readonly repository: FileRepository, +// private readonly remote: RemoteFileSystem +// ) {} + +// async run(file: File): Promise { +// await this.local.updateSyncStatus(file); +// } +// } diff --git a/src/context/virtual-drive/files/application/FileCreator.ts b/src/context/virtual-drive/files/application/FileCreator.ts index 3128d7828..ffab80019 100644 --- a/src/context/virtual-drive/files/application/FileCreator.ts +++ b/src/context/virtual-drive/files/application/FileCreator.ts @@ -22,15 +22,21 @@ export class FileCreator { private readonly ipc: SyncEngineIpc ) {} - async run(filePath: FilePath, contents: RemoteFileContents): Promise { + async run( + filePath: FilePath, + contents: RemoteFileContents, + existingFileAlreadyEvaluated = false + ): Promise { try { - const existingFile = this.repository.searchByPartial({ - path: PlatformPathConverter.winToPosix(filePath.value), - status: FileStatuses.EXISTS, - }); + if (!existingFileAlreadyEvaluated) { + const existingFile = this.repository.searchByPartial({ + path: PlatformPathConverter.winToPosix(filePath.value), + status: FileStatuses.EXISTS, + }); - if (existingFile) { - await this.fileDeleter.run(existingFile.contentsId); + if (existingFile) { + await this.fileDeleter.run(existingFile.contentsId); + } } const size = new FileSize(contents.size); diff --git a/src/context/virtual-drive/files/application/FileSyncronizer.ts b/src/context/virtual-drive/files/application/FileSyncronizer.ts new file mode 100644 index 000000000..93ed61687 --- /dev/null +++ b/src/context/virtual-drive/files/application/FileSyncronizer.ts @@ -0,0 +1,120 @@ +import { RemoteFileContents } from '../../contents/domain/RemoteFileContents'; +import { PlatformPathConverter } from '../../shared/application/PlatformPathConverter'; +import { FilePath } from '../domain/FilePath'; +import { FileRepository } from '../domain/FileRepository'; +import { FileStatuses } from '../domain/FileStatus'; +import Logger from 'electron-log'; +import { FileCreator } from './FileCreator'; +import { AbsolutePathToRelativeConverter } from '../../shared/application/AbsolutePathToRelativeConverter'; +import { FolderNotFoundError } from '../../folders/domain/errors/FolderNotFoundError'; +import { FolderCreator } from '../../folders/application/FolderCreator'; +import { OfflineFolderCreator } from '../../folders/application/Offline/OfflineFolderCreator'; +import { Folder } from '../../folders/domain/Folder'; +import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; +import * as fs from 'fs'; +import { File } from '../domain/File'; + +export class FileSyncronizer { + constructor( + private readonly repository: FileRepository, + private readonly localFileSystem: LocalFileSystem, + private readonly fileCreator: FileCreator, + private readonly absolutePathToRelativeConverter: AbsolutePathToRelativeConverter, + private readonly folderCreator: FolderCreator, + private readonly offlineFolderCreator: OfflineFolderCreator + ) {} + + async run( + absolutePath: string, + upload: (path: string) => Promise + ) { + const win32RelativePath = + this.absolutePathToRelativeConverter.run(absolutePath); + + const posixRelativePath = + PlatformPathConverter.winToPosix(win32RelativePath); + + const path = new FilePath(posixRelativePath); + + const existingFile = this.repository.searchByPartial({ + path: PlatformPathConverter.winToPosix(path.value), + status: FileStatuses.EXISTS, + }); + + Logger.debug(`Updating sync status file ${posixRelativePath}`); + if (existingFile) { + if (this.hasDifferentSize(existingFile, absolutePath)) { + Logger.debug( + `[${posixRelativePath}] has different size update content` + ); + return; + } + Logger.debug(`[${posixRelativePath}] already exists`); + await this.localFileSystem.updateSyncStatus(existingFile); + } else { + Logger.debug(`[${posixRelativePath}] does not exist`); + await this.retryCreation(posixRelativePath, path, upload); + } + } + + private async retryCreation( + posixRelativePath: string, + filePath: FilePath, + upload: (path: string) => Promise, + attemps = 3 + ) { + try { + const fileContents = await upload(posixRelativePath); + const createdFile = await this.fileCreator.run(filePath, fileContents); + await this.localFileSystem.updateSyncStatus(createdFile); + } catch (error: unknown) { + if (error instanceof FolderNotFoundError) { + await this.createFolderFather(posixRelativePath); + } + if (attemps > 0) { + await new Promise((resolve) => setTimeout(resolve, 2000)); + await this.retryCreation( + posixRelativePath, + filePath, + upload, + attemps - 1 + ); + return; + } + } + } + + private async runFolderCreator(posixRelativePath: string): Promise { + const offlineFolder = this.offlineFolderCreator.run(posixRelativePath); + return this.folderCreator.run(offlineFolder); + } + + private async createFolderFather(posixRelativePath: string) { + Logger.info('posixRelativePath', posixRelativePath); + const posixDir = + PlatformPathConverter.getFatherPathPosix(posixRelativePath); + Logger.info('posixDir', posixDir); + try { + await this.runFolderCreator(posixDir); + } catch (error) { + Logger.error('Error creating folder father creation:', error); + if (error instanceof FolderNotFoundError) { + // father created + await this.createFolderFather(posixDir); + // child created + await this.runFolderCreator(posixDir); + } else { + Logger.error( + 'Error creating folder father creation inside catch:', + error + ); + throw error; + } + } + } + + private hasDifferentSize(file: File, absoulthePath: string) { + const stats = fs.statSync(absoulthePath); + return file.size !== stats.size; + } +} diff --git a/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts b/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts index acdddbb1d..602e40a8c 100644 --- a/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts +++ b/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts @@ -4,4 +4,6 @@ export interface LocalFileSystem { createPlaceHolder(file: File): Promise; getLocalFileId(file: File): Promise<`${string}-${string}`>; + + updateSyncStatus(file: File): Promise; } diff --git a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts index 49e7e5cf1..27725ce86 100644 --- a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts @@ -4,6 +4,7 @@ import { File } from '../domain/File'; import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; import { RelativePathToAbsoluteConverter } from '../../shared/application/RelativePathToAbsoluteConverter'; import fs from 'fs/promises'; +import Logger from 'electron-log'; export class NodeWinLocalFileSystem implements LocalFileSystem { constructor( @@ -34,4 +35,12 @@ export class NodeWinLocalFileSystem implements LocalFileSystem { file.updatedAt.getTime() ); } + + async updateSyncStatus(file: File): Promise { + const win32AbsolutePath = this.relativePathToAbsoluteConverter.run( + file.path + ); + Logger.debug(`Updating sync status file ${win32AbsolutePath}`); + return this.virtualDrive.updateSyncStatus(win32AbsolutePath, false); + } } diff --git a/src/context/virtual-drive/files/infrastructure/dtos/ReplaceFileDTO.ts b/src/context/virtual-drive/files/infrastructure/dtos/ReplaceFileDTO.ts new file mode 100644 index 000000000..be1cd6457 --- /dev/null +++ b/src/context/virtual-drive/files/infrastructure/dtos/ReplaceFileDTO.ts @@ -0,0 +1,6 @@ +import { File } from '../../domain/File'; + +export interface ReplaceFileDTO { + fileId: File['contentsId']; + size: File['size']; +} diff --git a/src/context/virtual-drive/shared/domain/PollingMonitor.ts b/src/context/virtual-drive/shared/domain/PollingMonitor.ts index 9683e8386..ef232677a 100644 --- a/src/context/virtual-drive/shared/domain/PollingMonitor.ts +++ b/src/context/virtual-drive/shared/domain/PollingMonitor.ts @@ -1,8 +1,6 @@ export type MonitorFn = () => Promise; export class PollingMonitor { - constructor( - private readonly delay: number // private readonly fn: () => Promise - ) {} + constructor(private readonly delay: number) {} private timeout: NodeJS.Timeout | null = null; diff --git a/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts b/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts index 6495b0307..3ec74e1a0 100644 --- a/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts +++ b/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts @@ -4,6 +4,7 @@ import { File } from '../../../../../src/context/virtual-drive/files/domain/File export class LocalFileSystemMock implements LocalFileSystem { public readonly createPlaceHolderMock = jest.fn(); public readonly getLocalFileIdMock = jest.fn(); + public readonly updateSyncStatusMock = jest.fn(); createPlaceHolder(file: File): Promise { return this.createPlaceHolder(file); @@ -12,4 +13,8 @@ export class LocalFileSystemMock implements LocalFileSystem { getLocalFileId(file: File): Promise<`${string}-${string}`> { return this.getLocalFileIdMock(file); } + + updateSyncStatus(file: File): Promise { + throw this.updateSyncStatusMock(file); + } } From a75c9b198224b6a469fa07cc5ea08914b60383e3 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Wed, 7 Feb 2024 21:29:36 -0500 Subject: [PATCH 09/32] add placeholder converter --- src/apps/sync-engine/BindingManager.ts | 3 +- .../controllers/AddController.ts | 1 + .../files/FilesContainer.ts | 4 +++ .../dependency-injection/files/builder.ts | 16 +++++++-- .../folders/FoldersContainer.ts | 6 ++++ .../dependency-injection/folders/builder.ts | 25 +++++++++++-- .../application/FIlePlaceholderConverter.ts | 10 ++++++ .../application/FileSyncStatusUpdater.ts | 10 ++++++ .../files/application/FileSyncronizer.ts | 36 ++++++++++--------- .../domain/file-systems/LocalFileSystem.ts | 2 ++ .../infrastructure/NodeWinLocalFileSystem.ts | 13 +++++-- .../folders/application/FolderCreator.ts | 6 +++- .../application/FolderPlaceholderConverter.ts | 10 ++++++ .../application/FolderSyncStatusUpdater.ts | 10 ++++++ .../FoldersFatherSyncStatusUpdater.ts | 30 ++++++++++++++++ .../domain/file-systems/LocalFileSystem.ts | 4 +++ .../infrastructure/NodeWinLocalFileSystem.ts | 26 +++++++++++++- 17 files changed, 187 insertions(+), 25 deletions(-) create mode 100644 src/context/virtual-drive/files/application/FIlePlaceholderConverter.ts create mode 100644 src/context/virtual-drive/files/application/FileSyncStatusUpdater.ts create mode 100644 src/context/virtual-drive/folders/application/FolderPlaceholderConverter.ts create mode 100644 src/context/virtual-drive/folders/application/FolderSyncStatusUpdater.ts create mode 100644 src/context/virtual-drive/folders/application/FoldersFatherSyncStatusUpdater.ts diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index eb2d9e2a0..0c112941e 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -37,7 +37,7 @@ export class BindingsManager { async start(version: string, providerId: string) { await this.stop(); - this.pollingStart(); + await this.pollingStart(); const controllers = buildControllers(this.container); @@ -287,6 +287,7 @@ export class BindingsManager { Logger.info('[SYNC ENGINE] Monitoring polling...'); const fileInPendingPaths = (await this.container.virtualDrive.getPlaceholderWithStatePending()) as Array; + Logger.info('[SYNC ENGINE] fileInPendingPaths', fileInPendingPaths); await this.container.fileSyncOrchestrator.run(fileInPendingPaths); } } diff --git a/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts b/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts index c56cc9ecc..2645c6ce4 100644 --- a/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts +++ b/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts @@ -89,6 +89,7 @@ export class AddController extends CallbackController { const posixDir = PlatformPathConverter.getFatherPathPosix(posixRelativePath); try { + await new Promise((resolve) => setTimeout(resolve, 1000)); await this.runFolderCreator(posixDir); } catch (error) { Logger.error('Error creating folder father creation:', error); diff --git a/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts b/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts index 0106ae090..12fa6eab3 100644 --- a/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts +++ b/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts @@ -10,6 +10,8 @@ import { RepositoryPopulator } from '../../../../context/virtual-drive/files/app import { RetrieveAllFiles } from '../../../../context/virtual-drive/files/application/RetrieveAllFiles'; import { SameFileWasMoved } from '../../../../context/virtual-drive/files/application/SameFileWasMoved'; import { FileSyncronizer } from '../../../../context/virtual-drive/files/application/FileSyncronizer'; +import { FilePlaceholderConverter } from '../../../../context/virtual-drive/files/application/FIlePlaceholderConverter'; +import { FileSyncStatusUpdater } from '../../../../context/virtual-drive/files/application/FileSyncStatusUpdater'; export interface FilesContainer { fileFinderByContentsId: FileFinderByContentsId; @@ -24,4 +26,6 @@ export interface FilesContainer { repositoryPopulator: RepositoryPopulator; filesPlaceholderCreator: FilesPlaceholderCreator; filesPlaceholderUpdater: FilesPlaceholderUpdater; + filePlaceholderConverter: FilePlaceholderConverter; + fileSyncStatusUpdater: FileSyncStatusUpdater; } diff --git a/src/apps/sync-engine/dependency-injection/files/builder.ts b/src/apps/sync-engine/dependency-injection/files/builder.ts index 6256d0a7a..63f758eee 100644 --- a/src/apps/sync-engine/dependency-injection/files/builder.ts +++ b/src/apps/sync-engine/dependency-injection/files/builder.ts @@ -25,6 +25,8 @@ import { NodeWinLocalFileSystem } from '../../../../context/virtual-drive/files/ import { LocalFileIdProvider } from '../../../../context/virtual-drive/shared/application/LocalFileIdProvider'; import { DependencyInjectionHttpClientsProvider } from '../common/clients'; import { FileSyncronizer } from '../../../../context/virtual-drive/files/application/FileSyncronizer'; +import { FilePlaceholderConverter } from '../../../../context/virtual-drive/files/application/FIlePlaceholderConverter'; +import { FileSyncStatusUpdater } from '../../../../context/virtual-drive/files/application/FileSyncStatusUpdater'; export async function buildFilesContainer( folderContainer: FoldersContainer, @@ -116,13 +118,21 @@ export async function buildFilesContainer( eventHistory ); + const filePlaceholderConverter = new FilePlaceholderConverter( + localFileSystem + ); + + const fileSyncStatusUpdater = new FileSyncStatusUpdater(localFileSystem); + const fileSyncronizer = new FileSyncronizer( repository, - localFileSystem, + fileSyncStatusUpdater, + filePlaceholderConverter, fileCreator, sharedContainer.absolutePathToRelativeConverter, folderContainer.folderCreator, - folderContainer.offline.folderCreator + folderContainer.offline.folderCreator, + folderContainer.foldersFatherSyncStatusUpdater ); const container: FilesContainer = { @@ -139,6 +149,8 @@ export async function buildFilesContainer( repositoryPopulator: repositoryPopulator, filesPlaceholderCreator, filesPlaceholderUpdater, + filePlaceholderConverter, + fileSyncStatusUpdater, }; return { container, subscribers: [] }; diff --git a/src/apps/sync-engine/dependency-injection/folders/FoldersContainer.ts b/src/apps/sync-engine/dependency-injection/folders/FoldersContainer.ts index bcb774e3d..de4e028f0 100644 --- a/src/apps/sync-engine/dependency-injection/folders/FoldersContainer.ts +++ b/src/apps/sync-engine/dependency-injection/folders/FoldersContainer.ts @@ -12,6 +12,9 @@ import { RetrieveAllFolders } from '../../../../context/virtual-drive/folders/ap import { SynchronizeOfflineModifications } from '../../../../context/virtual-drive/folders/application/SynchronizeOfflineModifications'; import { SynchronizeOfflineModificationsOnFolderCreated } from '../../../../context/virtual-drive/folders/application/SynchronizeOfflineModificationsOnFolderCreated'; import { FolderPlaceholderUpdater } from '../../../../context/virtual-drive/folders/application/UpdatePlaceholderFolder'; +import { FolderPlaceholderConverter } from '../../../../context/virtual-drive/folders/application/FolderPlaceholderConverter'; +import { FolderSyncStatusUpdater } from '../../../../context/virtual-drive/folders/application/FolderSyncStatusUpdater'; +import { FoldersFatherSyncStatusUpdater } from '../../../../context/virtual-drive/folders/application/FoldersFatherSyncStatusUpdater'; export interface FoldersContainer { folderCreator: FolderCreator; @@ -30,4 +33,7 @@ export interface FoldersContainer { folderRepositoryInitiator: FolderRepositoryInitiator; folderPlaceholderUpdater: FolderPlaceholderUpdater; foldersPlaceholderCreator: FoldersPlaceholderCreator; + folderPlaceholderConverter: FolderPlaceholderConverter; + folderSyncStatusUpdater: FolderSyncStatusUpdater; + foldersFatherSyncStatusUpdater: FoldersFatherSyncStatusUpdater; } diff --git a/src/apps/sync-engine/dependency-injection/folders/builder.ts b/src/apps/sync-engine/dependency-injection/folders/builder.ts index b40d16044..233dd2b74 100644 --- a/src/apps/sync-engine/dependency-injection/folders/builder.ts +++ b/src/apps/sync-engine/dependency-injection/folders/builder.ts @@ -27,6 +27,9 @@ import { DependencyInjectionEventRepository } from '../common/eventRepository'; import { DependencyInjectionVirtualDrive } from '../common/virtualDrive'; import { SharedContainer } from '../shared/SharedContainer'; import { FoldersContainer } from './FoldersContainer'; +import { FolderPlaceholderConverter } from '../../../../context/virtual-drive/folders/application/FolderPlaceholderConverter'; +import { FolderSyncStatusUpdater } from '../../../../context/virtual-drive/folders/application/FolderSyncStatusUpdater'; +import { FoldersFatherSyncStatusUpdater } from '../../../../context/virtual-drive/folders/application/FoldersFatherSyncStatusUpdater'; export async function buildFoldersContainer( shredContainer: SharedContainer @@ -38,12 +41,21 @@ export async function buildFoldersContainer( const repository = new InMemoryFolderRepository(); - const localFileSystem = new NodeWinLocalFileSystem(virtualDrive); + const localFileSystem = new NodeWinLocalFileSystem( + virtualDrive, + shredContainer.relativePathToAbsoluteConverter + ); const remoteFileSystem = new HttpRemoteFileSystem( clients.drive, clients.newDrive ); + const folderPlaceholderConverter = new FolderPlaceholderConverter( + localFileSystem + ); + + const folderSyncStatusUpdater = new FolderSyncStatusUpdater(localFileSystem); + const folderFinder = new FolderFinder(repository); const allParentFoldersStatusIsExists = new AllParentFoldersStatusIsExists( @@ -61,7 +73,8 @@ export async function buildFoldersContainer( repository, remoteFileSystem, ipcRendererSyncEngine, - eventBus + eventBus, + folderPlaceholderConverter ); const folderMover = new FolderMover( @@ -124,6 +137,11 @@ export async function buildFoldersContainer( shredContainer.relativePathToAbsoluteConverter ); + const foldersFatherSyncStatusUpdater = new FoldersFatherSyncStatusUpdater( + localFileSystem, + repository + ); + return { folderCreator, folderFinder, @@ -141,5 +159,8 @@ export async function buildFoldersContainer( folderRepositoryInitiator, foldersPlaceholderCreator, folderPlaceholderUpdater, + folderPlaceholderConverter, + folderSyncStatusUpdater, + foldersFatherSyncStatusUpdater, }; } diff --git a/src/context/virtual-drive/files/application/FIlePlaceholderConverter.ts b/src/context/virtual-drive/files/application/FIlePlaceholderConverter.ts new file mode 100644 index 000000000..763a52482 --- /dev/null +++ b/src/context/virtual-drive/files/application/FIlePlaceholderConverter.ts @@ -0,0 +1,10 @@ +import { File } from '../domain/File'; +import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; + +export class FilePlaceholderConverter { + constructor(private readonly localFileSystem: LocalFileSystem) {} + + async run(file: File) { + await this.localFileSystem.convertToPlaceholder(file); + } +} diff --git a/src/context/virtual-drive/files/application/FileSyncStatusUpdater.ts b/src/context/virtual-drive/files/application/FileSyncStatusUpdater.ts new file mode 100644 index 000000000..a03a284e8 --- /dev/null +++ b/src/context/virtual-drive/files/application/FileSyncStatusUpdater.ts @@ -0,0 +1,10 @@ +import { File } from '../domain/File'; +import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; + +export class FileSyncStatusUpdater { + constructor(private readonly localFileSystem: LocalFileSystem) {} + + async run(file: File) { + await this.localFileSystem.updateSyncStatus(file); + } +} diff --git a/src/context/virtual-drive/files/application/FileSyncronizer.ts b/src/context/virtual-drive/files/application/FileSyncronizer.ts index 93ed61687..f38038945 100644 --- a/src/context/virtual-drive/files/application/FileSyncronizer.ts +++ b/src/context/virtual-drive/files/application/FileSyncronizer.ts @@ -10,18 +10,22 @@ import { FolderNotFoundError } from '../../folders/domain/errors/FolderNotFoundE import { FolderCreator } from '../../folders/application/FolderCreator'; import { OfflineFolderCreator } from '../../folders/application/Offline/OfflineFolderCreator'; import { Folder } from '../../folders/domain/Folder'; -import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; import * as fs from 'fs'; import { File } from '../domain/File'; +import { FileSyncStatusUpdater } from './FileSyncStatusUpdater'; +import { FilePlaceholderConverter } from './FIlePlaceholderConverter'; +import { FoldersFatherSyncStatusUpdater } from '../../folders/application/FoldersFatherSyncStatusUpdater'; export class FileSyncronizer { constructor( private readonly repository: FileRepository, - private readonly localFileSystem: LocalFileSystem, + private readonly fileSyncStatusUpdater: FileSyncStatusUpdater, + private readonly filePlaceholderConverter: FilePlaceholderConverter, private readonly fileCreator: FileCreator, private readonly absolutePathToRelativeConverter: AbsolutePathToRelativeConverter, private readonly folderCreator: FolderCreator, - private readonly offlineFolderCreator: OfflineFolderCreator + private readonly offlineFolderCreator: OfflineFolderCreator, + private readonly foldersFatherSyncStatusUpdater: FoldersFatherSyncStatusUpdater ) {} async run( @@ -41,33 +45,28 @@ export class FileSyncronizer { status: FileStatuses.EXISTS, }); - Logger.debug(`Updating sync status file ${posixRelativePath}`); if (existingFile) { if (this.hasDifferentSize(existingFile, absolutePath)) { - Logger.debug( - `[${posixRelativePath}] has different size update content` - ); return; } - Logger.debug(`[${posixRelativePath}] already exists`); - await this.localFileSystem.updateSyncStatus(existingFile); + await this.convertAndUpdateSyncStatus(existingFile); } else { - Logger.debug(`[${posixRelativePath}] does not exist`); await this.retryCreation(posixRelativePath, path, upload); } } - private async retryCreation( + private retryCreation = async ( posixRelativePath: string, filePath: FilePath, upload: (path: string) => Promise, attemps = 3 - ) { + ) => { try { const fileContents = await upload(posixRelativePath); const createdFile = await this.fileCreator.run(filePath, fileContents); - await this.localFileSystem.updateSyncStatus(createdFile); + await this.convertAndUpdateSyncStatus(createdFile); } catch (error: unknown) { + Logger.error('Error creating file:', error); if (error instanceof FolderNotFoundError) { await this.createFolderFather(posixRelativePath); } @@ -82,7 +81,7 @@ export class FileSyncronizer { return; } } - } + }; private async runFolderCreator(posixRelativePath: string): Promise { const offlineFolder = this.offlineFolderCreator.run(posixRelativePath); @@ -93,8 +92,8 @@ export class FileSyncronizer { Logger.info('posixRelativePath', posixRelativePath); const posixDir = PlatformPathConverter.getFatherPathPosix(posixRelativePath); - Logger.info('posixDir', posixDir); try { + await new Promise((resolve) => setTimeout(resolve, 1000)); await this.runFolderCreator(posixDir); } catch (error) { Logger.error('Error creating folder father creation:', error); @@ -115,6 +114,11 @@ export class FileSyncronizer { private hasDifferentSize(file: File, absoulthePath: string) { const stats = fs.statSync(absoulthePath); - return file.size !== stats.size; + return Math.abs(file.size - stats.size) > 0.001; + } + + private async convertAndUpdateSyncStatus(file: File) { + await this.filePlaceholderConverter.run(file); + await this.fileSyncStatusUpdater.run(file); } } diff --git a/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts b/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts index 602e40a8c..a7fb66102 100644 --- a/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts +++ b/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts @@ -6,4 +6,6 @@ export interface LocalFileSystem { getLocalFileId(file: File): Promise<`${string}-${string}`>; updateSyncStatus(file: File): Promise; + + convertToPlaceholder(file: File): Promise; } diff --git a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts index 27725ce86..21682a31f 100644 --- a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts @@ -4,7 +4,6 @@ import { File } from '../domain/File'; import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; import { RelativePathToAbsoluteConverter } from '../../shared/application/RelativePathToAbsoluteConverter'; import fs from 'fs/promises'; -import Logger from 'electron-log'; export class NodeWinLocalFileSystem implements LocalFileSystem { constructor( @@ -40,7 +39,17 @@ export class NodeWinLocalFileSystem implements LocalFileSystem { const win32AbsolutePath = this.relativePathToAbsoluteConverter.run( file.path ); - Logger.debug(`Updating sync status file ${win32AbsolutePath}`); return this.virtualDrive.updateSyncStatus(win32AbsolutePath, false); } + + async convertToPlaceholder(file: File): Promise { + const win32AbsolutePath = this.relativePathToAbsoluteConverter.run( + file.path + ); + + return this.virtualDrive.convertToPlaceholder( + win32AbsolutePath, + file.placeholderId + ); + } } diff --git a/src/context/virtual-drive/folders/application/FolderCreator.ts b/src/context/virtual-drive/folders/application/FolderCreator.ts index 1e3a83f36..0b64008dc 100644 --- a/src/context/virtual-drive/folders/application/FolderCreator.ts +++ b/src/context/virtual-drive/folders/application/FolderCreator.ts @@ -4,13 +4,15 @@ import { Folder } from '../domain/Folder'; import { FolderRepository } from '../domain/FolderRepository'; import { OfflineFolder } from '../domain/OfflineFolder'; import { RemoteFileSystem } from '../domain/file-systems/RemoteFileSystem'; +import { FolderPlaceholderConverter } from './FolderPlaceholderConverter'; export class FolderCreator { constructor( private readonly repository: FolderRepository, private readonly remote: RemoteFileSystem, private readonly ipc: SyncEngineIpc, - private readonly eventBus: EventBus + private readonly eventBus: EventBus, + private readonly folderPlaceholderConverter: FolderPlaceholderConverter ) {} async run(offlineFolder: OfflineFolder): Promise { @@ -27,6 +29,8 @@ export class FolderCreator { const events = folder.pullDomainEvents(); this.eventBus.publish(events); + await this.folderPlaceholderConverter.run(folder); + this.ipc.send('FOLDER_CREATED', { name: offlineFolder.name, }); diff --git a/src/context/virtual-drive/folders/application/FolderPlaceholderConverter.ts b/src/context/virtual-drive/folders/application/FolderPlaceholderConverter.ts new file mode 100644 index 000000000..b3d84f30c --- /dev/null +++ b/src/context/virtual-drive/folders/application/FolderPlaceholderConverter.ts @@ -0,0 +1,10 @@ +import { Folder } from '../domain/Folder'; +import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; + +export class FolderPlaceholderConverter { + constructor(private readonly localFileSystem: LocalFileSystem) {} + + async run(folder: Folder) { + await this.localFileSystem.convertToPlaceholder(folder); + } +} diff --git a/src/context/virtual-drive/folders/application/FolderSyncStatusUpdater.ts b/src/context/virtual-drive/folders/application/FolderSyncStatusUpdater.ts new file mode 100644 index 000000000..d333caf4c --- /dev/null +++ b/src/context/virtual-drive/folders/application/FolderSyncStatusUpdater.ts @@ -0,0 +1,10 @@ +import { Folder } from '../domain/Folder'; +import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; + +export class FolderSyncStatusUpdater { + constructor(private readonly localFileSystem: LocalFileSystem) {} + + async run(folder: Folder) { + await this.localFileSystem.updateSyncStatus(folder); + } +} diff --git a/src/context/virtual-drive/folders/application/FoldersFatherSyncStatusUpdater.ts b/src/context/virtual-drive/folders/application/FoldersFatherSyncStatusUpdater.ts new file mode 100644 index 000000000..6b94883df --- /dev/null +++ b/src/context/virtual-drive/folders/application/FoldersFatherSyncStatusUpdater.ts @@ -0,0 +1,30 @@ +import { File } from '../../files/domain/File'; +import { PlatformPathConverter } from '../../shared/application/PlatformPathConverter'; +import { Folder } from '../domain/Folder'; +import { FolderRepository } from '../domain/FolderRepository'; +import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; +import Logger from 'electron-log'; + +export class FoldersFatherSyncStatusUpdater { + constructor( + private readonly localFileSystem: LocalFileSystem, + private readonly repository: FolderRepository + ) {} + + async run(file: File): Promise { + return this.update(file.path); + } + + private async update(path: File['path'] | Folder['path']) { + const posixDir = PlatformPathConverter.getFatherPathPosix(path); + if (posixDir === '/') { + return; + } + const folder = await this.repository.searchByPartial({ path: posixDir }); + if (folder) { + Logger.debug(`Updating sync status for ${folder.path}`); + await this.localFileSystem.updateSyncStatus(folder); + } + this.update(posixDir); + } +} diff --git a/src/context/virtual-drive/folders/domain/file-systems/LocalFileSystem.ts b/src/context/virtual-drive/folders/domain/file-systems/LocalFileSystem.ts index b16434a46..e686e68ca 100644 --- a/src/context/virtual-drive/folders/domain/file-systems/LocalFileSystem.ts +++ b/src/context/virtual-drive/folders/domain/file-systems/LocalFileSystem.ts @@ -2,4 +2,8 @@ import { Folder } from '../Folder'; export interface LocalFileSystem { createPlaceHolder(folder: Folder): Promise; + + updateSyncStatus(folder: Folder): Promise; + + convertToPlaceholder(folder: Folder): Promise; } diff --git a/src/context/virtual-drive/folders/infrastructure/NodeWinLocalFileSystem.ts b/src/context/virtual-drive/folders/infrastructure/NodeWinLocalFileSystem.ts index 1c1a2d5b2..f644ea8fa 100644 --- a/src/context/virtual-drive/folders/infrastructure/NodeWinLocalFileSystem.ts +++ b/src/context/virtual-drive/folders/infrastructure/NodeWinLocalFileSystem.ts @@ -2,9 +2,13 @@ import { VirtualDrive } from 'virtual-drive/dist'; import { Folder } from '../domain/Folder'; import { FolderStatuses } from '../domain/FolderStatus'; import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; +import { RelativePathToAbsoluteConverter } from '../../shared/application/RelativePathToAbsoluteConverter'; export class NodeWinLocalFileSystem implements LocalFileSystem { - constructor(private readonly virtualDrive: VirtualDrive) {} + constructor( + private readonly virtualDrive: VirtualDrive, + private readonly relativePathToAbsoluteConverter: RelativePathToAbsoluteConverter + ) {} async createPlaceHolder(folder: Folder): Promise { if (!folder.hasStatus(FolderStatuses.EXISTS)) { @@ -21,4 +25,24 @@ export class NodeWinLocalFileSystem implements LocalFileSystem { folder.updatedAt.getTime() ); } + + async updateSyncStatus(folder: Folder): Promise { + const folderPath = `${folder.path}/`; + const win32AbsolutePath = + this.relativePathToAbsoluteConverter.run(folderPath); + + return this.virtualDrive.updateSyncStatus(win32AbsolutePath, true); + } + + async convertToPlaceholder(folder: Folder): Promise { + const folderPath = `${folder.path}/`; + + const win32AbsolutePath = + this.relativePathToAbsoluteConverter.run(folderPath); + + return this.virtualDrive.convertToPlaceholder( + win32AbsolutePath, + folder.placeholderId + ); + } } From 85d88cdb935c8fd610c03af1fe4337f6d9da6200 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 10:27:06 -0500 Subject: [PATCH 10/32] add uuid on file --- src/context/virtual-drive/files/domain/File.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/context/virtual-drive/files/domain/File.ts b/src/context/virtual-drive/files/domain/File.ts index e8f831024..427d84f09 100644 --- a/src/context/virtual-drive/files/domain/File.ts +++ b/src/context/virtual-drive/files/domain/File.ts @@ -15,6 +15,7 @@ import { FileRenamedDomainEvent } from './events/FileRenamedDomainEvent'; import { FilePlaceholderId, createFilePlaceholderId } from './PlaceholderId'; export type FileAttributes = { + uuid: string; contentsId: string; folderId: number; createdAt: string; @@ -27,6 +28,7 @@ export type FileAttributes = { export class File extends AggregateRoot { private constructor( + private _uuid: string, private _contentsId: ContentsId, private _folderId: number, private _path: FilePath, @@ -38,6 +40,10 @@ export class File extends AggregateRoot { super(); } + public get uuid(): string { + return this._uuid; + } + public get contentsId() { return this._contentsId.value; } @@ -80,6 +86,7 @@ export class File extends AggregateRoot { static from(attributes: FileAttributes): File { return new File( + attributes.uuid, new ContentsId(attributes.contentsId), attributes.folderId, new FilePath(attributes.path), @@ -97,6 +104,7 @@ export class File extends AggregateRoot { path: FilePath ): File { const file = new File( + '', // we should generate a uuid here new ContentsId(contentsId), folder.id, path, @@ -185,6 +193,7 @@ export class File extends AggregateRoot { attributes(): FileAttributes { return { + uuid: this._uuid, contentsId: this.contentsId, folderId: this.folderId, createdAt: this.createdAt.toISOString(), From da1b0f10b290c300ddbee366ae96cb75476d27c5 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 10:29:06 -0500 Subject: [PATCH 11/32] add replace function --- src/context/virtual-drive/files/domain/File.ts | 2 +- .../files/domain/file-systems/RemoteFileSystem.ts | 2 ++ .../files/infrastructure/SDKRemoteFileSystem.ts | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/context/virtual-drive/files/domain/File.ts b/src/context/virtual-drive/files/domain/File.ts index 427d84f09..f002cb2fe 100644 --- a/src/context/virtual-drive/files/domain/File.ts +++ b/src/context/virtual-drive/files/domain/File.ts @@ -15,7 +15,7 @@ import { FileRenamedDomainEvent } from './events/FileRenamedDomainEvent'; import { FilePlaceholderId, createFilePlaceholderId } from './PlaceholderId'; export type FileAttributes = { - uuid: string; + uuid?: string; contentsId: string; folderId: number; createdAt: string; diff --git a/src/context/virtual-drive/files/domain/file-systems/RemoteFileSystem.ts b/src/context/virtual-drive/files/domain/file-systems/RemoteFileSystem.ts index 9573f252c..992b1a01d 100644 --- a/src/context/virtual-drive/files/domain/file-systems/RemoteFileSystem.ts +++ b/src/context/virtual-drive/files/domain/file-systems/RemoteFileSystem.ts @@ -9,4 +9,6 @@ export interface RemoteFileSystem { move(file: File): Promise; rename(file: File): Promise; + + replace(file: File): Promise; } diff --git a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts index 6208baced..a367493c3 100644 --- a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts @@ -85,4 +85,14 @@ export class SDKRemoteFileSystem implements RemoteFileSystem { bucketId: this.bucket, }); } + + async replace(file: File): Promise { + await this.clients.newDrive.put( + `${process.env.DRIVE_URL}/api/files/${file.uuid}`, + { + fileId: file.contentsId, + size: this.bucket, + } + ); + } } From b2fa8e2c2bd95ad17c303dba77397aa06564bc04 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 10:41:33 -0500 Subject: [PATCH 12/32] add replace contents function --- .../files/application/FileContentsUpdater.ts | 30 +++++++++----- .../virtual-drive/files/domain/File.ts | 20 +++++++++- .../files/domain/FileRepository.ts | 6 +++ .../events/FileUpdateContentDomainEvent.ts | 40 +++++++++++++++++++ .../domain/file-systems/RemoteFileSystem.ts | 6 ++- .../infrastructure/InMemoryFileRepository.ts | 14 +++++++ .../infrastructure/SDKRemoteFileSystem.ts | 10 +++-- 7 files changed, 109 insertions(+), 17 deletions(-) create mode 100644 src/context/virtual-drive/files/domain/events/FileUpdateContentDomainEvent.ts diff --git a/src/context/virtual-drive/files/application/FileContentsUpdater.ts b/src/context/virtual-drive/files/application/FileContentsUpdater.ts index e0a189bc9..d9c61a1ad 100644 --- a/src/context/virtual-drive/files/application/FileContentsUpdater.ts +++ b/src/context/virtual-drive/files/application/FileContentsUpdater.ts @@ -1,13 +1,21 @@ -// import { FileRepository } from '../domain/FileRepository'; -// import { RemoteFileSystem } from '../domain/file-systems/RemoteFileSystem'; +import { File } from '../domain/File'; +import { FileRepository } from '../domain/FileRepository'; +import { RemoteFileSystem } from '../domain/file-systems/RemoteFileSystem'; -// export class FileContentsUpdater { -// constructor( -// private readonly repository: FileRepository, -// private readonly remote: RemoteFileSystem -// ) {} +export class FileContentsUpdater { + constructor( + private readonly repository: FileRepository, + private readonly remote: RemoteFileSystem + ) {} -// async run(file: File): Promise { -// await this.local.updateSyncStatus(file); -// } -// } + async run( + file: File, + contentsId: File['contentsId'], + size: File['size'] + ): Promise { + await Promise.all([ + this.remote.replace(file, contentsId, size), + this.repository.updateContentsAndSize(file, contentsId, size), + ]); + } +} diff --git a/src/context/virtual-drive/files/domain/File.ts b/src/context/virtual-drive/files/domain/File.ts index f002cb2fe..c9f3a64f3 100644 --- a/src/context/virtual-drive/files/domain/File.ts +++ b/src/context/virtual-drive/files/domain/File.ts @@ -32,7 +32,7 @@ export class File extends AggregateRoot { private _contentsId: ContentsId, private _folderId: number, private _path: FilePath, - private readonly _size: FileSize, + private _size: FileSize, public createdAt: Date, public updatedAt: Date, private _status: FileStatus @@ -86,7 +86,7 @@ export class File extends AggregateRoot { static from(attributes: FileAttributes): File { return new File( - attributes.uuid, + attributes.uuid || '', new ContentsId(attributes.contentsId), attributes.folderId, new FilePath(attributes.path), @@ -191,6 +191,22 @@ export class File extends AggregateRoot { return this._status.is(status); } + replaceContestsAndSize(contentsId: string, size: number) { + this._contentsId = new ContentsId(contentsId); + this._size = new FileSize(size); + this.updatedAt = new Date(); + + // this.record( + // new FileCreatedDomainEvent({ + // aggregateId: this.contentsId, + // size: this.size, + // type: this.type, + // path: this.path, + // }) + // ); + return this; + } + attributes(): FileAttributes { return { uuid: this._uuid, diff --git a/src/context/virtual-drive/files/domain/FileRepository.ts b/src/context/virtual-drive/files/domain/FileRepository.ts index db12ab343..6bb1592de 100644 --- a/src/context/virtual-drive/files/domain/FileRepository.ts +++ b/src/context/virtual-drive/files/domain/FileRepository.ts @@ -10,4 +10,10 @@ export interface FileRepository { add(file: File): Promise; update(file: File): Promise; + + updateContentsAndSize( + file: File, + newContentsId: File['contentsId'], + newSize: File['size'] + ): Promise; } diff --git a/src/context/virtual-drive/files/domain/events/FileUpdateContentDomainEvent.ts b/src/context/virtual-drive/files/domain/events/FileUpdateContentDomainEvent.ts new file mode 100644 index 000000000..d5651d7bc --- /dev/null +++ b/src/context/virtual-drive/files/domain/events/FileUpdateContentDomainEvent.ts @@ -0,0 +1,40 @@ +import { DomainEvent } from '../../../../shared/domain/DomainEvent'; + +export type CreatedWebdavFileDomainEventAttributes = { + readonly size: number; + readonly type: string; + readonly path: string; +}; + +export class FileUpdateContentDomainEvent extends DomainEvent { + static readonly EVENT_NAME = 'file.update.content'; + + readonly size: number; + readonly contentId: string; + + constructor({ + aggregateId, + eventId, + size, + contentId, + }: { + aggregateId: string; + eventId?: string; + size: number; + contentId: string; + }) { + super({ + eventName: FileUpdateContentDomainEvent.EVENT_NAME, + aggregateId, + eventId, + }); + this.size = size; + this.contentId = contentId; + } + + toPrimitives() { + const { size, contentId } = this; + + return { size, contentId }; + } +} diff --git a/src/context/virtual-drive/files/domain/file-systems/RemoteFileSystem.ts b/src/context/virtual-drive/files/domain/file-systems/RemoteFileSystem.ts index 992b1a01d..979af014c 100644 --- a/src/context/virtual-drive/files/domain/file-systems/RemoteFileSystem.ts +++ b/src/context/virtual-drive/files/domain/file-systems/RemoteFileSystem.ts @@ -10,5 +10,9 @@ export interface RemoteFileSystem { rename(file: File): Promise; - replace(file: File): Promise; + replace( + file: File, + newContentsId: File['contentsId'], + newSize: File['size'] + ): Promise; } diff --git a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts index cc7623820..8958ebcda 100644 --- a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts +++ b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts @@ -60,4 +60,18 @@ export class InMemoryFileRepository implements FileRepository { return this.add(file); } + + async updateContentsAndSize( + file: File, + newContentsId: File['contentsId'], + newSize: File['size'] + ): Promise { + if (!this.files.has(file.contentsId)) { + throw new Error('File not found'); + } + + const updatedFile = file.replaceContestsAndSize(newContentsId, newSize); + this.files.set(updatedFile.contentsId, updatedFile.attributes()); + this.files.delete(file.contentsId); + } } diff --git a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts index a367493c3..7aa8b74bb 100644 --- a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts @@ -86,12 +86,16 @@ export class SDKRemoteFileSystem implements RemoteFileSystem { }); } - async replace(file: File): Promise { + async replace( + file: File, + newContentsId: File['contentsId'], + newSize: File['size'] + ): Promise { await this.clients.newDrive.put( `${process.env.DRIVE_URL}/api/files/${file.uuid}`, { - fileId: file.contentsId, - size: this.bucket, + fileId: newContentsId, + size: newSize, } ); } From 72fbad1d8158e41034aaa1cc1f2992a45c5fbe28 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 10:48:39 -0500 Subject: [PATCH 13/32] add implementation to update content --- .../files/application/FileContentsUpdater.ts | 8 +++----- .../files/application/FileSyncronizer.ts | 17 +++++++++++++---- .../files/domain/FileRepository.ts | 2 +- .../infrastructure/InMemoryFileRepository.ts | 3 ++- .../files/infrastructure/SDKRemoteFileSystem.ts | 1 + 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/context/virtual-drive/files/application/FileContentsUpdater.ts b/src/context/virtual-drive/files/application/FileContentsUpdater.ts index d9c61a1ad..45a6969af 100644 --- a/src/context/virtual-drive/files/application/FileContentsUpdater.ts +++ b/src/context/virtual-drive/files/application/FileContentsUpdater.ts @@ -12,10 +12,8 @@ export class FileContentsUpdater { file: File, contentsId: File['contentsId'], size: File['size'] - ): Promise { - await Promise.all([ - this.remote.replace(file, contentsId, size), - this.repository.updateContentsAndSize(file, contentsId, size), - ]); + ): Promise { + await this.remote.replace(file, contentsId, size); + return this.repository.updateContentsAndSize(file, contentsId, size); } } diff --git a/src/context/virtual-drive/files/application/FileSyncronizer.ts b/src/context/virtual-drive/files/application/FileSyncronizer.ts index f38038945..85abb4755 100644 --- a/src/context/virtual-drive/files/application/FileSyncronizer.ts +++ b/src/context/virtual-drive/files/application/FileSyncronizer.ts @@ -15,6 +15,7 @@ import { File } from '../domain/File'; import { FileSyncStatusUpdater } from './FileSyncStatusUpdater'; import { FilePlaceholderConverter } from './FIlePlaceholderConverter'; import { FoldersFatherSyncStatusUpdater } from '../../folders/application/FoldersFatherSyncStatusUpdater'; +import { FileContentsUpdater } from './FileContentsUpdater'; export class FileSyncronizer { constructor( @@ -25,6 +26,7 @@ export class FileSyncronizer { private readonly absolutePathToRelativeConverter: AbsolutePathToRelativeConverter, private readonly folderCreator: FolderCreator, private readonly offlineFolderCreator: OfflineFolderCreator, + private readonly fileContentsUpdater: FileContentsUpdater, private readonly foldersFatherSyncStatusUpdater: FoldersFatherSyncStatusUpdater ) {} @@ -40,14 +42,19 @@ export class FileSyncronizer { const path = new FilePath(posixRelativePath); - const existingFile = this.repository.searchByPartial({ + let existingFile = this.repository.searchByPartial({ path: PlatformPathConverter.winToPosix(path.value), status: FileStatuses.EXISTS, }); if (existingFile) { if (this.hasDifferentSize(existingFile, absolutePath)) { - return; + const contents = await upload(posixRelativePath); + existingFile = await this.fileContentsUpdater.run( + existingFile, + contents.id, + contents.size + ); } await this.convertAndUpdateSyncStatus(existingFile); } else { @@ -118,7 +125,9 @@ export class FileSyncronizer { } private async convertAndUpdateSyncStatus(file: File) { - await this.filePlaceholderConverter.run(file); - await this.fileSyncStatusUpdater.run(file); + await Promise.all([ + this.filePlaceholderConverter.run(file), + this.fileSyncStatusUpdater.run(file), + ]); } } diff --git a/src/context/virtual-drive/files/domain/FileRepository.ts b/src/context/virtual-drive/files/domain/FileRepository.ts index 6bb1592de..ef83cda81 100644 --- a/src/context/virtual-drive/files/domain/FileRepository.ts +++ b/src/context/virtual-drive/files/domain/FileRepository.ts @@ -15,5 +15,5 @@ export interface FileRepository { file: File, newContentsId: File['contentsId'], newSize: File['size'] - ): Promise; + ): Promise; } diff --git a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts index 8958ebcda..92233f961 100644 --- a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts +++ b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts @@ -65,7 +65,7 @@ export class InMemoryFileRepository implements FileRepository { file: File, newContentsId: File['contentsId'], newSize: File['size'] - ): Promise { + ): Promise { if (!this.files.has(file.contentsId)) { throw new Error('File not found'); } @@ -73,5 +73,6 @@ export class InMemoryFileRepository implements FileRepository { const updatedFile = file.replaceContestsAndSize(newContentsId, newSize); this.files.set(updatedFile.contentsId, updatedFile.attributes()); this.files.delete(file.contentsId); + return updatedFile; } } diff --git a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts index 7aa8b74bb..512ee7411 100644 --- a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts @@ -91,6 +91,7 @@ export class SDKRemoteFileSystem implements RemoteFileSystem { newContentsId: File['contentsId'], newSize: File['size'] ): Promise { + //TODO: we don't have the file uuid, maybe we should use a different endpoint await this.clients.newDrive.put( `${process.env.DRIVE_URL}/api/files/${file.uuid}`, { From 7cd2aab966e3b639191ca5a31bb3db35320a81f1 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 10:49:57 -0500 Subject: [PATCH 14/32] update sdk version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 75f7653f6..a8827645d 100644 --- a/package.json +++ b/package.json @@ -264,7 +264,7 @@ "@iconscout/react-unicons": "^1.1.6", "@internxt/inxt-js": "^2.0.8", "@internxt/lib": "^1.1.6", - "@internxt/sdk": "^1.4.33", + "@internxt/sdk": "^1.4.68", "@phosphor-icons/react": "2.0.9", "@radix-ui/react-select": "^1.2.2", "@sentry/electron": "^4.5.0", From 2dd0007bad1c1087749506e7b7ab52cd70ee4a22 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 13:19:26 -0500 Subject: [PATCH 15/32] fix file builder --- src/apps/sync-engine/BindingManager.ts | 15 ++++++++++----- .../dependency-injection/files/builder.ts | 7 +++++++ .../application/FileSyncOrchestrator.ts | 11 ++++++----- .../files/application/FileContentsUpdater.ts | 3 +++ .../files/application/FileSyncronizer.ts | 5 +++-- .../items/application/RemoteItemsGenerator.ts | 1 + 6 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index 0c112941e..1cd02b814 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -284,10 +284,15 @@ export class BindingsManager { } private async polling(): Promise { - Logger.info('[SYNC ENGINE] Monitoring polling...'); - const fileInPendingPaths = - (await this.container.virtualDrive.getPlaceholderWithStatePending()) as Array; - Logger.info('[SYNC ENGINE] fileInPendingPaths', fileInPendingPaths); - await this.container.fileSyncOrchestrator.run(fileInPendingPaths); + try { + Logger.info('[SYNC ENGINE] Monitoring polling...'); + + const fileInPendingPaths = + (await this.container.virtualDrive.getPlaceholderWithStatePending()) as Array; + + await this.container.fileSyncOrchestrator.run(fileInPendingPaths); + } catch (error) { + Logger.error('[SYNC ENGINE] Polling', error); + } } } diff --git a/src/apps/sync-engine/dependency-injection/files/builder.ts b/src/apps/sync-engine/dependency-injection/files/builder.ts index 63f758eee..a1ff33e77 100644 --- a/src/apps/sync-engine/dependency-injection/files/builder.ts +++ b/src/apps/sync-engine/dependency-injection/files/builder.ts @@ -27,6 +27,7 @@ import { DependencyInjectionHttpClientsProvider } from '../common/clients'; import { FileSyncronizer } from '../../../../context/virtual-drive/files/application/FileSyncronizer'; import { FilePlaceholderConverter } from '../../../../context/virtual-drive/files/application/FIlePlaceholderConverter'; import { FileSyncStatusUpdater } from '../../../../context/virtual-drive/files/application/FileSyncStatusUpdater'; +import { FileContentsUpdater } from '../../../../context/virtual-drive/files/application/FileContentsUpdater'; export async function buildFilesContainer( folderContainer: FoldersContainer, @@ -124,6 +125,11 @@ export async function buildFilesContainer( const fileSyncStatusUpdater = new FileSyncStatusUpdater(localFileSystem); + const fileContentsUpdater = new FileContentsUpdater( + repository, + remoteFileSystem + ); + const fileSyncronizer = new FileSyncronizer( repository, fileSyncStatusUpdater, @@ -132,6 +138,7 @@ export async function buildFilesContainer( sharedContainer.absolutePathToRelativeConverter, folderContainer.folderCreator, folderContainer.offline.folderCreator, + fileContentsUpdater, folderContainer.foldersFatherSyncStatusUpdater ); diff --git a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts index 80e9afc13..299c837ff 100644 --- a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts +++ b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts @@ -1,5 +1,6 @@ import { RetryContentsUploader } from '../../contents/application/RetryContentsUploader'; import { FileSyncronizer } from '../../files/application/FileSyncronizer'; +import Logger from 'electron-log'; export class FileSyncOrchestrator { constructor( @@ -7,14 +8,14 @@ export class FileSyncOrchestrator { private readonly fileSyncronizer: FileSyncronizer ) {} - run(absolutePaths: string[]): Promise { - return Promise.all( - absolutePaths.map(async (absolutePath) => { + async run(absolutePaths: string[]): Promise { + const promises = absolutePaths.map( + async (absolutePath) => await this.fileSyncronizer.run( absolutePath, this.contentsUploader.run.bind(this.contentsUploader) - ); - }) + ) ); + return Promise.all(promises); } } diff --git a/src/context/virtual-drive/files/application/FileContentsUpdater.ts b/src/context/virtual-drive/files/application/FileContentsUpdater.ts index 45a6969af..809b368a7 100644 --- a/src/context/virtual-drive/files/application/FileContentsUpdater.ts +++ b/src/context/virtual-drive/files/application/FileContentsUpdater.ts @@ -1,6 +1,7 @@ import { File } from '../domain/File'; import { FileRepository } from '../domain/FileRepository'; import { RemoteFileSystem } from '../domain/file-systems/RemoteFileSystem'; +import Logger from 'electron-log'; export class FileContentsUpdater { constructor( @@ -13,7 +14,9 @@ export class FileContentsUpdater { contentsId: File['contentsId'], size: File['size'] ): Promise { + Logger.info('Replace', file, contentsId, size); await this.remote.replace(file, contentsId, size); + Logger.info('Updated', file, contentsId, size); return this.repository.updateContentsAndSize(file, contentsId, size); } } diff --git a/src/context/virtual-drive/files/application/FileSyncronizer.ts b/src/context/virtual-drive/files/application/FileSyncronizer.ts index 85abb4755..f699a9b0c 100644 --- a/src/context/virtual-drive/files/application/FileSyncronizer.ts +++ b/src/context/virtual-drive/files/application/FileSyncronizer.ts @@ -33,7 +33,7 @@ export class FileSyncronizer { async run( absolutePath: string, upload: (path: string) => Promise - ) { + ): Promise { const win32RelativePath = this.absolutePathToRelativeConverter.run(absolutePath); @@ -55,6 +55,7 @@ export class FileSyncronizer { contents.id, contents.size ); + Logger.info('existingFile ', existingFile); } await this.convertAndUpdateSyncStatus(existingFile); } else { @@ -100,7 +101,7 @@ export class FileSyncronizer { const posixDir = PlatformPathConverter.getFatherPathPosix(posixRelativePath); try { - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 2000)); await this.runFolderCreator(posixDir); } catch (error) { Logger.error('Error creating folder father creation:', error); diff --git a/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts b/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts index fb09c825e..e4f2510e6 100644 --- a/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts +++ b/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts @@ -32,6 +32,7 @@ export class RemoteItemsGenerator { updatedAt: updatedFile.updatedAt, userId: updatedFile.userId, status: updatedFile.status as ServerFileStatus, + uuid: updatedFile.uuid, }; }); From 8a8628044f1f6fd280486f3e6ed3666a7774b073 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 19:30:49 -0500 Subject: [PATCH 16/32] add client update content --- src/apps/sync-engine/BindingManager.ts | 5 +++- src/context/shared/domain/ServerFile.ts | 1 + .../application/FileSyncOrchestrator.ts | 1 - .../application/FileCreatorFromServerFile.ts | 1 + .../files/application/FileSyncronizer.ts | 25 +++++++++++++++++-- .../infrastructure/SDKRemoteFileSystem.ts | 3 +-- .../Offline/OfflineFolderCreator.ts | 1 + .../items/application/RemoteItemsGenerator.ts | 1 + 8 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index 1cd02b814..ff47055de 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -37,8 +37,11 @@ export class BindingsManager { async start(version: string, providerId: string) { await this.stop(); - await this.pollingStart(); + // await this.pollingStart(); + setTimeout(async () => { + await this.polling(); + }, 10000); const controllers = buildControllers(this.container); const callbacks = { diff --git a/src/context/shared/domain/ServerFile.ts b/src/context/shared/domain/ServerFile.ts index 12e5e3a3e..5f70e3587 100644 --- a/src/context/shared/domain/ServerFile.ts +++ b/src/context/shared/domain/ServerFile.ts @@ -19,4 +19,5 @@ export type ServerFile = { userId: number; status: ServerFileStatus; plainName?: string; + uuid?: string; }; diff --git a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts index 299c837ff..300d8477a 100644 --- a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts +++ b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts @@ -1,6 +1,5 @@ import { RetryContentsUploader } from '../../contents/application/RetryContentsUploader'; import { FileSyncronizer } from '../../files/application/FileSyncronizer'; -import Logger from 'electron-log'; export class FileSyncOrchestrator { constructor( diff --git a/src/context/virtual-drive/files/application/FileCreatorFromServerFile.ts b/src/context/virtual-drive/files/application/FileCreatorFromServerFile.ts index 454a71441..dad957b45 100644 --- a/src/context/virtual-drive/files/application/FileCreatorFromServerFile.ts +++ b/src/context/virtual-drive/files/application/FileCreatorFromServerFile.ts @@ -14,5 +14,6 @@ export function createFileFromServerFile( updatedAt: server.updatedAt, path: relativePath, status: server.status, + uuid: server.uuid, }); } diff --git a/src/context/virtual-drive/files/application/FileSyncronizer.ts b/src/context/virtual-drive/files/application/FileSyncronizer.ts index f699a9b0c..ebcf4dd91 100644 --- a/src/context/virtual-drive/files/application/FileSyncronizer.ts +++ b/src/context/virtual-drive/files/application/FileSyncronizer.ts @@ -18,6 +18,9 @@ import { FoldersFatherSyncStatusUpdater } from '../../folders/application/Folder import { FileContentsUpdater } from './FileContentsUpdater'; export class FileSyncronizer { + // queue of files to be uploaded + private filePath: string | undefined; + private foldersPathQueue: string[] = []; constructor( private readonly repository: FileRepository, private readonly fileSyncStatusUpdater: FileSyncStatusUpdater, @@ -34,6 +37,7 @@ export class FileSyncronizer { absolutePath: string, upload: (path: string) => Promise ): Promise { + this.filePath = absolutePath; const win32RelativePath = this.absolutePathToRelativeConverter.run(absolutePath); @@ -70,6 +74,7 @@ export class FileSyncronizer { attemps = 3 ) => { try { + await new Promise((resolve) => setTimeout(resolve, 2000)); const fileContents = await upload(posixRelativePath); const createdFile = await this.fileCreator.run(filePath, fileContents); await this.convertAndUpdateSyncStatus(createdFile); @@ -101,15 +106,18 @@ export class FileSyncronizer { const posixDir = PlatformPathConverter.getFatherPathPosix(posixRelativePath); try { - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, 4000)); await this.runFolderCreator(posixDir); } catch (error) { Logger.error('Error creating folder father creation:', error); if (error instanceof FolderNotFoundError) { + this.foldersPathQueue.push(posixDir); // father created await this.createFolderFather(posixDir); // child created - await this.runFolderCreator(posixDir); + // await new Promise((resolve) => setTimeout(resolve, 4000)); + Logger.info('Creando hijo', posixDir); + await this.retryFolderCreation(posixDir); } else { Logger.error( 'Error creating folder father creation inside catch:', @@ -131,4 +139,17 @@ export class FileSyncronizer { this.fileSyncStatusUpdater.run(file), ]); } + + private retryFolderCreation = async (posixDir: string, attemps = 3) => { + try { + await new Promise((resolve) => setTimeout(resolve, 4000)); + await this.runFolderCreator(posixDir); + } catch (error) { + Logger.error('Error creating folder father creation:', error); + if (attemps > 0) { + await this.retryFolderCreation(posixDir, attemps - 1); + return; + } + } + }; } diff --git a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts index 512ee7411..7bc0facc9 100644 --- a/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/SDKRemoteFileSystem.ts @@ -91,9 +91,8 @@ export class SDKRemoteFileSystem implements RemoteFileSystem { newContentsId: File['contentsId'], newSize: File['size'] ): Promise { - //TODO: we don't have the file uuid, maybe we should use a different endpoint await this.clients.newDrive.put( - `${process.env.DRIVE_URL}/api/files/${file.uuid}`, + `${process.env.NEW_DRIVE_URL}/drive/files/${file.uuid}`, { fileId: newContentsId, size: newSize, diff --git a/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts b/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts index 30650be19..af91a8994 100644 --- a/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts +++ b/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts @@ -3,6 +3,7 @@ import { OfflineFolder } from '../../domain/OfflineFolder'; import { OfflineFolderRepository } from '../../domain/OfflineFolderRepository'; import { FolderFinder } from '../FolderFinder'; import { FolderRepository } from '../../domain/FolderRepository'; +import Logger from 'electron-log'; export class OfflineFolderCreator { constructor( diff --git a/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts b/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts index e4f2510e6..e82fab16d 100644 --- a/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts +++ b/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts @@ -1,4 +1,5 @@ import { SyncEngineIpc } from '../../../../apps/sync-engine/ipcRendererSyncEngine'; +import Logger from 'electron-log'; import { ServerFile, ServerFileStatus, From d1b83ffc5c2f9237fd5f91c9c187c0e5aa5b5dd3 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 22:49:22 -0500 Subject: [PATCH 17/32] add forof to iterate each path --- src/apps/sync-engine/BindingManager.ts | 7 ++---- .../application/FileSyncOrchestrator.ts | 23 +++++++++++-------- .../files/application/FileSyncronizer.ts | 21 ++++++++++++++--- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index ff47055de..220a0083e 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -37,11 +37,8 @@ export class BindingsManager { async start(version: string, providerId: string) { await this.stop(); - // await this.pollingStart(); + await this.pollingStart(); - setTimeout(async () => { - await this.polling(); - }, 10000); const controllers = buildControllers(this.container); const callbacks = { @@ -292,7 +289,7 @@ export class BindingsManager { const fileInPendingPaths = (await this.container.virtualDrive.getPlaceholderWithStatePending()) as Array; - + Logger.info('[SYNC ENGINE] fileInPendingPaths', fileInPendingPaths); await this.container.fileSyncOrchestrator.run(fileInPendingPaths); } catch (error) { Logger.error('[SYNC ENGINE] Polling', error); diff --git a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts index 300d8477a..e68618b9b 100644 --- a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts +++ b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts @@ -7,14 +7,19 @@ export class FileSyncOrchestrator { private readonly fileSyncronizer: FileSyncronizer ) {} - async run(absolutePaths: string[]): Promise { - const promises = absolutePaths.map( - async (absolutePath) => - await this.fileSyncronizer.run( - absolutePath, - this.contentsUploader.run.bind(this.contentsUploader) - ) - ); - return Promise.all(promises); + async run(absolutePaths: string[]): Promise { + // const promises = absolutePaths.map(async (absolutePath) => { + // return this.fileSyncronizer.run( + // absolutePath, + // this.contentsUploader.run.bind(this.contentsUploader) + // ); + // }); + // return await Promise.all(promises); + for (const absolutePath of absolutePaths) { + await this.fileSyncronizer.run( + absolutePath, + this.contentsUploader.run.bind(this.contentsUploader) + ); + } } } diff --git a/src/context/virtual-drive/files/application/FileSyncronizer.ts b/src/context/virtual-drive/files/application/FileSyncronizer.ts index ebcf4dd91..e3088f942 100644 --- a/src/context/virtual-drive/files/application/FileSyncronizer.ts +++ b/src/context/virtual-drive/files/application/FileSyncronizer.ts @@ -46,11 +46,27 @@ export class FileSyncronizer { const path = new FilePath(posixRelativePath); - let existingFile = this.repository.searchByPartial({ + const existingFile = this.repository.searchByPartial({ path: PlatformPathConverter.winToPosix(path.value), status: FileStatuses.EXISTS, }); + await this.sync( + existingFile, + absolutePath, + posixRelativePath, + path, + upload + ); + } + + private async sync( + existingFile: File | undefined, + absolutePath: string, + posixRelativePath: string, + path: FilePath, + upload: (path: string) => Promise + ) { if (existingFile) { if (this.hasDifferentSize(existingFile, absolutePath)) { const contents = await upload(posixRelativePath); @@ -115,8 +131,7 @@ export class FileSyncronizer { // father created await this.createFolderFather(posixDir); // child created - // await new Promise((resolve) => setTimeout(resolve, 4000)); - Logger.info('Creando hijo', posixDir); + Logger.info('Creating child', posixDir); await this.retryFolderCreation(posixDir); } else { Logger.error( From 0fc762cb386028d0218313abf5ca13e547f295fb Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 22:53:37 -0500 Subject: [PATCH 18/32] fix issues and smell code --- .../boundaryBridge/application/FileSyncOrchestrator.ts | 7 ------- src/context/virtual-drive/files/domain/File.ts | 10 +--------- .../application/Offline/OfflineFolderCreator.ts | 1 - .../items/application/RemoteItemsGenerator.ts | 1 - 4 files changed, 1 insertion(+), 18 deletions(-) diff --git a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts index e68618b9b..d6f342379 100644 --- a/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts +++ b/src/context/virtual-drive/boundaryBridge/application/FileSyncOrchestrator.ts @@ -8,13 +8,6 @@ export class FileSyncOrchestrator { ) {} async run(absolutePaths: string[]): Promise { - // const promises = absolutePaths.map(async (absolutePath) => { - // return this.fileSyncronizer.run( - // absolutePath, - // this.contentsUploader.run.bind(this.contentsUploader) - // ); - // }); - // return await Promise.all(promises); for (const absolutePath of absolutePaths) { await this.fileSyncronizer.run( absolutePath, diff --git a/src/context/virtual-drive/files/domain/File.ts b/src/context/virtual-drive/files/domain/File.ts index c9f3a64f3..0e501a749 100644 --- a/src/context/virtual-drive/files/domain/File.ts +++ b/src/context/virtual-drive/files/domain/File.ts @@ -86,7 +86,7 @@ export class File extends AggregateRoot { static from(attributes: FileAttributes): File { return new File( - attributes.uuid || '', + attributes.uuid ?? '', new ContentsId(attributes.contentsId), attributes.folderId, new FilePath(attributes.path), @@ -196,14 +196,6 @@ export class File extends AggregateRoot { this._size = new FileSize(size); this.updatedAt = new Date(); - // this.record( - // new FileCreatedDomainEvent({ - // aggregateId: this.contentsId, - // size: this.size, - // type: this.type, - // path: this.path, - // }) - // ); return this; } diff --git a/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts b/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts index af91a8994..30650be19 100644 --- a/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts +++ b/src/context/virtual-drive/folders/application/Offline/OfflineFolderCreator.ts @@ -3,7 +3,6 @@ import { OfflineFolder } from '../../domain/OfflineFolder'; import { OfflineFolderRepository } from '../../domain/OfflineFolderRepository'; import { FolderFinder } from '../FolderFinder'; import { FolderRepository } from '../../domain/FolderRepository'; -import Logger from 'electron-log'; export class OfflineFolderCreator { constructor( diff --git a/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts b/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts index e82fab16d..e4f2510e6 100644 --- a/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts +++ b/src/context/virtual-drive/items/application/RemoteItemsGenerator.ts @@ -1,5 +1,4 @@ import { SyncEngineIpc } from '../../../../apps/sync-engine/ipcRendererSyncEngine'; -import Logger from 'electron-log'; import { ServerFile, ServerFileStatus, From e5d858aebcb42aa411ae3a9879a53cedb33e160f Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 8 Feb 2024 23:19:01 -0500 Subject: [PATCH 19/32] **smeller code **rollback master-erb --- .../controllers/DeleteController.ts | 3 +-- .../application/FileFolderContainerDetector.ts | 13 +++++++++---- .../folders/application/FolderFinder.ts | 2 +- .../virtual-drive/shared/domain/DelayQueue.ts | 4 ++++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/apps/sync-engine/callbacks-controllers/controllers/DeleteController.ts b/src/apps/sync-engine/callbacks-controllers/controllers/DeleteController.ts index f6be1e79d..defc439df 100644 --- a/src/apps/sync-engine/callbacks-controllers/controllers/DeleteController.ts +++ b/src/apps/sync-engine/callbacks-controllers/controllers/DeleteController.ts @@ -89,8 +89,7 @@ export class DeleteController extends CallbackController { } private CleanQueueFolder(folderUuid: Folder['uuid']) { - const folders = this.foldersQueue.values; - const reversedFolders = folders.reverse(); + const reversedFolders = this.foldersQueue.reversedValues; reversedFolders.forEach((folder) => { const isParentFolder = this.folderContainerDetector.run( folder, diff --git a/src/context/virtual-drive/files/application/FileFolderContainerDetector.ts b/src/context/virtual-drive/files/application/FileFolderContainerDetector.ts index b9b2cd822..b27794f66 100644 --- a/src/context/virtual-drive/files/application/FileFolderContainerDetector.ts +++ b/src/context/virtual-drive/files/application/FileFolderContainerDetector.ts @@ -2,6 +2,7 @@ import { FolderFinder } from '../../folders/application/FolderFinder'; import { File } from '../../files/domain/File'; import { Folder } from '../../folders/domain/Folder'; import { FileRepository } from '../domain/FileRepository'; +import { FileNotFoundError } from '../domain/errors/FileNotFoundError'; export class FileFolderContainerDetector { constructor( @@ -10,9 +11,13 @@ export class FileFolderContainerDetector { ) {} run(contentId: File['contentsId'], folderContentId: Folder['uuid']): boolean { - const file = this.repository.searchByPartial({ contentsId: contentId }); - const folder = this.folderFinder.findFromId(file?.folderId); - const [_, folderUuid] = folder.placeholderId.split(':'); - return folderUuid === folderContentId; + const file = this.repository.searchByPartial({ + contentsId: contentId, + }); + if (!file) { + throw new FileNotFoundError(contentId); + } + const folder = this.folderFinder.findFromId(file.folderId); + return folder.uuid === folderContentId; } } diff --git a/src/context/virtual-drive/folders/application/FolderFinder.ts b/src/context/virtual-drive/folders/application/FolderFinder.ts index 6db686c50..5aaae4d39 100644 --- a/src/context/virtual-drive/folders/application/FolderFinder.ts +++ b/src/context/virtual-drive/folders/application/FolderFinder.ts @@ -26,7 +26,7 @@ export class FolderFinder { return folder; } - findFromId(id: Folder['id'] | undefined): Folder { + findFromId(id: Folder['id']): Folder { const folder = this.repository.searchByPartial({ id }); if (!folder) { throw new Error('Folder not found'); diff --git a/src/context/virtual-drive/shared/domain/DelayQueue.ts b/src/context/virtual-drive/shared/domain/DelayQueue.ts index 19b31f561..63f17a6e8 100644 --- a/src/context/virtual-drive/shared/domain/DelayQueue.ts +++ b/src/context/virtual-drive/shared/domain/DelayQueue.ts @@ -68,6 +68,10 @@ export class DelayQueue { return Array.from(this.queue.keys()); } + get reversedValues(): string[] { + return Array.from(this.queue.keys()).reverse(); + } + get isEmpty(): boolean { return this.queue.size === 0; } From 9c0e6b439db24c5419616bffdcf8708035a77797 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Fri, 9 Feb 2024 10:55:59 -0500 Subject: [PATCH 20/32] add enum of placeholder state --- src/apps/shared/types/PlaceholderStates.ts | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 src/apps/shared/types/PlaceholderStates.ts diff --git a/src/apps/shared/types/PlaceholderStates.ts b/src/apps/shared/types/PlaceholderStates.ts new file mode 100644 index 000000000..9e3e3c7d8 --- /dev/null +++ b/src/apps/shared/types/PlaceholderStates.ts @@ -0,0 +1,34 @@ +export enum SyncState { + /* status that occurs when an error occurs while reading the status*/ + Undefined = -1, + NotInSync = 0, + InSync = 1, +} + +export enum PinState { + /* The pin state is derived from the state of the parent folder. + * For example new remote files start out in this state, following the state of their parent folder. + * This state is used purely for resetting pin states to their derived value. + * The effective state for an item will never be "Inherited". */ + Inherited = 0, + + /* The file shall be available and up to date locally. Also known as "pinned". + * Pinned dehydrated files shall be hydrated as soon as possible. */ + AlwaysLocal = 1, + + /* File shall be a dehydrated placeholder, filled on demand. + * Also known as "unpinned". Unpinned hydrated files shall be dehydrated as soon as possible. + * If a unpinned file becomes hydrated + * (such as due to an implicit hydration where the user requested access to the file's data) + * Its pin state changes to Unspecified. */ + OnlineOnly = 2, + + /* The user hasn't made a decision. + * The client or platform may hydrate or dehydrate as they see fit. + * New remote files in unspecified directories start unspecified, and dehydrated (which is an arbitrary decision).*/ + Unspecified = 3, + + /* The file will never be synced to the cloud. + * Useful for ignored files to indicate to the OS the file will never besynced */ + Excluded = 4, +} From a63318a1e5dae1a1c2a90ed33668040712ad4941 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Fri, 9 Feb 2024 16:38:19 -0500 Subject: [PATCH 21/32] add new sync status and getplaceholderstate and translation --- src/apps/renderer/localize/locales/en.json | 3 ++- src/apps/renderer/localize/locales/es.json | 3 ++- src/apps/renderer/localize/locales/fr.json | 3 ++- src/context/desktop/sync/domain/SyncStatus.ts | 2 +- .../files/domain/file-systems/LocalFileSystem.ts | 2 ++ .../files/infrastructure/NodeWinLocalFileSystem.ts | 4 ++++ .../folders/domain/file-systems/LocalFileSystem.ts | 2 ++ .../folders/infrastructure/NodeWinLocalFileSystem.ts | 6 ++++++ 8 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/apps/renderer/localize/locales/en.json b/src/apps/renderer/localize/locales/en.json index d5d0c02a4..36b6b7042 100644 --- a/src/apps/renderer/localize/locales/en.json +++ b/src/apps/renderer/localize/locales/en.json @@ -174,7 +174,8 @@ "action-description": { "syncing": "Syncing your files", "updated": "Fully synced", - "failed": "Sync failed" + "failed": "Sync failed", + "sync-pending": "Sync pending" }, "errors": { "lock": "Sync locked by other device", diff --git a/src/apps/renderer/localize/locales/es.json b/src/apps/renderer/localize/locales/es.json index e66dd663f..b38ac1226 100644 --- a/src/apps/renderer/localize/locales/es.json +++ b/src/apps/renderer/localize/locales/es.json @@ -174,7 +174,8 @@ "action-description": { "syncing": "Sincronizando tus archivos", "updated": "Sincronizado", - "failed": "Sincronización fallida" + "failed": "Sincronización fallida", + "sync-pending": "Sincronización pendiente" }, "errors": { "lock": "Sincronización bloqueada por otro dispositivo", diff --git a/src/apps/renderer/localize/locales/fr.json b/src/apps/renderer/localize/locales/fr.json index b57c8821c..37684efa3 100644 --- a/src/apps/renderer/localize/locales/fr.json +++ b/src/apps/renderer/localize/locales/fr.json @@ -172,7 +172,8 @@ "footer": { "action-description": { "syncing": "Synchronisation de vos fichiers", - "updated": "Mis à jour il y a quelques instants" + "updated": "Mis à jour il y a quelques instants", + "sync-pending": "Synchronisation en attente" }, "errors": { "lock": "Synchronisation bloquée par un autre dispositif" diff --git a/src/context/desktop/sync/domain/SyncStatus.ts b/src/context/desktop/sync/domain/SyncStatus.ts index b76562463..f3501878e 100644 --- a/src/context/desktop/sync/domain/SyncStatus.ts +++ b/src/context/desktop/sync/domain/SyncStatus.ts @@ -1 +1 @@ -export type SyncStatus = 'STANDBY' | 'RUNNING' | 'FAILED'; +export type SyncStatus = 'STANDBY' | 'RUNNING' | 'FAILED' | 'SYNC PENDING'; diff --git a/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts b/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts index a7fb66102..e347c4e1e 100644 --- a/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts +++ b/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts @@ -8,4 +8,6 @@ export interface LocalFileSystem { updateSyncStatus(file: File): Promise; convertToPlaceholder(file: File): Promise; + + getPlaceholderState(file: File): Promise; } diff --git a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts index 21682a31f..ec96e1c56 100644 --- a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts @@ -52,4 +52,8 @@ export class NodeWinLocalFileSystem implements LocalFileSystem { file.placeholderId ); } + + getPlaceholderState(file: File): Promise { + return this.virtualDrive.getPlaceholderState(file.path); + } } diff --git a/src/context/virtual-drive/folders/domain/file-systems/LocalFileSystem.ts b/src/context/virtual-drive/folders/domain/file-systems/LocalFileSystem.ts index e686e68ca..41bde0c42 100644 --- a/src/context/virtual-drive/folders/domain/file-systems/LocalFileSystem.ts +++ b/src/context/virtual-drive/folders/domain/file-systems/LocalFileSystem.ts @@ -6,4 +6,6 @@ export interface LocalFileSystem { updateSyncStatus(folder: Folder): Promise; convertToPlaceholder(folder: Folder): Promise; + + getPlaceholderState(folder: Folder): Promise; } diff --git a/src/context/virtual-drive/folders/infrastructure/NodeWinLocalFileSystem.ts b/src/context/virtual-drive/folders/infrastructure/NodeWinLocalFileSystem.ts index f644ea8fa..f15c27818 100644 --- a/src/context/virtual-drive/folders/infrastructure/NodeWinLocalFileSystem.ts +++ b/src/context/virtual-drive/folders/infrastructure/NodeWinLocalFileSystem.ts @@ -45,4 +45,10 @@ export class NodeWinLocalFileSystem implements LocalFileSystem { folder.placeholderId ); } + + getPlaceholderState(folder: Folder): Promise { + const folderPath = `${folder.path}/`; + + return this.virtualDrive.getPlaceholderState(folderPath); + } } From 07718b1a1107de9b2155e72a75b9efb9f14743ff Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Mon, 12 Feb 2024 06:50:11 -0500 Subject: [PATCH 22/32] add sync-pending in status --- src/apps/main/remote-sync/helpers.ts | 7 ++++++- src/apps/renderer/hooks/useSyncStatus.tsx | 8 ++++++++ src/apps/renderer/pages/Widget/SyncAction.tsx | 16 ++++++++++++++++ .../dependency-injection/files/builder.ts | 3 +-- .../files/application/FileSyncronizer.ts | 9 +++------ .../files/__mocks__/FileRepositoryMock.ts | 8 ++++++++ .../files/__mocks__/LocalFileSystemMock.ts | 10 ++++++++++ .../files/__mocks__/RemoteFileSystemMock.ts | 5 +++++ .../__mocks__/FolderLocalFileSystemMock.ts | 13 +++++++++++++ .../__mocks__/FolderPlaceholderConverterMock.ts | 10 ++++++++++ .../folders/application/FolderCreator.test.ts | 17 +++++++++++++++-- 11 files changed, 95 insertions(+), 11 deletions(-) create mode 100644 tests/context/virtual-drive/folders/__mocks__/FolderPlaceholderConverterMock.ts diff --git a/src/apps/main/remote-sync/helpers.ts b/src/apps/main/remote-sync/helpers.ts index d8906e635..ee6b1fa30 100644 --- a/src/apps/main/remote-sync/helpers.ts +++ b/src/apps/main/remote-sync/helpers.ts @@ -87,7 +87,12 @@ export type RemoteSyncedFolder = { status: string; }; -export type RemoteSyncStatus = 'IDLE' | 'SYNCED' | 'SYNCING' | 'SYNC_FAILED'; +export type RemoteSyncStatus = + | 'IDLE' + | 'SYNCED' + | 'SYNCING' + | 'SYNC_FAILED' + | 'SYNC_PENDING'; export type SyncConfig = { retry: number; maxRetries: number; diff --git a/src/apps/renderer/hooks/useSyncStatus.tsx b/src/apps/renderer/hooks/useSyncStatus.tsx index 1775900ee..e832adbae 100644 --- a/src/apps/renderer/hooks/useSyncStatus.tsx +++ b/src/apps/renderer/hooks/useSyncStatus.tsx @@ -19,6 +19,10 @@ export default function useSyncStatus( if (status === 'SYNC_FAILED') { setSyncStatus('FAILED'); } + + if (status === 'SYNC_PENDING') { + setSyncStatus('SYNC PENDING'); + } }); const removeListener = window.electron.onRemoteSyncStatusChange( @@ -34,6 +38,10 @@ export default function useSyncStatus( if (newStatus === 'SYNC_FAILED') { setSyncStatus('FAILED'); } + + if (newStatus === 'SYNC_PENDING') { + setSyncStatus('SYNC PENDING'); + } } ); diff --git a/src/apps/renderer/pages/Widget/SyncAction.tsx b/src/apps/renderer/pages/Widget/SyncAction.tsx index 0baf69085..af24629e2 100644 --- a/src/apps/renderer/pages/Widget/SyncAction.tsx +++ b/src/apps/renderer/pages/Widget/SyncAction.tsx @@ -7,6 +7,7 @@ import { useTranslationContext } from '../../context/LocalContext'; import useVirtualDriveStatus from '../../hooks/VirtualDriveStatus'; import useSyncStatus from '../../hooks/useSyncStatus'; import useUsage from '../../hooks/useUsage'; +import { WarningCircle } from 'phosphor-react'; export default function SyncAction(props: { syncStatus: SyncStatus }) { const { translate } = useTranslationContext(); @@ -74,6 +75,21 @@ export default function SyncAction(props: { syncStatus: SyncStatus }) { )} + {isOnLine && props.syncStatus === 'SYNC PENDING' && ( + <> + {/* UP TO DATE */} +
+ +
+ + {translate('widget.footer.action-description.sync-pending')} + + + )} ) ) : ( diff --git a/src/apps/sync-engine/dependency-injection/files/builder.ts b/src/apps/sync-engine/dependency-injection/files/builder.ts index e68547d6e..f0fae2b59 100644 --- a/src/apps/sync-engine/dependency-injection/files/builder.ts +++ b/src/apps/sync-engine/dependency-injection/files/builder.ts @@ -144,8 +144,7 @@ export async function buildFilesContainer( sharedContainer.absolutePathToRelativeConverter, folderContainer.folderCreator, folderContainer.offline.folderCreator, - fileContentsUpdater, - folderContainer.foldersFatherSyncStatusUpdater + fileContentsUpdater ); const container: FilesContainer = { diff --git a/src/context/virtual-drive/files/application/FileSyncronizer.ts b/src/context/virtual-drive/files/application/FileSyncronizer.ts index e3088f942..6d97c3124 100644 --- a/src/context/virtual-drive/files/application/FileSyncronizer.ts +++ b/src/context/virtual-drive/files/application/FileSyncronizer.ts @@ -14,12 +14,10 @@ import * as fs from 'fs'; import { File } from '../domain/File'; import { FileSyncStatusUpdater } from './FileSyncStatusUpdater'; import { FilePlaceholderConverter } from './FIlePlaceholderConverter'; -import { FoldersFatherSyncStatusUpdater } from '../../folders/application/FoldersFatherSyncStatusUpdater'; import { FileContentsUpdater } from './FileContentsUpdater'; export class FileSyncronizer { // queue of files to be uploaded - private filePath: string | undefined; private foldersPathQueue: string[] = []; constructor( private readonly repository: FileRepository, @@ -29,15 +27,14 @@ export class FileSyncronizer { private readonly absolutePathToRelativeConverter: AbsolutePathToRelativeConverter, private readonly folderCreator: FolderCreator, private readonly offlineFolderCreator: OfflineFolderCreator, - private readonly fileContentsUpdater: FileContentsUpdater, - private readonly foldersFatherSyncStatusUpdater: FoldersFatherSyncStatusUpdater - ) {} + private readonly fileContentsUpdater: FileContentsUpdater + ) // private readonly foldersFatherSyncStatusUpdater: FoldersFatherSyncStatusUpdater + {} async run( absolutePath: string, upload: (path: string) => Promise ): Promise { - this.filePath = absolutePath; const win32RelativePath = this.absolutePathToRelativeConverter.run(absolutePath); diff --git a/tests/context/virtual-drive/files/__mocks__/FileRepositoryMock.ts b/tests/context/virtual-drive/files/__mocks__/FileRepositoryMock.ts index dfcb8359a..dfd9855fc 100644 --- a/tests/context/virtual-drive/files/__mocks__/FileRepositoryMock.ts +++ b/tests/context/virtual-drive/files/__mocks__/FileRepositoryMock.ts @@ -11,6 +11,7 @@ export class FileRepositoryMock implements FileRepository { public readonly deleteMock = jest.fn(); public readonly addMock = jest.fn(); public readonly updateMock = jest.fn(); + public readonly updateContentsAndSizeMock = jest.fn(); all(): Promise { return this.allMock(); @@ -27,4 +28,11 @@ export class FileRepositoryMock implements FileRepository { update(file: File): Promise { return this.updateMock(file); } + updateContentsAndSize( + file: File, + newContentsId: string, + newSize: number + ): Promise { + return this.updateContentsAndSizeMock(file, newContentsId, newSize); + } } diff --git a/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts b/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts index 3ec74e1a0..66374f322 100644 --- a/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts +++ b/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts @@ -5,6 +5,8 @@ export class LocalFileSystemMock implements LocalFileSystem { public readonly createPlaceHolderMock = jest.fn(); public readonly getLocalFileIdMock = jest.fn(); public readonly updateSyncStatusMock = jest.fn(); + public readonly convertToPlaceholderMock = jest.fn(); + public readonly getPlaceholderStateMock = jest.fn(); createPlaceHolder(file: File): Promise { return this.createPlaceHolder(file); @@ -17,4 +19,12 @@ export class LocalFileSystemMock implements LocalFileSystem { updateSyncStatus(file: File): Promise { throw this.updateSyncStatusMock(file); } + + convertToPlaceholder(file: File): Promise { + return this.convertToPlaceholder(file); + } + + getPlaceholderState(file: File): Promise { + return this.getPlaceholderStateMock(file); + } } diff --git a/tests/context/virtual-drive/files/__mocks__/RemoteFileSystemMock.ts b/tests/context/virtual-drive/files/__mocks__/RemoteFileSystemMock.ts index 76473803f..a1f9e6661 100644 --- a/tests/context/virtual-drive/files/__mocks__/RemoteFileSystemMock.ts +++ b/tests/context/virtual-drive/files/__mocks__/RemoteFileSystemMock.ts @@ -10,6 +10,7 @@ export class RemoteFileSystemMock implements RemoteFileSystem { public readonly trashMock = jest.fn(); public readonly moveMock = jest.fn(); public readonly renameMock = jest.fn(); + public readonly replaceMock = jest.fn(); persist(offline: OfflineFile): Promise { return this.persistMock(offline); @@ -26,4 +27,8 @@ export class RemoteFileSystemMock implements RemoteFileSystem { rename(file: File): Promise { return this.renameMock(file); } + + replace(file: File, newContentsId: string, newSize: number): Promise { + return this.replaceMock(file, newContentsId, newSize); + } } diff --git a/tests/context/virtual-drive/folders/__mocks__/FolderLocalFileSystemMock.ts b/tests/context/virtual-drive/folders/__mocks__/FolderLocalFileSystemMock.ts index eccabee32..f494e88bd 100644 --- a/tests/context/virtual-drive/folders/__mocks__/FolderLocalFileSystemMock.ts +++ b/tests/context/virtual-drive/folders/__mocks__/FolderLocalFileSystemMock.ts @@ -3,8 +3,21 @@ import { LocalFileSystem } from '../../../../../src/context/virtual-drive/folder export class FolderLocalFileSystemMock implements LocalFileSystem { public readonly createPlaceHolderMock = jest.fn(); + public readonly getLocalFolderIdMock = jest.fn(); + public readonly updateSyncStatusMock = jest.fn(); + public readonly convertToPlaceholderMock = jest.fn(); + public readonly getPlaceholderStateMock = jest.fn(); createPlaceHolder(folder: Folder): Promise { return this.createPlaceHolderMock(folder); } + updateSyncStatus(folder: Folder): Promise { + return this.updateSyncStatusMock(folder); + } + convertToPlaceholder(folder: Folder): Promise { + return this.convertToPlaceholderMock(folder); + } + getPlaceholderState(folder: Folder): Promise { + return this.getPlaceholderStateMock(folder); + } } diff --git a/tests/context/virtual-drive/folders/__mocks__/FolderPlaceholderConverterMock.ts b/tests/context/virtual-drive/folders/__mocks__/FolderPlaceholderConverterMock.ts new file mode 100644 index 000000000..66ea37d57 --- /dev/null +++ b/tests/context/virtual-drive/folders/__mocks__/FolderPlaceholderConverterMock.ts @@ -0,0 +1,10 @@ +import { FolderPlaceholderConverter } from '../../../../../src/context/virtual-drive/folders/application/FolderPlaceholderConverter'; +import { Folder } from '../../../../../src/context/virtual-drive/folders/domain/Folder'; + +export class FolderPlaceholderConverterMock extends FolderPlaceholderConverter { + public readonly runMock = jest.fn(); + + async run(folder: Folder) { + return this.runMock(folder); + } +} diff --git a/tests/context/virtual-drive/folders/application/FolderCreator.test.ts b/tests/context/virtual-drive/folders/application/FolderCreator.test.ts index d216e8f6e..eda9f8b2e 100644 --- a/tests/context/virtual-drive/folders/application/FolderCreator.test.ts +++ b/tests/context/virtual-drive/folders/application/FolderCreator.test.ts @@ -2,10 +2,12 @@ import { FolderCreator } from '../../../../../src/context/virtual-drive/folders/ import { Folder } from '../../../../../src/context/virtual-drive/folders/domain/Folder'; import { EventBusMock } from '../../shared/__mock__/EventBusMock'; import { IpcRendererSyncEngineMock } from '../../shared/__mock__/IpcRendererSyncEngineMock'; +import { FolderPlaceholderConverterMock } from '../__mocks__/FolderPlaceholderConverterMock'; import { FolderRemoteFileSystemMock } from '../__mocks__/FolderRemoteFileSystemMock'; import { FolderRepositoryMock } from '../__mocks__/FolderRepositoryMock'; import { FolderMother } from '../domain/FolderMother'; import { OfflineFolderMother } from '../domain/OfflineFolderMother'; +import { FolderLocalFileSystemMock } from '../__mocks__/FolderLocalFileSystemMock'; describe('Folder Creator', () => { let SUT: FolderCreator; @@ -14,6 +16,8 @@ describe('Folder Creator', () => { let remote: FolderRemoteFileSystemMock; let syncEngineIpc: IpcRendererSyncEngineMock; let eventBus: EventBusMock; + let folderPlaceholderConverter: FolderPlaceholderConverterMock; + let folderLocalFileSystemMock: FolderLocalFileSystemMock; beforeEach(() => { repository = new FolderRepositoryMock(); @@ -21,8 +25,17 @@ describe('Folder Creator', () => { remote = new FolderRemoteFileSystemMock(); eventBus = new EventBusMock(); - - SUT = new FolderCreator(repository, remote, syncEngineIpc, eventBus); + folderPlaceholderConverter = new FolderPlaceholderConverterMock( + folderLocalFileSystemMock + ); + + SUT = new FolderCreator( + repository, + remote, + syncEngineIpc, + eventBus, + folderPlaceholderConverter + ); }); it('creates on a folder from a offline folder', async () => { From ea40806f515c74a9dc384728a7c010eff8ce68cc Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Mon, 12 Feb 2024 22:09:10 -0500 Subject: [PATCH 23/32] add ipc to render sync pending --- .../main/remote-sync/RemoteSyncManager.ts | 23 +++++++- src/apps/main/remote-sync/handlers.ts | 16 ++++++ src/apps/sync-engine/BindingManager.ts | 15 +++++- .../controllers/AddController.ts | 29 ++++++++--- .../files/FilesContainer.ts | 2 + .../dependency-injection/files/builder.ts | 4 ++ src/apps/sync-engine/index.ts | 9 ++++ .../application/ContentsDownloader.ts | 2 + .../contents/application/ContentsUploader.ts | 4 ++ .../application/FileCheckerStatusInRoot.ts | 52 +++++++++++++++++++ .../files/application/FileCreator.ts | 2 + .../files/domain/FileRepository.ts | 2 + .../files/domain/PlaceholderState.ts | 9 ++++ .../domain/file-systems/LocalFileSystem.ts | 5 ++ .../infrastructure/InMemoryFileRepository.ts | 23 ++++++++ .../infrastructure/NodeWinLocalFileSystem.ts | 9 ++++ 16 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 src/context/virtual-drive/files/application/FileCheckerStatusInRoot.ts create mode 100644 src/context/virtual-drive/files/domain/PlaceholderState.ts diff --git a/src/apps/main/remote-sync/RemoteSyncManager.ts b/src/apps/main/remote-sync/RemoteSyncManager.ts index db87a1487..338d31031 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.ts @@ -17,6 +17,7 @@ import { DriveFile } from '../database/entities/DriveFile'; export class RemoteSyncManager { private foldersSyncStatus: RemoteSyncStatus = 'IDLE'; private filesSyncStatus: RemoteSyncStatus = 'IDLE'; + private _placeholdersStatus: RemoteSyncStatus = 'IDLE'; private status: RemoteSyncStatus = 'IDLE'; private onStatusChangeCallbacks: Array< (newStatus: RemoteSyncStatus) => void @@ -34,9 +35,14 @@ export class RemoteSyncManager { fetchFoldersLimitPerRequest: number; syncFiles: boolean; syncFolders: boolean; - } + } // , // private chekers: { // fileCheker: FileCheckerStatusInRoot; // } ) {} + set placeholderStatus(status: RemoteSyncStatus) { + this._placeholdersStatus = status; + this.checkRemoteSyncStatus(); + } + onStatusChange(callback: (newStatus: RemoteSyncStatus) => void) { if (typeof callback !== 'function') return; this.onStatusChangeCallbacks.push(callback); @@ -70,6 +76,12 @@ export class RemoteSyncManager { async startRemoteSync() { // const start = Date.now(); Logger.info('Starting remote to local sync'); + Logger.info('Checking if we are in a valid state to start the sync'); + + // Logger.debug( + // '[ this.chekers.fileCheker.run()]', + // await this.chekers.fileCheker.run() + // ); const testPassed = this.smokeTest(); @@ -151,6 +163,15 @@ export class RemoteSyncManager { } private checkRemoteSyncStatus() { + if (this._placeholdersStatus === 'SYNCING') { + this.changeStatus('SYNCING'); + return; + } + // placeholders are still sync-pending + if (this._placeholdersStatus === 'SYNC_PENDING') { + this.changeStatus('SYNC_PENDING'); + return; + } // We only syncing files if ( this.config.syncFiles && diff --git a/src/apps/main/remote-sync/handlers.ts b/src/apps/main/remote-sync/handlers.ts index 2426f0bc3..c4462f76c 100644 --- a/src/apps/main/remote-sync/handlers.ts +++ b/src/apps/main/remote-sync/handlers.ts @@ -101,3 +101,19 @@ eventBus.on('USER_LOGGED_OUT', () => { remoteSyncManager.resetRemoteSync(); clearRemoteSyncStore(); }); + +ipcMain.on('CHECK_SYNC', (event) => { + Logger.info('Checking sync'); + event.sender.send( + 'CHECK_SYNC_ENGINE_RESPONSE', + 'Dato obtenido del proceso de sincronización' + ); +}); + +ipcMain.on('CHECK_SYNC_CHANGE_STATUS', async (_, placeholderStates) => { + await sleep(2_000); + Logger.info('[SYNC ENGINE] Changing status'); + remoteSyncManager.placeholderStatus = 'SYNCING'; + await sleep(7_00); + remoteSyncManager.placeholderStatus = placeholderStates; +}); diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index 220a0083e..e2256f311 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -9,11 +9,18 @@ import { executeControllerWithFallback } from './callbacks-controllers/middlewar import { DependencyContainer } from './dependency-injection/DependencyContainer'; import { ipcRendererSyncEngine } from './ipcRendererSyncEngine'; import { ProcessIssue } from '../shared/types'; +import { ipcRenderer } from 'electron'; export type CallbackDownload = ( success: boolean, filePath: string ) => Promise<{ finished: boolean; progress: number }>; + +export type FileAddedCallback = ( + acknowledge: boolean, + id: string +) => Promise; + export class BindingsManager { private static readonly PROVIDER_NAME = 'Internxt'; private progressBuffer = 0; @@ -55,6 +62,7 @@ export class BindingsManager { Logger.error(error); callback(false); }); + ipcRenderer.send('CHECK_SYNC'); }, notifyDeleteCompletionCallback: () => { Logger.info('Deletion completed'); @@ -73,13 +81,15 @@ export class BindingsManager { ), }); fn(absolutePath, contentsId, callback); + ipcRenderer.send('CHECK_SYNC'); }, notifyFileAddedCallback: ( absolutePath: string, - callback: (acknowledge: boolean, id: string) => boolean + callback: FileAddedCallback ) => { Logger.debug('Path received from callback', absolutePath); controllers.addFile.execute(absolutePath, callback); + ipcRenderer.send('CHECK_SYNC'); }, fetchDataCallback: async ( contentsId: FilePlaceholderId, @@ -140,6 +150,7 @@ export class BindingsManager { }); fs.unlinkSync(path); + ipcRenderer.send('CHECK_SYNC'); } catch (error) { Logger.error(error); callback(false, ''); @@ -160,6 +171,7 @@ export class BindingsManager { process: 'SYNC', kind: 'LOCAL', }); + ipcRenderer.send('CHECK_SYNC'); } catch (error) { Logger.error(error); callback(false); @@ -291,6 +303,7 @@ export class BindingsManager { (await this.container.virtualDrive.getPlaceholderWithStatePending()) as Array; Logger.info('[SYNC ENGINE] fileInPendingPaths', fileInPendingPaths); await this.container.fileSyncOrchestrator.run(fileInPendingPaths); + ipcRenderer.send('CHECK_SYNC'); } catch (error) { Logger.error('[SYNC ENGINE] Polling', error); } diff --git a/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts b/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts index 2645c6ce4..cea5c6676 100644 --- a/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts +++ b/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts @@ -11,8 +11,8 @@ import { PathTypeChecker } from '../../../shared/fs/PathTypeChecker '; import { CallbackController } from './CallbackController'; import { FolderNotFoundError } from '../../../../context/virtual-drive/folders/domain/errors/FolderNotFoundError'; import { Folder } from '../../../../context/virtual-drive/folders/domain/Folder'; - -type CreationCallback = (acknowledge: boolean, id: string) => void; +import { ipcRenderer } from 'electron'; +import { FileAddedCallback } from '../../BindingManager'; export class AddController extends CallbackController { // Gets called when: @@ -31,14 +31,22 @@ export class AddController extends CallbackController { private createFile = async ( posixRelativePath: string, - callback: (acknowledge: boolean, id: string) => void, + callback: FileAddedCallback, attempts = 3 ) => { try { const contentsId = await this.fileCreationOrchestrator.run( posixRelativePath ); - callback(true, createFilePlaceholderId(contentsId)); + const confirmCreation = await callback( + true, + createFilePlaceholderId(contentsId) + ); + if (confirmCreation) { + Logger.info('File created', posixRelativePath); + // ipcRenderer.send('CHECK_SYNC'); + } + ipcRenderer.send('CHECK_SYNC'); } catch (error: unknown) { Logger.error('Error when adding a file: ' + posixRelativePath, error); if (error instanceof FolderNotFoundError) { @@ -57,12 +65,19 @@ export class AddController extends CallbackController { private createFolder = async ( offlineFolder: OfflineFolder, - callback: (acknowledge: boolean, id: string) => void, + callback: FileAddedCallback, attempts = 3 ) => { try { await this.folderCreator.run(offlineFolder); - callback(true, createFolderPlaceholderId(offlineFolder.uuid)); + const creationConfirm = await callback( + true, + createFolderPlaceholderId(offlineFolder.uuid) + ); + if (creationConfirm) { + Logger.info('creationConfirm', creationConfirm); + } + ipcRenderer.send('CHECK_SYNC'); } catch (error: unknown) { Logger.error('Error creating folder', error); if (attempts > 0) { @@ -128,7 +143,7 @@ export class AddController extends CallbackController { async execute( absolutePath: string, - callback: CreationCallback + callback: FileAddedCallback ): Promise { const win32RelativePath = this.absolutePathToRelativeConverter.run(absolutePath); diff --git a/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts b/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts index 5c1f423be..0e654e264 100644 --- a/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts +++ b/src/apps/sync-engine/dependency-injection/files/FilesContainer.ts @@ -13,6 +13,7 @@ import { FileFolderContainerDetector } from '../../../../context/virtual-drive/f import { FileSyncronizer } from '../../../../context/virtual-drive/files/application/FileSyncronizer'; import { FilePlaceholderConverter } from '../../../../context/virtual-drive/files/application/FIlePlaceholderConverter'; import { FileSyncStatusUpdater } from '../../../../context/virtual-drive/files/application/FileSyncStatusUpdater'; +import { FileCheckerStatusInRoot } from '../../../../context/virtual-drive/files/application/FileCheckerStatusInRoot'; export interface FilesContainer { fileFinderByContentsId: FileFinderByContentsId; @@ -30,4 +31,5 @@ export interface FilesContainer { filesPlaceholderUpdater: FilesPlaceholderUpdater; filePlaceholderConverter: FilePlaceholderConverter; fileSyncStatusUpdater: FileSyncStatusUpdater; + filesCheckerStatusInRoot: FileCheckerStatusInRoot; } diff --git a/src/apps/sync-engine/dependency-injection/files/builder.ts b/src/apps/sync-engine/dependency-injection/files/builder.ts index f0fae2b59..2f39d929d 100644 --- a/src/apps/sync-engine/dependency-injection/files/builder.ts +++ b/src/apps/sync-engine/dependency-injection/files/builder.ts @@ -29,6 +29,7 @@ import { FileSyncronizer } from '../../../../context/virtual-drive/files/applica import { FilePlaceholderConverter } from '../../../../context/virtual-drive/files/application/FIlePlaceholderConverter'; import { FileSyncStatusUpdater } from '../../../../context/virtual-drive/files/application/FileSyncStatusUpdater'; import { FileContentsUpdater } from '../../../../context/virtual-drive/files/application/FileContentsUpdater'; +import { FileCheckerStatusInRoot } from '../../../../context/virtual-drive/files/application/FileCheckerStatusInRoot'; export async function buildFilesContainer( folderContainer: FoldersContainer, @@ -147,6 +148,8 @@ export async function buildFilesContainer( fileContentsUpdater ); + const filesCheckerStatusInRoot = new FileCheckerStatusInRoot(localFileSystem); + const container: FilesContainer = { fileFinderByContentsId, fileDeleter, @@ -164,6 +167,7 @@ export async function buildFilesContainer( filesPlaceholderUpdater, filePlaceholderConverter, fileSyncStatusUpdater, + filesCheckerStatusInRoot, }; return { container, subscribers: [] }; diff --git a/src/apps/sync-engine/index.ts b/src/apps/sync-engine/index.ts index 61b7814aa..bebccce73 100644 --- a/src/apps/sync-engine/index.ts +++ b/src/apps/sync-engine/index.ts @@ -32,6 +32,13 @@ async function setUp() { icon: iconPath, }); + ipcRenderer.on('CHECK_SYNC_ENGINE_RESPONSE', async (event) => { + Logger.info('[SYNC ENGINE] Checking sync engine response'); + const placeholderStatuses = await container.filesCheckerStatusInRoot.run(); + const placeholderStates = placeholderStatuses; + event.sender.send('CHECK_SYNC_CHANGE_STATUS', placeholderStates); + }); + ipcRenderer.on('STOP_SYNC_ENGINE_PROCESS', async (event) => { Logger.info('[SYNC ENGINE] Stopping sync engine'); @@ -72,6 +79,8 @@ async function setUp() { ); bindings.watch(); + + ipcRenderer.send('CHECK_SYNC'); } setUp() diff --git a/src/context/virtual-drive/contents/application/ContentsDownloader.ts b/src/context/virtual-drive/contents/application/ContentsDownloader.ts index aa8353e6b..4ba3b3759 100644 --- a/src/context/virtual-drive/contents/application/ContentsDownloader.ts +++ b/src/context/virtual-drive/contents/application/ContentsDownloader.ts @@ -11,6 +11,7 @@ import { LocalFileContents } from '../domain/LocalFileContents'; import { LocalFileWriter } from '../domain/LocalFileWriter'; import { ContentFileDownloader } from '../domain/contentHandlers/ContentFileDownloader'; import { TemporalFolderProvider } from './temporalFolderProvider'; +import { ipcRenderer } from 'electron'; export class ContentsDownloader { private readableDownloader: Readable | null; @@ -60,6 +61,7 @@ export class ContentsDownloader { size: file.size, processInfo: { elapsedTime: downloader.elapsedTime() }, }); + ipcRenderer.send('CHECK_SYNC'); } else { this.ipc.send('FILE_DOWNLOADING', { name: file.name, diff --git a/src/context/virtual-drive/contents/application/ContentsUploader.ts b/src/context/virtual-drive/contents/application/ContentsUploader.ts index 3bd65c842..8cdb5f33d 100644 --- a/src/context/virtual-drive/contents/application/ContentsUploader.ts +++ b/src/context/virtual-drive/contents/application/ContentsUploader.ts @@ -6,6 +6,7 @@ import { LocalFileContents } from '../domain/LocalFileContents'; import { PlatformPathConverter } from '../../shared/application/PlatformPathConverter'; import { RelativePathToAbsoluteConverter } from '../../shared/application/RelativePathToAbsoluteConverter'; import { SyncEngineIpc } from '../../../../apps/sync-engine/ipcRendererSyncEngine'; +import { ipcRenderer } from 'electron'; export class ContentsUploader { constructor( @@ -37,6 +38,7 @@ export class ContentsUploader { size: localFileContents.size, processInfo: { elapsedTime: uploader.elapsedTime(), progress }, }); + ipcRenderer.send('CHECK_SYNC'); }); uploader.on('error', (error: Error) => { @@ -46,6 +48,7 @@ export class ContentsUploader { nameWithExtension: localFileContents.nameWithExtension, error: error.message, }); + ipcRenderer.send('CHECK_SYNC'); }); uploader.on('finish', () => { @@ -56,6 +59,7 @@ export class ContentsUploader { size: localFileContents.size, processInfo: { elapsedTime: uploader.elapsedTime() }, }); + ipcRenderer.send('CHECK_SYNC'); }); } diff --git a/src/context/virtual-drive/files/application/FileCheckerStatusInRoot.ts b/src/context/virtual-drive/files/application/FileCheckerStatusInRoot.ts new file mode 100644 index 000000000..80a10a329 --- /dev/null +++ b/src/context/virtual-drive/files/application/FileCheckerStatusInRoot.ts @@ -0,0 +1,52 @@ +import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; +import Logger from 'electron-log'; +import { PlaceholderState } from '../domain/PlaceholderState'; +import { + PinState, + SyncState, +} from '../../../../apps/shared/types/PlaceholderStates'; +import fs from 'fs'; +import { DependencyInjectionLocalRootFolderPath } from '../../../../apps/sync-engine/dependency-injection/common/localRootFolderPath'; + +export class FileCheckerStatusInRoot { + constructor(private readonly localFileSystem: LocalFileSystem) {} + async run() { + const rootFolderPath = DependencyInjectionLocalRootFolderPath.get(); + const itemsOfRoot = await this.getItemsRoot(rootFolderPath); + const status = await this.checkSync(itemsOfRoot); + return status; + } + + private async checkSync(itemsOfRoot: string[]) { + let finalStatus = 'SYNCED'; + for (const path of itemsOfRoot) { + const placeholderStatus = + (await this.localFileSystem.getPlaceholderStateByRelativePath( + path + )) as PlaceholderState; + + const ps = placeholderStatus.pinState; + const ss = placeholderStatus.syncState; + const status = + ps && + ss && + ps !== PinState.AlwaysLocal && + ps !== PinState.OnlineOnly && + ss !== SyncState.InSync; + + if (status) { + Logger.debug( + `[File Checker Status In Root] item ${path} with status: ${status}` + ); + finalStatus = 'SYNC_PENDING'; + break; + } + } + return finalStatus; + } + + private async getItemsRoot(absolutePath: string): Promise { + const items = fs.readdirSync(absolutePath); + return items; + } +} diff --git a/src/context/virtual-drive/files/application/FileCreator.ts b/src/context/virtual-drive/files/application/FileCreator.ts index ffab80019..9c179a85d 100644 --- a/src/context/virtual-drive/files/application/FileCreator.ts +++ b/src/context/virtual-drive/files/application/FileCreator.ts @@ -11,6 +11,7 @@ import { RemoteFileSystem } from '../domain/file-systems/RemoteFileSystem'; import { OfflineFile } from '../domain/OfflineFile'; import { SyncEngineIpc } from '../../../../apps/sync-engine/ipcRendererSyncEngine'; import { FileStatuses } from '../domain/FileStatus'; +import { ipcRenderer } from 'electron'; export class FileCreator { constructor( @@ -56,6 +57,7 @@ export class FileCreator { extension: file.type, nameWithExtension: file.nameWithExtension, }); + ipcRenderer.send('CHECK_SYNC'); return file; } catch (error: unknown) { diff --git a/src/context/virtual-drive/files/domain/FileRepository.ts b/src/context/virtual-drive/files/domain/FileRepository.ts index ef83cda81..4dfd48c0a 100644 --- a/src/context/virtual-drive/files/domain/FileRepository.ts +++ b/src/context/virtual-drive/files/domain/FileRepository.ts @@ -5,6 +5,8 @@ export interface FileRepository { searchByPartial(partial: Partial): File | undefined; + allSearchByPartial(partial: Partial): Promise>; + delete(id: File['contentsId']): Promise; add(file: File): Promise; diff --git a/src/context/virtual-drive/files/domain/PlaceholderState.ts b/src/context/virtual-drive/files/domain/PlaceholderState.ts new file mode 100644 index 000000000..f91abbeb6 --- /dev/null +++ b/src/context/virtual-drive/files/domain/PlaceholderState.ts @@ -0,0 +1,9 @@ +import { + PinState, + SyncState, +} from '../../../../apps/shared/types/PlaceholderStates'; + +export interface PlaceholderState { + pinState: PinState; + syncState: SyncState; +} diff --git a/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts b/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts index e347c4e1e..1204c926d 100644 --- a/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts +++ b/src/context/virtual-drive/files/domain/file-systems/LocalFileSystem.ts @@ -1,4 +1,5 @@ import { File } from '../File'; +import { PlaceholderState } from '../PlaceholderState'; export interface LocalFileSystem { createPlaceHolder(file: File): Promise; @@ -10,4 +11,8 @@ export interface LocalFileSystem { convertToPlaceholder(file: File): Promise; getPlaceholderState(file: File): Promise; + + getPlaceholderStateByRelativePath( + relativePath: string + ): Promise; } diff --git a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts index 92233f961..431850df0 100644 --- a/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts +++ b/src/context/virtual-drive/files/infrastructure/InMemoryFileRepository.ts @@ -19,6 +19,29 @@ export class InMemoryFileRepository implements FileRepository { return Promise.resolve(files); } + async allSearchByPartial( + partial: Partial + ): Promise> { + const keys = Object.keys(partial) as Array>; + + const files = this.values + .filter((attributes) => { + return keys.every((key: keyof FileAttributes) => { + if (key === 'contentsId') { + return ( + attributes[key].normalize() == + (partial[key] as string).normalize() + ); + } + + return attributes[key] == partial[key]; + }); + }) + .map((attributes) => File.from(attributes)); + + return files; + } + searchByPartial(partial: Partial): File | undefined { const keys = Object.keys(partial) as Array>; diff --git a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts index ec96e1c56..cc9906a52 100644 --- a/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts +++ b/src/context/virtual-drive/files/infrastructure/NodeWinLocalFileSystem.ts @@ -4,6 +4,7 @@ import { File } from '../domain/File'; import { LocalFileSystem } from '../domain/file-systems/LocalFileSystem'; import { RelativePathToAbsoluteConverter } from '../../shared/application/RelativePathToAbsoluteConverter'; import fs from 'fs/promises'; +import { PlaceholderState } from '../domain/PlaceholderState'; export class NodeWinLocalFileSystem implements LocalFileSystem { constructor( @@ -56,4 +57,12 @@ export class NodeWinLocalFileSystem implements LocalFileSystem { getPlaceholderState(file: File): Promise { return this.virtualDrive.getPlaceholderState(file.path); } + + getPlaceholderStateByRelativePath( + relativePath: string + ): Promise { + return this.virtualDrive.getPlaceholderState( + relativePath + ) as Promise; + } } From 9736ab18b9b0a3fc6b71ba24d0da45db01c0988a Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Mon, 12 Feb 2024 22:14:57 -0500 Subject: [PATCH 24/32] remove issues --- src/apps/main/remote-sync/RemoteSyncManager.ts | 5 ----- .../callbacks-controllers/controllers/AddController.ts | 1 - 2 files changed, 6 deletions(-) diff --git a/src/apps/main/remote-sync/RemoteSyncManager.ts b/src/apps/main/remote-sync/RemoteSyncManager.ts index 338d31031..76c8762a7 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.ts @@ -78,11 +78,6 @@ export class RemoteSyncManager { Logger.info('Starting remote to local sync'); Logger.info('Checking if we are in a valid state to start the sync'); - // Logger.debug( - // '[ this.chekers.fileCheker.run()]', - // await this.chekers.fileCheker.run() - // ); - const testPassed = this.smokeTest(); if (!testPassed) { diff --git a/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts b/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts index cea5c6676..060afab1a 100644 --- a/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts +++ b/src/apps/sync-engine/callbacks-controllers/controllers/AddController.ts @@ -44,7 +44,6 @@ export class AddController extends CallbackController { ); if (confirmCreation) { Logger.info('File created', posixRelativePath); - // ipcRenderer.send('CHECK_SYNC'); } ipcRenderer.send('CHECK_SYNC'); } catch (error: unknown) { From b157e7b2338c6a1d1b7904f5d7a11d17b8adb979 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Wed, 14 Feb 2024 16:50:33 -0500 Subject: [PATCH 25/32] modify all to exists filter to get remote item --- src/apps/main/remote-sync/RemoteSyncManager.ts | 4 ++-- .../virtual-drive/contents/application/ContentsDownloader.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/apps/main/remote-sync/RemoteSyncManager.ts b/src/apps/main/remote-sync/RemoteSyncManager.ts index 76c8762a7..cf157c15e 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.ts @@ -335,7 +335,7 @@ export class RemoteSyncManager { const params = { limit: this.config.fetchFilesLimitPerRequest, offset: 0, - status: 'ALL', + status: 'EXISTS', updatedAt: updatedAtCheckpoint ? updatedAtCheckpoint.toISOString() : undefined, @@ -399,7 +399,7 @@ export class RemoteSyncManager { const params = { limit: this.config.fetchFilesLimitPerRequest, offset: 0, - status: 'ALL', + status: 'EXISTS', updatedAt: updatedAtCheckpoint ? updatedAtCheckpoint.toISOString() : undefined, diff --git a/src/context/virtual-drive/contents/application/ContentsDownloader.ts b/src/context/virtual-drive/contents/application/ContentsDownloader.ts index 4ba3b3759..0ea20cc02 100644 --- a/src/context/virtual-drive/contents/application/ContentsDownloader.ts +++ b/src/context/virtual-drive/contents/application/ContentsDownloader.ts @@ -46,6 +46,7 @@ export class ContentsDownloader { }); downloader.on('progress', async () => { + Logger.debug('[Server] Download progress'); const result = await cb(true, filePath); const hydrationProgress = result.progress; From 0e9b312a5e8b39276dfc34dbc1c96c9497289997 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 15 Feb 2024 08:50:10 -0500 Subject: [PATCH 26/32] modify the removing of files when logout and quit and update tests --- src/apps/sync-engine/BindingManager.ts | 35 +++++++++---------- .../files/__mocks__/FileRepositoryMock.ts | 4 +++ .../files/__mocks__/LocalFileSystemMock.ts | 8 +++++ 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index e2256f311..c69644daf 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -256,26 +256,25 @@ export class BindingsManager { Logger.debug('remainingItems', remainingItems); Logger.debug('win32AbsolutePaths', win32AbsolutePaths); + // find all common string in remainingItems and win32AbsolutePaths // and delete them - const commonItems = remainingItems.filter((item) => - win32AbsolutePaths.includes(item) - ); - - const toDeleteFolder: string[] = []; - - commonItems.forEach((item) => { - try { - const stat = fs.statSync(item); - if (stat.isDirectory()) { - toDeleteFolder.push(item); - } else if (stat.isFile()) { - fs.unlinkSync(item); - } - } catch (error) { - Logger.error(error); - } - }); + // const commonItems = remainingItems.filter((item) => + // win32AbsolutePaths.includes(item) + // ); + // const toDeleteFolder: string[] = []; + // commonItems.forEach((item) => { + // try { + // const stat = fs.statSync(item); + // if (stat.isDirectory()) { + // toDeleteFolder.push(item); + // } else if (stat.isFile()) { + // fs.unlinkSync(item); + // } + // } catch (error) { + // Logger.error(error); + // } + // }); } async update() { diff --git a/tests/context/virtual-drive/files/__mocks__/FileRepositoryMock.ts b/tests/context/virtual-drive/files/__mocks__/FileRepositoryMock.ts index dfd9855fc..1769e190a 100644 --- a/tests/context/virtual-drive/files/__mocks__/FileRepositoryMock.ts +++ b/tests/context/virtual-drive/files/__mocks__/FileRepositoryMock.ts @@ -12,6 +12,7 @@ export class FileRepositoryMock implements FileRepository { public readonly addMock = jest.fn(); public readonly updateMock = jest.fn(); public readonly updateContentsAndSizeMock = jest.fn(); + public readonly allSearchByPartialMock = jest.fn(); all(): Promise { return this.allMock(); @@ -35,4 +36,7 @@ export class FileRepositoryMock implements FileRepository { ): Promise { return this.updateContentsAndSizeMock(file, newContentsId, newSize); } + allSearchByPartial(partial: Partial): Promise { + return this.allSearchByPartialMock(partial); + } } diff --git a/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts b/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts index 66374f322..5ab6fc4d8 100644 --- a/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts +++ b/tests/context/virtual-drive/files/__mocks__/LocalFileSystemMock.ts @@ -1,5 +1,6 @@ import { LocalFileSystem } from '../../../../../src/context/virtual-drive/files/domain/file-systems/LocalFileSystem'; import { File } from '../../../../../src/context/virtual-drive/files/domain/File'; +import { PlaceholderState } from '../../../../../src/context/virtual-drive/files/domain/PlaceholderState'; export class LocalFileSystemMock implements LocalFileSystem { public readonly createPlaceHolderMock = jest.fn(); @@ -7,6 +8,7 @@ export class LocalFileSystemMock implements LocalFileSystem { public readonly updateSyncStatusMock = jest.fn(); public readonly convertToPlaceholderMock = jest.fn(); public readonly getPlaceholderStateMock = jest.fn(); + public readonly getPlaceholderStateByRelativePathMock = jest.fn(); createPlaceHolder(file: File): Promise { return this.createPlaceHolder(file); @@ -27,4 +29,10 @@ export class LocalFileSystemMock implements LocalFileSystem { getPlaceholderState(file: File): Promise { return this.getPlaceholderStateMock(file); } + + getPlaceholderStateByRelativePath( + relativePath: string + ): Promise { + return this.getPlaceholderStateByRelativePathMock(relativePath); + } } From 34e4234c6f5561094bd3b41acd8ca70f39118d45 Mon Sep 17 00:00:00 2001 From: joan vicens Date: Mon, 19 Feb 2024 17:10:24 +0100 Subject: [PATCH 27/32] patch: this awaits for files to download ? --- release/app/yarn.lock | 50 ++++++++++++++++++- src/apps/shared/fs/write-readable-to-file.ts | 23 ++++++++- .../application/ContentsDownloader.ts | 49 ++++++++---------- .../infrastructure/FSLocalFileWriter.ts | 2 +- yarn.lock | 11 +++- 5 files changed, 102 insertions(+), 33 deletions(-) diff --git a/release/app/yarn.lock b/release/app/yarn.lock index b4f8416f3..d89a9eb15 100644 --- a/release/app/yarn.lock +++ b/release/app/yarn.lock @@ -30,6 +30,18 @@ enabled "2.0.x" kuler "^2.0.0" +"@hapi/hoek@^9.0.0", "@hapi/hoek@^9.3.0": + version "9.3.0" + resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" + integrity sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ== + +"@hapi/topo@^5.1.0": + version "5.1.0" + resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.1.0.tgz#dc448e332c6c6e37a4dc02fd84ba8d44b9afb012" + integrity sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg== + dependencies: + "@hapi/hoek" "^9.0.0" + "@ioredis/commands@^1.1.1": version "1.2.0" resolved "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz" @@ -91,6 +103,23 @@ component-type "^1.2.1" join-component "^1.1.0" +"@sideway/address@^4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.5.tgz#4bc149a0076623ced99ca8208ba780d65a99b9d5" + integrity sha512-IqO/DUQHUkPeixNQ8n0JA6102hT9CmaljNTPmQ1u8MEhBo/R4Q8eKLN/vGZxuebwOroDB4cbpjheD4+/sKFK4Q== + dependencies: + "@hapi/hoek" "^9.0.0" + +"@sideway/formula@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.1.tgz#80fcbcbaf7ce031e0ef2dd29b1bfc7c3f583611f" + integrity sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg== + +"@sideway/pinpoint@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" + integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== + "@sqltools/formatter@^1.2.5": version "1.2.5" resolved "https://registry.npmjs.org/@sqltools/formatter/-/formatter-1.2.5.tgz" @@ -374,6 +403,11 @@ dotenv@^16.0.3: resolved "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz" integrity sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ== +dotenv@^16.4.1: + version "16.4.4" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.4.tgz#a26e7bb95ebd36272ebb56edb80b826aecf224c1" + integrity sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg== + emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" @@ -530,6 +564,17 @@ is-stream@^2.0.0: resolved "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== +joi@^17.12.0: + version "17.12.1" + resolved "https://registry.yarnpkg.com/joi/-/joi-17.12.1.tgz#3347ecf4cd3301962d42191c021b165eef1f395b" + integrity sha512-vtxmq+Lsc5SlfqotnfVjlViWfOL9nt/avKNbKYizwf6gsCfq9NYY/ceYRMFD8XDdrjJ9abJyScWmhmIiy+XRtQ== + dependencies: + "@hapi/hoek" "^9.3.0" + "@hapi/topo" "^5.1.0" + "@sideway/address" "^4.1.5" + "@sideway/formula" "^3.0.1" + "@sideway/pinpoint" "^2.0.0" + join-component@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/join-component/-/join-component-1.1.0.tgz" @@ -1031,7 +1076,10 @@ uuid@^9.0.0: virtual-drive@../../../node-win: version "1.0.1" dependencies: + dotenv "^16.4.1" + joi "^17.12.0" tsconfig-paths "^4.2.0" + yargs "^17.7.2" winston-transport@^4.5.0: version "4.5.0" @@ -1106,7 +1154,7 @@ yargs@^16.0.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@^17.6.2: +yargs@^17.6.2, yargs@^17.7.2: version "17.7.2" resolved "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== diff --git a/src/apps/shared/fs/write-readable-to-file.ts b/src/apps/shared/fs/write-readable-to-file.ts index 857cc509f..913fad799 100644 --- a/src/apps/shared/fs/write-readable-to-file.ts +++ b/src/apps/shared/fs/write-readable-to-file.ts @@ -1,14 +1,33 @@ import fs, { PathLike } from 'fs'; +import { stat } from 'fs/promises'; import { Readable } from 'stream'; export class WriteReadableToFile { - static write(readable: Readable, path: PathLike): Promise { + static write( + readable: Readable, + path: PathLike, + expectedSize?: number + ): Promise { const writableStream = fs.createWriteStream(path); readable.pipe(writableStream); return new Promise((resolve, reject) => { - writableStream.on('finish', resolve); + writableStream.on('finish', async () => { + const { size } = await stat(path); + + if (!expectedSize) { + resolve(); + return; + } + if (size !== expectedSize) { + reject('Wried file does not have expected size'); + return; + } + + resolve(); + }); + writableStream.on('error', reject); }); } diff --git a/src/context/virtual-drive/contents/application/ContentsDownloader.ts b/src/context/virtual-drive/contents/application/ContentsDownloader.ts index 0ea20cc02..c6ad91248 100644 --- a/src/context/virtual-drive/contents/application/ContentsDownloader.ts +++ b/src/context/virtual-drive/contents/application/ContentsDownloader.ts @@ -47,34 +47,18 @@ export class ContentsDownloader { downloader.on('progress', async () => { Logger.debug('[Server] Download progress'); - const result = await cb(true, filePath); - const hydrationProgress = result.progress; - - if (result.finished) { - downloader.forceStop(); - Logger.debug('Downloader force stop', this.readableDownloader); - this.readableDownloader?.destroy(); - this.readableDownloader?.emit('close'); - this.ipc.send('FILE_DOWNLOADED', { - name: file.name, - extension: file.type, - nameWithExtension: file.nameWithExtension, - size: file.size, - processInfo: { elapsedTime: downloader.elapsedTime() }, - }); - ipcRenderer.send('CHECK_SYNC'); - } else { - this.ipc.send('FILE_DOWNLOADING', { - name: file.name, - extension: file.type, - nameWithExtension: file.nameWithExtension, - size: file.size, - processInfo: { - elapsedTime: downloader.elapsedTime(), - progress: hydrationProgress, - }, - }); - } + const { progress } = await cb(true, filePath); + + this.ipc.send('FILE_DOWNLOADING', { + name: file.name, + extension: file.type, + nameWithExtension: file.nameWithExtension, + size: file.size, + processInfo: { + elapsedTime: downloader.elapsedTime(), + progress, + }, + }); }); downloader.on('error', (error: Error) => { @@ -99,6 +83,7 @@ export class ContentsDownloader { const readable = await downloader.download(file); this.readableDownloader = readable; + const localContents = LocalFileContents.downloadedFrom( file, readable, @@ -110,6 +95,14 @@ export class ContentsDownloader { const events = localContents.pullDomainEvents(); await this.eventBus.publish(events); + this.ipc.send('FILE_DOWNLOADED', { + name: file.name, + extension: file.type, + nameWithExtension: file.nameWithExtension, + size: file.size, + processInfo: { elapsedTime: downloader.elapsedTime() }, + }); + return write; } } diff --git a/src/context/virtual-drive/contents/infrastructure/FSLocalFileWriter.ts b/src/context/virtual-drive/contents/infrastructure/FSLocalFileWriter.ts index 51b26fcc2..e6aaf4f0c 100644 --- a/src/context/virtual-drive/contents/infrastructure/FSLocalFileWriter.ts +++ b/src/context/virtual-drive/contents/infrastructure/FSLocalFileWriter.ts @@ -16,7 +16,7 @@ export class FSLocalFileWriter implements LocalFileWriter { const filePath = path.join(location, contents.nameWithExtension); - await WriteReadableToFile.write(contents.stream, filePath); + await WriteReadableToFile.write(contents.stream, filePath, contents.size); return filePath; } diff --git a/yarn.lock b/yarn.lock index b08729e41..470a2aebd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1285,7 +1285,7 @@ resolved "https://npm.pkg.github.com/download/@internxt/lib/1.2.0/172d7929abb3dc34fda044ad9977875791f7a471#172d7929abb3dc34fda044ad9977875791f7a471" integrity sha512-14byNxSU0S0KSc3g+BWiJb7/fcHUfqxkHQRz9fwJhAHNmgOo+Bhx13xXk07KKUzeNX1JymSGEZkVCk0AOQiM+Q== -"@internxt/sdk@^1.4.27", "@internxt/sdk@^1.4.33": +"@internxt/sdk@^1.4.27": version "1.4.35" resolved "https://npm.pkg.github.com/download/@internxt/sdk/1.4.35/d947a5608ce9c16dbf22984f3b4ae3a0c4bd87cb#d947a5608ce9c16dbf22984f3b4ae3a0c4bd87cb" integrity sha512-WX5cJiRoj8buRobAq+zRPKNavZ09vbCNdw9PVtEG6A5okK4wYIaGqbWBTldffgrO4JgtyNbMqxUCDD0i0UUSCQ== @@ -1294,6 +1294,15 @@ query-string "^7.1.0" uuid "^8.3.2" +"@internxt/sdk@^1.4.68": + version "1.4.71" + resolved "https://npm.pkg.github.com/download/@internxt/sdk/1.4.71/eb6f1cd524f2a861afdb1f14483848910bc0d0a7#eb6f1cd524f2a861afdb1f14483848910bc0d0a7" + integrity sha512-pirbb8odDDdqJxXwpNapVdLlhSw9di2emklzVza81JuWzXeyOlwpgvu65JU0RFZ+AjSl+0mJR4fmdfbh3evTEA== + dependencies: + axios "^0.24.0" + query-string "^7.1.0" + uuid "^8.3.2" + "@isaacs/cliui@^8.0.2": version "8.0.2" resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" From 6026a501333339907bf5e0d99c9ce9d4e63c6000 Mon Sep 17 00:00:00 2001 From: joan vicens Date: Tue, 20 Feb 2024 12:25:25 +0100 Subject: [PATCH 28/32] fix: reject error --- src/apps/shared/fs/write-readable-to-file.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/shared/fs/write-readable-to-file.ts b/src/apps/shared/fs/write-readable-to-file.ts index 913fad799..113062658 100644 --- a/src/apps/shared/fs/write-readable-to-file.ts +++ b/src/apps/shared/fs/write-readable-to-file.ts @@ -21,7 +21,7 @@ export class WriteReadableToFile { return; } if (size !== expectedSize) { - reject('Wried file does not have expected size'); + reject(new Error('Wried file does not have expected size')); return; } From eb2fbe503da891c9f2aa108dce9d801f847255ec Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 22 Feb 2024 08:13:15 -0500 Subject: [PATCH 29/32] add remove listener when stream appears --- .../domain/contentHandlers/ContentFileDownloader.ts | 2 ++ .../download/EnvironmentContentFileDownloader.ts | 12 ++++++++++++ .../contents/__mocks__/ContentFileDownloaderMock.ts | 4 ++++ 3 files changed, 18 insertions(+) diff --git a/src/context/virtual-drive/contents/domain/contentHandlers/ContentFileDownloader.ts b/src/context/virtual-drive/contents/domain/contentHandlers/ContentFileDownloader.ts index 586057f85..c7ea3f877 100644 --- a/src/context/virtual-drive/contents/domain/contentHandlers/ContentFileDownloader.ts +++ b/src/context/virtual-drive/contents/domain/contentHandlers/ContentFileDownloader.ts @@ -19,4 +19,6 @@ export interface ContentFileDownloader { ): void; elapsedTime(): number; + + removeListeners(): void; } diff --git a/src/context/virtual-drive/contents/infrastructure/download/EnvironmentContentFileDownloader.ts b/src/context/virtual-drive/contents/infrastructure/download/EnvironmentContentFileDownloader.ts index 5744672bd..ba63f558a 100644 --- a/src/context/virtual-drive/contents/infrastructure/download/EnvironmentContentFileDownloader.ts +++ b/src/context/virtual-drive/contents/infrastructure/download/EnvironmentContentFileDownloader.ts @@ -8,6 +8,7 @@ import { File } from '../../../files/domain/File'; import { DownloadOneShardStrategy } from '@internxt/inxt-js/build/lib/core'; import { ActionState } from '@internxt/inxt-js/build/api'; import { Stopwatch } from '../../../../../apps/shared/types/Stopwatch'; +import Logger from 'electron-log'; export class EnvironmentContentFileDownloader implements ContentFileDownloader { private eventEmitter: EventEmitter; @@ -47,14 +48,21 @@ export class EnvironmentContentFileDownloader implements ContentFileDownloader { this.eventEmitter.emit('progress', progress); }, finishedCallback: async (err: Error, stream: Readable) => { + Logger.debug('[FinishedCallback] Stream is ready'); this.stopwatch.finish(); if (err) { this.eventEmitter.emit('error', err); return reject(err); } + this.eventEmitter.emit('finish'); + stream.on('close', () => { + Logger.debug('[FinishedCallback] Stream closed'); + this.removeListeners(); + }); + resolve(stream); }, }, @@ -79,4 +87,8 @@ export class EnvironmentContentFileDownloader implements ContentFileDownloader { elapsedTime(): number { return this.stopwatch.elapsedTime(); } + + removeListeners(): void { + this.eventEmitter.removeAllListeners(); + } } diff --git a/tests/context/virtual-drive/contents/__mocks__/ContentFileDownloaderMock.ts b/tests/context/virtual-drive/contents/__mocks__/ContentFileDownloaderMock.ts index 5fa88c343..d299891b8 100644 --- a/tests/context/virtual-drive/contents/__mocks__/ContentFileDownloaderMock.ts +++ b/tests/context/virtual-drive/contents/__mocks__/ContentFileDownloaderMock.ts @@ -9,6 +9,10 @@ export class ContentFileDownloaderMock implements ContentFileDownloader { onMock = jest.fn(); elapsedTimeMock = jest.fn(); + removeListeners(): void { + return; + } + download(): Promise { return this.mock(); } From 241c75db8387395cce2900ef9f10526bb0b0682f Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 22 Feb 2024 10:19:54 -0500 Subject: [PATCH 30/32] divide downloding and preparing process --- src/apps/sync-engine/BindingManager.ts | 57 ++++++++++--------- .../controllers/DownloadFileController.ts | 16 ++---- .../application/ContentsDownloader.ts | 20 +++---- .../application/ContentsDownloader._test.ts | 10 +--- 4 files changed, 48 insertions(+), 55 deletions(-) diff --git a/src/apps/sync-engine/BindingManager.ts b/src/apps/sync-engine/BindingManager.ts index c69644daf..04d975ac1 100644 --- a/src/apps/sync-engine/BindingManager.ts +++ b/src/apps/sync-engine/BindingManager.ts @@ -96,11 +96,8 @@ export class BindingsManager { callback: CallbackDownload ) => { try { - const path = await controllers.downloadFile.execute( - contentsId, - callback - ); - Logger.debug('Execute Fetch Data Callback, sending path:', path); + Logger.debug('[Fetch Data Callback] Donwloading begins'); + const path = await controllers.downloadFile.execute(contentsId); const file = controllers.downloadFile.fileFinderByContentsId( contentsId .replace( @@ -110,35 +107,43 @@ export class BindingsManager { ) .split(':')[1] ); + Logger.debug('[Fetch Data Callback] Preparing begins', path); let finished = false; - while (!finished) { - const result = await callback(true, path); - finished = result.finished; - if (this.progressBuffer == result.progress) { - break; - } else { - this.progressBuffer = result.progress; + try { + while (!finished) { + const result = await callback(true, path); + finished = result.finished; + Logger.debug('callback result', result); + + if (finished && result.progress === 0) { + throw new Error('Result progress is 0'); + } else if (this.progressBuffer == result.progress) { + break; + } else { + this.progressBuffer = result.progress; + } + Logger.debug('condition', finished); + ipcRendererSyncEngine.send('FILE_PREPARING', { + name: file.name, + extension: file.type, + nameWithExtension: file.nameWithExtension, + size: file.size, + processInfo: { + elapsedTime: 0, + progress: result.progress, + }, + }); } - Logger.debug('condition', finished); - ipcRendererSyncEngine.send('FILE_DOWNLOADING', { - name: file.name, - extension: file.type, - nameWithExtension: file.nameWithExtension, - size: file.size, - processInfo: { - elapsedTime: 0, - progress: result.progress, - }, - }); - } + this.progressBuffer = 0; - this.progressBuffer = 0; - try { await controllers.notifyPlaceholderHydrationFinished.execute( contentsId ); + + await this.container.virtualDrive.closeDownloadMutex(); } catch (error) { Logger.error('notify: ', error); + await this.container.virtualDrive.closeDownloadMutex(); } // Esperar hasta que la ejecución de fetchDataCallback esté completa antes de continuar diff --git a/src/apps/sync-engine/callbacks-controllers/controllers/DownloadFileController.ts b/src/apps/sync-engine/callbacks-controllers/controllers/DownloadFileController.ts index 6e5b8bfbc..aee6d1e49 100644 --- a/src/apps/sync-engine/callbacks-controllers/controllers/DownloadFileController.ts +++ b/src/apps/sync-engine/callbacks-controllers/controllers/DownloadFileController.ts @@ -2,7 +2,6 @@ import Logger from 'electron-log'; import { ContentsDownloader } from '../../../../context/virtual-drive/contents/application/ContentsDownloader'; import { FileFinderByContentsId } from '../../../../context/virtual-drive/files/application/FileFinderByContentsId'; import { FilePlaceholderId } from '../../../../context/virtual-drive/files/domain/PlaceholderId'; -import { CallbackDownload } from '../../BindingManager'; import { CallbackController } from './CallbackController'; export class DownloadFileController extends CallbackController { @@ -13,25 +12,21 @@ export class DownloadFileController extends CallbackController { super(); } - private async action(id: string, cb: CallbackDownload): Promise { + private async action(id: string): Promise { const file = this.fileFinder.run(id); - - return await this.downloader.run(file, cb); + return await this.downloader.run(file); } fileFinderByContentsId(contentsId: string) { return this.fileFinder.run(contentsId); } - async execute( - filePlaceholderId: FilePlaceholderId, - cb: CallbackDownload - ): Promise { + async execute(filePlaceholderId: FilePlaceholderId): Promise { const trimmedId = this.trim(filePlaceholderId); try { const [_, contentsId] = trimmedId.split(':'); - return await this.action(contentsId, cb); + return await this.action(contentsId); } catch (error: unknown) { Logger.error( 'Error downloading a file, going to refresh and retry: ', @@ -42,8 +37,7 @@ export class DownloadFileController extends CallbackController { setTimeout(async () => { try { const [_, contentsId] = trimmedId.split(':'); - Logger.debug('cb: ', cb); - const result = await this.action(contentsId, cb); + const result = await this.action(contentsId); resolve(result); } catch (error) { reject(error); diff --git a/src/context/virtual-drive/contents/application/ContentsDownloader.ts b/src/context/virtual-drive/contents/application/ContentsDownloader.ts index c6ad91248..39fa80ee5 100644 --- a/src/context/virtual-drive/contents/application/ContentsDownloader.ts +++ b/src/context/virtual-drive/contents/application/ContentsDownloader.ts @@ -11,7 +11,7 @@ import { LocalFileContents } from '../domain/LocalFileContents'; import { LocalFileWriter } from '../domain/LocalFileWriter'; import { ContentFileDownloader } from '../domain/contentHandlers/ContentFileDownloader'; import { TemporalFolderProvider } from './temporalFolderProvider'; -import { ipcRenderer } from 'electron'; +import * as fs from 'fs'; export class ContentsDownloader { private readableDownloader: Readable | null; @@ -25,11 +25,7 @@ export class ContentsDownloader { this.readableDownloader = null; } - private async registerEvents( - downloader: ContentFileDownloader, - file: File, - cb: CallbackDownload - ) { + private async registerEvents(downloader: ContentFileDownloader, file: File) { const location = await this.temporalFolderProvider(); ensureFolderExists(location); @@ -46,8 +42,11 @@ export class ContentsDownloader { }); downloader.on('progress', async () => { - Logger.debug('[Server] Download progress'); - const { progress } = await cb(true, filePath); + Logger.debug('[Server] Download progress', filePath); + + const stats = fs.statSync(filePath); + const fileSizeInBytes = stats.size; + const progress = fileSizeInBytes / file.size; this.ipc.send('FILE_DOWNLOADING', { name: file.name, @@ -71,15 +70,16 @@ export class ContentsDownloader { }); downloader.on('finish', () => { + // cb(true, filePath); // The file download being finished does not mean it has been hidratated // TODO: We might want to track this time instead of the whole completion time }); } - async run(file: File, cb: CallbackDownload): Promise { + async run(file: File): Promise { const downloader = this.managerFactory.downloader(); - this.registerEvents(downloader, file, cb); + await this.registerEvents(downloader, file); const readable = await downloader.download(file); this.readableDownloader = readable; diff --git a/tests/context/virtual-drive/contents/application/ContentsDownloader._test.ts b/tests/context/virtual-drive/contents/application/ContentsDownloader._test.ts index 10a2333ca..e2e4401b7 100644 --- a/tests/context/virtual-drive/contents/application/ContentsDownloader._test.ts +++ b/tests/context/virtual-drive/contents/application/ContentsDownloader._test.ts @@ -40,10 +40,7 @@ describe.skip('Contents Downloader', () => { new ReadableHelloWorld() ); - await SUT.run(FileMother.any(), async () => { - /* do nothing */ - return { finished: true, progress: 100 }; - }); + await SUT.run(FileMother.any()); expect(factory.mockDownloader.onMock).toBeCalledWith( event, @@ -56,10 +53,7 @@ describe.skip('Contents Downloader', () => { factory.mockDownloader.mock.mockResolvedValueOnce(new ReadableHelloWorld()); const file = FileMother.any(); - await SUT.run(file, async () => { - /* do nothing */ - return { finished: true, progress: 100 }; - }); + await SUT.run(file); expect(localWriter.writeMock).toBeCalledWith( expect.objectContaining({ From 6a4a1ac1ed753d44314af403582a78119a287919 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 22 Feb 2024 10:22:35 -0500 Subject: [PATCH 31/32] add preparing ux --- src/apps/main/fordwardToWindows.ts | 10 ++++++++++ src/apps/main/tray/handlers.ts | 4 ++++ src/apps/renderer/localize/locales/en.json | 1 + src/apps/renderer/localize/locales/es.json | 1 + src/apps/renderer/localize/locales/fr.json | 1 + src/apps/renderer/pages/Widget/Item.tsx | 3 +++ src/apps/shared/IPC/events/drive.ts | 1 + src/apps/shared/IPC/events/sync-engine.ts | 1 + src/apps/shared/types.ts | 7 ++++++- 9 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/apps/main/fordwardToWindows.ts b/src/apps/main/fordwardToWindows.ts index a592cef9a..651240815 100644 --- a/src/apps/main/fordwardToWindows.ts +++ b/src/apps/main/fordwardToWindows.ts @@ -22,6 +22,16 @@ ipcMainDrive.on('FILE_DOWNLOADING', (_, payload) => { }); }); +ipcMainDrive.on('FILE_PREPARING', (_, payload) => { + const { nameWithExtension, processInfo } = payload; + + broadcastToWindows('sync-info-update', { + action: 'PREPARING', + name: nameWithExtension, + progress: processInfo.progress, + }); +}); + ipcMainDrive.on('FILE_DOWNLOADED', (_, payload) => { const { nameWithExtension } = payload; diff --git a/src/apps/main/tray/handlers.ts b/src/apps/main/tray/handlers.ts index 40ee32769..f1ac01773 100644 --- a/src/apps/main/tray/handlers.ts +++ b/src/apps/main/tray/handlers.ts @@ -28,6 +28,10 @@ ipcMainDrive.on('FILE_DOWNLOADING', () => { setTrayStatus('SYNCING'); }); +ipcMainDrive.on('FILE_PREPARING', () => { + setTrayStatus('SYNCING'); +}); + ipcMainDrive.on('FILE_DOWNLOADED', () => { setTrayStatus('IDLE'); }); diff --git a/src/apps/renderer/localize/locales/en.json b/src/apps/renderer/localize/locales/en.json index 36b6b7042..59cfa584c 100644 --- a/src/apps/renderer/localize/locales/en.json +++ b/src/apps/renderer/localize/locales/en.json @@ -141,6 +141,7 @@ "activity": { "operation": { "downloading": "Downloading", + "preparing": "Preparing", "decrypting": "Decrypting", "uploading": "Uploading", "encrypting": "Encrypting", diff --git a/src/apps/renderer/localize/locales/es.json b/src/apps/renderer/localize/locales/es.json index b38ac1226..2ce257eaf 100644 --- a/src/apps/renderer/localize/locales/es.json +++ b/src/apps/renderer/localize/locales/es.json @@ -141,6 +141,7 @@ "activity": { "operation": { "downloading": "Descargando", + "preparing": "Preparando", "decrypting": "Desencriptando", "uploading": "Subiendo", "encrypting": "Encriptando", diff --git a/src/apps/renderer/localize/locales/fr.json b/src/apps/renderer/localize/locales/fr.json index 37684efa3..f873319a9 100644 --- a/src/apps/renderer/localize/locales/fr.json +++ b/src/apps/renderer/localize/locales/fr.json @@ -140,6 +140,7 @@ "activity": { "operation": { "downloading": "Téléchargement", + "preparing": "Préparation", "decrypting": "Décryptage", "uploading": "téléchargement", "encrypting": "Encryptage", diff --git a/src/apps/renderer/pages/Widget/Item.tsx b/src/apps/renderer/pages/Widget/Item.tsx index f1397d12a..1d2fca384 100644 --- a/src/apps/renderer/pages/Widget/Item.tsx +++ b/src/apps/renderer/pages/Widget/Item.tsx @@ -27,6 +27,8 @@ export function Item({ description = progress ? translate('widget.body.activity.operation.downloading') : translate('widget.body.activity.operation.decrypting'); + } else if (action === 'PREPARING') { + description = translate('widget.body.activity.operation.preparing'); } else if (action === 'UPLOADING') { description = progress ? translate('widget.body.activity.operation.uploading') @@ -94,6 +96,7 @@ export function Item({ {action && (action === 'UPLOADING' || action === 'DOWNLOADING' || + action === 'PREPARING' || action === 'RENAMING' || action === 'DELETING') && ( void; + FILE_PREPARING: (payload: FileProgressInfo) => void; FILE_DOWNLOADED: (payload: FileProgressInfo) => void; FILE_DOWNLOAD_ERROR: (payload: FileErrorInfo) => void; }; diff --git a/src/apps/shared/IPC/events/sync-engine.ts b/src/apps/shared/IPC/events/sync-engine.ts index 2b9494796..0a3dbd60e 100644 --- a/src/apps/shared/IPC/events/sync-engine.ts +++ b/src/apps/shared/IPC/events/sync-engine.ts @@ -66,6 +66,7 @@ export type FilesEvents = { }) => void; FILE_DOWNLOADING: (payload: FileUpdatePayload) => void; + FILE_PREPARING: (payload: FileUpdatePayload) => void; FILE_DOWNLOADED: (payload: FileUpdatePayload) => void; FILE_UPLOAD_ERROR: (payload: { name: string; diff --git a/src/apps/shared/types.ts b/src/apps/shared/types.ts index d77562c62..c15e316c5 100644 --- a/src/apps/shared/types.ts +++ b/src/apps/shared/types.ts @@ -227,7 +227,12 @@ export type ProcessInfoUpdatePayload = | (ProcessInfoBase & ( | { - action: 'UPLOADING' | 'DOWNLOADING' | 'RENAMING' | 'DELETING'; + action: + | 'UPLOADING' + | 'DOWNLOADING' + | 'PREPARING' + | 'RENAMING' + | 'DELETING'; progress: number; } | { From 60afafc4d281237267b42cae233ed3c27f7a47b4 Mon Sep 17 00:00:00 2001 From: migueldev01 Date: Thu, 22 Feb 2024 10:29:34 -0500 Subject: [PATCH 32/32] add version 2.0.5 and remove smell --- package.json | 2 +- release/app/package.json | 2 +- .../virtual-drive/contents/application/ContentsDownloader.ts | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index a8827645d..35a86fef6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "internxt-drive", - "version": "2.0.4", + "version": "2.0.5", "author": "Internxt ", "description": "Internxt Drive client UI", "license": "AGPL-3.0", diff --git a/release/app/package.json b/release/app/package.json index 27ab2e60f..5c1b2394b 100644 --- a/release/app/package.json +++ b/release/app/package.json @@ -1,6 +1,6 @@ { "name": "internxt-drive", - "version": "2.0.4", + "version": "2.0.5", "description": "Internxt Drive client UI", "main": "./dist/main/main.js", "author": "Internxt ", diff --git a/src/context/virtual-drive/contents/application/ContentsDownloader.ts b/src/context/virtual-drive/contents/application/ContentsDownloader.ts index 39fa80ee5..e5b03fd6b 100644 --- a/src/context/virtual-drive/contents/application/ContentsDownloader.ts +++ b/src/context/virtual-drive/contents/application/ContentsDownloader.ts @@ -2,7 +2,6 @@ import Logger from 'electron-log'; import path from 'path'; import { Readable } from 'stream'; import { ensureFolderExists } from '../../../../apps/shared/fs/ensure-folder-exists'; -import { CallbackDownload } from '../../../../apps/sync-engine/BindingManager'; import { SyncEngineIpc } from '../../../../apps/sync-engine/ipcRendererSyncEngine'; import { File } from '../../files/domain/File'; import { EventBus } from '../../shared/domain/EventBus';