diff --git a/src/apps/main/analytics/user-handlers.ts b/src/apps/main/analytics/user-handlers.ts index 2f878d385..6ccfa0521 100644 --- a/src/apps/main/analytics/user-handlers.ts +++ b/src/apps/main/analytics/user-handlers.ts @@ -6,7 +6,6 @@ import { userSignin, userSigninFailed, } from './service'; -import { clearRemoteSyncStore } from '../remote-sync/helpers'; import { clearTempFolder } from '../app-info/helpers'; eventBus.on('USER_LOGGED_IN', () => { @@ -16,7 +15,6 @@ eventBus.on('USER_LOGGED_IN', () => { eventBus.on('USER_LOGGED_OUT', () => { userLogout(); - clearRemoteSyncStore(); clearTempFolder(); }); diff --git a/src/apps/main/database/adapters/base.ts b/src/apps/main/database/adapters/base.ts index d91fec112..b0622e5e9 100644 --- a/src/apps/main/database/adapters/base.ts +++ b/src/apps/main/database/adapters/base.ts @@ -1,20 +1,20 @@ -export abstract class DatabaseCollectionAdapter { +export interface DatabaseCollectionAdapter { /** * Used to initialize the database adapter */ - abstract connect(): Promise<{ success: boolean }>; + connect(): Promise<{ success: boolean }>; /** * Gets an item from the database */ - abstract get( + get( itemId: string ): Promise<{ success: boolean; result: DatabaseItemType | null }>; /** * Updates an item in the database */ - abstract update( + update( itemId: string, updatePayload: Partial ): Promise<{ @@ -25,7 +25,7 @@ export abstract class DatabaseCollectionAdapter { /** * Creates an item in the database */ - abstract create(creationPayload: DatabaseItemType): Promise<{ + create(creationPayload: DatabaseItemType): Promise<{ success: boolean; result: DatabaseItemType | null; }>; @@ -33,7 +33,12 @@ export abstract class DatabaseCollectionAdapter { /** * Removes an item from the database */ - abstract remove(itemId: string): Promise<{ + remove(itemId: string): Promise<{ success: boolean; }>; + + getLastUpdated(): Promise<{ + success: boolean; + result: DatabaseItemType | null; + }>; } diff --git a/src/apps/main/database/collections/DriveFileCollection.ts b/src/apps/main/database/collections/DriveFileCollection.ts index c338afb2d..9c22ee07a 100644 --- a/src/apps/main/database/collections/DriveFileCollection.ts +++ b/src/apps/main/database/collections/DriveFileCollection.ts @@ -2,6 +2,9 @@ import { DatabaseCollectionAdapter } from '../adapters/base'; import { AppDataSource } from '../data-source'; import { DriveFile } from '../entities/DriveFile'; import { Repository } from 'typeorm'; +import * as Sentry from '@sentry/electron/main'; +import Logger from 'electron-log'; + export class DriveFilesCollection implements DatabaseCollectionAdapter { @@ -67,4 +70,28 @@ export class DriveFilesCollection success: result.affected ? true : false, }; } + + async getLastUpdated(): Promise<{ + success: boolean; + result: DriveFile | null; + }> { + try { + const queryResult = await this.repository + .createQueryBuilder('drive_file') + .orderBy('datetime(drive_file.updatedAt)', 'DESC') + .getOne(); + + return { + success: true, + result: queryResult, + }; + } catch (error) { + Sentry.captureException(error); + Logger.error('Error fetching newest drive file:', error); + return { + success: false, + result: null, + }; + } + } } diff --git a/src/apps/main/database/collections/DriveFolderCollection.ts b/src/apps/main/database/collections/DriveFolderCollection.ts index 6452cf312..c5fcf02a4 100644 --- a/src/apps/main/database/collections/DriveFolderCollection.ts +++ b/src/apps/main/database/collections/DriveFolderCollection.ts @@ -2,6 +2,9 @@ import { DatabaseCollectionAdapter } from '../adapters/base'; import { AppDataSource } from '../data-source'; import { DriveFolder } from '../entities/DriveFolder'; import { Repository } from 'typeorm'; +import * as Sentry from '@sentry/electron/main'; +import Logger from 'electron-log'; + export class DriveFoldersCollection implements DatabaseCollectionAdapter { @@ -66,4 +69,28 @@ export class DriveFoldersCollection success: result.affected ? true : false, }; } + + async getLastUpdated(): Promise<{ + success: boolean; + result: DriveFolder | null; + }> { + try { + const queryResult = await this.repository + .createQueryBuilder('drive_folder') + .orderBy('datetime(drive_folder.updatedAt)', 'DESC') + .getOne(); + + return { + success: true, + result: queryResult, + }; + } catch (error) { + Sentry.captureException(error); + Logger.error('Error fetching newest drive folder:', error); + return { + success: false, + result: null, + }; + } + } } diff --git a/src/apps/main/remote-sync/RemoteSyncManager.test.ts b/src/apps/main/remote-sync/RemoteSyncManager.test.ts index 6cce45eed..b5e6347e9 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.test.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.test.ts @@ -23,6 +23,7 @@ const inMemorySyncedFilesCollection: DatabaseCollectionAdapter = { update: jest.fn(), create: jest.fn(), remove: jest.fn(), + getLastUpdated: jest.fn(), }; const inMemorySyncedFoldersCollection: DatabaseCollectionAdapter = @@ -32,6 +33,7 @@ const inMemorySyncedFoldersCollection: DatabaseCollectionAdapter = update: jest.fn(), create: jest.fn(), remove: jest.fn(), + getLastUpdated: jest.fn(), }; const createRemoteSyncedFileFixture = ( @@ -94,6 +96,12 @@ describe('RemoteSyncManager', () => { syncFolders: true, } ); + + inMemorySyncedFilesCollection.getLastUpdated = () => + Promise.resolve({ success: false, result: null }); + inMemorySyncedFoldersCollection.getLastUpdated = () => + Promise.resolve({ success: false, result: null }); + beforeEach(() => { sut = new RemoteSyncManager( { @@ -249,6 +257,11 @@ describe('RemoteSyncManager', () => { }); it('Should fail the sync if some files or folders cannot be retrieved', async () => { + inMemorySyncedFilesCollection.getLastUpdated = () => + Promise.resolve({ success: false, result: null }); + inMemorySyncedFoldersCollection.getLastUpdated = () => + Promise.resolve({ success: false, result: null }); + const sut = new RemoteSyncManager( { folders: inMemorySyncedFoldersCollection, diff --git a/src/apps/main/remote-sync/RemoteSyncManager.ts b/src/apps/main/remote-sync/RemoteSyncManager.ts index 07e6030f5..c92f347c8 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.ts @@ -1,11 +1,11 @@ import Logger from 'electron-log'; -import * as helpers from './helpers'; import { RemoteSyncStatus, RemoteSyncedFolder, RemoteSyncedFile, SyncConfig, - SYNC_OFFSET_MS, + SIX_HOURS_IN_MILLISECONDS, + rewind, WAITING_AFTER_SYNCING_DEFAULT } from './helpers'; import { reportError } from '../bug-report/service'; @@ -14,6 +14,7 @@ import { DatabaseCollectionAdapter } from '../database/adapters/base'; import { Axios } from 'axios'; import { DriveFolder } from '../database/entities/DriveFolder'; import { DriveFile } from '../database/entities/DriveFile'; +import { Nullable } from '../../shared/types/Nullable'; export class RemoteSyncManager { private foldersSyncStatus: RemoteSyncStatus = 'IDLE'; @@ -38,7 +39,7 @@ export class RemoteSyncManager { fetchFoldersLimitPerRequest: number; syncFiles: boolean; syncFolders: boolean; - } // , // private chekers: { // fileCheker: FileCheckerStatusInRoot; // } + } ) {} set placeholderStatus(status: RemoteSyncStatus) { @@ -130,27 +131,8 @@ export class RemoteSyncManager { this.changeStatus('SYNC_FAILED'); reportError(error as Error); } finally { - // const totalDuration = Date.now() - start; - - // Logger.info('-----------------'); - // Logger.info('REMOTE SYNC STATS\n'); Logger.info('Total synced files: ', this.totalFilesSynced); Logger.info('Total synced folders: ', this.totalFoldersSynced); - - // Logger.info( - // `Files sync speed: ${ - // this.totalFilesSynced / (totalDuration / 1000) - // } files/second` - // ); - - // Logger.info('Total synced folders: ', this.totalFoldersSynced); - // Logger.info( - // `Folders sync speed: ${ - // this.totalFoldersSynced / (totalDuration / 1000) - // } folders/second` - // ); - // Logger.info(`Total remote to local sync time: ${totalDuration}ms`); - // Logger.info('-----------------'); } } @@ -233,30 +215,43 @@ export class RemoteSyncManager { } } + private async getFileCheckpoint(): Promise> { + const { success, result } = await this.db.files.getLastUpdated(); + + if (!success) return undefined; + + if (!result) return undefined; + + const updatedAt = new Date(result.updatedAt); + + return rewind(updatedAt, SIX_HOURS_IN_MILLISECONDS); + } + /** * Syncs all the remote files and saves them into the local db * @param syncConfig Config to execute the sync with * @returns */ - private async syncRemoteFiles(syncConfig: SyncConfig) { - const lastFilesSyncAt = await helpers.getLastFilesSyncAt(); + private async syncRemoteFiles(syncConfig: SyncConfig, from?: Date) { + const fileCheckpoint = from ?? (await this.getFileCheckpoint()); try { Logger.info( `Syncing files updated from ${ - lastFilesSyncAt ?? '(no last date provided)' + fileCheckpoint ?? '(no last date provided)' }` ); const { hasMore, result } = await this.fetchFilesFromRemote( - lastFilesSyncAt + fileCheckpoint ); + let lastFileSynced = null; + for (const remoteFile of result) { // eslint-disable-next-line no-await-in-loop await this.createOrUpdateSyncedFileEntry(remoteFile); - const fileUpdatedAt = new Date(remoteFile.updatedAt); - helpers.saveLastFilesSyncAt(fileUpdatedAt, SYNC_OFFSET_MS); this.totalFilesSynced++; + lastFileSynced = remoteFile; } if (!hasMore) { @@ -266,16 +261,19 @@ export class RemoteSyncManager { return; } Logger.info('Retrieving more files for sync'); - await this.syncRemoteFiles({ - retry: 1, - maxRetries: syncConfig.maxRetries, - }); + await this.syncRemoteFiles( + { + retry: 1, + maxRetries: syncConfig.maxRetries, + }, + lastFileSynced ? new Date(lastFileSynced.updatedAt) : undefined + ); } catch (error) { Logger.error('Remote files sync failed with error: ', error); reportError(error as Error, { - lastFilesSyncAt: lastFilesSyncAt - ? lastFilesSyncAt.toISOString() + lastFilesSyncAt: fileCheckpoint + ? fileCheckpoint.toISOString() : 'INITIAL_FILES_SYNC', }); if (syncConfig.retry >= syncConfig.maxRetries) { @@ -292,31 +290,43 @@ export class RemoteSyncManager { } } + private async getLastFolderSyncAt(): Promise> { + const { success, result } = await this.db.folders.getLastUpdated(); + + if (!success) return undefined; + + if (!result) return undefined; + + const updatedAt = new Date(result.updatedAt); + + return rewind(updatedAt, SIX_HOURS_IN_MILLISECONDS); + } + /** * Syncs all the remote folders and saves them into the local db * @param syncConfig Config to execute the sync with * @returns */ - private async syncRemoteFolders(syncConfig: SyncConfig) { - const lastFoldersSyncAt = await helpers.getLastFoldersSyncAt(); + private async syncRemoteFolders(syncConfig: SyncConfig, from?: Date) { + const lastFolderSyncAt = from ?? (await this.getLastFolderSyncAt()); try { Logger.info( `Syncing folders updated from ${ - lastFoldersSyncAt ?? '(no last date provided)' + lastFolderSyncAt ?? '(no last date provided)' }` ); const { hasMore, result } = await this.fetchFoldersFromRemote( - lastFoldersSyncAt + lastFolderSyncAt ); + let lastFolderSynced = null; + for (const remoteFolder of result) { // eslint-disable-next-line no-await-in-loop await this.createOrUpdateSyncedFolderEntry(remoteFolder); - const foldersUpdatedAt = new Date(remoteFolder.updatedAt); - Logger.info(`Saving folders updatedAt ${foldersUpdatedAt}`); - helpers.saveLastFoldersSyncAt(foldersUpdatedAt, SYNC_OFFSET_MS); this.totalFoldersSynced++; + lastFolderSynced = remoteFolder; } if (!hasMore) { @@ -326,15 +336,18 @@ export class RemoteSyncManager { } Logger.info('Retrieving more folders for sync'); - await this.syncRemoteFolders({ - retry: 1, - maxRetries: syncConfig.maxRetries, - }); + await this.syncRemoteFolders( + { + retry: 1, + maxRetries: syncConfig.maxRetries, + }, + lastFolderSynced ? new Date(lastFolderSynced.updatedAt) : undefined + ); } catch (error) { Logger.error('Remote folders sync failed with error: ', error); reportError(error as Error, { - lastFoldersSyncAt: lastFoldersSyncAt - ? lastFoldersSyncAt.toISOString() + lastFoldersSyncAt: lastFolderSyncAt + ? lastFolderSyncAt.toISOString() : 'INITIAL_FOLDERS_SYNC', }); if (syncConfig.retry >= syncConfig.maxRetries) { diff --git a/src/apps/main/remote-sync/handlers.ts b/src/apps/main/remote-sync/handlers.ts index 05f73dfdc..ec786c0a2 100644 --- a/src/apps/main/remote-sync/handlers.ts +++ b/src/apps/main/remote-sync/handlers.ts @@ -2,7 +2,7 @@ import eventBus from '../event-bus'; import { RemoteSyncManager } from './RemoteSyncManager'; import { DriveFilesCollection } from '../database/collections/DriveFileCollection'; import { DriveFoldersCollection } from '../database/collections/DriveFolderCollection'; -import { clearRemoteSyncStore, RemoteSyncStatus } from './helpers'; +import { RemoteSyncStatus } from './helpers'; import { getNewTokenClient } from '../../shared/HttpClient/main-process-client'; import Logger from 'electron-log'; import { ipcMain } from 'electron'; @@ -115,7 +115,6 @@ eventBus.on('USER_LOGGED_IN', async () => { eventBus.on('USER_LOGGED_OUT', () => { initialSyncReady = false; remoteSyncManager.resetRemoteSync(); - clearRemoteSyncStore(); }); ipcMain.on('CHECK_SYNC', (event) => { diff --git a/src/apps/main/remote-sync/helpers.ts b/src/apps/main/remote-sync/helpers.ts index 128f447d9..c5b79fb4e 100644 --- a/src/apps/main/remote-sync/helpers.ts +++ b/src/apps/main/remote-sync/helpers.ts @@ -1,59 +1,5 @@ -import Store from 'electron-store'; - -let store: Store<{ - lastFilesSyncAt?: string; - lastFoldersSyncAt?: string; -}> | null = null; -export const getRemoteSyncStore = () => { - if (!store) { - store = new Store<{ - lastFilesSyncAt?: string; - lastFoldersSyncAt?: string; - }>({ - defaults: { - lastFilesSyncAt: undefined, - lastFoldersSyncAt: undefined, - }, - }); - - return store; - } - - return store; -}; - -export const clearRemoteSyncStore = () => getRemoteSyncStore().clear(); -export function getLastFilesSyncAt(): Date | undefined { - const value = getRemoteSyncStore().get('lastFilesSyncAt'); - - if (!value) return undefined; - - return new Date(value); -} - -export function saveLastFilesSyncAt(date: Date, offsetMs: number): Date { - getRemoteSyncStore().set( - 'lastFilesSyncAt', - new Date(date.getTime() - offsetMs).toISOString() - ); - return date; -} - -export function getLastFoldersSyncAt(): Date | undefined { - const value = getRemoteSyncStore().get('lastFoldersSyncAt'); - - if (!value) return undefined; - - return new Date(value); -} - -export function saveLastFoldersSyncAt(date: Date, offsetMs: number): Date { - getRemoteSyncStore().set( - 'lastFoldersSyncAt', - new Date(date.getTime() - offsetMs).toISOString() - ); - return date; -} +export const WAITING_AFTER_SYNCING = 1000 * 60 * 3; // 5 minutes +export const SIX_HOURS_IN_MILLISECONDS = 6 * 60 * 60 * 1000; export type RemoteSyncedFile = { id: number; @@ -108,3 +54,11 @@ export const lastSyncedAtIsNewer = ( ) => { return itemUpdatedAt.getTime() - offset > lastItemsSyncAt.getTime(); }; + +export function rewind(original: Date, milliseconds: number): Date { + const shallowCopy = new Date(original.getTime()); + + shallowCopy.setTime(shallowCopy.getTime() - milliseconds); + + return shallowCopy; +}