diff --git a/src/apps/main/database/adapters/base.ts b/src/apps/main/database/adapters/base.ts index d91fec112..b0622e5e9 100644 --- a/src/apps/main/database/adapters/base.ts +++ b/src/apps/main/database/adapters/base.ts @@ -1,20 +1,20 @@ -export abstract class DatabaseCollectionAdapter { +export interface DatabaseCollectionAdapter { /** * Used to initialize the database adapter */ - abstract connect(): Promise<{ success: boolean }>; + connect(): Promise<{ success: boolean }>; /** * Gets an item from the database */ - abstract get( + get( itemId: string ): Promise<{ success: boolean; result: DatabaseItemType | null }>; /** * Updates an item in the database */ - abstract update( + update( itemId: string, updatePayload: Partial ): Promise<{ @@ -25,7 +25,7 @@ export abstract class DatabaseCollectionAdapter { /** * Creates an item in the database */ - abstract create(creationPayload: DatabaseItemType): Promise<{ + create(creationPayload: DatabaseItemType): Promise<{ success: boolean; result: DatabaseItemType | null; }>; @@ -33,7 +33,12 @@ export abstract class DatabaseCollectionAdapter { /** * Removes an item from the database */ - abstract remove(itemId: string): Promise<{ + remove(itemId: string): Promise<{ success: boolean; }>; + + getLastUpdated(): Promise<{ + success: boolean; + result: DatabaseItemType | null; + }>; } diff --git a/src/apps/main/database/collections/DriveFileCollection.ts b/src/apps/main/database/collections/DriveFileCollection.ts index c338afb2d..0be3bfc0c 100644 --- a/src/apps/main/database/collections/DriveFileCollection.ts +++ b/src/apps/main/database/collections/DriveFileCollection.ts @@ -2,6 +2,9 @@ import { DatabaseCollectionAdapter } from '../adapters/base'; import { AppDataSource } from '../data-source'; import { DriveFile } from '../entities/DriveFile'; import { Repository } from 'typeorm'; +import * as Sentry from '@sentry/electron/main'; +import Logger from 'electron-log'; + export class DriveFilesCollection implements DatabaseCollectionAdapter { @@ -67,4 +70,32 @@ export class DriveFilesCollection success: result.affected ? true : false, }; } + + async getLastUpdated(): Promise<{ + success: boolean; + result: DriveFile | null; + }> { + try { + const queryResult = await this.repository + .createQueryBuilder('drive_file') + .orderBy( + // eslint-disable-next-line quotes + "STR_TO_DATE(drive_file.updatedAt, '%Y-%m-%dT%H:%i:%sZ')", + 'DESC' + ) + .getOne(); + + return { + success: true, + result: queryResult, + }; + } catch (error) { + Sentry.captureException(error); + Logger.error('Error fetching newest drive file:', error); + return { + success: false, + result: null, + }; + } + } } diff --git a/src/apps/main/database/collections/DriveFolderCollection.ts b/src/apps/main/database/collections/DriveFolderCollection.ts index 6452cf312..cf74bdac5 100644 --- a/src/apps/main/database/collections/DriveFolderCollection.ts +++ b/src/apps/main/database/collections/DriveFolderCollection.ts @@ -2,6 +2,9 @@ import { DatabaseCollectionAdapter } from '../adapters/base'; import { AppDataSource } from '../data-source'; import { DriveFolder } from '../entities/DriveFolder'; import { Repository } from 'typeorm'; +import * as Sentry from '@sentry/electron/main'; +import Logger from 'electron-log'; + export class DriveFoldersCollection implements DatabaseCollectionAdapter { @@ -66,4 +69,32 @@ export class DriveFoldersCollection success: result.affected ? true : false, }; } + + async getLastUpdated(): Promise<{ + success: boolean; + result: DriveFolder | null; + }> { + try { + const queryResult = await this.repository + .createQueryBuilder('drive_folder') + .orderBy( + // eslint-disable-next-line quotes + "STR_TO_DATE(drive_folder.updatedAt, '%Y-%m-%dT%H:%i:%sZ')", + 'DESC' + ) + .getOne(); + + return { + success: true, + result: queryResult, + }; + } catch (error) { + Sentry.captureException(error); + Logger.error('Error fetching newest drive folder:', error); + return { + success: false, + result: null, + }; + } + } } diff --git a/src/apps/main/remote-sync/RemoteSyncManager.ts b/src/apps/main/remote-sync/RemoteSyncManager.ts index 6fc982cb0..dbb972396 100644 --- a/src/apps/main/remote-sync/RemoteSyncManager.ts +++ b/src/apps/main/remote-sync/RemoteSyncManager.ts @@ -1,12 +1,12 @@ import Logger from 'electron-log'; -import * as helpers from './helpers'; import { RemoteSyncStatus, RemoteSyncedFolder, RemoteSyncedFile, SyncConfig, - SYNC_OFFSET_MS, WAITING_AFTER_SYNCING, + SIX_HOURS_IN_MILLISECONDS, + rewind, } from './helpers'; import { reportError } from '../bug-report/service'; @@ -14,6 +14,7 @@ import { DatabaseCollectionAdapter } from '../database/adapters/base'; import { Axios } from 'axios'; import { DriveFolder } from '../database/entities/DriveFolder'; import { DriveFile } from '../database/entities/DriveFile'; +import { Nullable } from '../../shared/types/Nullable'; export class RemoteSyncManager { private foldersSyncStatus: RemoteSyncStatus = 'IDLE'; @@ -38,7 +39,7 @@ export class RemoteSyncManager { fetchFoldersLimitPerRequest: number; syncFiles: boolean; syncFolders: boolean; - } // , // private chekers: { // fileCheker: FileCheckerStatusInRoot; // } + } ) {} set placeholderStatus(status: RemoteSyncStatus) { @@ -131,27 +132,8 @@ export class RemoteSyncManager { this.changeStatus('SYNC_FAILED'); reportError(error as Error); } finally { - // const totalDuration = Date.now() - start; - - // Logger.info('-----------------'); - // Logger.info('REMOTE SYNC STATS\n'); Logger.info('Total synced files: ', this.totalFilesSynced); Logger.info('Total synced folders: ', this.totalFoldersSynced); - - // Logger.info( - // `Files sync speed: ${ - // this.totalFilesSynced / (totalDuration / 1000) - // } files/second` - // ); - - // Logger.info('Total synced folders: ', this.totalFoldersSynced); - // Logger.info( - // `Folders sync speed: ${ - // this.totalFoldersSynced / (totalDuration / 1000) - // } folders/second` - // ); - // Logger.info(`Total remote to local sync time: ${totalDuration}ms`); - // Logger.info('-----------------'); } } @@ -234,21 +216,33 @@ export class RemoteSyncManager { } } + private async getFileCheckpoint(): Promise> { + const { success, result } = await this.db.files.getLastUpdated(); + + if (!success) return undefined; + + if (!result) return undefined; + + const updatedAt = new Date(result.updatedAt); + + return rewind(updatedAt, SIX_HOURS_IN_MILLISECONDS); + } + /** * Syncs all the remote files and saves them into the local db * @param syncConfig Config to execute the sync with * @returns */ private async syncRemoteFiles(syncConfig: SyncConfig, from?: Date) { - const lastFilesSyncAt = from ?? helpers.getLastFilesSyncAt(); + const fileCheckpoint = from ?? (await this.getFileCheckpoint()); try { Logger.info( `Syncing files updated from ${ - lastFilesSyncAt ?? '(no last date provided)' + fileCheckpoint ?? '(no last date provided)' }` ); const { hasMore, result } = await this.fetchFilesFromRemote( - lastFilesSyncAt + fileCheckpoint ); let lastFileSynced = null; @@ -256,9 +250,7 @@ export class RemoteSyncManager { for (const remoteFile of result) { // eslint-disable-next-line no-await-in-loop await this.createOrUpdateSyncedFileEntry(remoteFile); - const fileUpdatedAt = new Date(remoteFile.updatedAt); - helpers.saveLastFilesSyncAt(fileUpdatedAt, SYNC_OFFSET_MS); this.totalFilesSynced++; lastFileSynced = remoteFile; } @@ -281,8 +273,8 @@ export class RemoteSyncManager { Logger.error('Remote files sync failed with error: ', error); reportError(error as Error, { - lastFilesSyncAt: lastFilesSyncAt - ? lastFilesSyncAt.toISOString() + lastFilesSyncAt: fileCheckpoint + ? fileCheckpoint.toISOString() : 'INITIAL_FILES_SYNC', }); if (syncConfig.retry >= syncConfig.maxRetries) { @@ -299,21 +291,33 @@ export class RemoteSyncManager { } } + private async getLastFolderSyncAt(): Promise> { + const { success, result } = await this.db.folders.getLastUpdated(); + + if (!success) return undefined; + + if (!result) return undefined; + + const updatedAt = new Date(result.updatedAt); + + return rewind(updatedAt, SIX_HOURS_IN_MILLISECONDS); + } + /** * Syncs all the remote folders and saves them into the local db * @param syncConfig Config to execute the sync with * @returns */ private async syncRemoteFolders(syncConfig: SyncConfig, from?: Date) { - const lastFoldersSyncAt = from ?? helpers.getLastFoldersSyncAt(); + const lastFolderSyncAt = from ?? (await this.getLastFolderSyncAt()); try { Logger.info( `Syncing folders updated from ${ - lastFoldersSyncAt ?? '(no last date provided)' + lastFolderSyncAt ?? '(no last date provided)' }` ); const { hasMore, result } = await this.fetchFoldersFromRemote( - lastFoldersSyncAt + lastFolderSyncAt ); let lastFolderSynced = null; @@ -321,10 +325,7 @@ export class RemoteSyncManager { for (const remoteFolder of result) { // eslint-disable-next-line no-await-in-loop await this.createOrUpdateSyncedFolderEntry(remoteFolder); - const foldersUpdatedAt = new Date(remoteFolder.updatedAt); - Logger.info(`Saving folders updatedAt ${foldersUpdatedAt}`); - helpers.saveLastFoldersSyncAt(foldersUpdatedAt, SYNC_OFFSET_MS); this.totalFoldersSynced++; lastFolderSynced = remoteFolder; } @@ -346,8 +347,8 @@ export class RemoteSyncManager { } catch (error) { Logger.error('Remote folders sync failed with error: ', error); reportError(error as Error, { - lastFoldersSyncAt: lastFoldersSyncAt - ? lastFoldersSyncAt.toISOString() + lastFoldersSyncAt: lastFolderSyncAt + ? lastFolderSyncAt.toISOString() : 'INITIAL_FOLDERS_SYNC', }); if (syncConfig.retry >= syncConfig.maxRetries) { diff --git a/src/apps/main/remote-sync/handlers.ts b/src/apps/main/remote-sync/handlers.ts index d79e8a915..552f6d545 100644 --- a/src/apps/main/remote-sync/handlers.ts +++ b/src/apps/main/remote-sync/handlers.ts @@ -2,7 +2,7 @@ import eventBus from '../event-bus'; import { RemoteSyncManager } from './RemoteSyncManager'; import { DriveFilesCollection } from '../database/collections/DriveFileCollection'; import { DriveFoldersCollection } from '../database/collections/DriveFolderCollection'; -import { clearRemoteSyncStore, RemoteSyncStatus } from './helpers'; +import { RemoteSyncStatus } from './helpers'; import { getNewTokenClient } from '../../shared/HttpClient/main-process-client'; import Logger from 'electron-log'; import { ipcMain } from 'electron'; @@ -99,7 +99,6 @@ eventBus.on('USER_LOGGED_IN', async () => { eventBus.on('USER_LOGGED_OUT', () => { initialSyncReady = false; remoteSyncManager.resetRemoteSync(); - clearRemoteSyncStore(); }); ipcMain.on('CHECK_SYNC', (event) => { diff --git a/src/apps/main/remote-sync/helpers.ts b/src/apps/main/remote-sync/helpers.ts index d164a9fe9..d9fa66370 100644 --- a/src/apps/main/remote-sync/helpers.ts +++ b/src/apps/main/remote-sync/helpers.ts @@ -1,70 +1,5 @@ -import Store from 'electron-store'; - -const SIX_HOURS_IN_MILLISECONDS = 6 * 60 * 60 * 1000; - -let store: Store<{ - lastFilesSyncAt?: string; - lastFoldersSyncAt?: string; -}> | null = null; -export const getRemoteSyncStore = () => { - if (!store) { - store = new Store<{ - lastFilesSyncAt?: string; - lastFoldersSyncAt?: string; - }>({ - defaults: { - lastFilesSyncAt: undefined, - lastFoldersSyncAt: undefined, - }, - }); - - return store; - } - - return store; -}; - -export const clearRemoteSyncStore = () => getRemoteSyncStore().clear(); - -export function getLastFilesSyncAt(): Date | undefined { - const value = getRemoteSyncStore().get('lastFilesSyncAt'); - - if (!value) return undefined; - - const date = new Date(value); - - date.setTime(date.getTime() - SIX_HOURS_IN_MILLISECONDS); - - return date; -} - -export function saveLastFilesSyncAt(date: Date, offsetMs: number): Date { - getRemoteSyncStore().set( - 'lastFilesSyncAt', - new Date(date.getTime() - offsetMs).toISOString() - ); - return date; -} - -export function getLastFoldersSyncAt(): Date | undefined { - const value = getRemoteSyncStore().get('lastFoldersSyncAt'); - - if (!value) return undefined; - - const date = new Date(value); - - date.setTime(date.getTime() - SIX_HOURS_IN_MILLISECONDS); - - return date; -} - -export function saveLastFoldersSyncAt(date: Date, offsetMs: number): Date { - getRemoteSyncStore().set( - 'lastFoldersSyncAt', - new Date(date.getTime() - offsetMs).toISOString() - ); - return date; -} +export const WAITING_AFTER_SYNCING = 1000 * 60 * 3; // 5 minutes +export const SIX_HOURS_IN_MILLISECONDS = 6 * 60 * 60 * 1000; export type RemoteSyncedFile = { id: number; @@ -109,9 +44,6 @@ export type SyncConfig = { maxRetries: number; }; -export const SYNC_OFFSET_MS = 0; -export const WAITING_AFTER_SYNCING = 1000 * 60 * 3; // 5 minutes - export const lastSyncedAtIsNewer = ( itemUpdatedAt: Date, lastItemsSyncAt: Date, @@ -119,3 +51,11 @@ export const lastSyncedAtIsNewer = ( ) => { return itemUpdatedAt.getTime() - offset > lastItemsSyncAt.getTime(); }; + +export function rewind(original: Date, milliseconds: number): Date { + const shallowCopy = new Date(original.getTime()); + + shallowCopy.setTime(shallowCopy.getTime() - milliseconds); + + return shallowCopy; +}