[_] fix: ensure newer files are included

src/apps/main/analytics/user-handlers.ts

@@ -6,7 +6,6 @@ import {
   userSignin,
   userSigninFailed,
 } from './service';
-import { clearRemoteSyncStore } from '../remote-sync/helpers';
 import { clearTempFolder } from '../app-info/helpers';
 
 eventBus.on('USER_LOGGED_IN', () => {
@@ -16,7 +15,6 @@ eventBus.on('USER_LOGGED_IN', () => {
 
 eventBus.on('USER_LOGGED_OUT', () => {
   userLogout();
-  clearRemoteSyncStore();
   clearTempFolder();
 });
 

src/apps/main/database/adapters/base.ts

@@ -1,20 +1,20 @@
-export abstract class DatabaseCollectionAdapter<DatabaseItemType> {
+export interface DatabaseCollectionAdapter<DatabaseItemType> {
   /**
    * Used to initialize the database adapter
    */
-  abstract connect(): Promise<{ success: boolean }>;
+  connect(): Promise<{ success: boolean }>;
 
   /**
    * Gets an item from the database
    */
-  abstract get(
+  get(
     itemId: string
   ): Promise<{ success: boolean; result: DatabaseItemType | null }>;
 
   /**
    * Updates an item in the database
    */
-  abstract update(
+  update(
     itemId: string,
     updatePayload: Partial<DatabaseItemType>
   ): Promise<{
@@ -25,15 +25,20 @@ export abstract class DatabaseCollectionAdapter<DatabaseItemType> {
   /**
    * Creates an item in the database
    */
-  abstract create(creationPayload: DatabaseItemType): Promise<{
+  create(creationPayload: DatabaseItemType): Promise<{
     success: boolean;
     result: DatabaseItemType | null;
   }>;
 
   /**
    * Removes an item from the database
    */
-  abstract remove(itemId: string): Promise<{
+  remove(itemId: string): Promise<{
     success: boolean;
   }>;
+
+  getLastUpdated(): Promise<{
+    success: boolean;
+    result: DatabaseItemType | null;
+  }>;
 }

src/apps/main/database/collections/DriveFileCollection.ts

@@ -2,6 +2,9 @@ import { DatabaseCollectionAdapter } from '../adapters/base';
 import { AppDataSource } from '../data-source';
 import { DriveFile } from '../entities/DriveFile';
 import { Repository } from 'typeorm';
+import * as Sentry from '@sentry/electron/main';
+import Logger from 'electron-log';
+
 export class DriveFilesCollection
   implements DatabaseCollectionAdapter<DriveFile>
 {
@@ -67,4 +70,28 @@ export class DriveFilesCollection
       success: result.affected ? true : false,
     };
   }
+
+  async getLastUpdated(): Promise<{
+    success: boolean;
+    result: DriveFile | null;
+  }> {
+    try {
+      const queryResult = await this.repository
+        .createQueryBuilder('drive_file')
+        .orderBy('datetime(drive_file.updatedAt)', 'DESC')
+        .getOne();
+
+      return {
+        success: true,
+        result: queryResult,
+      };
+    } catch (error) {
+      Sentry.captureException(error);
+      Logger.error('Error fetching newest drive file:', error);
+      return {
+        success: false,
+        result: null,
+      };
+    }
+  }
 }

src/apps/main/database/collections/DriveFolderCollection.ts

@@ -2,6 +2,9 @@ import { DatabaseCollectionAdapter } from '../adapters/base';
 import { AppDataSource } from '../data-source';
 import { DriveFolder } from '../entities/DriveFolder';
 import { Repository } from 'typeorm';
+import * as Sentry from '@sentry/electron/main';
+import Logger from 'electron-log';
+
 export class DriveFoldersCollection
   implements DatabaseCollectionAdapter<DriveFolder>
 {
@@ -66,4 +69,28 @@ export class DriveFoldersCollection
       success: result.affected ? true : false,
     };
   }
+
+  async getLastUpdated(): Promise<{
+    success: boolean;
+    result: DriveFolder | null;
+  }> {
+    try {
+      const queryResult = await this.repository
+        .createQueryBuilder('drive_folder')
+        .orderBy('datetime(drive_folder.updatedAt)', 'DESC')
+        .getOne();
+
+      return {
+        success: true,
+        result: queryResult,
+      };
+    } catch (error) {
+      Sentry.captureException(error);
+      Logger.error('Error fetching newest drive folder:', error);
+      return {
+        success: false,
+        result: null,
+      };
+    }
+  }
 }

src/apps/main/remote-sync/RemoteSyncManager.test.ts

@@ -23,6 +23,7 @@ const inMemorySyncedFilesCollection: DatabaseCollectionAdapter<DriveFile> = {
   update: jest.fn(),
   create: jest.fn(),
   remove: jest.fn(),
+  getLastUpdated: jest.fn(),
 };
 
 const inMemorySyncedFoldersCollection: DatabaseCollectionAdapter<DriveFolder> =
@@ -32,6 +33,7 @@ const inMemorySyncedFoldersCollection: DatabaseCollectionAdapter<DriveFolder> =
     update: jest.fn(),
     create: jest.fn(),
     remove: jest.fn(),
+    getLastUpdated: jest.fn(),
   };
 
 const createRemoteSyncedFileFixture = (
@@ -94,6 +96,12 @@ describe('RemoteSyncManager', () => {
       syncFolders: true,
     }
   );
+
+  inMemorySyncedFilesCollection.getLastUpdated = () =>
+    Promise.resolve({ success: false, result: null });
+  inMemorySyncedFoldersCollection.getLastUpdated = () =>
+    Promise.resolve({ success: false, result: null });
+
   beforeEach(() => {
     sut = new RemoteSyncManager(
       {
@@ -249,6 +257,11 @@ describe('RemoteSyncManager', () => {
     });
 
     it('Should fail the sync if some files or folders cannot be retrieved', async () => {
+      inMemorySyncedFilesCollection.getLastUpdated = () =>
+        Promise.resolve({ success: false, result: null });
+      inMemorySyncedFoldersCollection.getLastUpdated = () =>
+        Promise.resolve({ success: false, result: null });
+
       const sut = new RemoteSyncManager(
         {
           folders: inMemorySyncedFoldersCollection,

src/apps/main/remote-sync/RemoteSyncManager.ts

@@ -1,11 +1,11 @@
 import Logger from 'electron-log';
-import * as helpers from './helpers';
 import {
   RemoteSyncStatus,
   RemoteSyncedFolder,
   RemoteSyncedFile,
   SyncConfig,
-  SYNC_OFFSET_MS,
+  SIX_HOURS_IN_MILLISECONDS,
+  rewind,
   WAITING_AFTER_SYNCING_DEFAULT
 } from './helpers';
 import { reportError } from '../bug-report/service';
@@ -14,6 +14,7 @@ import { DatabaseCollectionAdapter } from '../database/adapters/base';
 import { Axios } from 'axios';
 import { DriveFolder } from '../database/entities/DriveFolder';
 import { DriveFile } from '../database/entities/DriveFile';
+import { Nullable } from '../../shared/types/Nullable';
 
 export class RemoteSyncManager {
   private foldersSyncStatus: RemoteSyncStatus = 'IDLE';
@@ -38,7 +39,7 @@ export class RemoteSyncManager {
       fetchFoldersLimitPerRequest: number;
       syncFiles: boolean;
       syncFolders: boolean;
-    } // , // private chekers: { //   fileCheker: FileCheckerStatusInRoot; // }
+    }
   ) {}
 
   set placeholderStatus(status: RemoteSyncStatus) {
@@ -130,27 +131,8 @@ export class RemoteSyncManager {
       this.changeStatus('SYNC_FAILED');
       reportError(error as Error);
     } finally {
-      // const totalDuration = Date.now() - start;
-
-      // Logger.info('-----------------');
-      // Logger.info('REMOTE SYNC STATS\n');
       Logger.info('Total synced files: ', this.totalFilesSynced);
       Logger.info('Total synced folders: ', this.totalFoldersSynced);
-
-      // Logger.info(
-      //   `Files sync speed: ${
-      //     this.totalFilesSynced / (totalDuration / 1000)
-      //   } files/second`
-      // );
-
-      // Logger.info('Total synced folders: ', this.totalFoldersSynced);
-      // Logger.info(
-      //   `Folders sync speed: ${
-      //     this.totalFoldersSynced / (totalDuration / 1000)
-      //   } folders/second`
-      // );
-      // Logger.info(`Total remote to local sync time: ${totalDuration}ms`);
-      // Logger.info('-----------------');
     }
   }
 
@@ -233,30 +215,43 @@ export class RemoteSyncManager {
     }
   }
 
+  private async getFileCheckpoint(): Promise<Nullable<Date>> {
+    const { success, result } = await this.db.files.getLastUpdated();
+
+    if (!success) return undefined;
+
+    if (!result) return undefined;
+
+    const updatedAt = new Date(result.updatedAt);
+
+    return rewind(updatedAt, SIX_HOURS_IN_MILLISECONDS);
+  }
+
   /**
    * Syncs all the remote files and saves them into the local db
    * @param syncConfig Config to execute the sync with
    * @returns
    */
-  private async syncRemoteFiles(syncConfig: SyncConfig) {
-    const lastFilesSyncAt = await helpers.getLastFilesSyncAt();
+  private async syncRemoteFiles(syncConfig: SyncConfig, from?: Date) {
+    const fileCheckpoint = from ?? (await this.getFileCheckpoint());
     try {
       Logger.info(
         `Syncing files updated from ${
-          lastFilesSyncAt ?? '(no last date provided)'
+          fileCheckpoint ?? '(no last date provided)'
         }`
       );
       const { hasMore, result } = await this.fetchFilesFromRemote(
-        lastFilesSyncAt
+        fileCheckpoint
       );
 
+      let lastFileSynced = null;
+
       for (const remoteFile of result) {
         // eslint-disable-next-line no-await-in-loop
         await this.createOrUpdateSyncedFileEntry(remoteFile);
-        const fileUpdatedAt = new Date(remoteFile.updatedAt);
 
-        helpers.saveLastFilesSyncAt(fileUpdatedAt, SYNC_OFFSET_MS);
         this.totalFilesSynced++;
+        lastFileSynced = remoteFile;
       }
 
       if (!hasMore) {
@@ -266,16 +261,19 @@ export class RemoteSyncManager {
         return;
       }
       Logger.info('Retrieving more files for sync');
-      await this.syncRemoteFiles({
-        retry: 1,
-        maxRetries: syncConfig.maxRetries,
-      });
+      await this.syncRemoteFiles(
+        {
+          retry: 1,
+          maxRetries: syncConfig.maxRetries,
+        },
+        lastFileSynced ? new Date(lastFileSynced.updatedAt) : undefined
+      );
     } catch (error) {
       Logger.error('Remote files sync failed with error: ', error);
 
       reportError(error as Error, {
-        lastFilesSyncAt: lastFilesSyncAt
-          ? lastFilesSyncAt.toISOString()
+        lastFilesSyncAt: fileCheckpoint
+          ? fileCheckpoint.toISOString()
           : 'INITIAL_FILES_SYNC',
       });
       if (syncConfig.retry >= syncConfig.maxRetries) {
@@ -292,31 +290,43 @@ export class RemoteSyncManager {
     }
   }
 
+  private async getLastFolderSyncAt(): Promise<Nullable<Date>> {
+    const { success, result } = await this.db.folders.getLastUpdated();
+
+    if (!success) return undefined;
+
+    if (!result) return undefined;
+
+    const updatedAt = new Date(result.updatedAt);
+
+    return rewind(updatedAt, SIX_HOURS_IN_MILLISECONDS);
+  }
+
   /**
    * Syncs all the remote folders and saves them into the local db
    * @param syncConfig Config to execute the sync with
    * @returns
    */
-  private async syncRemoteFolders(syncConfig: SyncConfig) {
-    const lastFoldersSyncAt = await helpers.getLastFoldersSyncAt();
+  private async syncRemoteFolders(syncConfig: SyncConfig, from?: Date) {
+    const lastFolderSyncAt = from ?? (await this.getLastFolderSyncAt());
     try {
       Logger.info(
         `Syncing folders updated from ${
-          lastFoldersSyncAt ?? '(no last date provided)'
+          lastFolderSyncAt ?? '(no last date provided)'
         }`
       );
       const { hasMore, result } = await this.fetchFoldersFromRemote(
-        lastFoldersSyncAt
+        lastFolderSyncAt
       );
 
+      let lastFolderSynced = null;
+
       for (const remoteFolder of result) {
         // eslint-disable-next-line no-await-in-loop
         await this.createOrUpdateSyncedFolderEntry(remoteFolder);
-        const foldersUpdatedAt = new Date(remoteFolder.updatedAt);
 
-        Logger.info(`Saving folders updatedAt ${foldersUpdatedAt}`);
-        helpers.saveLastFoldersSyncAt(foldersUpdatedAt, SYNC_OFFSET_MS);
         this.totalFoldersSynced++;
+        lastFolderSynced = remoteFolder;
       }
 
       if (!hasMore) {
@@ -326,15 +336,18 @@ export class RemoteSyncManager {
       }
 
       Logger.info('Retrieving more folders for sync');
-      await this.syncRemoteFolders({
-        retry: 1,
-        maxRetries: syncConfig.maxRetries,
-      });
+      await this.syncRemoteFolders(
+        {
+          retry: 1,
+          maxRetries: syncConfig.maxRetries,
+        },
+        lastFolderSynced ? new Date(lastFolderSynced.updatedAt) : undefined
+      );
     } catch (error) {
       Logger.error('Remote folders sync failed with error: ', error);
       reportError(error as Error, {
-        lastFoldersSyncAt: lastFoldersSyncAt
-          ? lastFoldersSyncAt.toISOString()
+        lastFoldersSyncAt: lastFolderSyncAt
+          ? lastFolderSyncAt.toISOString()
           : 'INITIAL_FOLDERS_SYNC',
       });
       if (syncConfig.retry >= syncConfig.maxRetries) {