From 9414e9028699a8ea9c1919f2777a1360b4e363e7 Mon Sep 17 00:00:00 2001 From: Dwynr Date: Sun, 30 Jun 2024 22:12:24 +0200 Subject: [PATCH] feat: smoke tests, more ipc events, update/remove syncs --- package-lock.json | 12 +- package.json | 2 +- src/index.ts | 9 + src/lib/filesystems/local.ts | 42 +++-- src/lib/filesystems/remote.ts | 14 ++ src/lib/sync.ts | 66 +++++++- src/lib/tasks.ts | 300 ++++++++++++++++++++++++++++++---- src/types.ts | 173 ++++++++++++++++++++ 8 files changed, 555 insertions(+), 63 deletions(-) diff --git a/package-lock.json b/package-lock.json index c4c80d0..239a4f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "@filen/sync", - "version": "0.1.2", + "version": "0.1.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@filen/sync", - "version": "0.1.2", + "version": "0.1.3", "license": "AGPLv3", "dependencies": { - "@filen/sdk": "^0.1.129", + "@filen/sdk": "^0.1.130", "@parcel/watcher": "^2.4.1", "fs-extra": "^11.2.0", "ignore": "^5.3.1", @@ -1036,9 +1036,9 @@ } }, "node_modules/@filen/sdk": { - "version": "0.1.129", - "resolved": "https://registry.npmjs.org/@filen/sdk/-/sdk-0.1.129.tgz", - "integrity": "sha512-4dSsI7bsCZXbw8NXKYPx33viinNFzM9wlpNc90V6zGU0xbFRp8HEEOsgANaMQJZjn8ksVoa4xsmNpRzp6xPn4Q==", + "version": "0.1.130", + "resolved": "https://registry.npmjs.org/@filen/sdk/-/sdk-0.1.130.tgz", + "integrity": "sha512-A07WOu3OJ3OfXZvLQmCFIIEz4t3tOIuKm5i8uaO/xGF/99AWDofY4d1fN7oX1Ps2hPUfLCvJ8YwnxQCC6PzKGw==", "dependencies": { "agentkeepalive": "^4.5.0", "axios": "^1.6.7", diff --git a/package.json b/package.json index ad61155..c30f80e 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "wait-on": "^7.2.0" }, "dependencies": { - "@filen/sdk": "^0.1.129", + "@filen/sdk": "^0.1.130", "@parcel/watcher": "^2.4.1", "fs-extra": "^11.2.0", "ignore": "^5.3.1", diff --git a/src/index.ts b/src/index.ts index 718b2c3..41ee96e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -120,6 +120,9 @@ export class SyncWorker { private async initSyncPairs(pairs: SyncPair[]): Promise { await this.initSyncPairsMutex.acquire() + const currentSyncPairsUUIDs = this.syncPairs.map(pair => pair.uuid) + const newSyncPairsUUIDs = pairs.map(pair => pair.uuid) + try { const promises: Promise[] = [] @@ -135,6 +138,12 @@ export class SyncWorker { } await Promise.all(promises) + + for (const uuid of currentSyncPairsUUIDs) { + if (!newSyncPairsUUIDs.includes(uuid) && this.syncs[uuid]) { + this.syncs[uuid]!.removed = true + } + } } catch (e) { this.logger.log("error", e, "index.initSyncPairs") diff --git a/src/lib/filesystems/local.ts b/src/lib/filesystems/local.ts index a08218d..2d060de 100644 --- a/src/lib/filesystems/local.ts +++ b/src/lib/filesystems/local.ts @@ -321,17 +321,6 @@ export class LocalFileSystem { return await fs.stat(localPath) } - /** - * Delete a file/directory inside the local sync path. - * @date 3/3/2024 - 10:05:55 PM - * - * @public - * @async - * @param {{ relativePath: string; permanent?: boolean }} param0 - * @param {string} param0.relativePath - * @param {boolean} [param0.permanent=false] - * @returns {Promise} - */ public async unlink({ relativePath, permanent = false }: { relativePath: string; permanent?: boolean }): Promise { const localPath = pathModule.join(this.sync.syncPair.localPath, relativePath) @@ -355,17 +344,6 @@ export class LocalFileSystem { }) } - /** - * Rename a file/directory inside the local sync path. Recursively creates intermediate directories if needed. - * @date 3/2/2024 - 12:41:15 PM - * - * @public - * @async - * @param {{ fromRelativePath: string; toRelativePath: string }} param0 - * @param {string} param0.fromRelativePath - * @param {string} param0.toRelativePath - * @returns {Promise} - */ public async rename({ fromRelativePath, toRelativePath }: { fromRelativePath: string; toRelativePath: string }): Promise { const fromLocalPath = pathModule.join(this.sync.syncPair.localPath, fromRelativePath) const toLocalPath = pathModule.join(this.sync.syncPair.localPath, toRelativePath) @@ -535,6 +513,26 @@ export class LocalFileSystem { delete this.sync.abortControllers[signalKey] } } + + public async isPathWritable(path: string): Promise { + try { + await fs.access(path, fs.constants.W_OK | fs.constants.R_OK) + + return true + } catch { + return false + } + } + + public async isPathReadable(path: string): Promise { + try { + await fs.access(path, fs.constants.R_OK) + + return true + } catch { + return false + } + } } export default LocalFileSystem diff --git a/src/lib/filesystems/remote.ts b/src/lib/filesystems/remote.ts index 4e31f82..cbec530 100644 --- a/src/lib/filesystems/remote.ts +++ b/src/lib/filesystems/remote.ts @@ -891,6 +891,20 @@ export class RemoteFileSystem { delete this.sync.abortControllers[signalKey] } } + + public async remotePathExisting(): Promise { + try { + const present = await this.sync.sdk.api(3).dir().present({ uuid: this.sync.syncPair.remoteParentUUID }) + + if (!present.present || present.trash) { + return false + } + + return true + } catch { + return false + } + } } export default RemoteFileSystem diff --git a/src/lib/sync.ts b/src/lib/sync.ts index 227dc76..f8463d2 100644 --- a/src/lib/sync.ts +++ b/src/lib/sync.ts @@ -46,6 +46,7 @@ export class Sync { public mode: SyncMode public excludeDotFiles: boolean public readonly worker: SyncWorker + public removed: boolean = false /** * Creates an instance of Sync. @@ -138,7 +139,14 @@ export class Sync { this.isInitialized = true try { - //local/remote smoke test + const [localSmokeTest, remoteSmokeTest] = await Promise.all([ + this.localFileSystem.isPathWritable(this.syncPair.localPath), + this.remoteFileSystem.remotePathExisting() + ]) + + if (!localSmokeTest || !remoteSmokeTest) { + throw new Error("Smoke tests failed. Either the local or remote path is not readable/writable") + } await Promise.all([ this.localFileSystem.startDirectoryWatcher(), @@ -158,6 +166,15 @@ export class Sync { } private async run(): Promise { + if (this.removed) { + postMessageToMain({ + type: "syncPairRemoved", + syncPair: this.syncPair + }) + + return + } + if (this.paused) { postMessageToMain({ type: "cyclePaused", @@ -181,6 +198,53 @@ export class Sync { syncPair: this.syncPair }) + const [localSmokeTest, remoteSmokeTest] = await Promise.all([ + this.localFileSystem.isPathWritable(this.syncPair.localPath), + this.remoteFileSystem.remotePathExisting() + ]) + + if (!localSmokeTest) { + setTimeout(() => { + this.run() + }, SYNC_INTERVAL) + + postMessageToMain({ + type: "cycleLocalSmokeTestFailed", + syncPair: this.syncPair, + data: { + error: serializeError(new Error("Local path is not writable.")) + } + }) + + postMessageToMain({ + type: "cycleRestarting", + syncPair: this.syncPair + }) + + return + } + + if (!remoteSmokeTest) { + setTimeout(() => { + this.run() + }, SYNC_INTERVAL) + + postMessageToMain({ + type: "cycleLocalSmokeTestFailed", + syncPair: this.syncPair, + data: { + error: serializeError(new Error("Remote path is not present or in the trash.")) + } + }) + + postMessageToMain({ + type: "cycleRestarting", + syncPair: this.syncPair + }) + + return + } + try { postMessageToMain({ type: "cycleWaitingForLocalDirectoryChangesStarted", diff --git a/src/lib/tasks.ts b/src/lib/tasks.ts index 1c44175..dd5b3da 100644 --- a/src/lib/tasks.ts +++ b/src/lib/tasks.ts @@ -1,9 +1,11 @@ import type Sync from "./sync" import { type Delta } from "./deltas" -import { promiseAllSettledChunked } from "../utils" +import { promiseAllSettledChunked, serializeError } from "../utils" import { type CloudItem } from "@filen/sdk" import fs from "fs-extra" import { Semaphore } from "../semaphore" +import { postMessageToMain } from "./ipc" +import pathModule from "path" export type TaskError = { path: string @@ -109,75 +111,307 @@ export class Tasks { private async processTask({ delta }: { delta: Delta }): Promise { switch (delta.type) { case "createLocalDirectory": { - const stats = await this.sync.localFileSystem.mkdir({ relativePath: delta.path }) + try { + const stats = await this.sync.localFileSystem.mkdir({ relativePath: delta.path }) - return { - ...delta, - stats + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "success", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path) + } + }) + + return { + ...delta, + stats + } + } catch (e) { + if (e instanceof Error) { + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "error", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path), + error: serializeError(e) + } + }) + } + + throw e } } case "createRemoteDirectory": { - const uuid = await this.sync.remoteFileSystem.mkdir({ relativePath: delta.path }) + try { + const uuid = await this.sync.remoteFileSystem.mkdir({ relativePath: delta.path }) - return { - ...delta, - uuid + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "success", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path) + } + }) + + return { + ...delta, + uuid + } + } catch (e) { + if (e instanceof Error) { + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "error", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path), + error: serializeError(e) + } + }) + } + + throw e } } case "deleteLocalDirectory": case "deleteLocalFile": { - await this.sync.localFileSystem.unlink({ relativePath: delta.path }) + try { + await this.sync.localFileSystem.unlink({ relativePath: delta.path }) - return delta + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "success", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path) + } + }) + + return delta + } catch (e) { + if (e instanceof Error) { + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "error", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path), + error: serializeError(e) + } + }) + } + + throw e + } } case "deleteRemoteDirectory": case "deleteRemoteFile": { - await this.sync.remoteFileSystem.unlink({ relativePath: delta.path }) + try { + await this.sync.remoteFileSystem.unlink({ relativePath: delta.path }) + + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "success", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path) + } + }) + + return delta + } catch (e) { + if (e instanceof Error) { + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "error", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path), + error: serializeError(e) + } + }) + } - return delta + throw e + } } case "renameLocalDirectory": case "renameLocalFile": { - const stats = await this.sync.localFileSystem.rename({ - fromRelativePath: delta.from, - toRelativePath: delta.to - }) + try { + const stats = await this.sync.localFileSystem.rename({ + fromRelativePath: delta.from, + toRelativePath: delta.to + }) - return { - ...delta, - stats + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "success", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path) + } + }) + + return { + ...delta, + stats + } + } catch (e) { + if (e instanceof Error) { + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "error", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path), + error: serializeError(e) + } + }) + } + + throw e } } case "renameRemoteDirectory": case "renameRemoteFile": { - await this.sync.remoteFileSystem.rename({ - fromRelativePath: delta.from, - toRelativePath: delta.to - }) + try { + await this.sync.remoteFileSystem.rename({ + fromRelativePath: delta.from, + toRelativePath: delta.to + }) - return delta + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "success", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path) + } + }) + + return delta + } catch (e) { + if (e instanceof Error) { + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "error", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path), + error: serializeError(e) + } + }) + } + + throw e + } } case "downloadFile": { - const stats = await this.sync.remoteFileSystem.download({ relativePath: delta.path }) + try { + const stats = await this.sync.remoteFileSystem.download({ relativePath: delta.path }) - return { - ...delta, - stats + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "success", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path) + } + }) + + return { + ...delta, + stats + } + } catch (e) { + if (e instanceof Error) { + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "error", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path), + error: serializeError(e) + } + }) + } + + throw e } } case "uploadFile": { - const item = await this.sync.localFileSystem.upload({ relativePath: delta.path }) + try { + const item = await this.sync.localFileSystem.upload({ relativePath: delta.path }) + + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "success", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path) + } + }) + + return { + ...delta, + item + } + } catch (e) { + if (e instanceof Error) { + postMessageToMain({ + type: "transfer", + syncPair: this.sync.syncPair, + data: { + of: delta.type, + type: "error", + relativePath: delta.path, + localPath: pathModule.join(this.sync.syncPair.localPath, delta.path), + error: serializeError(e) + } + }) + } - return { - ...delta, - item + throw e } } } diff --git a/src/types.ts b/src/types.ts index 730d59a..ca39fee 100644 --- a/src/types.ts +++ b/src/types.ts @@ -7,6 +7,7 @@ import { type SerializedError } from "./utils" export type SyncMode = "twoWay" | "localToCloud" | "localBackup" | "cloudToLocal" | "cloudBackup" export type SyncPair = { + name: string uuid: string localPath: string remotePath: string @@ -81,6 +82,162 @@ export type SyncMessage = localPath: string error: SerializedError } + | { + of: "createLocalDirectory" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "createLocalDirectory" + type: "success" + relativePath: string + localPath: string + } + | { + of: "createRemoteDirectory" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "createRemoteDirectory" + type: "success" + relativePath: string + localPath: string + } + | { + of: "deleteLocalFile" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "deleteLocalFile" + type: "success" + relativePath: string + localPath: string + } + | { + of: "deleteLocalDirectory" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "deleteLocalDirectory" + type: "success" + relativePath: string + localPath: string + } + | { + of: "deleteRemoteFile" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "deleteRemoteFile" + type: "success" + relativePath: string + localPath: string + } + | { + of: "deleteRemoteDirectory" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "deleteRemoteDirectory" + type: "success" + relativePath: string + localPath: string + } + | { + of: "renameLocalFile" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "renameLocalFile" + type: "success" + relativePath: string + localPath: string + } + | { + of: "renameLocalDirectory" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "renameLocalDirectory" + type: "success" + relativePath: string + localPath: string + } + | { + of: "renameRemoteDirectory" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "renameRemoteDirectory" + type: "success" + relativePath: string + localPath: string + } + | { + of: "renameRemoteFile" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "renameRemoteFile" + type: "success" + relativePath: string + localPath: string + } + | { + of: "downloadFile" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "downloadFile" + type: "success" + relativePath: string + localPath: string + } + | { + of: "uploadFile" + type: "error" + relativePath: string + localPath: string + error: SerializedError + } + | { + of: "uploadFile" + type: "success" + relativePath: string + localPath: string + } } | { type: "localTreeErrors" @@ -146,6 +303,18 @@ export type SyncMessage = | { type: "cycleProcessingDeltasDone" } + | { + type: "cycleLocalSmokeTestFailed" + data: { + error: SerializedError + } + } + | { + type: "cycleRemoteSmokeTestFailed" + data: { + error: SerializedError + } + } | { type: "cycleNoChanges" } @@ -251,3 +420,7 @@ export type SyncMessage = type: "syncPairIncludeDotFiles" syncPair: SyncPair } + | { + type: "syncPairRemoved" + syncPair: SyncPair + }