diff --git a/package-lock.json b/package-lock.json index 1251a63..5252425 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,17 @@ { "name": "@filen/sync", - "version": "0.1.62", + "version": "0.1.66", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@filen/sync", - "version": "0.1.62", + "version": "0.1.66", "license": "AGPLv3", "dependencies": { "@filen/sdk": "^0.1.167", "@parcel/watcher": "^2.4.1", + "fast-glob": "^3.3.2", "fs-extra": "^11.2.0", "ignore": "^5.3.1", "node-watch": "^0.7.4", @@ -2049,7 +2050,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -2062,7 +2062,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "engines": { "node": ">= 8" } @@ -2071,7 +2070,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -4378,7 +4376,7 @@ "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -4394,7 +4392,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -4427,7 +4424,6 @@ "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", - "dev": true, "dependencies": { "reusify": "^1.0.4" } @@ -7149,7 +7145,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, "engines": { "node": ">= 8" } @@ -7715,7 +7710,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -7844,7 +7838,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true, "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" @@ -7906,7 +7899,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", diff --git a/package.json b/package.json index 046a169..4bdd9f9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@filen/sync", - "version": "0.1.66", + "version": "0.1.67", "description": "Filen Sync", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -13,6 +13,7 @@ "clear": "rimraf ./dist", "build": "npm run clear && npm run lint && npm run tsc", "dev": "tsx ./dev/index.ts", + "dev:test": "tsx ./dev/test.ts", "yalc": "npm run build && yalc push", "install:filen": "npm install @filen/sdk@latest" }, @@ -54,6 +55,7 @@ "dependencies": { "@filen/sdk": "^0.1.167", "@parcel/watcher": "^2.4.1", + "fast-glob": "^3.3.2", "fs-extra": "^11.2.0", "ignore": "^5.3.1", "node-watch": "^0.7.4", diff --git a/src/lib/filesystems/local.ts b/src/lib/filesystems/local.ts index 8ada2be..1e4ab50 100644 --- a/src/lib/filesystems/local.ts +++ b/src/lib/filesystems/local.ts @@ -1,7 +1,6 @@ import fs from "fs-extra" import watcher from "@parcel/watcher" import { - promiseAllChunked, isRelativePathIgnoredByDefault, serializeError, replacePathStartWithFromAndTo, @@ -20,6 +19,7 @@ import { postMessageToMain } from "../ipc" import { Semaphore } from "../../semaphore" import { v4 as uuidv4 } from "uuid" import { type Watcher } from "node-watch" +import FastGlob, { type Entry } from "fast-glob" const pipelineAsync = promisify(pipeline) @@ -114,76 +114,117 @@ export class LocalFileSystem { ignored: LocalTreeIgnored[] changed: boolean }> { - if ( - this.lastDirectoryChangeTimestamp > 0 && - this.getDirectoryTreeCache.timestamp > 0 && - this.lastDirectoryChangeTimestamp < this.getDirectoryTreeCache.timestamp - ) { - return { - result: { - tree: this.getDirectoryTreeCache.tree, - inodes: this.getDirectoryTreeCache.inodes - }, - errors: this.getDirectoryTreeCache.errors, - ignored: this.getDirectoryTreeCache.ignored, - changed: false - } - } + return new Promise<{ + result: LocalTree + errors: LocalTreeError[] + ignored: LocalTreeIgnored[] + changed: boolean + // eslint-disable-next-line no-async-promise-executor + }>(async (resolve, reject) => { + try { + if ( + this.lastDirectoryChangeTimestamp > 0 && + this.getDirectoryTreeCache.timestamp > 0 && + this.lastDirectoryChangeTimestamp < this.getDirectoryTreeCache.timestamp + ) { + resolve({ + result: { + tree: this.getDirectoryTreeCache.tree, + inodes: this.getDirectoryTreeCache.inodes + }, + errors: this.getDirectoryTreeCache.errors, + ignored: this.getDirectoryTreeCache.ignored, + changed: false + }) - const isWindows = process.platform === "win32" - const pathsAdded: Record = {} + return + } - const dir = await fs.readdir(this.sync.syncPair.localPath, { - recursive: true, - encoding: "utf-8" - }) + this.getDirectoryTreeCache.tree = {} + this.getDirectoryTreeCache.inodes = {} + this.getDirectoryTreeCache.ignored = [] + this.getDirectoryTreeCache.errors = [] + + const pathsAdded: Record = {} + let didError = false + let didErrorErr: Error = new Error("Could not read local directory.") + const stream = FastGlob.stream("**/*", { + dot: true, + onlyDirectories: false, + onlyFiles: false, + throwErrorOnBrokenSymbolicLink: false, + cwd: this.sync.syncPair.localPath, + followSymbolicLinks: false, + deep: Infinity, + fs, + ignore: [".filen.trash.local/**/*"], + suppressErrors: false, + stats: true, + unique: true, + objectMode: true + }) + + stream.on("error", err => { + didError = true + didErrorErr = err + + reject(err) + }) - this.getDirectoryTreeCache.tree = {} - this.getDirectoryTreeCache.inodes = {} - this.getDirectoryTreeCache.ignored = [] - this.getDirectoryTreeCache.errors = [] + if (didError) { + this.getDirectoryTreeCache.tree = {} + this.getDirectoryTreeCache.inodes = {} + this.getDirectoryTreeCache.ignored = [] + this.getDirectoryTreeCache.errors = [] + this.getDirectoryTreeCache.timestamp = 0 + + reject(didErrorErr) + + return + } - await promiseAllChunked( - dir.map(async entry => { - await this.listSemaphore.acquire() + for await (const entry of stream) { + if (didError) { + break + } - try { - const entryPath = `/${isWindows ? entry.replace(/\\/g, "/") : entry}` + const entryItem = entry as unknown as Required + const entryPath = "/" + entryItem.path if (entryPath.includes(LOCAL_TRASH_NAME)) { - return + continue } - const itemPath = pathModule.join(this.sync.syncPair.localPath, entry) + const itemPath = pathModule.join(this.sync.syncPair.localPath, entryItem.path) if (isRelativePathIgnoredByDefault(entryPath)) { this.getDirectoryTreeCache.ignored.push({ localPath: itemPath, - relativePath: entry, + relativePath: entryItem.path, reason: "defaultIgnore" }) - return + continue } if (this.sync.excludeDotFiles && pathIncludesDotFile(entryPath)) { this.getDirectoryTreeCache.ignored.push({ localPath: itemPath, - relativePath: entry, + relativePath: entryItem.path, reason: "dotFile" }) - return + continue } - if (this.sync.ignorer.ignores(entry)) { + if (this.sync.ignorer.ignores(entryItem.path)) { this.getDirectoryTreeCache.ignored.push({ localPath: itemPath, - relativePath: entry, + relativePath: entryItem.path, reason: "filenIgnore" }) - return + continue } try { @@ -191,108 +232,109 @@ export class LocalFileSystem { } catch { this.getDirectoryTreeCache.ignored.push({ localPath: itemPath, - relativePath: entry, + relativePath: entryItem.path, reason: "permissions" }) - return + continue } - try { - const stats = await fs.lstat(itemPath) + if ( + entryItem.dirent.isBlockDevice() || + entryItem.dirent.isCharacterDevice() || + entryItem.dirent.isFIFO() || + entryItem.dirent.isSocket() + ) { + this.getDirectoryTreeCache.ignored.push({ + localPath: itemPath, + relativePath: entryItem.path, + reason: "invalidType" + }) - if (stats.isBlockDevice() || stats.isCharacterDevice() || stats.isFIFO() || stats.isSocket()) { - this.getDirectoryTreeCache.ignored.push({ - localPath: itemPath, - relativePath: entry, - reason: "invalidType" - }) + continue + } - return - } + if (entryItem.dirent.isSymbolicLink()) { + this.getDirectoryTreeCache.ignored.push({ + localPath: itemPath, + relativePath: entryItem.path, + reason: "symlink" + }) - if (stats.isSymbolicLink()) { - this.getDirectoryTreeCache.ignored.push({ - localPath: itemPath, - relativePath: entry, - reason: "symlink" - }) + continue + } - return - } + if (entryItem.dirent.isFile() && entryItem.stats.size <= 0) { + this.getDirectoryTreeCache.ignored.push({ + localPath: itemPath, + relativePath: entryItem.path, + reason: "empty" + }) - if (stats.isFile() && stats.size <= 0) { - this.getDirectoryTreeCache.ignored.push({ - localPath: itemPath, - relativePath: entry, - reason: "empty" - }) + continue + } - return - } + const lowercasePath = entryPath.toLowerCase() - const lowercasePath = entryPath.toLowerCase() + if (pathsAdded[lowercasePath]) { + this.getDirectoryTreeCache.ignored.push({ + localPath: itemPath, + relativePath: entryItem.path, + reason: "duplicate" + }) - if (pathsAdded[lowercasePath]) { - this.getDirectoryTreeCache.ignored.push({ - localPath: itemPath, - relativePath: entry, - reason: "duplicate" - }) + continue + } - return - } + pathsAdded[lowercasePath] = true - pathsAdded[lowercasePath] = true + const item: LocalItem = { + lastModified: normalizeUTime(entryItem.stats.mtimeMs), // Sometimes comes as a float, but we need an int + type: entryItem.dirent.isDirectory() ? "directory" : "file", + path: entryPath, + creation: normalizeUTime(entryItem.stats.birthtimeMs), // Sometimes comes as a float, but we need an int + size: entryItem.stats.size, + inode: entryItem.stats.ino + } - const item: LocalItem = { - lastModified: normalizeUTime(stats.mtimeMs), // Sometimes comes as a float, but we need an int - type: stats.isDirectory() ? "directory" : "file", - path: entryPath, - creation: normalizeUTime(stats.birthtimeMs), // Sometimes comes as a float, but we need an int - size: stats.size, - inode: stats.ino - } + this.getDirectoryTreeCache.tree[entryPath] = item + this.getDirectoryTreeCache.inodes[item.inode] = item + } - this.getDirectoryTreeCache.tree[entryPath] = item - this.getDirectoryTreeCache.inodes[item.inode] = item - } catch (e) { - this.sync.worker.logger.log("error", e, "filesystems.local.getDirectoryTree") - this.sync.worker.logger.log("error", e) - - if (e instanceof Error) { - this.getDirectoryTreeCache.errors.push({ - localPath: itemPath, - relativePath: entryPath, - error: e, - uuid: uuidv4() - }) - } - } - } finally { - this.listSemaphore.release() + if (didError) { + this.getDirectoryTreeCache.tree = {} + this.getDirectoryTreeCache.inodes = {} + this.getDirectoryTreeCache.ignored = [] + this.getDirectoryTreeCache.errors = [] + this.getDirectoryTreeCache.timestamp = 0 + + reject(didErrorErr) + + return } - }) - ) - this.getDirectoryTreeCache.timestamp = Date.now() + this.getDirectoryTreeCache.timestamp = Date.now() - // Clear old local file hashes that are not present anymore - for (const path in this.sync.localFileHashes) { - if (!this.getDirectoryTreeCache.tree[path]) { - delete this.sync.localFileHashes[path] - } - } + // Clear old local file hashes that are not present anymore + for (const path in this.sync.localFileHashes) { + if (!this.getDirectoryTreeCache.tree[path]) { + delete this.sync.localFileHashes[path] + } + } - return { - result: { - tree: this.getDirectoryTreeCache.tree, - inodes: this.getDirectoryTreeCache.inodes - }, - errors: this.getDirectoryTreeCache.errors, - ignored: this.getDirectoryTreeCache.ignored, - changed: true - } + resolve({ + result: { + tree: this.getDirectoryTreeCache.tree, + inodes: this.getDirectoryTreeCache.inodes + }, + errors: this.getDirectoryTreeCache.errors, + ignored: this.getDirectoryTreeCache.ignored, + changed: true + }) + } catch (e) { + reject(e) + } + }) } /** diff --git a/src/lib/state.ts b/src/lib/state.ts index 8dfaf97..ee1356a 100644 --- a/src/lib/state.ts +++ b/src/lib/state.ts @@ -7,6 +7,7 @@ import { type DoneTask } from "./tasks" import { replacePathStartWithFromAndTo, normalizeUTime } from "../utils" import readline from "readline" import { v4 as uuidv4 } from "uuid" +import FastGlob from "fast-glob" const STATE_VERSION = 2 @@ -442,15 +443,24 @@ export class State { public async loadPreviousTrees(): Promise { await fs.ensureDir(this.statePath) - // Clear leftover .tmp files from previous runs - const dir = await fs.readdir(this.statePath, { - recursive: false, - encoding: "utf-8" + const dir = await FastGlob.async("**/*", { + dot: true, + onlyDirectories: false, + onlyFiles: true, + throwErrorOnBrokenSymbolicLink: false, + cwd: this.statePath, + followSymbolicLinks: false, + deep: 0, + fs, + suppressErrors: false, + stats: false, + unique: true, + objectMode: false }) - for (const file of dir) { - if (file.endsWith(".tmp")) { - await fs.rm(pathModule.join(this.statePath, file), { + for (const entry of dir) { + if (entry.trim().endsWith(".tmp")) { + await fs.rm(pathModule.join(this.statePath, entry), { force: true, maxRetries: 60 * 10, recursive: true, diff --git a/src/lib/sync.ts b/src/lib/sync.ts index 54261bb..fb08e4a 100644 --- a/src/lib/sync.ts +++ b/src/lib/sync.ts @@ -8,12 +8,13 @@ import Tasks, { type TaskError } from "./tasks" import State from "./state" import { postMessageToMain } from "./ipc" import Ignorer from "../ignorer" -import { serializeError, promiseAllChunked } from "../utils" +import { serializeError } from "../utils" import type SyncWorker from ".." import Lock from "./lock" import pathModule from "path" import fs from "fs-extra" import { v4 as uuidv4 } from "uuid" +import FastGlob from "fast-glob" /** * Sync @@ -138,26 +139,31 @@ export class Sync { if (await fs.exists(localTrashPath)) { const now = Date.now() - const dir = await fs.readdir(localTrashPath, { - recursive: false, - encoding: "utf-8" + const dir = await FastGlob.async("**/*", { + dot: true, + onlyDirectories: false, + onlyFiles: true, + throwErrorOnBrokenSymbolicLink: false, + cwd: localTrashPath, + followSymbolicLinks: false, + deep: 0, + fs, + suppressErrors: false, + stats: true, + unique: true, + objectMode: true }) - await promiseAllChunked( - dir.map(async entry => { - const entryPath = pathModule.join(localTrashPath, entry) - const stat = await fs.stat(entryPath) - - if (stat.atimeMs + 86400000 * 30 < now) { - await fs.rm(entryPath, { - force: true, - maxRetries: 60 * 10, - recursive: true, - retryDelay: 100 - }) - } - }) - ) + for (const entry of dir) { + if (entry.stats && entry.stats.atimeMs + 86400000 * 30 < now) { + await fs.rm(pathModule.join(localTrashPath, entry.path), { + force: true, + maxRetries: 60 * 10, + recursive: true, + retryDelay: 100 + }) + } + } } } catch (e) { this.worker.logger.log("error", e, "sync.cleanupLocalTrash")