diff --git a/package-lock.json b/package-lock.json index 37738b3..63ab97b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "@filen/sync", - "version": "0.1.91", + "version": "0.1.92", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@filen/sync", - "version": "0.1.91", + "version": "0.1.92", "license": "AGPLv3", "dependencies": { - "@filen/sdk": "^0.1.189", + "@filen/sdk": "^0.1.191", "@parcel/watcher": "^2.5.0", "fast-glob": "^3.3.2", "fs-extra": "^11.2.0", @@ -1053,9 +1053,9 @@ } }, "node_modules/@filen/sdk": { - "version": "0.1.189", - "resolved": "https://registry.npmjs.org/@filen/sdk/-/sdk-0.1.189.tgz", - "integrity": "sha512-GVU/xAH1Rs0A0haQl9HpR47nIGjwYCsiapjy8d9WB7cK5te867qTkNqQXecRS+mNJ7V27vRhmaVOWvcBbdEuEg==", + "version": "0.1.191", + "resolved": "https://registry.npmjs.org/@filen/sdk/-/sdk-0.1.191.tgz", + "integrity": "sha512-JwlDuUKaGMKgX54AHkSVCHKbEappi4S/3qX1CcWY76ZZt/+RDr3AdzMKIvt7hgM9YFAEzByBsnHk/6GloIR6rA==", "license": "AGPLv3", "dependencies": { "agentkeepalive": "^4.5.0", diff --git a/package.json b/package.json index e72ae54..c50e98d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@filen/sync", - "version": "0.1.91", + "version": "0.1.92", "description": "Filen Sync", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -54,7 +54,7 @@ "wait-on": "^7.2.0" }, "dependencies": { - "@filen/sdk": "^0.1.189", + "@filen/sdk": "^0.1.191", "@parcel/watcher": "^2.5.0", "fast-glob": "^3.3.2", "fs-extra": "^11.2.0", diff --git a/src/constants.ts b/src/constants.ts index 4ae222c..753aebb 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -9,7 +9,6 @@ export const DEFAULT_IGNORED = { "desktop.ini", "thumbs.db", "ntuser.dat", - ".trash", ".filen.trash.local", "AUX", "PRN", @@ -46,15 +45,7 @@ export const DEFAULT_IGNORED = { "*:/ProgramData/**/*", "*:/Program Files/**/*", "*:/Program Files (x86)/**/*", - "/share/Trash/**/*", "*:/System Volume Information/**/*" ], - relativeGlobs: [ - ".filen.trash.local/**/*", - "$RECYCLE.BIN/**/*", - ".Trash/**/*", - ".local/share/Trash/**/*", - "local/share/Trash/**/*", - "System Volume Information/**/*" - ] + relativeGlobs: [".filen.trash.local/**/*", "$RECYCLE.BIN/**/*", "System Volume Information/**/*"] } diff --git a/src/lib/filesystems/local.ts b/src/lib/filesystems/local.ts index dfd638b..2270f12 100644 --- a/src/lib/filesystems/local.ts +++ b/src/lib/filesystems/local.ts @@ -14,7 +14,7 @@ import { import pathModule from "path" import process from "process" import type Sync from "../sync" -import { SYNC_INTERVAL, LOCAL_TRASH_NAME, DEFAULT_IGNORED } from "../../constants" +import { SYNC_INTERVAL, LOCAL_TRASH_NAME } from "../../constants" import crypto from "crypto" import { pipeline } from "stream" import { promisify } from "util" @@ -23,7 +23,7 @@ import { postMessageToMain } from "../ipc" import { Semaphore } from "../../semaphore" import { v4 as uuidv4 } from "uuid" import { type Watcher } from "node-watch" -import FastGlob, { type Entry } from "fast-glob" +import FastGlob from "fast-glob" const pipelineAsync = promisify(pipeline) @@ -118,16 +118,16 @@ export class LocalFileSystem { this.sync = sync } - public isPathIgnored(entry: Required): { ignored: true; reason: LocalTreeIgnoredReason } | { ignored: false } { - if (this.ignoredCache.get(entry.path)) { - return this.ignoredCache.get(entry.path)! + public isPathIgnored( + relativePath: string, + absolutePath: string + ): { ignored: true; reason: LocalTreeIgnoredReason } | { ignored: false } { + if (this.ignoredCache.get(relativePath)) { + return this.ignoredCache.get(relativePath)! } - const entryPath = entry.path.startsWith("/") ? entry.path : "/" + entry.path - const absolutePath = pathModule.join(this.sync.syncPair.localPath, entry.path) - if (isPathOverMaxLength(absolutePath)) { - this.ignoredCache.set(entry.path, { + this.ignoredCache.set(relativePath, { ignored: true, reason: "pathLength" }) @@ -138,8 +138,8 @@ export class LocalFileSystem { } } - if (isNameOverMaxLength(entry.name)) { - this.ignoredCache.set(entry.path, { + if (isNameOverMaxLength(pathModule.basename(absolutePath))) { + this.ignoredCache.set(relativePath, { ignored: true, reason: "nameLength" }) @@ -151,7 +151,7 @@ export class LocalFileSystem { } if (!isValidPath(absolutePath)) { - this.ignoredCache.set(entry.path, { + this.ignoredCache.set(relativePath, { ignored: true, reason: "invalidPath" }) @@ -162,8 +162,8 @@ export class LocalFileSystem { } } - if (isRelativePathIgnoredByDefault(entryPath) || isAbsolutePathIgnoredByDefault(absolutePath)) { - this.ignoredCache.set(entry.path, { + if (isRelativePathIgnoredByDefault(relativePath) || isAbsolutePathIgnoredByDefault(absolutePath)) { + this.ignoredCache.set(relativePath, { ignored: true, reason: "defaultIgnore" }) @@ -174,8 +174,8 @@ export class LocalFileSystem { } } - if (this.sync.excludeDotFiles && pathIncludesDotFile(entryPath)) { - this.ignoredCache.set(entry.path, { + if (this.sync.excludeDotFiles && pathIncludesDotFile(relativePath)) { + this.ignoredCache.set(relativePath, { ignored: true, reason: "dotFile" }) @@ -186,8 +186,8 @@ export class LocalFileSystem { } } - if (this.sync.ignorer.ignores(entry.path)) { - this.ignoredCache.set(entry.path, { + if (this.sync.ignorer.ignores(relativePath)) { + this.ignoredCache.set(relativePath, { ignored: true, reason: "filenIgnore" }) @@ -198,7 +198,7 @@ export class LocalFileSystem { } } - this.ignoredCache.set(entry.path, { + this.ignoredCache.set(relativePath, { ignored: false }) @@ -259,21 +259,10 @@ export class LocalFileSystem { followSymbolicLinks: false, deep: Infinity, fs, - ignore: [ - "**/.filen.trash.local/**/*", - "**/$RECYCLE.BIN/**/*", - "**/System Volume Information/**/*", - ...DEFAULT_IGNORED.relativeGlobs.map(glob => `**/${glob}`), - ...DEFAULT_IGNORED.absoluteGlobs.map(glob => { - const normalized = glob.replace("*:", "") - - return `**${normalized.startsWith("/") ? "" : "/"}${normalized}` - }) - ], - suppressErrors: false, - stats: true, + suppressErrors: true, + stats: false, unique: false, - objectMode: true + objectMode: false }) stream.on("error", err => { @@ -300,14 +289,19 @@ export class LocalFileSystem { break } - const entryItem = entry as unknown as Required - const entryPath = "/" + entryItem.path + const entryItem = entry as unknown as string | null + + if (!entryItem) { + continue + } + + const entryPath = "/" + entryItem if (entryPath.includes(LOCAL_TRASH_NAME)) { continue } - const absolutePath = pathModule.join(this.sync.syncPair.localPath, entryItem.path) + const absolutePath = pathModule.join(this.sync.syncPair.localPath, entryItem) const lowercasePath = entryPath.toLowerCase() if (pathsAdded[lowercasePath]) { @@ -322,12 +316,33 @@ export class LocalFileSystem { pathsAdded[lowercasePath] = true - if ( - entryItem.dirent.isBlockDevice() || - entryItem.dirent.isCharacterDevice() || - entryItem.dirent.isFIFO() || - entryItem.dirent.isSocket() - ) { + const ignored = this.isPathIgnored(entryItem, absolutePath) + + if (ignored.ignored) { + this.getDirectoryTreeCache.ignored.push({ + localPath: absolutePath, + relativePath: entryPath, + reason: ignored.reason + }) + + continue + } + + let stats: fs.Stats | null = null + + try { + stats = await fs.stat(absolutePath) + } catch { + this.getDirectoryTreeCache.ignored.push({ + localPath: absolutePath, + relativePath: entryPath, + reason: "permissions" + }) + + continue + } + + if (stats.isBlockDevice() || stats.isCharacterDevice() || stats.isFIFO() || stats.isSocket()) { this.getDirectoryTreeCache.ignored.push({ localPath: absolutePath, relativePath: entryPath, @@ -337,7 +352,7 @@ export class LocalFileSystem { continue } - if (entryItem.dirent.isSymbolicLink()) { + if (stats.isSymbolicLink()) { this.getDirectoryTreeCache.ignored.push({ localPath: absolutePath, relativePath: entryPath, @@ -347,7 +362,7 @@ export class LocalFileSystem { continue } - if (entryItem.dirent.isFile() && entryItem.stats.size <= 0) { + if (stats.isFile() && stats.size <= 0) { this.getDirectoryTreeCache.ignored.push({ localPath: absolutePath, relativePath: entryPath, @@ -374,25 +389,13 @@ export class LocalFileSystem { continue } - const ignored = this.isPathIgnored(entry as unknown as Required) - - if (ignored.ignored) { - this.getDirectoryTreeCache.ignored.push({ - localPath: absolutePath, - relativePath: entryPath, - reason: ignored.reason - }) - - continue - } - const item: LocalItem = { - lastModified: normalizeUTime(entryItem.stats.mtimeMs), // Sometimes comes as a float, but we need an int - type: entryItem.dirent.isDirectory() ? "directory" : "file", + lastModified: normalizeUTime(stats.mtimeMs), // Sometimes comes as a float, but we need an int + type: stats.isDirectory() ? "directory" : "file", path: entryPath, - creation: normalizeUTime(entryItem.stats.birthtimeMs), // Sometimes comes as a float, but we need an int - size: entryItem.stats.size, - inode: entryItem.stats.ino + creation: normalizeUTime(stats.birthtimeMs), // Sometimes comes as a float, but we need an int + size: stats.size, + inode: stats.ino } this.getDirectoryTreeCache.tree[entryPath] = item diff --git a/src/lib/filesystems/remote.ts b/src/lib/filesystems/remote.ts index f73388f..fd783eb 100644 --- a/src/lib/filesystems/remote.ts +++ b/src/lib/filesystems/remote.ts @@ -71,7 +71,7 @@ export class RemoteFileSystem { private readonly mutex = new Semaphore(1) private readonly mkdirMutex = new Semaphore(1) public readonly itemsMutex = new Semaphore(1) - public readonly listSemaphore = new Semaphore(128) + public readonly listSemaphore = new Semaphore(1024) private deviceIdCache: string = "" public ignoredCache = new Map() diff --git a/src/lib/state.ts b/src/lib/state.ts index 234a7fe..f4ce695 100644 --- a/src/lib/state.ts +++ b/src/lib/state.ts @@ -452,13 +452,17 @@ export class State { followSymbolicLinks: false, deep: 0, fs, - suppressErrors: false, + suppressErrors: true, stats: false, unique: true, objectMode: false }) for (const entry of dir) { + if (!entry) { + continue + } + if (entry.trim().endsWith(".tmp")) { await fs.rm(pathModule.join(this.statePath, entry), { force: true, diff --git a/src/lib/sync.ts b/src/lib/sync.ts index ebd74e3..4fe0ff7 100644 --- a/src/lib/sync.ts +++ b/src/lib/sync.ts @@ -159,13 +159,17 @@ export class Sync { followSymbolicLinks: false, deep: 0, fs, - suppressErrors: false, + suppressErrors: true, stats: true, unique: true, objectMode: true }) for (const entry of dir) { + if (!entry) { + return + } + if (entry.stats && entry.stats.atimeMs + 86400000 * 30 < now) { await fs.rm(pathModule.join(localTrashPath, entry.path), { force: true,