Skip to content

Commit

Permalink
feat: perf
Browse files Browse the repository at this point in the history
  • Loading branch information
@Dwynr
Dwynr committed Aug 15, 2024
1 parent 718cf5f commit 95bda2d
Show file tree
Hide file tree
Showing 9 changed files with 199 additions and 151 deletions.
4 changes: 2 additions & 2 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@filen/sync",
"version": "0.1.42",
"version": "0.1.43",
"description": "Filen Sync",
"main": "dist/index.js",
"types": "dist/index.d.ts",
Expand Down
3 changes: 2 additions & 1 deletion src/lib/deltas.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,8 @@ export class Deltas {
}
}

return deltas
// Work on deltas from "left to right" (ascending order, path length).
return deltas.sort((a, b) => a.path.split("/").length - b.path.split("/").length)
}
}

Expand Down
46 changes: 21 additions & 25 deletions src/lib/filesystems/local.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ export class LocalFileSystem {
public readonly itemsMutex = new Semaphore(1)
public readonly mutex = new Semaphore(1)
public readonly mkdirMutex = new Semaphore(1)
public readonly listSemaphore = new Semaphore(1024)
public readonly listSemaphore = new Semaphore(256)
public readonly watcherMutex = new Semaphore(1)

/**
Expand Down Expand Up @@ -124,16 +124,18 @@ export class LocalFileSystem {
}

const isWindows = process.platform === "win32"
const tree: LocalDirectoryTree = {}
const inodes: LocalDirectoryINodes = {}
const errors: LocalTreeError[] = []
const ignored: LocalTreeIgnored[] = []
const pathsAdded: Record<string, boolean> = {}

const dir = await fs.readdir(this.sync.syncPair.localPath, {
recursive: true,
encoding: "utf-8"
})

this.getDirectoryTreeCache.tree = {}
this.getDirectoryTreeCache.inodes = {}
this.getDirectoryTreeCache.ignored = []
this.getDirectoryTreeCache.errors = []

await promiseAllSettledChunked(
dir.map(async entry => {
await this.listSemaphore.acquire()
Expand All @@ -148,7 +150,7 @@ export class LocalFileSystem {
const itemPath = pathModule.join(this.sync.syncPair.localPath, entry)

if (isRelativePathIgnoredByDefault(entryPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath: itemPath,
relativePath: entry,
reason: "defaultIgnore"
Expand All @@ -158,7 +160,7 @@ export class LocalFileSystem {
}

if (this.sync.excludeDotFiles && pathIncludesDotFile(entryPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath: itemPath,
relativePath: entry,
reason: "dotFile"
Expand All @@ -168,7 +170,7 @@ export class LocalFileSystem {
}

if (this.sync.ignorer.ignores(entry)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath: itemPath,
relativePath: entry,
reason: "filenIgnore"
Expand All @@ -180,7 +182,7 @@ export class LocalFileSystem {
try {
await fs.access(itemPath, fs.constants.R_OK | fs.constants.W_OK)
} catch {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath: itemPath,
relativePath: entry,
reason: "permissions"
Expand All @@ -193,7 +195,7 @@ export class LocalFileSystem {
const stats = await fs.lstat(itemPath)

if (stats.isBlockDevice() || stats.isCharacterDevice() || stats.isFIFO() || stats.isSocket()) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath: itemPath,
relativePath: entry,
reason: "invalidType"
Expand All @@ -203,7 +205,7 @@ export class LocalFileSystem {
}

if (stats.isSymbolicLink()) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath: itemPath,
relativePath: entry,
reason: "symlink"
Expand All @@ -213,7 +215,7 @@ export class LocalFileSystem {
}

if (stats.isFile() && stats.size <= 0) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath: itemPath,
relativePath: entry,
reason: "empty"
Expand All @@ -225,7 +227,7 @@ export class LocalFileSystem {
const lowercasePath = entryPath.toLowerCase()

if (pathsAdded[lowercasePath]) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath: itemPath,
relativePath: entry,
reason: "duplicate"
Expand All @@ -245,14 +247,14 @@ export class LocalFileSystem {
inode: stats.ino
}

tree[entryPath] = item
inodes[item.inode] = item
this.getDirectoryTreeCache.tree[entryPath] = item
this.getDirectoryTreeCache.inodes[item.inode] = item
} catch (e) {
this.sync.worker.logger.log("error", e, "filesystems.local.getDirectoryTree")
this.sync.worker.logger.log("error", e)

if (e instanceof Error) {
errors.push({
this.getDirectoryTreeCache.errors.push({
localPath: itemPath,
relativePath: entryPath,
error: e,
Expand All @@ -266,21 +268,15 @@ export class LocalFileSystem {
})
)

this.getDirectoryTreeCache = {
timestamp: Date.now(),
tree,
inodes,
errors,
ignored
}
this.getDirectoryTreeCache.timestamp = Date.now()

return {
result: {
tree: this.getDirectoryTreeCache.tree,
inodes: this.getDirectoryTreeCache.inodes
},
errors,
ignored,
errors: this.getDirectoryTreeCache.errors,
ignored: this.getDirectoryTreeCache.ignored,
changed: true
}
}
Expand Down
54 changes: 25 additions & 29 deletions src/lib/filesystems/remote.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,11 @@ export class RemoteFileSystem {
}

const folderNames: Record<string, string> = { base: "/" }
const tree: RemoteDirectoryTree = {}
const uuids: RemoteDirectoryUUIDs = {}
const pathsAdded: Record<string, boolean> = {}
const ignored: RemoteTreeIgnored[] = []

this.getDirectoryTreeCache.ignored = []
this.getDirectoryTreeCache.tree = {}
this.getDirectoryTreeCache.uuids = {}

for (const folder of dir.folders) {
try {
Expand All @@ -180,7 +181,7 @@ export class RemoteFileSystem {
}

if (isPathOverMaxLength(localPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: folderPath,
reason: "pathLength"
Expand All @@ -190,7 +191,7 @@ export class RemoteFileSystem {
}

if (isNameOverMaxLength(decrypted.name)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: folderPath,
reason: "nameLength"
Expand All @@ -200,7 +201,7 @@ export class RemoteFileSystem {
}

if (!isValidPath(localPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: folderPath,
reason: "invalidPath"
Expand All @@ -210,7 +211,7 @@ export class RemoteFileSystem {
}

if (isRelativePathIgnoredByDefault(folderPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: folderPath,
reason: "defaultIgnore"
Expand All @@ -220,7 +221,7 @@ export class RemoteFileSystem {
}

if (this.sync.ignorer.ignores(folderPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: folderPath,
reason: "filenIgnore"
Expand All @@ -230,7 +231,7 @@ export class RemoteFileSystem {
}

if (this.sync.excludeDotFiles && pathIncludesDotFile(folderPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: folderPath,
reason: "dotFile"
Expand All @@ -242,7 +243,7 @@ export class RemoteFileSystem {
const lowercasePath = folderPath.toLowerCase()

if (pathsAdded[lowercasePath]) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: folderPath,
reason: "duplicate"
Expand All @@ -265,8 +266,8 @@ export class RemoteFileSystem {
path: folderPath
}

tree[folderPath] = item
uuids[folder[0]] = item
this.getDirectoryTreeCache.tree[folderPath] = item
this.getDirectoryTreeCache.uuids[folder[0]] = item
} catch (e) {
this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree")
this.sync.worker.logger.log("error", e)
Expand Down Expand Up @@ -295,7 +296,7 @@ export class RemoteFileSystem {
}

if (decrypted.size <= 0) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: filePath,
reason: "empty"
Expand All @@ -305,7 +306,7 @@ export class RemoteFileSystem {
}

if (isPathOverMaxLength(localPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: filePath,
reason: "pathLength"
Expand All @@ -315,7 +316,7 @@ export class RemoteFileSystem {
}

if (isNameOverMaxLength(decrypted.name)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: filePath,
reason: "nameLength"
Expand All @@ -325,7 +326,7 @@ export class RemoteFileSystem {
}

if (!isValidPath(localPath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: filePath,
reason: "invalidPath"
Expand All @@ -335,7 +336,7 @@ export class RemoteFileSystem {
}

if (isRelativePathIgnoredByDefault(filePath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: filePath,
reason: "defaultIgnore"
Expand All @@ -345,7 +346,7 @@ export class RemoteFileSystem {
}

if (this.sync.ignorer.ignores(filePath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: filePath,
reason: "filenIgnore"
Expand All @@ -355,7 +356,7 @@ export class RemoteFileSystem {
}

if (this.sync.excludeDotFiles && pathIncludesDotFile(filePath)) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: filePath,
reason: "dotFile"
Expand All @@ -367,7 +368,7 @@ export class RemoteFileSystem {
const lowercasePath = filePath.toLowerCase()

if (pathsAdded[lowercasePath]) {
ignored.push({
this.getDirectoryTreeCache.ignored.push({
localPath,
relativePath: filePath,
reason: "duplicate"
Expand Down Expand Up @@ -395,8 +396,8 @@ export class RemoteFileSystem {
path: filePath
}

tree[filePath] = item
uuids[item.uuid] = item
this.getDirectoryTreeCache.tree[filePath] = item
this.getDirectoryTreeCache.uuids[item.uuid] = item
} catch (e) {
this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree")
this.sync.worker.logger.log("error", e)
Expand All @@ -405,19 +406,14 @@ export class RemoteFileSystem {
)

this.previousTreeRawResponse = rawEx.length === 2 ? rawEx[0] ?? "" : ""
this.getDirectoryTreeCache = {
timestamp: now,
tree,
uuids,
ignored
}
this.getDirectoryTreeCache.timestamp = now

return {
result: {
tree: this.getDirectoryTreeCache.tree,
uuids: this.getDirectoryTreeCache.uuids
},
ignored,
ignored: this.getDirectoryTreeCache.ignored,
changed: true
}
}
Expand Down
Loading

0 comments on commit 95bda2d

Please sign in to comment.