Skip to content

Commit

Permalink
rm: default ignore, ipc throttle, unused
Browse files Browse the repository at this point in the history
  • Loading branch information
Dwynr committed Jul 25, 2024
1 parent 27cc70d commit 6a6bc7e
Show file tree
Hide file tree
Showing 7 changed files with 37 additions and 229 deletions.
47 changes: 1 addition & 46 deletions src/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,50 +2,5 @@ export const SYNC_INTERVAL = 5000
export const LOCAL_TRASH_NAME: string = ".filen.trash.local"
export const DEFAULT_IGNORED = {
names: [".ds_store"],
extensions: [".tmp", ".temp", ".ffs_tmp", ".temporary", ".crdownload", ".~cr", ".thumbdata"],
directories: [
LOCAL_TRASH_NAME,
"$RECYCLE.BIN",
".Trash",
".local/share/Trash",
"/share/Trash",
"local/share/Trash",
"/AppData/Local",
"/AppData/Roaming"
],
system: [
"C:\\$WINDOWS.~BT",
"C:\\$Windows.~WS",
"C:\\$WinREAgent",
"C:\\Windows",
"C:\\OneDriveTemp",
"C:\\PerfLogs",
"C:\\ProgramData",
"C:\\Program Files\\Uninstall Information",
"C:\\Program Files\\WindowsApps",
"C:\\Program Files\\Windows Defender",
"C:\\Program Files\\Windows Mail",
"C:\\Program Files\\Windows Media Player",
"C:\\Program Files\\Windows Multimedia Platform",
"C:\\Program Files\\Windows NT",
"C:\\Program Files\\Windows Photo Viewer",
"C:\\Program Files\\Windows Portable Devices",
"C:\\Program Files\\Windows Security",
"C:\\Program Files\\WindowsPowerShell",
"C:\\Program Files (x86)\\Uninstall Information",
"C:\\Program Files (x86)\\WindowsApps",
"C:\\Program Files (x86)\\Windows Defender",
"C:\\Program Files (x86)\\Windows Mail",
"C:\\Program Files (x86)\\Windows Media Player",
"C:\\Program Files (x86)\\Windows Multimedia Platform",
"C:\\Program Files (x86)\\Windows NT",
"C:\\Program Files (x86)\\Windows Photo Viewer",
"C:\\Program Files (x86)\\Windows Portable Devices",
"C:\\Program Files (x86)\\Windows Security",
"C:\\Program Files (x86)\\WindowsPowerShell",
"C:\\Program Files (x86)\\Internet Explorer",
"C:\\Program Files (x86)\\Microsoft",
"C:\\Program Files (x86)\\WindowsPowerShell",
"C:\\Program Files (x86)\\Reference Assemblies"
]
extensions: [".tmp", ".temp", ".ffs_tmp", ".temporary", ".crdownload", ".~cr", ".thumbdata"]
}
25 changes: 12 additions & 13 deletions src/lib/filesystems/local.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ import watcher from "@parcel/watcher"
import {
promiseAllSettledChunked,
isRelativePathIgnoredByDefault,
isDirectoryPathIgnoredByDefault,
isSystemPathIgnoredByDefault,
serializeError,
replacePathStartWithFromAndTo,
pathIncludesDotFile
Expand Down Expand Up @@ -138,19 +136,14 @@ export class LocalFileSystem {

try {
const entryPath = `/${isWindows ? entry.replace(/\\/g, "/") : entry}`
const itemName = pathModule.posix.basename(entryPath)

if (itemName.startsWith(LOCAL_TRASH_NAME)) {
if (entryPath.includes(LOCAL_TRASH_NAME)) {
return
}

const itemPath = pathModule.join(this.sync.syncPair.localPath, entry)

if (
isDirectoryPathIgnoredByDefault(entryPath) ||
isRelativePathIgnoredByDefault(entryPath) ||
isSystemPathIgnoredByDefault(itemPath)
) {
if (isRelativePathIgnoredByDefault(entryPath)) {
ignored.push({
localPath: itemPath,
relativePath: entry,
Expand Down Expand Up @@ -299,11 +292,17 @@ export class LocalFileSystem {
return
}

this.watcherInstance = await watcher.subscribe(this.sync.syncPair.localPath, (err, events) => {
if (!err && events && events.length > 0) {
this.lastDirectoryChangeTimestamp = Date.now()
this.watcherInstance = await watcher.subscribe(
this.sync.syncPair.localPath,
(err, events) => {
if (!err && events && events.length > 0) {
this.lastDirectoryChangeTimestamp = Date.now()
}
},
{
ignore: [".filen.trash.local"]
}
})
)
}

/**
Expand Down
5 changes: 2 additions & 3 deletions src/lib/filesystems/remote.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import {
isPathOverMaxLength,
isNameOverMaxLength,
isValidPath,
isDirectoryPathIgnoredByDefault,
isRelativePathIgnoredByDefault,
serializeError,
replacePathStartWithFromAndTo,
Expand Down Expand Up @@ -198,7 +197,7 @@ export class RemoteFileSystem {
continue
}

if (isDirectoryPathIgnoredByDefault(folderPath) || isRelativePathIgnoredByDefault(folderPath)) {
if (isRelativePathIgnoredByDefault(folderPath)) {
ignored.push({
localPath,
relativePath: folderPath,
Expand Down Expand Up @@ -322,7 +321,7 @@ export class RemoteFileSystem {
return
}

if (isDirectoryPathIgnoredByDefault(filePath) || isRelativePathIgnoredByDefault(filePath)) {
if (isRelativePathIgnoredByDefault(filePath)) {
ignored.push({
localPath,
relativePath: filePath,
Expand Down
80 changes: 11 additions & 69 deletions src/lib/ipc.ts
Original file line number Diff line number Diff line change
@@ -1,84 +1,26 @@
import { isMainThread, parentPort } from "worker_threads"
import { type SyncMessage } from "../types"

const postMessageToMainProgressThrottle: Record<
string,
{
next: number
storedBytes: number
}
> = {}

// We have to throttle the "progress" events of the "download"/"upload" message type. The SDK sends too many events for the IPC to handle properly.
// It freezes the main process if we don't throttle it.
export function throttlePostMessageToMain(message: SyncMessage, callback: (message: SyncMessage) => void) {
const now = Date.now()
let key = ""

if (message.type === "transfer" && message.data.type === "progress") {
key = `${message.type}:${message.data.relativePath}:${message.data.localPath}:${message.data.type}`

if (!postMessageToMainProgressThrottle[key]) {
postMessageToMainProgressThrottle[key] = {
next: 0,
storedBytes: 0
}
}

postMessageToMainProgressThrottle[key]!.storedBytes += message.data.bytes

if (postMessageToMainProgressThrottle[key]!.next > now) {
return
}

message = {
...message,
data: {
...message.data,
bytes: postMessageToMainProgressThrottle[key]!.storedBytes
}
}
}

callback(message)

if (key.length > 0 && postMessageToMainProgressThrottle[key] && message.type === "transfer") {
postMessageToMainProgressThrottle[key]!.storedBytes = 0
postMessageToMainProgressThrottle[key]!.next = now + 100

if (
message.data.type === "error" ||
message.data.type === "queued" ||
// TODO: Stopped event (message.data.type === "stopped")
message.data.type === "finished"
) {
delete postMessageToMainProgressThrottle[key]
}
}
}

/**
* Send a message to the main thread if we are running inside a worker thread, otherwise send it to the process interface.
*
* @export
* @param {SyncMessage} message
*/
export function postMessageToMain(message: SyncMessage): void {
throttlePostMessageToMain(message, throttledMessage => {
if (process.onMessage) {
process.onMessage(message)
if (process.onMessage) {
process.onMessage(message)

return
}

if (isMainThread || !parentPort) {
if (process.send) {
process.send(throttledMessage)
}
return
}

return
if (isMainThread || !parentPort) {
if (process.send) {
process.send(message)
}

parentPort.postMessage(throttledMessage)
})
return
}

parentPort.postMessage(message)
}
15 changes: 10 additions & 5 deletions src/lib/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,11 +143,6 @@ export class Sync {

private async run(): Promise<void> {
if (this.removed) {
postMessageToMain({
type: "syncPairRemoved",
syncPair: this.syncPair
})

return
}

Expand Down Expand Up @@ -190,6 +185,11 @@ export class Sync {
syncPair: this.syncPair
})

postMessageToMain({
type: "cycleSuccess",
syncPair: this.syncPair
})

postMessageToMain({
type: "cycleRestarting",
syncPair: this.syncPair
Expand Down Expand Up @@ -276,6 +276,11 @@ export class Sync {
})
}

postMessageToMain({
type: "cycleSuccess",
syncPair: this.syncPair
})

postMessageToMain({
type: "cycleNoChanges",
syncPair: this.syncPair
Expand Down
68 changes: 0 additions & 68 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -372,27 +372,6 @@ export type SyncMessage =
| {
type: "cyclePaused"
}
| {
type: "stopTransfer"
data: {
of: "upload" | "download"
relativePath: string
}
}
| {
type: "pauseTransfer"
data: {
of: "upload" | "download"
relativePath: string
}
}
| {
type: "resumeTransfer"
data: {
of: "upload" | "download"
relativePath: string
}
}
))
| {
type: "updateSyncPairs"
Expand All @@ -407,50 +386,3 @@ export type SyncMessage =
error: SerializedError
}
}
| {
type: "syncPairsUpdated"
}
| {
type: "updateLocalIgnorer"
syncPair: SyncPair
data?: {
content?: string
}
}
| {
type: "updateRemoteIgnorer"
syncPair: SyncPair
data?: {
content?: string
}
}
| {
type: "resetSyncPairCache"
}
| {
type: "pauseSyncPair"
syncPair: SyncPair
}
| {
type: "resumeSyncPair"
syncPair: SyncPair
}
| {
type: "changeSyncPairMode"
syncPair: SyncPair
data: {
mode: SyncMode
}
}
| {
type: "syncPairExcludeDotFiles"
syncPair: SyncPair
}
| {
type: "syncPairIncludeDotFiles"
syncPair: SyncPair
}
| {
type: "syncPairRemoved"
syncPair: SyncPair
}
26 changes: 1 addition & 25 deletions src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,16 +123,12 @@ export const isValidPath = memoize((path: string): boolean => {
})

export const isNameIgnoredByDefault = memoize((name: string): boolean => {
const nameLowercase = name.toLowerCase()
const nameLowercase = name.toLowerCase().trim()
const extension = pathModule.extname(name)
const extensionLowercase = extension.toLowerCase()

if (
name.length === 0 ||
name.startsWith(" ") ||
name.endsWith(" ") ||
name.includes("\n") ||
name.includes("\r") ||
nameLowercase.startsWith(".~lock.") ||
nameLowercase.startsWith(".~lock") ||
nameLowercase.startsWith("~$") ||
Expand All @@ -149,26 +145,6 @@ export const isRelativePathIgnoredByDefault = memoize((path: string): boolean =>
return path.split("/").some(part => part.length > 0 && isNameIgnoredByDefault(part))
})

export const isSystemPathIgnoredByDefault = memoize((path: string): boolean => {
for (const systemPath of DEFAULT_IGNORED.system) {
if (path.toLowerCase().includes(systemPath.toLowerCase())) {
return true
}
}

return false
})

export const isDirectoryPathIgnoredByDefault = memoize((path: string): boolean => {
for (const directoryPath of DEFAULT_IGNORED.directories) {
if (path.toLowerCase().includes(directoryPath.toLowerCase())) {
return true
}
}

return false
})

export type SerializedError = {
name: string
message: string
Expand Down

0 comments on commit 6a6bc7e

Please sign in to comment.