From 9cb0f37cc66bfa2f16caf14d246415a0c9a21b4e Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 15 Dec 2024 00:30:49 -0800 Subject: [PATCH] experiment: parallelize chmod and \r\n normalization --- src/install/bin.zig | 72 +++++++++++++++++++++++++++++++++++++++-- src/install/install.zig | 12 +++++++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/src/install/bin.zig b/src/install/bin.zig index d0162e63839de4..b6dd8ffbf4d573 100644 --- a/src/install/bin.zig +++ b/src/install/bin.zig @@ -598,8 +598,62 @@ pub const Bin = extern struct { } if (comptime !Environment.isWindows) { + const pm = Install.PackageManager.get(); + const task = CRLFNormalizerTask.init(abs_target, pm); + _ = pm.incrementPendingTasks(1); + _ = pm.pending_linking_tasks.fetchAdd(1, .monotonic); + pm.thread_pool.schedule(bun.ThreadPool.Batch.from(&task.task)); + } + } + + const ChmodTask = struct { + abs_target: [:0]const u8, + package_manager: *bun.install.PackageManager, + task: bun.ThreadPool.Task = .{ .callback = run }, + + pub usingnamespace bun.New(@This()); + + pub fn init(abs_target: [:0]const u8, package_manager: *bun.install.PackageManager) *ChmodTask { + return ChmodTask.new(.{ + .abs_target = bun.default_allocator.dupeZ(u8, abs_target) catch bun.outOfMemory(), + .package_manager = package_manager, + }); + } + + pub fn run(task: *bun.ThreadPool.Task) void { + const this: *ChmodTask = @fieldParentPtr("task", task); + defer this.deinit(); + _ = bun.sys.chmod(this.abs_target, umask | 0o777); + } + + pub fn deinit(this: *ChmodTask) void { + _ = this.package_manager.decrementPendingTasks(); + _ = this.package_manager.pending_linking_tasks.fetchSub(1, .monotonic); + this.package_manager.wake(); + bun.default_allocator.free(this.abs_target); + this.destroy(); + } + }; + + const CRLFNormalizerTask = struct { + abs_target: [:0]const u8, + package_manager: *bun.install.PackageManager, + task: bun.ThreadPool.Task = .{ .callback = run }, + + pub usingnamespace bun.New(@This()); + + pub fn init(abs_target: [:0]const u8, package_manager: *bun.install.PackageManager) *CRLFNormalizerTask { + return CRLFNormalizerTask.new(.{ + .abs_target = bun.default_allocator.dupeZ(u8, abs_target) catch bun.outOfMemory(), + .package_manager = package_manager, + }); + } + + pub fn run(task: *bun.ThreadPool.Task) void { + const this: *CRLFNormalizerTask = @fieldParentPtr("task", task); + defer this.deinit(); // any error here is ignored - const bin = bun.sys.File.openat(bun.invalid_fd, abs_target, bun.O.RDWR, 0o664).unwrap() catch return; + const bin = bun.sys.File.openat(bun.invalid_fd, this.abs_target, bun.O.RDWR, 0o664).unwrap() catch return; defer bin.close(); var shebang_buf: [1024]u8 = undefined; @@ -617,7 +671,16 @@ pub const Bin = extern struct { } } } - } + + pub fn deinit(this: *CRLFNormalizerTask) void { + _ = this.package_manager.decrementPendingTasks(); + _ = this.package_manager.pending_linking_tasks.fetchSub(1, .monotonic); + this.package_manager.wake(); + + bun.default_allocator.free(this.abs_target); + this.destroy(); + } + }; fn createWindowsShim(this: *Linker, target: bun.FileDescriptor, abs_target: [:0]const u8, abs_dest: [:0]const u8, global: bool) void { const WinBinLinkingShim = @import("./windows-shim/BinLinkingShim.zig"); @@ -708,7 +771,10 @@ pub const Bin = extern struct { fn createSymlink(this: *Linker, abs_target: [:0]const u8, abs_dest: [:0]const u8, global: bool) void { defer { if (this.err == null) { - _ = bun.sys.chmod(abs_target, umask | 0o777); + const task = ChmodTask.init(abs_target, Install.PackageManager.get()); + _ = task.package_manager.pending_linking_tasks.fetchAdd(1, .monotonic); + _ = task.package_manager.incrementPendingTasks(1); + task.package_manager.thread_pool.schedule(bun.ThreadPool.Batch.from(&task.task)); } } diff --git a/src/install/install.zig b/src/install/install.zig index f7fcd9d3563090..ba6a39d02b8a6b 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -2692,6 +2692,7 @@ pub const PackageManager = struct { /// every single time, because someone could edit the patchfile at anytime pending_pre_calc_hashes: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), pending_tasks: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), + pending_linking_tasks: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), total_tasks: u32 = 0, preallocated_network_tasks: PreallocatedNetworkTasks = PreallocatedNetworkTasks.init(bun.default_allocator), preallocated_resolve_tasks: PreallocatedTaskStore = PreallocatedTaskStore.init(bun.default_allocator), @@ -14010,7 +14011,9 @@ pub const PackageManager = struct { // need to make sure bins are linked before completing any remaining scripts. // this can happen if a package fails to download + installer.linkRemainingBins(log_level); + this.sleepUntilLinkingFinishes(); installer.completeRemainingScripts(log_level); while (this.pending_lifecycle_script_tasks.load(.monotonic) > 0) { @@ -14033,6 +14036,15 @@ pub const PackageManager = struct { return manager.pending_tasks.load(.monotonic); } + pub fn sleepUntilLinkingFinishes(manager: *PackageManager) void { + const Closure = struct { + pub fn isDone(pm: *PackageManager) bool { + return pm.pending_linking_tasks.load(.monotonic) == 0; + } + }; + manager.sleepUntil(manager, &Closure.isDone); + } + pub inline fn incrementPendingTasks(manager: *PackageManager, count: u32) u32 { manager.total_tasks += count; return manager.pending_tasks.fetchAdd(count, .monotonic);