HomePhorge
Diffusion Ant (master) Code

ant/src/pkg/linker.zig
8755416ca476master

Last Change
Raw File

linker.zig
View Options

const std = @import("std");
const builtin = @import("builtin");
const json = @import("json.zig");
const debug = @import("debug.zig");
const PARALLEL_LINK_THRESHOLD = 500;
const LINK_THREAD_COUNT = 8;
pub fn createSymlinkOrCopy(dir: std.fs.Dir, target: []const u8, link_name: []const u8) !void {
if (comptime builtin.os.tag == .windows) {
try createSymlinkWindows(dir, target, link_name);
} else try dir.symLink(target, link_name, .{});
}
pub fn createSymlinkAbsolute(target: []const u8, link_path: []const u8) void {
if (comptime builtin.os.tag == .windows) {
createSymlinkAbsoluteWindows(target, link_path);
} else std.posix.symlink(target, link_path) catch {};
}
fn createSymlinkWindows(dir: std.fs.Dir, target: []const u8, link_name: []const u8) !void {
if (comptime builtin.os.tag != .windows) return;
var target_utf16: [std.fs.max_path_bytes]u16 = undefined;
var link_utf16: [std.fs.max_path_bytes]u16 = undefined;
const target_len = try std.unicode.utf8ToUtf16Le(&target_utf16, target);
const link_len = try std.unicode.utf8ToUtf16Le(&link_utf16, link_name);
target_utf16[target_len] = 0;
_ = try std.os.windows.CreateSymbolicLink(
dir.fd, link_utf16[0..link_len],
target_utf16[0..target_len :0], false,
);
}
fn createSymlinkAbsoluteWindows(target: []const u8, link_path: []const u8) void {
if (comptime builtin.os.tag != .windows) return;
var target_utf16: [std.fs.max_path_bytes]u16 = undefined;
var link_utf16: [std.fs.max_path_bytes]u16 = undefined;
const target_len = std.unicode.utf8ToUtf16Le(&target_utf16, target) catch return;
const link_len = std.unicode.utf8ToUtf16Le(&link_utf16, link_path) catch return;
target_utf16[target_len] = 0;
_ = std.os.windows.CreateSymbolicLink(
null, link_utf16[0..link_len],
target_utf16[0..target_len :0], false,
) catch {};
}
pub const LinkError = error{
IoError,
PathNotFound,
CrossDevice,
PermissionDenied,
OutOfMemory,
PathTooLong,
};
pub const StatsSnapshot = struct {
files_linked: u32,
files_copied: u32,
files_cloned: u32,
bytes_linked: u64,
bytes_copied: u64,
dirs_created: u32,
bins_linked: u32,
packages_installed: u32,
packages_skipped: u32,
};
pub const LinkStats = struct {
files_linked: std.atomic.Value(u32),
files_copied: std.atomic.Value(u32),
files_cloned: std.atomic.Value(u32),
bytes_linked: std.atomic.Value(u64),
bytes_copied: std.atomic.Value(u64),
dirs_created: std.atomic.Value(u32),
bins_linked: std.atomic.Value(u32),
packages_installed: std.atomic.Value(u32),
packages_skipped: std.atomic.Value(u32),
pub fn init() LinkStats {
return .{
.files_linked = std.atomic.Value(u32).init(0),
.files_copied = std.atomic.Value(u32).init(0),
.files_cloned = std.atomic.Value(u32).init(0),
.bytes_linked = std.atomic.Value(u64).init(0),
.bytes_copied = std.atomic.Value(u64).init(0),
.dirs_created = std.atomic.Value(u32).init(0),
.bins_linked = std.atomic.Value(u32).init(0),
.packages_installed = std.atomic.Value(u32).init(0),
.packages_skipped = std.atomic.Value(u32).init(0),
};
}
pub fn snapshot(self: *const LinkStats) StatsSnapshot {
return .{
.files_linked = self.files_linked.load(.acquire),
.files_copied = self.files_copied.load(.acquire),
.files_cloned = self.files_cloned.load(.acquire),
.bytes_linked = self.bytes_linked.load(.acquire),
.bytes_copied = self.bytes_copied.load(.acquire),
.dirs_created = self.dirs_created.load(.acquire),
.bins_linked = self.bins_linked.load(.acquire),
.packages_installed = self.packages_installed.load(.acquire),
.packages_skipped = self.packages_skipped.load(.acquire),
};
}
};
pub const PackageLink = struct {
cache_path: []const u8,
node_modules_path: []const u8,
name: []const u8,
parent_path: ?[]const u8 = null,
file_count: u32 = 0,
has_bin: bool = true,
};
pub const Linker = struct {
allocator: std.mem.Allocator,
stats: LinkStats,
node_modules_dir: ?std.fs.Dir,
bin_dir: ?std.fs.Dir,
node_modules_path: []const u8,
cross_device: std.atomic.Value(bool),
pub fn init(allocator: std.mem.Allocator) Linker {
return .{
.allocator = allocator,
.stats = LinkStats.init(),
.node_modules_dir = null,
.bin_dir = null,
.node_modules_path = "",
.cross_device = std.atomic.Value(bool).init(false),
};
}
pub fn deinit(self: *Linker) void {
if (self.node_modules_dir) |*d| d.close();
if (self.bin_dir) |*d| d.close();
if (self.node_modules_path.len > 0) self.allocator.free(self.node_modules_path);
}
pub fn setNodeModulesPath(self: *Linker, path: []const u8) !void {
std.fs.cwd().makePath(path) catch |err| switch (err) {
error.PathAlreadyExists => {},
else => return error.IoError,
};
var new_nm_dir = try std.fs.cwd().openDir(path, .{});
errdefer new_nm_dir.close();
const new_path = try self.allocator.dupe(u8, path);
errdefer self.allocator.free(new_path);
const bin_path = try std.fmt.allocPrint(self.allocator, "{s}/.bin", .{path});
defer self.allocator.free(bin_path);
std.fs.cwd().makePath(bin_path) catch |err| switch (err) {
error.PathAlreadyExists => {},
else => return error.IoError,
};
const new_bin_dir = try std.fs.cwd().openDir(bin_path, .{});
if (self.bin_dir) |*d| d.close();
if (self.node_modules_dir) |*d| d.close();
if (self.node_modules_path.len > 0) self.allocator.free(self.node_modules_path);
self.node_modules_dir = new_nm_dir;
self.node_modules_path = new_path;
self.bin_dir = new_bin_dir;
}
pub fn linkPackage(self: *Linker, pkg: PackageLink) !void {
const node_modules = self.node_modules_dir orelse return error.IoError;
const install_path = if (pkg.parent_path) |parent|
try std.fmt.allocPrint(self.allocator, "{s}/node_modules/{s}", .{ parent, pkg.name })
else
try self.allocator.dupe(u8, pkg.name);
defer self.allocator.free(install_path);
var source_dir = std.fs.cwd().openDir(pkg.cache_path, .{ .iterate = true }) catch {
return error.PathNotFound;
};
defer source_dir.close();
const source_version = readPackageVersion(self.allocator, source_dir);
defer if (source_version) |v| self.allocator.free(v);
var should_skip = false;
var has_existing_install = false;
var has_existing_package = false;
{
const existing = node_modules.openDir(install_path, .{}) catch null;
if (existing) |dir| {
has_existing_install = true;
var installed_dir = dir;
defer installed_dir.close();
const installed_version = readPackageVersion(self.allocator, installed_dir);
defer if (installed_version) |v| self.allocator.free(v);
has_existing_package = installed_version != null;
should_skip = packageVersionsMatch(source_version, installed_version);
}
}
if (should_skip) {
_ = self.stats.packages_skipped.fetchAdd(1, .release);
return;
}
if (has_existing_install and has_existing_package) {
node_modules.deleteTree(install_path) catch return error.IoError;
}
node_modules.makePath(install_path) catch |err| switch (err) {
error.PathAlreadyExists => {},
else => return error.IoError,
};
_ = self.stats.dirs_created.fetchAdd(1, .release);
var dest_dir = node_modules.openDir(install_path, .{}) catch return error.IoError;
defer dest_dir.close();
self.linkDirectoryWithHint(source_dir, dest_dir, pkg.file_count, pkg.name) catch |err| return err;
if (pkg.parent_path == null and pkg.has_bin) try self.linkBinaries(pkg.name);
_ = self.stats.packages_installed.fetchAdd(1, .release);
}
fn linkBinaries(self: *Linker, pkg_name: []const u8) !void {
const bin_dir = self.bin_dir orelse return;
const node_modules = self.node_modules_dir orelse return;
var pkg_dir = node_modules.openDir(pkg_name, .{}) catch return;
defer pkg_dir.close();
const pkg_json_file = pkg_dir.openFile("package.json", .{}) catch return;
defer pkg_json_file.close();
const content = pkg_json_file.readToEndAlloc(self.allocator, 1024 * 1024) catch return;
defer self.allocator.free(content);
var doc = json.JsonDoc.parse(content) catch return;
defer doc.deinit();
const root_val = doc.root();
if (root_val.getObject("bin")) |bin_obj| {
var iter = bin_obj.objectIterator() orelse return;
while (iter.next()) |entry| {
const bin_path = entry.value.asString() orelse continue;
self.createBinSymlink(pkg_name, entry.key, bin_path, bin_dir) catch continue;
}
} else if (root_val.getString("bin")) |bin_path| {
const simple_name = if (std.mem.indexOf(u8, pkg_name, "/")) |slash|
pkg_name[slash + 1 ..]
else
pkg_name;
self.createBinSymlink(pkg_name, simple_name, bin_path, bin_dir) catch {};
}
}
fn readPackageVersion(allocator: std.mem.Allocator, dir: std.fs.Dir) ?[]const u8 {
const pkg_json = dir.openFile("package.json", .{}) catch return null;
defer pkg_json.close();
const content = pkg_json.readToEndAlloc(allocator, 256 * 1024) catch return null;
defer allocator.free(content);
var doc = json.JsonDoc.parse(content) catch return null;
defer doc.deinit();
const version = doc.root().getString("version") orelse return null;
return allocator.dupe(u8, version) catch null;
}
fn packageVersionsMatch(source_version: ?[]const u8, installed_version: ?[]const u8) bool {
const src = source_version orelse return false;
const dst = installed_version orelse return false;
return std.mem.eql(u8, src, dst);
}
fn createBinSymlink(self: *Linker, pkg_name: []const u8, cmd_name: []const u8, bin_path: []const u8, bin_dir: std.fs.Dir) !void {
var normalized_path = bin_path;
if (std.mem.startsWith(u8, normalized_path, "./")) {
normalized_path = normalized_path[2..];
}
const target = try std.fmt.allocPrint(self.allocator, "../{s}/{s}", .{ pkg_name, normalized_path });
defer self.allocator.free(target);
bin_dir.deleteFile(cmd_name) catch {};
try createSymlinkOrCopy(bin_dir, target, cmd_name);
_ = self.stats.bins_linked.fetchAdd(1, .release);
}
const FileWorkItem = struct {
source_path: []const u8,
dest_path: []const u8,
kind: std.fs.Dir.Entry.Kind,
link_target: ?[]const u8,
};
fn linkDirectory(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void {
try self.linkDirectorySequential(source, dest);
}
pub fn linkDirectoryWithHint(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir, file_count: u32, name: []const u8) !void {
if (file_count >= PARALLEL_LINK_THRESHOLD) {
debug.log(" parallel link: {s} ({d} files)", .{ name, file_count });
try self.linkDirectoryParallel(source, dest);
} else try self.linkDirectorySequential(source, dest);
}
fn linkDirectorySequential(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void {
var iter = source.iterate();
while (try iter.next()) |entry| {
switch (entry.kind) {
.directory => {
dest.makePath(entry.name) catch |err| switch (err) {
error.PathAlreadyExists => {},
else => return error.IoError,
};
_ = self.stats.dirs_created.fetchAdd(1, .release);
var child_source = source.openDir(entry.name, .{ .iterate = true }) catch continue;
defer child_source.close();
var child_dest = dest.openDir(entry.name, .{}) catch continue;
defer child_dest.close();
try self.linkDirectorySequential(child_source, child_dest);
},
.file => try self.linkFile(source, dest, entry.name),
.sym_link => {
var link_buf: [std.fs.max_path_bytes]u8 = undefined;
const target = source.readLink(entry.name, &link_buf) catch continue;
createSymlinkOrCopy(dest, target, entry.name) catch {};
},
else => {},
}
}
}
const ParallelThreadContext = struct {
linker: *Linker,
items: []const FileWorkItem,
source_base: std.fs.Dir,
dest_base: std.fs.Dir,
};
fn linkDirectoryParallel(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir) !void {
var work_items = std.ArrayListUnmanaged(FileWorkItem){};
defer {
for (work_items.items) |item| {
self.allocator.free(item.source_path);
self.allocator.free(item.dest_path);
if (item.link_target) |t| self.allocator.free(t);
}
work_items.deinit(self.allocator);
}
try self.collectWorkItems(source, dest, "", &work_items);
if (work_items.items.len == 0) return;
const items_slice = work_items.items;
const chunk_size = (items_slice.len + LINK_THREAD_COUNT - 1) / LINK_THREAD_COUNT;
var threads: [LINK_THREAD_COUNT]?std.Thread = undefined;
for (&threads) |*t| t.* = null;
var contexts: [LINK_THREAD_COUNT]ParallelThreadContext = undefined;
var offset: usize = 0;
for (0..LINK_THREAD_COUNT) |i| {
if (offset >= items_slice.len) break;
const end = @min(offset + chunk_size, items_slice.len);
contexts[i] = .{
.linker = self,
.items = items_slice[offset..end],
.source_base = source,
.dest_base = dest,
};
threads[i] = std.Thread.spawn(.{}, processWorkItems, .{&contexts[i]}) catch |err| blk: {
debug.log("Thread spawn failed for chunk {d}-{d}: {s}", .{ offset, end, @errorName(err) });
processWorkItems(&contexts[i]);
break :blk null;
}; offset = end;
}
for (&threads) |*t| if (t.*) |thread| thread.join();
}
fn processWorkItems(ctx: *const ParallelThreadContext) void {
for (ctx.items) |item| {
switch (item.kind) {
.file => {
const src_dir_path = std.fs.path.dirname(item.source_path) orelse "";
const dst_dir_path = std.fs.path.dirname(item.dest_path) orelse "";
const filename = std.fs.path.basename(item.source_path);
var src_dir = if (src_dir_path.len > 0)
ctx.source_base.openDir(src_dir_path, .{}) catch continue
else
ctx.source_base;
defer if (src_dir_path.len > 0) src_dir.close();
var dst_dir = if (dst_dir_path.len > 0)
ctx.dest_base.openDir(dst_dir_path, .{}) catch continue
else
ctx.dest_base;
defer if (dst_dir_path.len > 0) dst_dir.close();
ctx.linker.linkFile(src_dir, dst_dir, filename) catch {};
},
.sym_link => {
if (item.link_target) |target| {
const dst_dir_path = std.fs.path.dirname(item.dest_path) orelse "";
const filename = std.fs.path.basename(item.dest_path);
var dst_dir = if (dst_dir_path.len > 0)
ctx.dest_base.openDir(dst_dir_path, .{}) catch continue
else
ctx.dest_base;
defer if (dst_dir_path.len > 0) dst_dir.close();
createSymlinkOrCopy(dst_dir, target, filename) catch {};
}
},
else => {},
}
}
}
fn collectWorkItems(self: *Linker, source: std.fs.Dir, dest: std.fs.Dir, prefix: []const u8, work_items: *std.ArrayListUnmanaged(FileWorkItem)) !void {
var iter = source.iterate();
while (try iter.next()) |entry| {
const rel_path = if (prefix.len > 0)
try std.fmt.allocPrint(self.allocator, "{s}/{s}", .{ prefix, entry.name })
else
try self.allocator.dupe(u8, entry.name);
errdefer self.allocator.free(rel_path);
switch (entry.kind) {
.directory => {
dest.makePath(rel_path) catch |err| switch (err) {
error.PathAlreadyExists => {},
else => {
self.allocator.free(rel_path);
return error.IoError;
},
};
_ = self.stats.dirs_created.fetchAdd(1, .release);
var child_source = source.openDir(entry.name, .{ .iterate = true }) catch {
self.allocator.free(rel_path);
continue;
};
defer child_source.close();
try self.collectWorkItems(child_source, dest, rel_path, work_items);
self.allocator.free(rel_path);
},
.file => {
try work_items.append(self.allocator, .{
.source_path = rel_path,
.dest_path = try self.allocator.dupe(u8, rel_path),
.kind = .file,
.link_target = null,
});
},
.sym_link => {
var link_buf: [std.fs.max_path_bytes]u8 = undefined;
const target = source.readLink(entry.name, &link_buf) catch {
self.allocator.free(rel_path);
continue;
};
try work_items.append(self.allocator, .{
.source_path = rel_path,
.dest_path = try self.allocator.dupe(u8, rel_path),
.kind = .sym_link,
.link_target = try self.allocator.dupe(u8, target),
});
},
else => self.allocator.free(rel_path),
}
}
}
fn linkFile(self: *Linker, source_dir: std.fs.Dir, dest_dir: std.fs.Dir, name: []const u8) !void {
dest_dir.deleteFile(name) catch {};
if (comptime builtin.os.tag != .windows) {
if (!self.cross_device.load(.acquire)) {
if (linkAt(source_dir, name, dest_dir, name)) {
_ = self.stats.files_linked.fetchAdd(1, .release);
return;
} else |err| {
if (err == error.CrossDevice) {
self.cross_device.store(true, .release);
} else if (err != error.IoError) return err;
}
}
}
if (comptime builtin.os.tag == .macos) {
if (fclonefileat(source_dir.fd, name, dest_dir.fd, name)) {
_ = self.stats.files_cloned.fetchAdd(1, .release);
return;
} else |_| {}
}
try self.copyFile(source_dir, dest_dir, name);
_ = self.stats.files_copied.fetchAdd(1, .release);
}
fn linkAt(source_dir: std.fs.Dir, source_name: []const u8, dest_dir: std.fs.Dir, dest_name: []const u8) !void {
if (comptime builtin.os.tag == .windows) return error.IoError;
var source_buf: [256]u8 = undefined;
var dest_buf: [256]u8 = undefined;
if (source_name.len >= source_buf.len or dest_name.len >= dest_buf.len) {
return error.PathTooLong;
}
@memcpy(source_buf[0..source_name.len], source_name);
source_buf[source_name.len] = 0;
@memcpy(dest_buf[0..dest_name.len], dest_name);
dest_buf[dest_name.len] = 0;
const source_z: [*:0]const u8 = source_buf[0..source_name.len :0];
const dest_z: [*:0]const u8 = dest_buf[0..dest_name.len :0];
const result = std.c.linkat(source_dir.fd, source_z, dest_dir.fd, dest_z, 0);
if (result != 0) {
const errno = std.posix.errno(result);
return switch (errno) {
.XDEV => error.CrossDevice,
.PERM, .ACCES => error.PermissionDenied,
else => error.IoError,
};
}
}
fn fclonefileat(src_fd: std.posix.fd_t, src_name: []const u8, dst_fd: std.posix.fd_t, dst_name: []const u8) !void {
var src_buf: [256]u8 = undefined;
var dst_buf: [256]u8 = undefined;
if (src_name.len >= src_buf.len or dst_name.len >= dst_buf.len) {
return error.PathTooLong;
}
@memcpy(src_buf[0..src_name.len], src_name);
src_buf[src_name.len] = 0;
@memcpy(dst_buf[0..dst_name.len], dst_name);
dst_buf[dst_name.len] = 0;
const src_z: [*:0]const u8 = src_buf[0..src_name.len :0];
const dst_z: [*:0]const u8 = dst_buf[0..dst_name.len :0];
const fclonefileat_fn = struct {
extern "c" fn fclonefileat(c_int, [*:0]const u8, c_int, [*:0]const u8, c_uint) c_int;
}.fclonefileat;
const result = fclonefileat_fn(src_fd, src_z, dst_fd, dst_z, 0);
if (result != 0) {
return error.IoError;
}
}
fn copyFile(self: *Linker, source_dir: std.fs.Dir, dest_dir: std.fs.Dir, name: []const u8) !void {
_ = self;
var source = source_dir.openFile(name, .{}) catch return error.IoError;
defer source.close();
const stat = source.stat() catch return error.IoError;
var dest = dest_dir.createFile(name,
if (comptime builtin.os.tag != .windows) .{ .mode = stat.mode } else .{}
) catch return error.IoError; defer dest.close();
var buf: [64 * 1024]u8 = undefined;
while (true) {
const bytes_read = source.read(&buf) catch return error.IoError;
if (bytes_read == 0) break;
dest.writeAll(buf[0..bytes_read]) catch return error.IoError;
}
}
pub fn getStats(self: *const Linker) StatsSnapshot {
return self.stats.snapshot();
}
};

Owner Packages

  • No Owners