From 3e4a3fa5b7faadaae0a57088baa392e2bb52fe38 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 17 Jul 2018 18:36:47 -0400 Subject: [PATCH] self-hosted: find libc on linux --- src-self-hosted/compilation.zig | 35 ++-- src-self-hosted/libc_installation.zig | 234 ++++++++++++++++++++++++++ src-self-hosted/main.zig | 65 ++++--- std/fmt/index.zig | 8 +- std/os/file.zig | 45 +++-- 5 files changed, 326 insertions(+), 61 deletions(-) create mode 100644 src-self-hosted/libc_installation.zig diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig index 741324c871..1eb4339bbb 100644 --- a/src-self-hosted/compilation.zig +++ b/src-self-hosted/compilation.zig @@ -28,6 +28,7 @@ const Span = errmsg.Span; const codegen = @import("codegen.zig"); const Package = @import("package.zig").Package; const link = @import("link.zig").link; +const LibCInstallation = @import("libc_installation.zig").LibCInstallation; /// Data that is local to the event loop. pub const EventLoopLocal = struct { @@ -37,6 +38,8 @@ pub const EventLoopLocal = struct { /// TODO pool these so that it doesn't have to lock prng: event.Locked(std.rand.DefaultPrng), + native_libc: event.Future(LibCInstallation), + var lazy_init_targets = std.lazyInit(void); fn init(loop: *event.Loop) !EventLoopLocal { @@ -52,6 +55,7 @@ pub const EventLoopLocal = struct { .loop = loop, .llvm_handle_pool = std.atomic.Stack(llvm.ContextRef).init(), .prng = event.Locked(std.rand.DefaultPrng).init(loop, std.rand.DefaultPrng.init(seed)), + .native_libc = event.Future(LibCInstallation).init(loop), }; } @@ -78,6 +82,13 @@ pub const EventLoopLocal = struct { return LlvmHandle{ .node = node }; } + + pub async fn getNativeLibC(self: *EventLoopLocal) !*LibCInstallation { + if (await (async self.native_libc.start() catch unreachable)) |ptr| return ptr; + try await (async self.native_libc.data.findNative(self.loop) catch unreachable); + self.native_libc.resolve(); + return &self.native_libc.data; + } }; pub const LlvmHandle = struct { @@ -109,11 +120,6 @@ pub const Compilation = struct { linker_script: ?[]const u8, cache_dir: []const u8, - libc_lib_dir: ?[]const u8, - libc_static_lib_dir: ?[]const u8, - libc_include_dir: ?[]const u8, - msvc_lib_dir: ?[]const u8, - kernel32_lib_dir: ?[]const u8, dynamic_linker: ?[]const u8, out_h_path: ?[]const u8, @@ -318,11 +324,6 @@ pub const Compilation = struct { .verbose_link = false, .linker_script = null, - .libc_lib_dir = null, - .libc_static_lib_dir = null, - .libc_include_dir = null, - .msvc_lib_dir = null, - .kernel32_lib_dir = null, .dynamic_linker = null, .out_h_path = null, .is_test = false, @@ -762,10 +763,24 @@ pub const Compilation = struct { try self.link_libs_list.append(link_lib); if (is_libc) { self.libc_link_lib = link_lib; + + // get a head start on looking for the native libc + if (self.target == Target.Native) { + try async self.startFindingNativeLibC(); + } } return link_lib; } + /// cancels itself so no need to await or cancel the promise. + async fn startFindingNativeLibC(self: *Compilation) void { + // we don't care if it fails, we're just trying to kick off the future resolution + _ = (await (async self.loop.call(EventLoopLocal.getNativeLibC, self.event_loop_local) catch unreachable)) catch {}; + suspend |p| { + cancel p; + } + } + /// General Purpose Allocator. Must free when done. fn gpa(self: Compilation) *mem.Allocator { return self.loop.allocator; diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig new file mode 100644 index 0000000000..5606a467e9 --- /dev/null +++ b/src-self-hosted/libc_installation.zig @@ -0,0 +1,234 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const event = std.event; + +pub const LibCInstallation = struct { + /// The directory that contains `stdlib.h`. + /// On Linux, can be found with: `cc -E -Wp,-v -xc /dev/null` + include_dir: []const u8, + + /// The directory that contains `crt1.o`. + /// On Linux, can be found with `cc -print-file-name=crt1.o`. + /// Not needed when targeting MacOS. + lib_dir: ?[]const u8, + + /// The directory that contains `crtbegin.o`. + /// On Linux, can be found with `cc -print-file-name=crt1.o`. + /// Not needed when targeting MacOS or Windows. + static_lib_dir: ?[]const u8, + + /// The directory that contains `vcruntime.lib`. + /// Only needed when targeting Windows. + msvc_lib_dir: ?[]const u8, + + /// The directory that contains `kernel32.lib`. + /// Only needed when targeting Windows. + kernel32_lib_dir: ?[]const u8, + + pub const Error = error{ + OutOfMemory, + FileSystem, + UnableToSpawnCCompiler, + CCompilerExitCode, + CCompilerCrashed, + CCompilerCannotFindHeaders, + CCompilerCannotFindCRuntime, + LibCStdLibHeaderNotFound, + }; + + /// Finds the default, native libc. + pub async fn findNative(self: *LibCInstallation, loop: *event.Loop) !void { + self.* = LibCInstallation{ + .lib_dir = null, + .include_dir = ([*]const u8)(undefined)[0..0], + .static_lib_dir = null, + .msvc_lib_dir = null, + .kernel32_lib_dir = null, + }; + var group = event.Group(Error!void).init(loop); + switch (builtin.os) { + builtin.Os.windows => { + try group.call(findNativeIncludeDirWindows, self, loop); + try group.call(findNativeLibDirWindows, self, loop); + try group.call(findNativeMsvcLibDir, self, loop); + try group.call(findNativeKernel32LibDir, self, loop); + }, + builtin.Os.linux => { + try group.call(findNativeIncludeDirLinux, self, loop); + try group.call(findNativeLibDirLinux, self, loop); + try group.call(findNativeStaticLibDir, self, loop); + }, + builtin.Os.macosx => { + try group.call(findNativeIncludeDirMacOS, self, loop); + }, + else => @compileError("unimplemented: find libc for this OS"), + } + return await (async group.wait() catch unreachable); + } + + async fn findNativeIncludeDirLinux(self: *LibCInstallation, loop: *event.Loop) !void { + const cc_exe = std.os.getEnvPosix("CC") orelse "cc"; + const argv = []const []const u8{ + cc_exe, + "-E", + "-Wp,-v", + "-xc", + "/dev/null", + }; + // TODO make this use event loop + const errorable_result = std.os.ChildProcess.exec(loop.allocator, argv, null, null, 1024 * 1024); + const exec_result = if (std.debug.runtime_safety) blk: { + break :blk errorable_result catch unreachable; + } else blk: { + break :blk errorable_result catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return error.UnableToSpawnCCompiler, + }; + }; + defer { + loop.allocator.free(exec_result.stdout); + loop.allocator.free(exec_result.stderr); + } + + switch (exec_result.term) { + std.os.ChildProcess.Term.Exited => |code| { + if (code != 0) return error.CCompilerExitCode; + }, + else => { + return error.CCompilerCrashed; + }, + } + + var it = std.mem.split(exec_result.stderr, "\n\r"); + var search_paths = std.ArrayList([]const u8).init(loop.allocator); + defer search_paths.deinit(); + while (it.next()) |line| { + if (line.len != 0 and line[0] == ' ') { + try search_paths.append(line); + } + } + if (search_paths.len == 0) { + return error.CCompilerCannotFindHeaders; + } + + // search in reverse order + var path_i: usize = 0; + while (path_i < search_paths.len) : (path_i += 1) { + const search_path_untrimmed = search_paths.at(search_paths.len - path_i - 1); + const search_path = std.mem.trimLeft(u8, search_path_untrimmed, " "); + const stdlib_path = try std.os.path.join(loop.allocator, search_path, "stdlib.h"); + defer loop.allocator.free(stdlib_path); + + if (std.os.File.access(loop.allocator, stdlib_path)) |_| { + self.include_dir = try std.mem.dupe(loop.allocator, u8, search_path); + return; + } else |err| switch (err) { + error.NotFound, error.PermissionDenied => continue, + error.OutOfMemory => return error.OutOfMemory, + else => return error.FileSystem, + } + } + + return error.LibCStdLibHeaderNotFound; + } + + async fn findNativeIncludeDirWindows(self: *LibCInstallation, loop: *event.Loop) !void { + // TODO + //ZigWindowsSDK *sdk = get_windows_sdk(g); + //g->libc_include_dir = buf_alloc(); + //if (os_get_win32_ucrt_include_path(sdk, g->libc_include_dir)) { + // fprintf(stderr, "Unable to determine libc include path. --libc-include-dir"); + // exit(1); + //} + @panic("TODO"); + } + + async fn findNativeIncludeDirMacOS(self: *LibCInstallation, loop: *event.Loop) !void { + self.include_dir = try std.mem.dupe(loop.allocator, u8, "/usr/include"); + } + + async fn findNativeLibDirWindows(self: *LibCInstallation, loop: *event.Loop) Error!void { + // TODO + //ZigWindowsSDK *sdk = get_windows_sdk(g); + + //if (g->msvc_lib_dir == nullptr) { + // Buf* vc_lib_dir = buf_alloc(); + // if (os_get_win32_vcruntime_path(vc_lib_dir, g->zig_target.arch.arch)) { + // fprintf(stderr, "Unable to determine vcruntime path. --msvc-lib-dir"); + // exit(1); + // } + // g->msvc_lib_dir = vc_lib_dir; + //} + + //if (g->libc_lib_dir == nullptr) { + // Buf* ucrt_lib_path = buf_alloc(); + // if (os_get_win32_ucrt_lib_path(sdk, ucrt_lib_path, g->zig_target.arch.arch)) { + // fprintf(stderr, "Unable to determine ucrt path. --libc-lib-dir"); + // exit(1); + // } + // g->libc_lib_dir = ucrt_lib_path; + //} + + //if (g->kernel32_lib_dir == nullptr) { + // Buf* kern_lib_path = buf_alloc(); + // if (os_get_win32_kern32_path(sdk, kern_lib_path, g->zig_target.arch.arch)) { + // fprintf(stderr, "Unable to determine kernel32 path. --kernel32-lib-dir"); + // exit(1); + // } + // g->kernel32_lib_dir = kern_lib_path; + //} + @panic("TODO"); + } + + async fn findNativeLibDirLinux(self: *LibCInstallation, loop: *event.Loop) Error!void { + self.lib_dir = try await (async ccPrintFileNameDir(loop, "crt1.o") catch unreachable); + } + + async fn findNativeStaticLibDir(self: *LibCInstallation, loop: *event.Loop) Error!void { + self.static_lib_dir = try await (async ccPrintFileNameDir(loop, "crtbegin.o") catch unreachable); + } + + async fn findNativeMsvcLibDir(self: *LibCInstallation, loop: *event.Loop) Error!void { + @panic("TODO"); + } + + async fn findNativeKernel32LibDir(self: *LibCInstallation, loop: *event.Loop) Error!void { + @panic("TODO"); + } +}; + +/// caller owns returned memory +async fn ccPrintFileNameDir(loop: *event.Loop, o_file: []const u8) ![]u8 { + const cc_exe = std.os.getEnvPosix("CC") orelse "cc"; + const arg1 = try std.fmt.allocPrint(loop.allocator, "-print-file-name={}", o_file); + defer loop.allocator.free(arg1); + const argv = []const []const u8{ cc_exe, arg1 }; + + // TODO evented I/O + const errorable_result = std.os.ChildProcess.exec(loop.allocator, argv, null, null, 1024 * 1024); + const exec_result = if (std.debug.runtime_safety) blk: { + break :blk errorable_result catch unreachable; + } else blk: { + break :blk errorable_result catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return error.UnableToSpawnCCompiler, + }; + }; + defer { + loop.allocator.free(exec_result.stdout); + loop.allocator.free(exec_result.stderr); + } + switch (exec_result.term) { + std.os.ChildProcess.Term.Exited => |code| { + if (code != 0) return error.CCompilerExitCode; + }, + else => { + return error.CCompilerCrashed; + }, + } + var it = std.mem.split(exec_result.stdout, "\n\r"); + const line = it.next() orelse return error.CCompilerCannotFindCRuntime; + const dirname = std.os.path.dirname(line) orelse return error.CCompilerCannotFindCRuntime; + + return std.mem.dupe(loop.allocator, u8, dirname); +} diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 8b668e35bd..ff24677b6d 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -31,6 +31,7 @@ const usage = \\ build-exe [source] Create executable from source or object files \\ build-lib [source] Create library from source or object files \\ build-obj [source] Create object from source or assembly + \\ find-libc Show native libc installation paths \\ fmt [source] Parse file and render in canonical zig format \\ targets List available compilation targets \\ version Print version number and exit @@ -81,6 +82,10 @@ pub fn main() !void { .name = "build-obj", .exec = cmdBuildObj, }, + Command{ + .name = "find-libc", + .exec = cmdFindLibc, + }, Command{ .name = "fmt", .exec = cmdFmt, @@ -134,7 +139,6 @@ const usage_build_generic = \\ --cache-dir [path] Override the cache directory \\ --emit [filetype] Emit a specific file format as compilation output \\ --enable-timing-info Print timing diagnostics - \\ --libc-include-dir [path] Directory where libc stdlib.h resides \\ --name [name] Override output name \\ --output [file] Override destination path \\ --output-h [file] Override generated header file path @@ -165,10 +169,6 @@ const usage_build_generic = \\ --ar-path [path] Set the path to ar \\ --dynamic-linker [path] Set the path to ld.so \\ --each-lib-rpath Add rpath for each used dynamic library - \\ --libc-lib-dir [path] Directory where libc crt1.o resides - \\ --libc-static-lib-dir [path] Directory where libc crtbegin.o resides - \\ --msvc-lib-dir [path] (windows) directory where vcruntime.lib resides - \\ --kernel32-lib-dir [path] (windows) directory where kernel32.lib resides \\ --library [lib] Link against lib \\ --forbid-library [lib] Make it an error to link against lib \\ --library-path [dir] Add a directory to the library search path @@ -210,7 +210,6 @@ const args_build_generic = []Flag{ "llvm-ir", }), Flag.Bool("--enable-timing-info"), - Flag.Arg1("--libc-include-dir"), Flag.Arg1("--name"), Flag.Arg1("--output"), Flag.Arg1("--output-h"), @@ -236,10 +235,6 @@ const args_build_generic = []Flag{ Flag.Arg1("--ar-path"), Flag.Arg1("--dynamic-linker"), Flag.Bool("--each-lib-rpath"), - Flag.Arg1("--libc-lib-dir"), - Flag.Arg1("--libc-static-lib-dir"), - Flag.Arg1("--msvc-lib-dir"), - Flag.Arg1("--kernel32-lib-dir"), Flag.ArgMergeN("--library", 1), Flag.ArgMergeN("--forbid-library", 1), Flag.ArgMergeN("--library-path", 1), @@ -430,21 +425,6 @@ fn buildOutputType(allocator: *Allocator, args: []const []const u8, out_type: Co comp.strip = flags.present("strip"); - if (flags.single("libc-lib-dir")) |libc_lib_dir| { - comp.libc_lib_dir = libc_lib_dir; - } - if (flags.single("libc-static-lib-dir")) |libc_static_lib_dir| { - comp.libc_static_lib_dir = libc_static_lib_dir; - } - if (flags.single("libc-include-dir")) |libc_include_dir| { - comp.libc_include_dir = libc_include_dir; - } - if (flags.single("msvc-lib-dir")) |msvc_lib_dir| { - comp.msvc_lib_dir = msvc_lib_dir; - } - if (flags.single("kernel32-lib-dir")) |kernel32_lib_dir| { - comp.kernel32_lib_dir = kernel32_lib_dir; - } if (flags.single("dynamic-linker")) |dynamic_linker| { comp.dynamic_linker = dynamic_linker; } @@ -579,6 +559,41 @@ const Fmt = struct { } }; +fn cmdFindLibc(allocator: *Allocator, args: []const []const u8) !void { + var loop: event.Loop = undefined; + try loop.initMultiThreaded(allocator); + defer loop.deinit(); + + var event_loop_local = try EventLoopLocal.init(&loop); + defer event_loop_local.deinit(); + + const handle = try async findLibCAsync(&event_loop_local); + defer cancel handle; + + loop.run(); +} + +async fn findLibCAsync(event_loop_local: *EventLoopLocal) void { + const libc = (await (async event_loop_local.getNativeLibC() catch unreachable)) catch |err| { + stderr.print("unable to find libc: {}\n", @errorName(err)) catch os.exit(1); + os.exit(1); + }; + stderr.print( + \\include_dir={} + \\lib_dir={} + \\static_lib_dir={} + \\msvc_lib_dir={} + \\kernel32_lib_dir={} + \\ + , + libc.include_dir, + libc.lib_dir, + libc.static_lib_dir orelse "", + libc.msvc_lib_dir orelse "", + libc.kernel32_lib_dir orelse "", + ) catch os.exit(1); +} + fn cmdFmt(allocator: *Allocator, args: []const []const u8) !void { var flags = try Args.parse(allocator, args_fmt_spec, args); defer flags.deinit(); diff --git a/std/fmt/index.zig b/std/fmt/index.zig index c3c17f5322..2188cc5803 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -785,11 +785,15 @@ pub fn bufPrint(buf: []u8, comptime fmt: []const u8, args: ...) ![]u8 { return buf[0 .. buf.len - context.remaining.len]; } -pub fn allocPrint(allocator: *mem.Allocator, comptime fmt: []const u8, args: ...) ![]u8 { +pub const AllocPrintError = error{OutOfMemory}; + +pub fn allocPrint(allocator: *mem.Allocator, comptime fmt: []const u8, args: ...) AllocPrintError![]u8 { var size: usize = 0; format(&size, error{}, countSize, fmt, args) catch |err| switch (err) {}; const buf = try allocator.alloc(u8, size); - return bufPrint(buf, fmt, args); + return bufPrint(buf, fmt, args) catch |err| switch (err) { + error.BufferTooSmall => unreachable, // we just counted the size above + }; } fn countSize(size: *usize, bytes: []const u8) (error{}!void) { diff --git a/std/os/file.zig b/std/os/file.zig index 055f185121..f468615ec0 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -109,43 +109,40 @@ pub const File = struct { Unexpected, }; - pub fn access(allocator: *mem.Allocator, path: []const u8, file_mode: os.FileMode) AccessError!bool { + pub fn access(allocator: *mem.Allocator, path: []const u8) AccessError!void { const path_with_null = try std.cstr.addNullByte(allocator, path); defer allocator.free(path_with_null); if (is_posix) { - // mode is ignored and is always F_OK for now const result = posix.access(path_with_null.ptr, posix.F_OK); const err = posix.getErrno(result); - if (err > 0) { - return switch (err) { - posix.EACCES => error.PermissionDenied, - posix.EROFS => error.PermissionDenied, - posix.ELOOP => error.PermissionDenied, - posix.ETXTBSY => error.PermissionDenied, - posix.ENOTDIR => error.NotFound, - posix.ENOENT => error.NotFound, + switch (err) { + 0 => return, + posix.EACCES => return error.PermissionDenied, + posix.EROFS => return error.PermissionDenied, + posix.ELOOP => return error.PermissionDenied, + posix.ETXTBSY => return error.PermissionDenied, + posix.ENOTDIR => return error.NotFound, + posix.ENOENT => return error.NotFound, - posix.ENAMETOOLONG => error.NameTooLong, - posix.EINVAL => error.BadMode, - posix.EFAULT => error.BadPathName, - posix.EIO => error.Io, - posix.ENOMEM => error.SystemResources, - else => os.unexpectedErrorPosix(err), - }; + posix.ENAMETOOLONG => return error.NameTooLong, + posix.EINVAL => unreachable, + posix.EFAULT => return error.BadPathName, + posix.EIO => return error.Io, + posix.ENOMEM => return error.SystemResources, + else => return os.unexpectedErrorPosix(err), } - return true; } else if (is_windows) { if (os.windows.GetFileAttributesA(path_with_null.ptr) != os.windows.INVALID_FILE_ATTRIBUTES) { - return true; + return; } const err = windows.GetLastError(); - return switch (err) { - windows.ERROR.FILE_NOT_FOUND => error.NotFound, - windows.ERROR.ACCESS_DENIED => error.PermissionDenied, - else => os.unexpectedErrorWindows(err), - }; + switch (err) { + windows.ERROR.FILE_NOT_FOUND => return error.NotFound, + windows.ERROR.ACCESS_DENIED => return error.PermissionDenied, + else => return os.unexpectedErrorWindows(err), + } } else { @compileError("TODO implement access for this OS"); }