From 9e519727014fcc484714d8230a0a8bce8744998d Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sun, 26 Feb 2023 05:27:26 -0500 Subject: [PATCH 01/25] llvm: fix untagged struct names in debug info for llvm (again) At least lldb misbehaves with all these same-named unions, so just generate a unique name. --- src/codegen/llvm.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 558556f108..6f240b88f5 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -2214,11 +2214,16 @@ pub const Object = struct { )); } - const union_name = if (layout.tag_size == 0) name.ptr else "AnonUnion"; + var union_name_buf: ?[:0]const u8 = null; + defer if (union_name_buf) |buf| gpa.free(buf); + const union_name = if (layout.tag_size == 0) name else name: { + union_name_buf = try std.fmt.allocPrintZ(gpa, "{s}:Payload", .{name}); + break :name union_name_buf.?; + }; const union_di_ty = dib.createUnionType( compile_unit_scope, - union_name, + union_name.ptr, null, // file 0, // line ty.abiSize(target) * 8, // size in bits From a3529c2dea579e0a15dcde1b8ee71d9ec9c75330 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sun, 26 Feb 2023 22:11:41 -0500 Subject: [PATCH 02/25] tools: implement more lldb pretty printers --- lib/std/hash_map.zig | 9 +- lib/std/multi_array_list.zig | 29 +- src/type.zig | 30 ++ src/value.zig | 30 ++ tools/lldb_pretty_printers.py | 575 +++++++++++++++++++++++++++ tools/stage2_lldb_pretty_printers.py | 59 --- tools/std_gdb_pretty_printers.py | 4 +- 7 files changed, 665 insertions(+), 71 deletions(-) create mode 100644 tools/lldb_pretty_printers.py delete mode 100644 tools/stage2_lldb_pretty_printers.py diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig index 164b81d651..af0ecc5993 100644 --- a/lib/std/hash_map.zig +++ b/lib/std/hash_map.zig @@ -1595,16 +1595,17 @@ pub fn HashMapUnmanaged( self.available = 0; } - /// This function is used in tools/zig-gdb.py to fetch the header type to facilitate - /// fancy debug printing for this type. - fn gdbHelper(self: *Self, hdr: *Header) void { + /// This function is used in the debugger pretty formatters in tools/ to fetch the + /// header type to facilitate fancy debug printing for this type. + fn dbHelper(self: *Self, hdr: *Header, entry: *Entry) void { _ = self; _ = hdr; + _ = entry; } comptime { if (builtin.mode == .Debug) { - _ = gdbHelper; + _ = dbHelper; } } }; diff --git a/lib/std/multi_array_list.zig b/lib/std/multi_array_list.zig index 56b36aaa81..6965205b1e 100644 --- a/lib/std/multi_array_list.zig +++ b/lib/std/multi_array_list.zig @@ -131,8 +131,8 @@ pub fn MultiArrayList(comptime S: type) type { .capacity = self.capacity, }; var ptr: [*]u8 = self.bytes; - for (sizes.bytes, 0..) |field_size, i| { - result.ptrs[sizes.fields[i]] = ptr; + for (sizes.bytes, sizes.fields) |field_size, i| { + result.ptrs[i] = ptr; ptr += field_size * self.capacity; } return result; @@ -446,16 +446,33 @@ pub fn MultiArrayList(comptime S: type) type { return meta.fieldInfo(S, field).type; } - /// This function is used in tools/zig-gdb.py to fetch the child type to facilitate - /// fancy debug printing for this type. - fn gdbHelper(self: *Self, child: *S) void { + const Entry = entry: { + var entry_fields: [fields.len]std.builtin.Type.StructField = undefined; + for (&entry_fields, sizes.fields) |*entry_field, i| entry_field.* = .{ + .name = fields[i].name ++ "_ptr", + .type = *fields[i].type, + .default_value = null, + .is_comptime = fields[i].is_comptime, + .alignment = fields[i].alignment, + }; + break :entry @Type(.{ .Struct = .{ + .layout = .Extern, + .fields = &entry_fields, + .decls = &.{}, + .is_tuple = false, + } }); + }; + /// This function is used in the debugger pretty formatters in tools/ to fetch the + /// child type to facilitate fancy debug printing for this type. + fn dbHelper(self: *Self, child: *S, entry: *Entry) void { _ = self; _ = child; + _ = entry; } comptime { if (builtin.mode == .Debug) { - _ = gdbHelper; + _ = dbHelper; } } }; diff --git a/src/type.zig b/src/type.zig index ec4db8689f..15525f14eb 100644 --- a/src/type.zig +++ b/src/type.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const Value = @import("value.zig").Value; const assert = std.debug.assert; const Allocator = std.mem.Allocator; @@ -6694,4 +6695,33 @@ pub const Type = extern union { /// This is only used for comptime asserts. Bump this number when you make a change /// to packed struct layout to find out all the places in the codebase you need to edit! pub const packed_struct_layout_version = 2; + + /// This function is used in the debugger pretty formatters in tools/ to fetch the + /// Tag to Payload mapping to facilitate fancy debug printing for this type. + fn dbHelper(self: *Type, tag_to_payload_map: *map: { + const tags = @typeInfo(Tag).Enum.fields; + var fields: [tags.len]std.builtin.Type.StructField = undefined; + for (&fields, tags) |*field, t| field.* = .{ + .name = t.name, + .type = *if (t.value < Tag.no_payload_count) void else @field(Tag, t.name).Type(), + .default_value = null, + .is_comptime = false, + .alignment = 0, + }; + break :map @Type(.{ .Struct = .{ + .layout = .Extern, + .fields = &fields, + .decls = &.{}, + .is_tuple = false, + } }); + }) void { + _ = self; + _ = tag_to_payload_map; + } + + comptime { + if (builtin.mode == .Debug) { + _ = dbHelper; + } + } }; diff --git a/src/value.zig b/src/value.zig index 5646a837ad..4a5683df36 100644 --- a/src/value.zig +++ b/src/value.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const Type = @import("type.zig").Type; const log2 = std.math.log2; const assert = std.debug.assert; @@ -5584,6 +5585,35 @@ pub const Value = extern union { ri.* = @intToEnum(RuntimeIndex, @enumToInt(ri.*) + 1); } }; + + /// This function is used in the debugger pretty formatters in tools/ to fetch the + /// Tag to Payload mapping to facilitate fancy debug printing for this type. + fn dbHelper(self: *Value, tag_to_payload_map: *map: { + const tags = @typeInfo(Tag).Enum.fields; + var fields: [tags.len]std.builtin.Type.StructField = undefined; + for (&fields, tags) |*field, t| field.* = .{ + .name = t.name, + .type = *if (t.value < Tag.no_payload_count) void else @field(Tag, t.name).Type(), + .default_value = null, + .is_comptime = false, + .alignment = 0, + }; + break :map @Type(.{ .Struct = .{ + .layout = .Extern, + .fields = &fields, + .decls = &.{}, + .is_tuple = false, + } }); + }) void { + _ = self; + _ = tag_to_payload_map; + } + + comptime { + if (builtin.mode == .Debug) { + _ = dbHelper; + } + } }; var negative_one_payload: Value.Payload.I64 = .{ diff --git a/tools/lldb_pretty_printers.py b/tools/lldb_pretty_printers.py new file mode 100644 index 0000000000..3013fbb43e --- /dev/null +++ b/tools/lldb_pretty_printers.py @@ -0,0 +1,575 @@ +# pretty printing for the zig language, zig standard library, and zig stage 2 compiler. +# put commands in ~/.lldbinit to run them automatically when starting lldb +# `command script /path/to/stage2_lldb_pretty_printers.py` to import this file +# `type category enable zig` to enable pretty printing for the zig language +# `type category enable zig.std` to enable pretty printing for the zig standard library +# `type category enable zig.stage2` to enable pretty printing for the zig stage 2 compiler +import lldb +import re + +page_size = 1 << 12 + +def log2_int(i): return i.bit_length() - 1 + +# Define Zig Language + +zig_keywords = { + 'addrspace', + 'align', + 'allowzero', + 'and', + 'anyframe', + 'anytype', + 'asm', + 'async', + 'await', + 'break', + 'callconv', + 'catch', + 'comptime', + 'const', + 'continue', + 'defer', + 'else', + 'enum', + 'errdefer', + 'error', + 'export', + 'extern', + 'fn', + 'for', + 'if', + 'inline', + 'noalias', + 'noinline', + 'nosuspend', + 'opaque', + 'or', + 'orelse', + 'packed', + 'pub', + 'resume', + 'return', + 'linksection', + 'struct', + 'suspend', + 'switch', + 'test', + 'threadlocal', + 'try', + 'union', + 'unreachable', + 'usingnamespace', + 'var', + 'volatile', + 'while', +} +zig_primitives = { + 'anyerror', + 'anyframe', + 'anyopaque', + 'bool', + 'c_int', + 'c_long', + 'c_longdouble', + 'c_longlong', + 'c_short', + 'c_uint', + 'c_ulong', + 'c_ulonglong', + 'c_ushort', + 'comptime_float', + 'comptime_int', + 'f128', + 'f16', + 'f32', + 'f64', + 'f80', + 'false', + 'isize', + 'noreturn', + 'null', + 'true', + 'type', + 'undefined', + 'usize', + 'void', +} +zig_integer_type = re.compile('[iu][1-9][0-9]+') +zig_identifier_regex = re.compile('[A-Z_a-z][0-9A-Z_a-z]*') +def zig_IsVariableName(string): return string != '_' and string not in zig_keywords and string not in zig_primitives and not zig_integer_type.fullmatch(string) and zig_identifier_regex.fullmatch(string) +def zig_IsFieldName(string): return string not in zig_keywords and zig_identifier_regex.fullmatch(string) + +class zig_Slice_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + self.ptr = self.value.GetChildMemberWithName('ptr') + self.len = self.value.GetChildMemberWithName('len').unsigned if self.ptr.unsigned > page_size else 0 + self.elem_type = self.ptr.type.GetPointeeType() + self.elem_size = self.elem_type.size + except: pass + def has_children(self): return True + def num_children(self): return self.len or 0 + def get_child_index(self, name): + try: return int(name.removeprefix('[').removesuffix(']')) + except: return -1 + def get_child_at_index(self, index): + if index < 0 or index >= self.len: return None + try: return self.ptr.CreateChildAtOffset('[%d]' % index, index * self.elem_size, self.elem_type) + except: return None + +def zig_String_decode(value, offset=0, length=None): + try: + value = value.GetNonSyntheticValue() + data = value.GetChildMemberWithName('ptr').GetPointeeData(offset, length if length is not None else value.GetChildMemberWithName('len').unsigned) + b = bytes(data.uint8) + b = b.replace(b'\\', b'\\\\') + b = b.replace(b'\n', b'\\n') + b = b.replace(b'\r', b'\\r') + b = b.replace(b'\t', b'\\t') + b = b.replace(b'"', b'\\"') + b = b.replace(b'\'', b'\\\'') + s = b.decode(encoding='ascii', errors='backslashreplace') + return s if s.isprintable() else ''.join((c if c.isprintable() else '\\x%02x' % ord(c) for c in s)) + except: return None +def zig_String_SummaryProvider(value, _=None): return '"%s"' % zig_String_decode(value) +def zig_String_AsIdentifier(value, pred): + string = zig_String_decode(value) + return string if pred(string) else '@"%s"' % string + +class zig_Optional_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + self.child = self.value.GetChildMemberWithName('some').unsigned == 1 and self.value.GetChildMemberWithName('data').Clone('child') + except: pass + def has_children(self): return bool(self.child) + def num_children(self): return int(self.child) + def get_child_index(self, name): return 0 if self.child and (name == 'child' or name == '?') else -1 + def get_child_at_index(self, index): return self.child if self.child and index == 0 else None +def zig_Optional_SummaryProvider(value, _=None): + child = value.GetChildMemberWithName('child') + return child or 'null' + +class zig_ErrorUnion_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + self.error_set = self.value.GetChildMemberWithName('tag').Clone('error_set') + self.payload = self.value.GetChildMemberWithName('value').Clone('payload') if self.error_set.unsigned == 0 else None + except: pass + def has_children(self): return True + def num_children(self): return 1 + def get_child_index(self, name): return 0 if name == ('payload' if self.payload else 'error_set') else -1 + def get_child_at_index(self, index): return self.payload or self.error_set if index == 0 else None + +# Define Zig Standard Library + +class std_SegmentedList_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + self.prealloc_segment = self.value.GetChildMemberWithName('prealloc_segment') + self.dynamic_segments = zig_Slice_SynthProvider(self.value.GetChildMemberWithName('dynamic_segments')) + self.dynamic_segments.update() + self.len = self.value.GetChildMemberWithName('len').unsigned + except: pass + def has_children(self): return True + def num_children(self): return self.len + def get_child_index(self, name): + try: return int(name.removeprefix('[').removesuffix(']')) + except: return -1 + def get_child_at_index(self, index): + try: + if index < 0 or index >= self.len: return None + prealloc_item_count = len(self.prealloc_segment) + if index < prealloc_item_count: return self.prealloc_segment.child[index] + prealloc_exp = prealloc_item_count.bit_length() - 1 + shelf_index = log2_int(index + 1) if prealloc_item_count == 0 else log2_int(index + prealloc_item_count) - prealloc_exp - 1 + shelf = self.dynamic_segments.get_child_at_index(shelf_index) + box_index = (index + 1) - (1 << shelf_index) if prealloc_item_count == 0 else index + prealloc_item_count - (1 << ((prealloc_exp + 1) + shelf_index)) + elem_type = shelf.type.GetPointeeType() + return shelf.CreateChildAtOffset('[%d]' % index, box_index * elem_type.size, elem_type) + except: return None + +class std_MultiArrayList_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + self.len = 0 + + value_type = self.value.type + for helper in self.value.target.FindFunctions('%s.dbHelper' % value_type.name, lldb.eFunctionNameTypeFull): + ptr_self_type, ptr_child_type, ptr_entry_type = helper.function.type.GetFunctionArgumentTypes() + if ptr_self_type.GetPointeeType() == value_type: break + else: return + + self.entry_type = ptr_entry_type.GetPointeeType() + self.bytes = self.value.GetChildMemberWithName('bytes') + self.len = self.value.GetChildMemberWithName('len').unsigned + self.capacity = self.value.GetChildMemberWithName('capacity').unsigned + except: pass + def has_children(self): return True + def num_children(self): return self.len + def get_child_index(self, name): + try: return int(name.removeprefix('[').removesuffix(']')) + except: return -1 + def get_child_at_index(self, index): + try: + if index < 0 or index >= self.len: return None + offset = 0 + data = lldb.SBData() + for field in self.entry_type.fields: + ptr_field_type = field.type + field_size = ptr_field_type.GetPointeeType().size + data.Append(self.bytes.CreateChildAtOffset(field.name, offset + index * field_size, ptr_field_type).address_of.data) + offset += self.capacity * field_size + return self.bytes.CreateValueFromData('[%d]' % index, data, self.entry_type) + except: return None + +class std_HashMapUnmanaged_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + self.capacity = 0 + self.indices = tuple() + + self.metadata = self.value.GetChildMemberWithName('metadata') + if not self.metadata.unsigned: return + + value_type = self.value.type + for helper in self.value.target.FindFunctions('%s.dbHelper' % value_type.name, lldb.eFunctionNameTypeFull): + ptr_self_type, ptr_hdr_type, ptr_entry_type = helper.function.type.GetFunctionArgumentTypes() + if ptr_self_type.GetPointeeType() == value_type: break + else: return + self.entry_type = ptr_entry_type.GetPointeeType() + + hdr_type = ptr_hdr_type.GetPointeeType() + hdr = self.metadata.CreateValueFromAddress('header', self.metadata.deref.load_addr - hdr_type.size, hdr_type) + self.values = hdr.GetChildMemberWithName('values') + self.keys = hdr.GetChildMemberWithName('keys') + self.capacity = hdr.GetChildMemberWithName('capacity').unsigned + + self.indices = tuple(i for i, value in enumerate(self.metadata.GetPointeeData(0, self.capacity).sint8) if value < 0) + except: pass + def has_children(self): return True + def num_children(self): return len(self.indices) + def get_capacity(self): return self.capacity + def get_child_index(self, name): + try: return int(name.removeprefix('[').removesuffix(']')) + except: return -1 + def get_child_at_index(self, index): + try: + fields = {name: base.CreateChildAtOffset(name, self.indices[index] * pointee_type.size, pointee_type).address_of.data for name, base, pointee_type in ((name, base, base.type.GetPointeeType()) for name, base in (('key_ptr', self.keys), ('value_ptr', self.values)))} + data = lldb.SBData() + for field in self.entry_type.fields: data.Append(fields[field.name]) + return self.metadata.CreateValueFromData('[%d]' % index, data, self.entry_type) + except: return None +def std_HashMapUnmanaged_SummaryProvider(value, _=None): + synth = std_HashMapUnmanaged_SynthProvider(value.GetNonSyntheticValue(), _) + synth.update() + return 'len=%d capacity=%d' % (synth.num_children(), synth.get_capacity()) + +# formats a struct of fields of the form `name_ptr: *Type` by auto dereferencing its fields +class std_Entry_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + self.children = tuple(child.Clone(child.name.removesuffix('_ptr')) for child in self.value.children if child.type.GetPointeeType().size != 0) + self.indices = {child.name: i for i, child in enumerate(self.children)} + except: pass + def has_children(self): return self.num_children() != 0 + def num_children(self): return len(self.children) + def get_child_index(self, name): return self.indices.get(name) + def get_child_at_index(self, index): return self.children[index].deref if index >= 0 and index < len(self.children) else None + +# Define Zig Stage2 Compiler + +class TagAndPayload_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + self.tag = self.value.GetChildMemberWithName('tag') or self.value.GetChildMemberWithName('tag_ptr').deref.Clone('tag') + data = self.value.GetChildMemberWithName('data_ptr') or self.value.GetChildMemberWithName('data') + self.payload = data.GetChildMemberWithName('payload').GetChildMemberWithName(data.GetChildMemberWithName('tag').value) + except: pass + def has_children(self): return True + def num_children(self): return 2 + def get_child_index(self, name): + try: return ('tag', 'payload').index(name) + except: return -1 + def get_child_at_index(self, index): return (self.tag, self.payload)[index] if index >= 0 and index < 2 else None + +def Inst_Ref_SummaryProvider(value, _=None): + members = value.type.enum_members + return value if any(value.unsigned == member.unsigned for member in members) else 'instructions[%d]' % (value.unsigned - len(members)) + +class Module_Decl__Module_Decl_Index_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + for frame in self.value.thread: + mod = frame.FindVariable('mod') or frame.FindVariable('module') + if mod: break + else: return + self.ptr = mod.GetChildMemberWithName('allocated_decls').GetChildAtIndex(self.value.unsigned).Clone('decl') + except: pass + def has_children(self): return True + def num_children(self): return 1 + def get_child_index(self, name): return 0 if name == 'decl' else -1 + def get_child_at_index(self, index): return self.ptr if index == 0 else None + +class TagOrPayloadPtr_SynthProvider: + def __init__(self, value, _=None): self.value = value + def update(self): + try: + value_type = self.value.type + for helper in self.value.target.FindFunctions('%s.dbHelper' % value_type.name, lldb.eFunctionNameTypeFull): + ptr_self_type, ptr_tag_to_payload_map_type = helper.function.type.GetFunctionArgumentTypes() + self_type = ptr_self_type.GetPointeeType() + if self_type == value_type: break + else: return + tag_to_payload_map = {field.name: field.type for field in ptr_tag_to_payload_map_type.GetPointeeType().fields} + + tag = self.value.GetChildMemberWithName('tag_if_small_enough') + if tag.unsigned < page_size: + self.tag = tag.Clone('tag') + self.payload = None + else: + ptr_otherwise = self.value.GetChildMemberWithName('ptr_otherwise') + self.tag = ptr_otherwise.GetChildMemberWithName('tag') + self.payload = ptr_otherwise.Cast(tag_to_payload_map[self.tag.value]).GetChildMemberWithName('data').Clone('payload') + except: pass + def has_children(self): return True + def num_children(self): return 1 + (self.payload is not None) + def get_child_index(self, name): + try: return ('tag', 'payload').index(name) + except: return -1 + def get_child_at_index(self, index): return (self.tag, self.payload)[index] if index >= 0 and index < 2 else None + +def Module_Decl_name(decl): + error = lldb.SBError() + return decl.process.ReadCStringFromMemory(decl.GetChildMemberWithName('name').deref.load_addr, 256, error) + +def Module_Namespace_RenderFullyQualifiedName(namespace): + parent = namespace.GetChildMemberWithName('parent') + if parent.unsigned < page_size: return zig_String_decode(namespace.GetChildMemberWithName('file_scope').GetChildMemberWithName('sub_file_path')).removesuffix('.zig').replace('/', '.') + return '.'.join((Module_Namespace_RenderFullyQualifiedName(parent), Module_Decl_name(namespace.GetChildMemberWithName('ty').GetChildMemberWithName('payload').GetChildMemberWithName('owner_decl').GetChildMemberWithName('decl')))) + +def Module_Decl_RenderFullyQualifiedName(decl): return '.'.join((Module_Namespace_RenderFullyQualifiedName(decl.GetChildMemberWithName('src_namespace')), Module_Decl_name(decl))) + +def OwnerDecl_RenderFullyQualifiedName(payload): return Module_Decl_RenderFullyQualifiedName(payload.GetChildMemberWithName('owner_decl').GetChildMemberWithName('decl')) + +def type_Type_pointer(payload): + pointee_type = payload.GetChildMemberWithName('pointee_type') + sentinel = payload.GetChildMemberWithName('sentinel').GetChildMemberWithName('child') + align = payload.GetChildMemberWithName('align').unsigned + addrspace = payload.GetChildMemberWithName('addrspace').value + bit_offset = payload.GetChildMemberWithName('bit_offset').unsigned + host_size = payload.GetChildMemberWithName('host_size').unsigned + vector_index = payload.GetChildMemberWithName('vector_index') + allowzero = payload.GetChildMemberWithName('allowzero').unsigned + const = not payload.GetChildMemberWithName('mutable').unsigned + volatile = payload.GetChildMemberWithName('volatile').unsigned + size = payload.GetChildMemberWithName('size').value + + if size == 'One': summary = '*' + elif size == 'Many': summary = '[*' + elif size == 'Slice': summary = '[' + elif size == 'C': summary = '[*c' + if sentinel: summary += ':%s' % value_Value_SummaryProvider(sentinel) + if size != 'One': summary += ']' + if allowzero: summary += 'allowzero ' + if align != 0 or host_size != 0 or vector_index.value != 'none': summary += 'align(%d%s%s) ' % (align, ':%d:%d' % (bit_offset, host_size) if bit_offset != 0 or host_size != 0 else '', ':?' if vector_index.value == 'runtime' else ':%d' % vector_index.unsigned if vector_index.value != 'none' else '') + if addrspace != 'generic': summary += 'addrspace(.%s) ' % addrspace + if const: summary += 'const ' + if volatile: summary += 'volatile ' + summary += type_Type_SummaryProvider(pointee_type) + return summary + +def type_Type_function(payload): + param_types = payload.GetChildMemberWithName('param_types').children + comptime_params = payload.GetChildMemberWithName('comptime_params').GetPointeeData(0, len(param_types)).uint8 + return_type = payload.GetChildMemberWithName('return_type') + alignment = payload.GetChildMemberWithName('alignment').unsigned + noalias_bits = payload.GetChildMemberWithName('noalias_bits').unsigned + cc = payload.GetChildMemberWithName('cc').value + is_var_args = payload.GetChildMemberWithName('is_var_args').unsigned + + return 'fn(%s)%s%s %s' % (', '.join(tuple(''.join(('comptime ' if comptime_param else '', 'noalias ' if noalias_bits & 1 << i else '', type_Type_SummaryProvider(param_type))) for i, (comptime_param, param_type) in enumerate(zip(comptime_params, param_types))) + (('...',) if is_var_args else ())), ' align(%d)' % alignment if alignment != 0 else '', ' callconv(.%s)' % cc if cc != 'Unspecified' else '', type_Type_SummaryProvider(return_type)) + +def type_Type_SummaryProvider(value, _=None): + tag = value.GetChildMemberWithName('tag').value + return type_tag_handlers.get(tag, lambda payload: tag)(value.GetChildMemberWithName('payload')) + +type_tag_handlers = { + 'atomic_order': lambda payload: 'std.builtin.AtomicOrder', + 'atomic_rmw_op': lambda payload: 'std.builtin.AtomicRmwOp', + 'calling_convention': lambda payload: 'std.builtin.CallingConvention', + 'address_space': lambda payload: 'std.builtin.AddressSpace', + 'float_mode': lambda payload: 'std.builtin.FloatMode', + 'reduce_op': lambda payload: 'std.builtin.ReduceOp', + 'modifier': lambda payload: 'std.builtin.CallModifier', + 'prefetch_options': lambda payload: 'std.builtin.PrefetchOptions', + 'export_options': lambda payload: 'std.builtin.ExportOptions', + 'extern_options': lambda payload: 'std.builtin.ExternOptions', + 'type_info': lambda payload: 'std.builtin.Type', + + 'enum_literal': lambda payload: '@TypeOf(.enum_literal)', + 'null': lambda payload: '@TypeOf(null)', + 'undefined': lambda payload: '@TypeOf(undefined)', + 'empty_struct_literal': lambda payload: '@TypeOf(.{})', + + 'anyerror_void_error_union': lambda payload: 'anyerror!void', + 'const_slice_u8': lambda payload: '[]const u8', + 'const_slice_u8_sentinel_0': lambda payload: '[:0]const u8', + 'fn_noreturn_no_args': lambda payload: 'fn() noreturn', + 'fn_void_no_args': lambda payload: 'fn() void', + 'fn_naked_noreturn_no_args': lambda payload: 'fn() callconv(.Naked) noreturn', + 'fn_ccc_void_no_args': lambda payload: 'fn() callconv(.C) void', + 'single_const_pointer_to_comptime_int': lambda payload: '*const comptime_int', + 'manyptr_u8': lambda payload: '[*]u8', + 'manyptr_const_u8': lambda payload: '[*]const u8', + 'manyptr_const_u8_sentinel_0': lambda payload: '[*:0]const u8', + + 'function': type_Type_function, + 'error_union': lambda payload: '%s!%s' % (type_Type_SummaryProvider(payload.GetChildMemberWithName('error_set')), type_Type_SummaryProvider(payload.GetChildMemberWithName('payload'))), + 'array_u8': lambda payload: '[%d]u8' % payload.unsigned, + 'array_u8_sentinel_0': lambda payload: '[%d:0]u8' % payload.unsigned, + 'vector': lambda payload: '@Vector(%d, %s)' % (payload.GetChildMemberWithName('len').unsigned, type_Type_SummaryProvider(payload.GetChildMemberWithName('elem_type'))), + 'array': lambda payload: '[%d]%s' % (payload.GetChildMemberWithName('len').unsigned, type_Type_SummaryProvider(payload.GetChildMemberWithName('elem_type'))), + 'array_sentinel': lambda payload: '[%d:%s]%s' % (payload.GetChildMemberWithName('len').unsigned, value_Value_SummaryProvider(payload.GetChildMemberWithName('sentinel')), type_Type_SummaryProvider(payload.GetChildMemberWithName('elem_type'))), + 'tuple': lambda payload: 'tuple{%s}' % ', '.join(('comptime %%s = %s' % value_Value_SummaryProvider(value) if value.GetChildMemberWithName('tag').value != 'unreachable_value' else '%s') % type_Type_SummaryProvider(type) for type, value in zip(payload.GetChildMemberWithName('types').children, payload.GetChildMemberWithName('values').children)), + 'anon_struct': lambda payload: 'struct{%s}' % ', '.join(('comptime %%s: %%s = %s' % value_Value_SummaryProvider(value) if value.GetChildMemberWithName('tag').value != 'unreachable_value' else '%s: %s') % (zig_String_AsIdentifier(name, zig_IsFieldName), type_Type_SummaryProvider(type)) for name, type, value in zip(payload.GetChildMemberWithName('names').children, payload.GetChildMemberWithName('types').children, payload.GetChildMemberWithName('values').children)), + 'pointer': type_Type_pointer, + 'single_const_pointer': lambda payload: '*const %s' % type_Type_SummaryProvider(payload), + 'single_mut_pointer': lambda payload: '*%s' % type_Type_SummaryProvider(payload), + 'many_const_pointer': lambda payload: '[*]const %s' % type_Type_SummaryProvider(payload), + 'many_mut_pointer': lambda payload: '[*]%s' % type_Type_SummaryProvider(payload), + 'c_const_pointer': lambda payload: '[*c]const %s' % type_Type_SummaryProvider(payload), + 'c_mut_pointer': lambda payload: '[*c]%s' % type_Type_SummaryProvider(payload), + 'const_slice': lambda payload: '[]const %s' % type_Type_SummaryProvider(payload), + 'mut_slice': lambda payload: '[]%s' % type_Type_SummaryProvider(payload), + 'int_signed': lambda payload: 'i%d' % payload.unsigned, + 'int_unsigned': lambda payload: 'u%d' % payload.unsigned, + 'optional': lambda payload: '?%s' % type_Type_SummaryProvider(payload), + 'optional_single_mut_pointer': lambda payload: '?*%s' % type_Type_SummaryProvider(payload), + 'optional_single_const_pointer': lambda payload: '?*const %s' % type_Type_SummaryProvider(payload), + 'anyframe_T': lambda payload: 'anyframe->%s' % type_Type_SummaryProvider(payload), + 'error_set': lambda payload: type_tag_handlers['error_set_merged'](payload.GetChildMemberWithName('names')), + 'error_set_single': lambda payload: 'error{%s}' % zig_String_AsIdentifier(payload, zig_IsFieldName), + 'error_set_merged': lambda payload: 'error{%s}' % ','.join(zig_String_AsIdentifier(child.GetChildMemberWithName('key'), zig_IsFieldName) for child in payload.GetChildMemberWithName('entries').children), + 'error_set_inferred': lambda payload: '@typeInfo(@typeInfo(@TypeOf(%s)).Fn.return_type.?).ErrorUnion.error_set' % OwnerDecl_RenderFullyQualifiedName(payload.GetChildMemberWithName('func')), + + 'enum_full': OwnerDecl_RenderFullyQualifiedName, + 'enum_nonexhaustive': OwnerDecl_RenderFullyQualifiedName, + 'enum_numbered': OwnerDecl_RenderFullyQualifiedName, + 'enum_simple': OwnerDecl_RenderFullyQualifiedName, + 'struct': OwnerDecl_RenderFullyQualifiedName, + 'union': OwnerDecl_RenderFullyQualifiedName, + 'union_safety_tagged': OwnerDecl_RenderFullyQualifiedName, + 'union_tagged': OwnerDecl_RenderFullyQualifiedName, + 'opaque': OwnerDecl_RenderFullyQualifiedName, +} + +def value_Value_str_lit(payload): + for frame in payload.thread: + mod = frame.FindVariable('mod') or frame.FindVariable('module') + if mod: break + else: return + return '"%s"' % zig_String_decode(mod.GetChildMemberWithName('string_literal_bytes').GetChildMemberWithName('items'), payload.GetChildMemberWithName('index').unsigned, payload.GetChildMemberWithName('len').unsigned) + +def value_Value_SummaryProvider(value, _=None): + tag = value.GetChildMemberWithName('tag').value + return value_tag_handlers.get(tag, lambda payload: tag.removesuffix('_type'))(value.GetChildMemberWithName('payload')) + +value_tag_handlers = { + 'undef': lambda payload: 'undefined', + 'zero': lambda payload: '0', + 'one': lambda payload: '1', + 'void_value': lambda payload: '{}', + 'unreachable_value': lambda payload: 'unreachable', + 'null_value': lambda payload: 'null', + 'bool_true': lambda payload: 'true', + 'bool_false': lambda payload: 'false', + + 'empty_struct_value': lambda payload: '.{}', + 'empty_array': lambda payload: '.{}', + + 'ty': type_Type_SummaryProvider, + 'int_type': lambda payload: '%c%d' % (payload.GetChildMemberWithName('bits').unsigned, 's' if payload.GetChildMemberWithName('signed').unsigned == 1 else 'u'), + 'int_u64': lambda payload: '%d' % payload.unsigned, + 'int_i64': lambda payload: '%d' % payload.signed, + 'int_big_positive': lambda payload: sum(child.unsigned << i * child.type.size * 8 for i, child in enumerate(payload.children)), + 'int_big_negative': lambda payload: '-%s' % value_tag_handlers['int_big_positive'](payload), + 'function': OwnerDecl_RenderFullyQualifiedName, + 'extern_fn': OwnerDecl_RenderFullyQualifiedName, + 'variable': lambda payload: value_Value_SummaryProvider(payload.GetChildMemberWithName('decl').GetChildMemberWithName('val')), + 'runtime_value': value_Value_SummaryProvider, + 'decl_ref': lambda payload: value_Value_SummaryProvider(payload.GetChildMemberWithName('decl').GetChildMemberWithName('val')), + 'decl_ref_mut': lambda payload: value_Value_SummaryProvider(payload.GetChildMemberWithName('decl_index').GetChildMemberWithName('decl').GetChildMemberWithName('val')), + 'comptime_field_ptr': lambda payload: '&%s' % value_Value_SummaryProvider(payload.GetChildMemberWithName('field_val')), + 'elem_ptr': lambda payload: '(%s)[%d]' % (value_Value_SummaryProvider(payload.GetChildMemberWithName('array_ptr')), payload.GetChildMemberWithName('index').unsigned), + 'field_ptr': lambda payload: '(%s).field[%d]' % (value_Value_SummaryProvider(payload.GetChildMemberWithName('container_ptr')), payload.GetChildMemberWithName('field_index').unsigned), + 'bytes': lambda payload: '"%s"' % zig_String_decode(payload), + 'str_lit': value_Value_str_lit, + 'repeated': lambda payload: '.{%s} ** _' % value_Value_SummaryProvider(payload), + 'empty_array_sentinel': lambda payload: '.{%s}' % value_Value_SummaryProvider(payload), + 'slice': lambda payload: '(%s)[0..%s]' % tuple(value_Value_SummaryProvider(payload.GetChildMemberWithName(name)) for name in ('ptr', 'len')), + 'float_16': lambda payload: payload.value, + 'float_32': lambda payload: payload.value, + 'float_64': lambda payload: payload.value, + 'float_80': lambda payload: payload.value, + 'float_128': lambda payload: payload.value, + 'enum_literal': lambda payload: '.%s' % zig_String_AsIdentifier(payload, zig_IsFieldName), + 'enum_field_index': lambda payload: 'field[%d]' % payload.unsigned, + 'error': lambda payload: 'error.%s' % zig_String_AsIdentifier(payload.GetChildMemberWithName('name'), zig_IsFieldName), + 'eu_payload': value_Value_SummaryProvider, + 'eu_payload_ptr': lambda payload: '&((%s).* catch unreachable)' % value_Value_SummaryProvider(payload.GetChildMemberWithName('container_ptr')), + 'opt_payload': value_Value_SummaryProvider, + 'opt_payload_ptr': lambda payload: '&(%s).*.?' % value_Value_SummaryProvider(payload.GetChildMemberWithName('container_ptr')), + 'aggregate': lambda payload: '.{%s}' % ', '.join(map(value_Value_SummaryProvider, payload.children)), + 'union': lambda payload: '.{.%s = %s}' % tuple(value_Value_SummaryProvider(payload.GetChildMemberWithName(name)) for name in ('tag', 'val')), + + 'lazy_align': lambda payload: '@alignOf(%s)' % type_Type_SummaryProvider(payload), + 'lazy_size': lambda payload: '@sizeOf(%s)' % type_Type_SummaryProvider(payload), +} + +# Initialize + +def add(debugger, *, category, regex=False, type, identifier=None, synth=False, inline_children=False, expand=False, summary=False): + prefix = '.'.join((__name__, (identifier or type).replace('.', '_').replace(':', '_'))) + if summary: debugger.HandleCommand('type summary add --category %s%s%s "%s"' % (category, ' --inline-children' if inline_children else ''.join((' --expand' if expand else '', ' --python-function %s_SummaryProvider' % prefix if summary == True else ' --summary-string "%s"' % summary)), ' --regex' if regex else '', type)) + if synth: debugger.HandleCommand('type synthetic add --category %s%s --python-class %s_SynthProvider "%s"' % (category, ' --regex' if regex else '', prefix, type)) + +def MultiArrayList_Entry(type): return '^multi_array_list\\.MultiArrayList\\(%s\\)\\.Entry__struct_[1-9][0-9]*$' % type + +def __lldb_init_module(debugger, _=None): + # Initialize Zig Language + add(debugger, category='zig', regex=True, type='^\\[\\]', identifier='zig_Slice', synth=True, expand=True, summary='len=${svar%#}') + add(debugger, category='zig', type='[]u8', identifier='zig_String', summary=True) + add(debugger, category='zig', regex=True, type='^\\?', identifier='zig_Optional', synth=True, summary=True) + add(debugger, category='zig', regex=True, type='^(error{.*}|anyerror)!', identifier='zig_ErrorUnion', synth=True, inline_children=True, summary=True) + + # Initialize Zig Standard Library + add(debugger, category='zig.std', type='mem.Allocator', summary='${var.ptr}') + add(debugger, category='zig.std', regex=True, type='^segmented_list\\.SegmentedList\\(.*\\)$', identifier='std_SegmentedList', synth=True, expand=True, summary='len=${var.len}') + add(debugger, category='zig.std', regex=True, type='^multi_array_list\\.MultiArrayList\\(.*\\)$', identifier='std_MultiArrayList', synth=True, expand=True, summary='len=${var.len} capacity=${var.capacity}') + add(debugger, category='zig.std', regex=True, type=MultiArrayList_Entry('.*'), identifier='std_Entry', synth=True, inline_children=True, summary=True) + add(debugger, category='zig.std', regex=True, type='^hash_map\\.HashMapUnmanaged\\(.*\\)$', identifier='std_HashMapUnmanaged', synth=True, expand=True, summary=True) + add(debugger, category='zig.std', regex=True, type='^hash_map\\.HashMapUnmanaged\\(.*\\)\\.Entry$', identifier = 'std_Entry', synth=True, inline_children=True, summary=True) + + # Initialize Zig Stage2 Compiler + add(debugger, category='zig.stage2', type='Zir.Inst', identifier='TagAndPayload', synth=True, inline_children=True, summary=True) + add(debugger, category='zig.stage2', regex=True, type=MultiArrayList_Entry('Zir\\.Inst'), identifier='TagAndPayload', synth=True, inline_children=True, summary=True) + add(debugger, category='zig.stage2', regex=True, type='^Zir\\.Inst\\.Data\\.Data__struct_[1-9][0-9]*$', inline_children=True, summary=True) + add(debugger, category='zig.stage2', type='Zir.Inst::Zir.Inst.Ref', identifier='Inst_Ref', summary=True) + add(debugger, category='zig.stage2', type='Air.Inst', identifier='TagAndPayload', synth=True, inline_children=True, summary=True) + add(debugger, category='zig.stage2', regex=True, type=MultiArrayList_Entry('Air\\.Inst'), identifier='TagAndPayload', synth=True, inline_children=True, summary=True) + add(debugger, category='zig.stage2', regex=True, type='^Air\\.Inst\\.Data\\.Data__struct_[1-9][0-9]*$', inline_children=True, summary=True) + add(debugger, category='zig.stage2', type='Module.Decl::Module.Decl.Index', synth=True) + add(debugger, category='zig.stage2', type='type.Type', identifier='TagOrPayloadPtr', synth=True) + add(debugger, category='zig.stage2', type='type.Type', summary=True) + add(debugger, category='zig.stage2', type='value.Value', identifier='TagOrPayloadPtr', synth=True) + add(debugger, category='zig.stage2', type='value.Value', summary=True) diff --git a/tools/stage2_lldb_pretty_printers.py b/tools/stage2_lldb_pretty_printers.py deleted file mode 100644 index 3dcb06f538..0000000000 --- a/tools/stage2_lldb_pretty_printers.py +++ /dev/null @@ -1,59 +0,0 @@ -# pretty printing for stage 2. -# put "command script /path/to/stage2_lldb_pretty_printers.py" and "type category enable stage2" in ~/.lldbinit to load it automatically. -import lldb -import stage2_pretty_printers_common as common - -category = 'stage2' -module = category + '_lldb_pretty_printers' - -class type_Type_SynthProvider: - def __init__(self, type, _=None): - self.type = type - - def update(self): - self.tag = self.type.GetChildMemberWithName('tag_if_small_enough').Clone('tag') - self.payload = None - if self.tag.GetValueAsUnsigned() >= common.Type.no_payload_count: - ptr_otherwise = self.type.GetChildMemberWithName('ptr_otherwise') - self.tag = ptr_otherwise.Dereference().GetChildMemberWithName('tag') - payload_type = self.type.target.FindFirstType('type.' + common.Type.payload_type_names[self.tag.GetValue()]) - self.payload = ptr_otherwise.Cast(payload_type.GetPointerType()).Dereference().GetChildMemberWithName('data').Clone('payload') - - def num_children(self): - return 1 + (self.payload is not None) - - def get_child_index(self, name): - return ['tag', 'payload'].index(name) - - def get_child_at_index(self, index): - return [self.tag, self.payload][index] - -class value_Value_SynthProvider: - def __init__(self, value, _=None): - self.value = value - - def update(self): - self.tag = self.value.GetChildMemberWithName('tag_if_small_enough').Clone('tag') - self.payload = None - if self.tag.GetValueAsUnsigned() >= common.Value.no_payload_count: - ptr_otherwise = self.value.GetChildMemberWithName('ptr_otherwise') - self.tag = ptr_otherwise.Dereference().GetChildMemberWithName('tag') - payload_type = self.value.target.FindFirstType('value.' + common.Value.payload_type_names[self.tag.GetValue()]) - self.payload = ptr_otherwise.Cast(payload_type.GetPointerType()).Dereference().GetChildMemberWithName('data').Clone('payload') - - def num_children(self): - return 1 + (self.payload is not None) - - def get_child_index(self, name): - return ['tag', 'payload'].index(name) - - def get_child_at_index(self, index): - return [self.tag, self.payload][index] - -def add(debugger, type, summary=False, synth=False): - if summary: debugger.HandleCommand('type summary add --python-function ' + module + '.' + type.replace('.', '_') + '_SummaryProvider "' + type + '" --category ' + category) - if synth: debugger.HandleCommand('type synthetic add --python-class ' + module + '.' + type.replace('.', '_') + '_SynthProvider "' + type + '" --category ' + category) - -def __lldb_init_module(debugger, _=None): - add(debugger, 'type.Type', synth=True) - add(debugger, 'value.Value', synth=True) diff --git a/tools/std_gdb_pretty_printers.py b/tools/std_gdb_pretty_printers.py index c89de56fa6..a564de7c18 100644 --- a/tools/std_gdb_pretty_printers.py +++ b/tools/std_gdb_pretty_printers.py @@ -26,7 +26,7 @@ class MultiArrayListPrinter: self.val = val def child_type(self): - (helper_fn, _) = gdb.lookup_symbol('%s.gdbHelper' % self.val.type.name) + (helper_fn, _) = gdb.lookup_symbol('%s.dbHelper' % self.val.type.name) return helper_fn.type.fields()[1].type.target() def to_string(self): @@ -65,7 +65,7 @@ class HashMapPrinter: self.val = val['unmanaged'] if is_managed else val def header_ptr_type(self): - (helper_fn, _) = gdb.lookup_symbol('%s.gdbHelper' % self.val.type.name) + (helper_fn, _) = gdb.lookup_symbol('%s.dbHelper' % self.val.type.name) return helper_fn.type.fields()[1].type def header(self): From 57236961e6fc9da02623f79131f58113e3fa0531 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sun, 26 Feb 2023 22:18:33 -0500 Subject: [PATCH 03/25] tools: revert sharing of stage2 pretty printer info Partial revert of 1e963053d0ff67361b587b046a917375e963d5e9. Now that lldb has a new method for accessing this data that does not require manual updating, there is no longer any reason to share this data. --- tools/stage2_gdb_pretty_printers.py | 119 ++++++++++++++++++++++--- tools/stage2_pretty_printers_common.py | 98 -------------------- 2 files changed, 105 insertions(+), 112 deletions(-) delete mode 100644 tools/stage2_pretty_printers_common.py diff --git a/tools/stage2_gdb_pretty_printers.py b/tools/stage2_gdb_pretty_printers.py index 215b27699f..bd64916536 100644 --- a/tools/stage2_gdb_pretty_printers.py +++ b/tools/stage2_gdb_pretty_printers.py @@ -3,13 +3,55 @@ import re import gdb.printing -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent)) -import stage2_pretty_printers_common as common - - class TypePrinter: + no_payload_count = 4096 + + # Keep in sync with src/type.zig + # Types which have no payload do not need to be entered here. + payload_type_names = { + 'array_u8': 'Type.Payload.Len', + 'array_u8_sentinel_0': 'Type.Payload.Len', + + 'single_const_pointer': 'Type.Payload.ElemType', + 'single_mut_pointer': 'Type.Payload.ElemType', + 'many_const_pointer': 'Type.Payload.ElemType', + 'many_mut_pointer': 'Type.Payload.ElemType', + 'c_const_pointer': 'Type.Payload.ElemType', + 'c_mut_pointer': 'Type.Payload.ElemType', + 'const_slice': 'Type.Payload.ElemType', + 'mut_slice': 'Type.Payload.ElemType', + 'optional': 'Type.Payload.ElemType', + 'optional_single_mut_pointer': 'Type.Payload.ElemType', + 'optional_single_const_pointer': 'Type.Payload.ElemType', + 'anyframe_T': 'Type.Payload.ElemType', + + 'int_signed': 'Type.Payload.Bits', + 'int_unsigned': 'Type.Payload.Bits', + + 'error_set': 'Type.Payload.ErrorSet', + 'error_set_inferred': 'Type.Payload.ErrorSetInferred', + 'error_set_merged': 'Type.Payload.ErrorSetMerged', + + 'array': 'Type.Payload.Array', + 'vector': 'Type.Payload.Array', + + 'array_sentinel': 'Type.Payload.ArraySentinel', + 'pointer': 'Type.Payload.Pointer', + 'function': 'Type.Payload.Function', + 'error_union': 'Type.Payload.ErrorUnion', + 'error_set_single': 'Type.Payload.Name', + 'opaque': 'Type.Payload.Opaque', + 'struct': 'Type.Payload.Struct', + 'union': 'Type.Payload.Union', + 'union_tagged': 'Type.Payload.Union', + 'enum_full, .enum_nonexhaustive': 'Type.Payload.EnumFull', + 'enum_simple': 'Type.Payload.EnumSimple', + 'enum_numbered': 'Type.Payload.EnumNumbered', + 'empty_struct': 'Type.Payload.ContainerScope', + 'tuple': 'Type.Payload.Tuple', + 'anon_struct': 'Type.Payload.AnonStruct', + } + def __init__(self, val): self.val = val @@ -17,7 +59,7 @@ class TypePrinter: tag_if_small_enough = self.val['tag_if_small_enough'] tag_type = tag_if_small_enough.type - if tag_if_small_enough < common.Type.no_payload_count: + if tag_if_small_enough < TypePrinter.no_payload_count: return tag_if_small_enough else: return self.val['ptr_otherwise'].dereference()['tag'] @@ -27,7 +69,7 @@ class TypePrinter: if tag is None: return None - type_name = common.Type.payload_type_names.get(str(tag)) + type_name = TypePrinter.payload_type_names.get(str(tag)) if type_name is None: return None return gdb.lookup_type('struct type.%s' % type_name) @@ -36,12 +78,12 @@ class TypePrinter: tag = self.tag() if tag is None: return '(invalid type)' - if self.val['tag_if_small_enough'] < common.Type.no_payload_count: + if self.val['tag_if_small_enough'] < TypePrinter.no_payload_count: return '.%s' % str(tag) return None def children(self): - if self.val['tag_if_small_enough'] < common.Type.no_payload_count: + if self.val['tag_if_small_enough'] < TypePrinter.no_payload_count: return yield ('tag', '.%s' % str(self.tag())) @@ -51,6 +93,55 @@ class TypePrinter: yield ('payload', self.val['ptr_otherwise'].cast(payload_type.pointer()).dereference()['data']) class ValuePrinter: + no_payload_count = 4096 + + # Keep in sync with src/value.zig + # Values which have no payload do not need to be entered here. + payload_type_names = { + 'big_int_positive': 'Value.Payload.BigInt', + 'big_int_negative': 'Value.Payload.BigInt', + + 'extern_fn': 'Value.Payload.ExternFn', + + 'decl_ref': 'Value.Payload.Decl', + + 'repeated': 'Value.Payload.SubValue', + 'eu_payload': 'Value.Payload.SubValue', + 'opt_payload': 'Value.Payload.SubValue', + 'empty_array_sentinel': 'Value.Payload.SubValue', + + 'eu_payload_ptr': 'Value.Payload.PayloadPtr', + 'opt_payload_ptr': 'Value.Payload.PayloadPtr', + + 'bytes': 'Value.Payload.Bytes', + 'enum_literal': 'Value.Payload.Bytes', + + 'slice': 'Value.Payload.Slice', + + 'enum_field_index': 'Value.Payload.U32', + + 'ty': 'Value.Payload.Ty', + 'int_type': 'Value.Payload.IntType', + 'int_u64': 'Value.Payload.U64', + 'int_i64': 'Value.Payload.I64', + 'function': 'Value.Payload.Function', + 'variable': 'Value.Payload.Variable', + 'decl_ref_mut': 'Value.Payload.DeclRefMut', + 'elem_ptr': 'Value.Payload.ElemPtr', + 'field_ptr': 'Value.Payload.FieldPtr', + 'float_16': 'Value.Payload.Float_16', + 'float_32': 'Value.Payload.Float_32', + 'float_64': 'Value.Payload.Float_64', + 'float_80': 'Value.Payload.Float_80', + 'float_128': 'Value.Payload.Float_128', + 'error': 'Value.Payload.Error', + 'inferred_alloc': 'Value.Payload.InferredAlloc', + 'inferred_alloc_comptime': 'Value.Payload.InferredAllocComptime', + 'aggregate': 'Value.Payload.Aggregate', + 'union': 'Value.Payload.Union', + 'bound_fn': 'Value.Payload.BoundFn', + } + def __init__(self, val): self.val = val @@ -58,7 +149,7 @@ class ValuePrinter: tag_if_small_enough = self.val['tag_if_small_enough'] tag_type = tag_if_small_enough.type - if tag_if_small_enough < common.Value.no_payload_count: + if tag_if_small_enough < ValuePrinter.no_payload_count: return tag_if_small_enough else: return self.val['ptr_otherwise'].dereference()['tag'] @@ -68,7 +159,7 @@ class ValuePrinter: if tag is None: return None - type_name = Comman.Value.payload_type_names.get(str(tag)) + type_name = ValuePrinter.payload_type_names.get(str(tag)) if type_name is None: return None return gdb.lookup_type('struct value.%s' % type_name) @@ -77,12 +168,12 @@ class ValuePrinter: tag = self.tag() if tag is None: return '(invalid value)' - if self.val['tag_if_small_enough'] < common.Value.no_payload_count: + if self.val['tag_if_small_enough'] < ValuePrinter.no_payload_count: return '.%s' % str(tag) return None def children(self): - if self.val['tag_if_small_enough'] < common.Value.no_payload_count: + if self.val['tag_if_small_enough'] < ValuePrinter.no_payload_count: return yield ('tag', '.%s' % str(self.tag())) diff --git a/tools/stage2_pretty_printers_common.py b/tools/stage2_pretty_printers_common.py deleted file mode 100644 index eab369d3a8..0000000000 --- a/tools/stage2_pretty_printers_common.py +++ /dev/null @@ -1,98 +0,0 @@ -class Type: - no_payload_count = 4096 - - # Keep in sync with src/type.zig - # Types which have no payload do not need to be entered here. - payload_type_names = { - 'array_u8': 'Type.Payload.Len', - 'array_u8_sentinel_0': 'Type.Payload.Len', - - 'single_const_pointer': 'Type.Payload.ElemType', - 'single_mut_pointer': 'Type.Payload.ElemType', - 'many_const_pointer': 'Type.Payload.ElemType', - 'many_mut_pointer': 'Type.Payload.ElemType', - 'c_const_pointer': 'Type.Payload.ElemType', - 'c_mut_pointer': 'Type.Payload.ElemType', - 'const_slice': 'Type.Payload.ElemType', - 'mut_slice': 'Type.Payload.ElemType', - 'optional': 'Type.Payload.ElemType', - 'optional_single_mut_pointer': 'Type.Payload.ElemType', - 'optional_single_const_pointer': 'Type.Payload.ElemType', - 'anyframe_T': 'Type.Payload.ElemType', - - 'int_signed': 'Type.Payload.Bits', - 'int_unsigned': 'Type.Payload.Bits', - - 'error_set': 'Type.Payload.ErrorSet', - 'error_set_inferred': 'Type.Payload.ErrorSetInferred', - 'error_set_merged': 'Type.Payload.ErrorSetMerged', - - 'array': 'Type.Payload.Array', - 'vector': 'Type.Payload.Array', - - 'array_sentinel': 'Type.Payload.ArraySentinel', - 'pointer': 'Type.Payload.Pointer', - 'function': 'Type.Payload.Function', - 'error_union': 'Type.Payload.ErrorUnion', - 'error_set_single': 'Type.Payload.Name', - 'opaque': 'Type.Payload.Opaque', - 'struct': 'Type.Payload.Struct', - 'union': 'Type.Payload.Union', - 'union_tagged': 'Type.Payload.Union', - 'enum_full, .enum_nonexhaustive': 'Type.Payload.EnumFull', - 'enum_simple': 'Type.Payload.EnumSimple', - 'enum_numbered': 'Type.Payload.EnumNumbered', - 'empty_struct': 'Type.Payload.ContainerScope', - 'tuple': 'Type.Payload.Tuple', - 'anon_struct': 'Type.Payload.AnonStruct', - } - -class Value: - no_payload_count = 4096 - - # Keep in sync with src/value.zig - # Values which have no payload do not need to be entered here. - payload_type_names = { - 'big_int_positive': 'Value.Payload.BigInt', - 'big_int_negative': 'Value.Payload.BigInt', - - 'extern_fn': 'Value.Payload.ExternFn', - - 'decl_ref': 'Value.Payload.Decl', - - 'repeated': 'Value.Payload.SubValue', - 'eu_payload': 'Value.Payload.SubValue', - 'opt_payload': 'Value.Payload.SubValue', - 'empty_array_sentinel': 'Value.Payload.SubValue', - - 'eu_payload_ptr': 'Value.Payload.PayloadPtr', - 'opt_payload_ptr': 'Value.Payload.PayloadPtr', - - 'bytes': 'Value.Payload.Bytes', - 'enum_literal': 'Value.Payload.Bytes', - - 'slice': 'Value.Payload.Slice', - - 'enum_field_index': 'Value.Payload.U32', - - 'ty': 'Value.Payload.Ty', - 'int_type': 'Value.Payload.IntType', - 'int_u64': 'Value.Payload.U64', - 'int_i64': 'Value.Payload.I64', - 'function': 'Value.Payload.Function', - 'variable': 'Value.Payload.Variable', - 'decl_ref_mut': 'Value.Payload.DeclRefMut', - 'elem_ptr': 'Value.Payload.ElemPtr', - 'field_ptr': 'Value.Payload.FieldPtr', - 'float_16': 'Value.Payload.Float_16', - 'float_32': 'Value.Payload.Float_32', - 'float_64': 'Value.Payload.Float_64', - 'float_80': 'Value.Payload.Float_80', - 'float_128': 'Value.Payload.Float_128', - 'error': 'Value.Payload.Error', - 'inferred_alloc': 'Value.Payload.InferredAlloc', - 'inferred_alloc_comptime': 'Value.Payload.InferredAllocComptime', - 'aggregate': 'Value.Payload.Aggregate', - 'union': 'Value.Payload.Union', - 'bound_fn': 'Value.Payload.BoundFn', - } From 6bed45b87358975da97fec45fd30eb095ad1dfc9 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Mon, 27 Feb 2023 16:12:02 -0500 Subject: [PATCH 04/25] codegen: fix test failures Various backend were mismatching arg instructions and function args. --- src/arch/aarch64/CodeGen.zig | 6 ++++-- src/arch/arm/CodeGen.zig | 6 ++++-- src/arch/x86_64/CodeGen.zig | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index e7fef20a4f..818b04f890 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -4177,8 +4177,10 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { } fn airArg(self: *Self, inst: Air.Inst.Index) !void { - const arg_index = self.arg_index; - self.arg_index += 1; + // skip zero-bit arguments as they don't have a corresponding arg instruction + var arg_index = self.arg_index; + while (self.args[arg_index] == .none) arg_index += 1; + self.arg_index = arg_index + 1; const ty = self.air.typeOfIndex(inst); const tag = self.air.instructions.items(.tag)[inst]; diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index 01a1d6b7eb..ceabe70438 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -4125,8 +4125,10 @@ fn genInlineMemsetCode( } fn airArg(self: *Self, inst: Air.Inst.Index) !void { - const arg_index = self.arg_index; - self.arg_index += 1; + // skip zero-bit arguments as they don't have a corresponding arg instruction + var arg_index = self.arg_index; + while (self.args[arg_index] == .none) arg_index += 1; + self.arg_index = arg_index + 1; const ty = self.air.typeOfIndex(inst); const tag = self.air.instructions.items(.tag)[inst]; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 20e443b83c..53d38f520a 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -3827,8 +3827,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M } fn airArg(self: *Self, inst: Air.Inst.Index) !void { - const arg_index = self.arg_index; - self.arg_index += 1; + // skip zero-bit arguments as they don't have a corresponding arg instruction + var arg_index = self.arg_index; + while (self.args[arg_index] == .none) arg_index += 1; + self.arg_index = arg_index + 1; const ty = self.air.typeOfIndex(inst); const mcv = self.args[arg_index]; From f33af7af40faccc05f66417f1ba5caa4eb158769 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Feb 2023 20:42:13 -0700 Subject: [PATCH 05/25] delete a subtly incorrect Haiku collectOutput implementation It's not OK to half-ass this function. Please implement it correctly, or not at all. --- lib/std/child_process.zig | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index c3bd53b880..55e375b93b 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -210,13 +210,7 @@ pub const ChildProcess = struct { ) !void { debug.assert(child.stdout_behavior == .Pipe); debug.assert(child.stderr_behavior == .Pipe); - if (builtin.os.tag == .haiku) { - const stdout_in = child.stdout.?.reader(); - const stderr_in = child.stderr.?.reader(); - - try stdout_in.readAllArrayList(stdout, max_output_bytes); - try stderr_in.readAllArrayList(stderr, max_output_bytes); - } else if (builtin.os.tag == .windows) { + if (builtin.os.tag == .windows) { try collectOutputWindows(child, stdout, stderr, max_output_bytes); } else { try collectOutputPosix(child, stdout, stderr, max_output_bytes); From 5236842a9d0f7b0a82c440bcc08d315db4051d48 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Feb 2023 22:04:29 -0700 Subject: [PATCH 06/25] std.heap.GeneralPurposeAllocator: add doc comment for deinit --- lib/std/heap/general_purpose_allocator.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig index 452480dc7a..fed6eba47b 100644 --- a/lib/std/heap/general_purpose_allocator.zig +++ b/lib/std/heap/general_purpose_allocator.zig @@ -423,6 +423,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type { } } else struct {}; + /// Returns true if there were leaks; false otherwise. pub fn deinit(self: *Self) bool { const leaks = if (config.safety) self.detectLeaks() else false; if (config.retain_metadata) { From 814de45bd2e6711056c634b089b2ebf0c96f50b9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Feb 2023 22:06:18 -0700 Subject: [PATCH 07/25] add std.io.poll and implement it for POSIX I think having inputs is problematic here, it should only be for outputs. --- lib/std/io.zig | 133 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/lib/std/io.zig b/lib/std/io.zig index a61f2a4e0e..a6a1791760 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -168,6 +168,139 @@ test "null_writer" { null_writer.writeAll("yay" ** 10) catch |err| switch (err) {}; } +pub fn poll( + allocator: std.mem.Allocator, + comptime StreamEnum: type, + files: PollFiles(StreamEnum), +) Poller(StreamEnum) { + const enum_fields = @typeInfo(StreamEnum).Enum.fields; + var result: Poller(StreamEnum) = undefined; + inline for (0..enum_fields.len) |i| { + result.fifos[i] = .{ + .allocator = allocator, + .buf = &.{}, + .head = 0, + .count = 0, + }; + result.poll_fds[i] = .{ + .fd = @field(files, enum_fields[i].name).file.handle, + .events = switch (@field(files, enum_fields[i].name).direction) { + .in => os.POLL.IN, + .out => os.POLL.OUT, + }, + .revents = undefined, + }; + } + return result; +} + +pub fn Poller(comptime StreamEnum: type) type { + return struct { + const enum_fields = @typeInfo(StreamEnum).Enum.fields; + const Fifo = std.fifo.LinearFifo(u8, .Dynamic); + + fifos: [enum_fields.len]Fifo, + //directions: [enum_fields.len]PollFile.Direction, + //handles: [enum_fields.len]std.fs.File.Handle, + poll_fds: [enum_fields.len]std.os.pollfd, + + const Self = @This(); + + pub fn poll(self: *Self) !void { + if (builtin.os.tag == .windows) { + return pollWindows(self); + } else { + return pollPosix(self); + } + } + + pub inline fn fifo(self: *Self, comptime which: StreamEnum) *Fifo { + return &self.fifos[@enumToInt(which)]; + } + + pub fn done(self: Self) bool { + for (self.poll_fds) |poll_fd| { + if (poll_fd.fd != -1) return false; + } else return true; + } + + fn pollWindows(self: *Self) !void { + _ = self; + @compileError("TODO"); + } + + fn pollPosix(self: *Self) !void { + // We ask for ensureUnusedCapacity with this much extra space. This + // has more of an effect on small reads because once the reads + // start to get larger the amount of space an ArrayList will + // allocate grows exponentially. + const bump_amt = 512; + + const err_mask = os.POLL.ERR | os.POLL.NVAL | os.POLL.HUP; + + const events_len = try os.poll(&self.poll_fds, std.math.maxInt(i32)); + if (events_len == 0) return; + + inline for (0..enum_fields.len) |i| { + // Try reading whatever is available before checking the error + // conditions. + // It's still possible to read after a POLL.HUP is received, + // always check if there's some data waiting to be read first. + if (self.poll_fds[i].revents & os.POLL.IN != 0) { + const q = &self.fifos[i]; + const buf = try q.writableWithSize(bump_amt); + const amt = try os.read(self.poll_fds[i].fd, buf); + q.update(amt); + std.debug.print("read {d} bytes\n", .{amt}); + if (amt == 0) { + // Remove the fd when the EOF condition is met. + self.poll_fds[i].fd = -1; + } + } else if (self.poll_fds[i].revents & err_mask != 0) { + // Exclude the fds that signaled an error. + self.poll_fds[i].fd = -1; + } else if (self.poll_fds[i].revents & os.POLL.OUT != 0) { + const q = &self.fifos[i]; + const amt = try os.write(self.poll_fds[i].fd, q.readableSlice(0)); + q.discard(amt); + if (amt == 0) { + self.poll_fds[i].fd = -1; + } + } + } + } + }; +} + +/// Given an enum, returns a struct with fields of that enum, each field +/// representing an I/O stream for polling. +pub fn PollFiles(comptime StreamEnum: type) type { + const enum_fields = @typeInfo(StreamEnum).Enum.fields; + var struct_fields: [enum_fields.len]std.builtin.Type.StructField = undefined; + for (&struct_fields, enum_fields) |*struct_field, enum_field| { + struct_field.* = .{ + .name = enum_field.name, + .type = PollFile, + .default_value = null, + .is_comptime = false, + .alignment = @alignOf(PollFile), + }; + } + return @Type(.{ .Struct = .{ + .layout = .Auto, + .fields = &struct_fields, + .decls = &.{}, + .is_tuple = false, + } }); +} + +pub const PollFile = struct { + file: File, + direction: Direction, + + pub const Direction = enum { in, out }; +}; + test { _ = @import("io/bit_reader.zig"); _ = @import("io/bit_writer.zig"); From d8c3738e21611f0292c205f7b5c5c0db23a23262 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 27 Feb 2023 22:39:47 -0700 Subject: [PATCH 08/25] redo std.io.poll with only outputs --- lib/std/io.zig | 46 +++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/lib/std/io.zig b/lib/std/io.zig index a6a1791760..8563fd50d0 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -183,11 +183,8 @@ pub fn poll( .count = 0, }; result.poll_fds[i] = .{ - .fd = @field(files, enum_fields[i].name).file.handle, - .events = switch (@field(files, enum_fields[i].name).direction) { - .in => os.POLL.IN, - .out => os.POLL.OUT, - }, + .fd = @field(files, enum_fields[i].name).handle, + .events = os.POLL.IN, .revents = undefined, }; } @@ -200,12 +197,15 @@ pub fn Poller(comptime StreamEnum: type) type { const Fifo = std.fifo.LinearFifo(u8, .Dynamic); fifos: [enum_fields.len]Fifo, - //directions: [enum_fields.len]PollFile.Direction, - //handles: [enum_fields.len]std.fs.File.Handle, poll_fds: [enum_fields.len]std.os.pollfd, const Self = @This(); + pub fn deinit(self: *Self) void { + inline for (&self.fifos) |*q| q.deinit(); + self.* = undefined; + } + pub fn poll(self: *Self) !void { if (builtin.os.tag == .windows) { return pollWindows(self); @@ -241,31 +241,22 @@ pub fn Poller(comptime StreamEnum: type) type { const events_len = try os.poll(&self.poll_fds, std.math.maxInt(i32)); if (events_len == 0) return; - inline for (0..enum_fields.len) |i| { + inline for (&self.poll_fds, &self.fifos) |*poll_fd, *q| { // Try reading whatever is available before checking the error // conditions. // It's still possible to read after a POLL.HUP is received, // always check if there's some data waiting to be read first. - if (self.poll_fds[i].revents & os.POLL.IN != 0) { - const q = &self.fifos[i]; + if (poll_fd.revents & os.POLL.IN != 0) { const buf = try q.writableWithSize(bump_amt); - const amt = try os.read(self.poll_fds[i].fd, buf); + const amt = try os.read(poll_fd.fd, buf); q.update(amt); - std.debug.print("read {d} bytes\n", .{amt}); if (amt == 0) { // Remove the fd when the EOF condition is met. - self.poll_fds[i].fd = -1; + poll_fd.fd = -1; } - } else if (self.poll_fds[i].revents & err_mask != 0) { + } else if (poll_fd.revents & err_mask != 0) { // Exclude the fds that signaled an error. - self.poll_fds[i].fd = -1; - } else if (self.poll_fds[i].revents & os.POLL.OUT != 0) { - const q = &self.fifos[i]; - const amt = try os.write(self.poll_fds[i].fd, q.readableSlice(0)); - q.discard(amt); - if (amt == 0) { - self.poll_fds[i].fd = -1; - } + poll_fd.fd = -1; } } } @@ -280,10 +271,10 @@ pub fn PollFiles(comptime StreamEnum: type) type { for (&struct_fields, enum_fields) |*struct_field, enum_field| { struct_field.* = .{ .name = enum_field.name, - .type = PollFile, + .type = fs.File, .default_value = null, .is_comptime = false, - .alignment = @alignOf(PollFile), + .alignment = @alignOf(fs.File), }; } return @Type(.{ .Struct = .{ @@ -294,13 +285,6 @@ pub fn PollFiles(comptime StreamEnum: type) type { } }); } -pub const PollFile = struct { - file: File, - direction: Direction, - - pub const Direction = enum { in, out }; -}; - test { _ = @import("io/bit_reader.zig"); _ = @import("io/bit_writer.zig"); From ef72cd6698e808c731320617ea62c8a33a5ecd87 Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Tue, 28 Feb 2023 12:28:53 -0700 Subject: [PATCH 09/25] std.io.poll initial windows implementation --- lib/std/io.zig | 154 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 143 insertions(+), 11 deletions(-) diff --git a/lib/std/io.zig b/lib/std/io.zig index 8563fd50d0..b125265449 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -175,6 +175,19 @@ pub fn poll( ) Poller(StreamEnum) { const enum_fields = @typeInfo(StreamEnum).Enum.fields; var result: Poller(StreamEnum) = undefined; + + if (builtin.os.tag == .windows) result.windows = .{ + .first_read_done = false, + .overlapped = [1]os.windows.OVERLAPPED { + mem.zeroes(os.windows.OVERLAPPED), + } ** enum_fields.len, + .active = .{ + .count = 0, + .handles_buf = undefined, + .stream_map = undefined, + }, + }; + inline for (0..enum_fields.len) |i| { result.fifos[i] = .{ .allocator = allocator, @@ -182,26 +195,56 @@ pub fn poll( .head = 0, .count = 0, }; - result.poll_fds[i] = .{ - .fd = @field(files, enum_fields[i].name).handle, - .events = os.POLL.IN, - .revents = undefined, - }; + if (builtin.os.tag == .windows) { + result.windows.active.handles_buf[i] = @field(files, enum_fields[i].name).handle; + } else { + result.poll_fds[i] = .{ + .fd = @field(files, enum_fields[i].name).handle, + .events = os.POLL.IN, + .revents = undefined, + }; + } } return result; } +pub const PollFifo = std.fifo.LinearFifo(u8, .Dynamic); + pub fn Poller(comptime StreamEnum: type) type { return struct { const enum_fields = @typeInfo(StreamEnum).Enum.fields; - const Fifo = std.fifo.LinearFifo(u8, .Dynamic); + const PollFd = if (builtin.os.tag == .windows) void else std.os.pollfd; - fifos: [enum_fields.len]Fifo, - poll_fds: [enum_fields.len]std.os.pollfd, + fifos: [enum_fields.len]PollFifo, + poll_fds: [enum_fields.len]PollFd, + windows: if (builtin.os.tag == .windows) struct { + first_read_done: bool, + overlapped: [enum_fields.len]os.windows.OVERLAPPED, + active: struct { + count: math.IntFittingRange(0, enum_fields.len), + handles_buf: [enum_fields.len]os.windows.HANDLE, + stream_map: [enum_fields.len]StreamEnum, + + pub fn removeAt(self: *@This(), index: u32) void { + std.debug.assert(index < self.count); + for (index + 1 .. self.count) |i| { + self.handles_buf[i - 1] = self.handles_buf[i]; + self.stream_map[i - 1] = self.stream_map[i]; + } + self.count -= 1; + } + }, + } else void, const Self = @This(); pub fn deinit(self: *Self) void { + if (builtin.os.tag == .windows) { + // cancel any pending IO to prevent clobbering OVERLAPPED value + for (self.windows.active.handles_buf[0 .. self.windows.active.count]) |h| { + _ = os.windows.kernel32.CancelIo(h); + } + } inline for (&self.fifos) |*q| q.deinit(); self.* = undefined; } @@ -214,19 +257,89 @@ pub fn Poller(comptime StreamEnum: type) type { } } - pub inline fn fifo(self: *Self, comptime which: StreamEnum) *Fifo { + pub inline fn fifo(self: *Self, comptime which: StreamEnum) *PollFifo { return &self.fifos[@enumToInt(which)]; } pub fn done(self: Self) bool { + if (builtin.os.tag == .windows) + return self.windows.first_read_done and self.windows.active.count == 0; + for (self.poll_fds) |poll_fd| { if (poll_fd.fd != -1) return false; } else return true; } fn pollWindows(self: *Self) !void { - _ = self; - @compileError("TODO"); + const bump_amt = 512; + + if (!self.windows.first_read_done) { + // Windows Async IO requires an initial call to ReadFile before waiting on the handle + for (0..enum_fields.len) |i| { + const handle = self.windows.active.handles_buf[i]; + switch (try windowsAsyncRead( + handle, + &self.windows.overlapped[i], + &self.fifos[i], + bump_amt, + )) { + .pending => { + self.windows.active.handles_buf[self.windows.active.count] = handle; + self.windows.active.stream_map[self.windows.active.count] = @intToEnum(StreamEnum, i); + self.windows.active.count += 1; + }, + .closed => {}, // don't add to the wait_objects list + } + } + self.windows.first_read_done = true; + } + + while (true) { + if (self.windows.active.count == 0) return; + + const status = os.windows.kernel32.WaitForMultipleObjects( + self.windows.active.count, + &self.windows.active.handles_buf, + 0, + os.windows.INFINITE, + ); + if (status == os.windows.WAIT_FAILED) + return os.windows.unexpectedError(os.windows.kernel32.GetLastError()); + + if (status < os.windows.WAIT_OBJECT_0 or status > os.windows.WAIT_OBJECT_0 + enum_fields.len - 1) + unreachable; + + const active_idx = status - os.windows.WAIT_OBJECT_0; + + const handle = self.windows.active.handles_buf[active_idx]; + const stream_idx = @enumToInt(self.windows.active.stream_map[active_idx]); + var read_bytes: u32 = undefined; + if (0 == os.windows.kernel32.GetOverlappedResult( + handle, + &self.windows.overlapped[stream_idx], + &read_bytes, + 0, + )) switch (os.windows.kernel32.GetLastError()) { + .BROKEN_PIPE => { + self.windows.active.removeAt(active_idx); + continue; + }, + else => |err| return os.windows.unexpectedError(err), + }; + + self.fifos[stream_idx].update(read_bytes); + + switch (try windowsAsyncRead( + handle, + &self.windows.overlapped[stream_idx], + &self.fifos[stream_idx], + bump_amt, + )) { + .pending => {}, + .closed => self.windows.active.removeAt(active_idx), + } + return; + } } fn pollPosix(self: *Self) !void { @@ -263,6 +376,25 @@ pub fn Poller(comptime StreamEnum: type) type { }; } +fn windowsAsyncRead( + handle: os.windows.HANDLE, + overlapped: *os.windows.OVERLAPPED, + fifo: *PollFifo, + bump_amt: usize, +) !enum{ pending, closed } { + while (true) { + const buf = try fifo.writableWithSize(bump_amt); + var read_bytes: u32 = undefined; + const read_result = os.windows.kernel32.ReadFile(handle, buf.ptr, math.cast(u32, buf.len) orelse math.maxInt(u32), &read_bytes, overlapped); + if (read_result == 0) return switch (os.windows.kernel32.GetLastError()) { + .IO_PENDING => .pending, + .BROKEN_PIPE => .closed, + else => |err| os.windows.unexpectedError(err), + }; + fifo.update(read_bytes); + } +} + /// Given an enum, returns a struct with fields of that enum, each field /// representing an I/O stream for polling. pub fn PollFiles(comptime StreamEnum: type) type { From e41bc640c6c4277385236d3dd90b4db566550509 Mon Sep 17 00:00:00 2001 From: John Schmidt Date: Tue, 28 Feb 2023 11:27:56 +0100 Subject: [PATCH 10/25] astgen: do not discard result location in for/while loops If we use the discard result location any break with a value will be ignored and not checked for usage. Closes https://github.com/ziglang/zig/issues/14684. --- src/AstGen.zig | 4 +-- .../for_loop_break_value_ignored.zig | 15 +++++++++++ .../while_loop_break_value_ignored.zig | 26 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 test/cases/compile_errors/for_loop_break_value_ignored.zig create mode 100644 test/cases/compile_errors/while_loop_break_value_ignored.zig diff --git a/src/AstGen.zig b/src/AstGen.zig index de259521bc..41a8ccadb2 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -2342,10 +2342,10 @@ fn blockExprStmts(gz: *GenZir, parent_scope: *Scope, statements: []const Ast.Nod .while_simple, .while_cont, - .@"while", => _ = try whileExpr(gz, scope, .{ .rl = .discard }, inner_node, tree.fullWhile(inner_node).?, true), + .@"while", => _ = try whileExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullWhile(inner_node).?, true), .for_simple, - .@"for", => _ = try forExpr(gz, scope, .{ .rl = .discard }, inner_node, tree.fullFor(inner_node).?, true), + .@"for", => _ = try forExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullFor(inner_node).?, true), else => noreturn_src_node = try unusedResultExpr(gz, scope, inner_node), // zig fmt: on diff --git a/test/cases/compile_errors/for_loop_break_value_ignored.zig b/test/cases/compile_errors/for_loop_break_value_ignored.zig new file mode 100644 index 0000000000..a1119ec651 --- /dev/null +++ b/test/cases/compile_errors/for_loop_break_value_ignored.zig @@ -0,0 +1,15 @@ +fn returns() usize { + return 2; +} + +export fn f1() void { + for ("hello") |_| { + break returns(); + } +} + +// error +// backend=stage2 +// target=native +// +// :6:5: error: incompatible types: 'usize' and 'void' diff --git a/test/cases/compile_errors/while_loop_break_value_ignored.zig b/test/cases/compile_errors/while_loop_break_value_ignored.zig new file mode 100644 index 0000000000..2d14693fe5 --- /dev/null +++ b/test/cases/compile_errors/while_loop_break_value_ignored.zig @@ -0,0 +1,26 @@ +fn returns() usize { + return 2; +} + +export fn f1() void { + var a: bool = true; + while (a) { + break returns(); + } +} + +export fn f2() void { + var x: bool = true; + outer: while (x) { + while (x) { + break :outer returns(); + } + } +} + +// error +// backend=stage2 +// target=native +// +// :7:5: error: incompatible types: 'usize' and 'void' +// :14:12: error: incompatible types: 'usize' and 'void' From a7a709aaa974c394c001f6d7d9be138cc44fd22d Mon Sep 17 00:00:00 2001 From: Binary Craft Date: Fri, 24 Feb 2023 11:03:27 +0800 Subject: [PATCH 11/25] Fixes #13893 - some standard library networking tests are failing on Windows --- lib/std/os/windows.zig | 2 +- lib/std/os/windows/test.zig | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index fe0ebc4951..b63fdb9f92 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -2068,7 +2068,7 @@ pub fn loadWinsockExtensionFunction(comptime T: type, sock: ws2_32.SOCKET, guid: ws2_32.SIO_GET_EXTENSION_FUNCTION_POINTER, @ptrCast(*const anyopaque, &guid), @sizeOf(GUID), - @intToPtr(?*anyopaque, @ptrToInt(function)), + @intToPtr(?*anyopaque, @ptrToInt(&function)), @sizeOf(T), &num_bytes, null, diff --git a/lib/std/os/windows/test.zig b/lib/std/os/windows/test.zig index 8e4d88615c..fa340da178 100644 --- a/lib/std/os/windows/test.zig +++ b/lib/std/os/windows/test.zig @@ -63,3 +63,28 @@ test "removeDotDirs" { try testRemoveDotDirs("a\\b\\..\\", "a\\"); try testRemoveDotDirs("a\\b\\..\\c", "a\\c"); } + +test "loadWinsockExtensionFunction" { + _ = try windows.WSAStartup(2, 2); + defer windows.WSACleanup() catch unreachable; + + const LPFN_CONNECTEX = *const fn ( + Socket: windows.ws2_32.SOCKET, + SockAddr: *const windows.ws2_32.sockaddr, + SockLen: std.os.socklen_t, + SendBuf: ?*const anyopaque, + SendBufLen: windows.DWORD, + BytesSent: *windows.DWORD, + Overlapped: *windows.OVERLAPPED, + ) callconv(windows.WINAPI) windows.BOOL; + + _ = windows.loadWinsockExtensionFunction( + LPFN_CONNECTEX, + try std.os.socket(std.os.AF.INET, std.os.SOCK.DGRAM, 0), + windows.ws2_32.WSAID_CONNECTEX, + ) catch |err| switch (err) { + error.OperationNotSupported => unreachable, + error.ShortRead => unreachable, + else => |e| return e, + }; +} From 4f58a80735b47e6b98ed6f73cc9a0a772cdc6fcd Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Tue, 28 Feb 2023 13:25:53 -0700 Subject: [PATCH 12/25] std.io.zig: fmt --- lib/std/io.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/std/io.zig b/lib/std/io.zig index b125265449..ea48b2d121 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -178,7 +178,7 @@ pub fn poll( if (builtin.os.tag == .windows) result.windows = .{ .first_read_done = false, - .overlapped = [1]os.windows.OVERLAPPED { + .overlapped = [1]os.windows.OVERLAPPED{ mem.zeroes(os.windows.OVERLAPPED), } ** enum_fields.len, .active = .{ @@ -227,7 +227,7 @@ pub fn Poller(comptime StreamEnum: type) type { pub fn removeAt(self: *@This(), index: u32) void { std.debug.assert(index < self.count); - for (index + 1 .. self.count) |i| { + for (index + 1..self.count) |i| { self.handles_buf[i - 1] = self.handles_buf[i]; self.stream_map[i - 1] = self.stream_map[i]; } @@ -241,7 +241,7 @@ pub fn Poller(comptime StreamEnum: type) type { pub fn deinit(self: *Self) void { if (builtin.os.tag == .windows) { // cancel any pending IO to prevent clobbering OVERLAPPED value - for (self.windows.active.handles_buf[0 .. self.windows.active.count]) |h| { + for (self.windows.active.handles_buf[0..self.windows.active.count]) |h| { _ = os.windows.kernel32.CancelIo(h); } } @@ -381,7 +381,7 @@ fn windowsAsyncRead( overlapped: *os.windows.OVERLAPPED, fifo: *PollFifo, bump_amt: usize, -) !enum{ pending, closed } { +) !enum { pending, closed } { while (true) { const buf = try fifo.writableWithSize(bump_amt); var read_bytes: u32 = undefined; From 138e8b162aeecc69cc62f0822e73b1862b7d07f6 Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Tue, 28 Feb 2023 14:10:44 -0700 Subject: [PATCH 13/25] std.child_process: use std.io.poll for collectOutput --- lib/std/child_process.zig | 208 ++++++-------------------------------- 1 file changed, 30 insertions(+), 178 deletions(-) diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 55e375b93b..440b512fd2 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -197,6 +197,19 @@ pub const ChildProcess = struct { stderr: []u8, }; + fn fifoToOwnedArrayList(fifo: *std.io.PollFifo) std.ArrayList(u8) { + if (fifo.head > 0) { + std.mem.copy(u8, fifo.buf[0..fifo.count], fifo.buf[fifo.head .. fifo.head + fifo.count]); + } + const result = std.ArrayList(u8){ + .items = fifo.buf[0..fifo.count], + .capacity = fifo.buf.len, + .allocator = fifo.allocator, + }; + fifo.* = std.io.PollFifo.init(fifo.allocator); + return result; + } + /// Collect the output from the process's stdout and stderr. Will return once all output /// has been collected. This does not mean that the process has ended. `wait` should still /// be called to wait for and clean up the process. @@ -210,189 +223,28 @@ pub const ChildProcess = struct { ) !void { debug.assert(child.stdout_behavior == .Pipe); debug.assert(child.stderr_behavior == .Pipe); - if (builtin.os.tag == .windows) { - try collectOutputWindows(child, stdout, stderr, max_output_bytes); - } else { - try collectOutputPosix(child, stdout, stderr, max_output_bytes); - } - } - fn collectOutputPosix( - child: ChildProcess, - stdout: *std.ArrayList(u8), - stderr: *std.ArrayList(u8), - max_output_bytes: usize, - ) !void { - var poll_fds = [_]os.pollfd{ - .{ .fd = child.stdout.?.handle, .events = os.POLL.IN, .revents = undefined }, - .{ .fd = child.stderr.?.handle, .events = os.POLL.IN, .revents = undefined }, - }; + // we could make this work with multiple allocators but YAGNI + if (stdout.allocator.ptr != stderr.allocator.ptr or + stdout.allocator.vtable != stderr.allocator.vtable) + @panic("ChildProcess.collectOutput only supports 1 allocator"); - var dead_fds: usize = 0; - // We ask for ensureTotalCapacity with this much extra space. This has more of an - // effect on small reads because once the reads start to get larger the amount - // of space an ArrayList will allocate grows exponentially. - const bump_amt = 512; + var poller = std.io.poll(stdout.allocator, enum { stdout, stderr }, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); - const err_mask = os.POLL.ERR | os.POLL.NVAL | os.POLL.HUP; - - while (dead_fds < poll_fds.len) { - const events = try os.poll(&poll_fds, std.math.maxInt(i32)); - if (events == 0) continue; - - var remove_stdout = false; - var remove_stderr = false; - // Try reading whatever is available before checking the error - // conditions. - // It's still possible to read after a POLL.HUP is received, always - // check if there's some data waiting to be read first. - if (poll_fds[0].revents & os.POLL.IN != 0) { - // stdout is ready. - const new_capacity = std.math.min(stdout.items.len + bump_amt, max_output_bytes); - try stdout.ensureTotalCapacity(new_capacity); - const buf = stdout.unusedCapacitySlice(); - if (buf.len == 0) return error.StdoutStreamTooLong; - const nread = try os.read(poll_fds[0].fd, buf); - stdout.items.len += nread; - - // Remove the fd when the EOF condition is met. - remove_stdout = nread == 0; - } else { - remove_stdout = poll_fds[0].revents & err_mask != 0; - } - - if (poll_fds[1].revents & os.POLL.IN != 0) { - // stderr is ready. - const new_capacity = std.math.min(stderr.items.len + bump_amt, max_output_bytes); - try stderr.ensureTotalCapacity(new_capacity); - const buf = stderr.unusedCapacitySlice(); - if (buf.len == 0) return error.StderrStreamTooLong; - const nread = try os.read(poll_fds[1].fd, buf); - stderr.items.len += nread; - - // Remove the fd when the EOF condition is met. - remove_stderr = nread == 0; - } else { - remove_stderr = poll_fds[1].revents & err_mask != 0; - } - - // Exclude the fds that signaled an error. - if (remove_stdout) { - poll_fds[0].fd = -1; - dead_fds += 1; - } - if (remove_stderr) { - poll_fds[1].fd = -1; - dead_fds += 1; - } - } - } - - const WindowsAsyncReadResult = enum { - pending, - closed, - full, - }; - - fn windowsAsyncRead( - handle: windows.HANDLE, - overlapped: *windows.OVERLAPPED, - buf: *std.ArrayList(u8), - bump_amt: usize, - max_output_bytes: usize, - ) !WindowsAsyncReadResult { - while (true) { - const new_capacity = std.math.min(buf.items.len + bump_amt, max_output_bytes); - try buf.ensureTotalCapacity(new_capacity); - const next_buf = buf.unusedCapacitySlice(); - if (next_buf.len == 0) return .full; - var read_bytes: u32 = undefined; - const read_result = windows.kernel32.ReadFile(handle, next_buf.ptr, math.cast(u32, next_buf.len) orelse maxInt(u32), &read_bytes, overlapped); - if (read_result == 0) return switch (windows.kernel32.GetLastError()) { - .IO_PENDING => .pending, - .BROKEN_PIPE => .closed, - else => |err| windows.unexpectedError(err), - }; - buf.items.len += read_bytes; - } - } - - fn collectOutputWindows(child: ChildProcess, stdout: *std.ArrayList(u8), stderr: *std.ArrayList(u8), max_output_bytes: usize) !void { - const bump_amt = 512; - const outs = [_]*std.ArrayList(u8){ - stdout, - stderr, - }; - const handles = [_]windows.HANDLE{ - child.stdout.?.handle, - child.stderr.?.handle, - }; - - var overlapped = [_]windows.OVERLAPPED{ - mem.zeroes(windows.OVERLAPPED), - mem.zeroes(windows.OVERLAPPED), - }; - - var wait_objects: [2]windows.HANDLE = undefined; - var wait_object_count: u2 = 0; - - // we need to cancel all pending IO before returning so our OVERLAPPED values don't go out of scope - defer for (wait_objects[0..wait_object_count]) |o| { - _ = windows.kernel32.CancelIo(o); - }; - - // Windows Async IO requires an initial call to ReadFile before waiting on the handle - for ([_]u1{ 0, 1 }) |i| { - switch (try windowsAsyncRead(handles[i], &overlapped[i], outs[i], bump_amt, max_output_bytes)) { - .pending => { - wait_objects[wait_object_count] = handles[i]; - wait_object_count += 1; - }, - .closed => {}, // don't add to the wait_objects list - .full => return if (i == 0) error.StdoutStreamTooLong else error.StderrStreamTooLong, - } + while (!poller.done()) { + try poller.poll(); + if (poller.fifo(.stdout).count > max_output_bytes) + return error.StdoutStreamTooLong; + if (poller.fifo(.stderr).count > max_output_bytes) + return error.StderrStreamTooLong; } - while (wait_object_count > 0) { - const status = windows.kernel32.WaitForMultipleObjects(wait_object_count, &wait_objects, 0, windows.INFINITE); - if (status == windows.WAIT_FAILED) { - switch (windows.kernel32.GetLastError()) { - else => |err| return windows.unexpectedError(err), - } - } - if (status < windows.WAIT_OBJECT_0 or status > windows.WAIT_OBJECT_0 + wait_object_count - 1) - unreachable; - - const wait_idx = status - windows.WAIT_OBJECT_0; - - // this extra `i` index is needed to map the wait handle back to the stdout or stderr - // values since the wait_idx can change which handle it corresponds with - const i: u1 = if (wait_objects[wait_idx] == handles[0]) 0 else 1; - - // remove completed event from the wait list - wait_object_count -= 1; - if (wait_idx == 0) - wait_objects[0] = wait_objects[1]; - - var read_bytes: u32 = undefined; - if (windows.kernel32.GetOverlappedResult(handles[i], &overlapped[i], &read_bytes, 0) == 0) { - switch (windows.kernel32.GetLastError()) { - .BROKEN_PIPE => continue, - else => |err| return windows.unexpectedError(err), - } - } - - outs[i].items.len += read_bytes; - - switch (try windowsAsyncRead(handles[i], &overlapped[i], outs[i], bump_amt, max_output_bytes)) { - .pending => { - wait_objects[wait_object_count] = handles[i]; - wait_object_count += 1; - }, - .closed => {}, // don't add to the wait_objects list - .full => return if (i == 0) error.StdoutStreamTooLong else error.StderrStreamTooLong, - } - } + stdout.* = fifoToOwnedArrayList(poller.fifo(.stdout)); + stderr.* = fifoToOwnedArrayList(poller.fifo(.stderr)); } /// Spawns a child process, waits for it, collecting stdout and stderr, and then returns. From f2b15420ad595f77b6a3575dd7a6e85411dc69e9 Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Tue, 28 Feb 2023 21:00:10 -0700 Subject: [PATCH 14/25] std.io.poll: remove done function --- lib/std/child_process.zig | 3 +-- lib/std/io.zig | 31 ++++++++++++++++--------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 440b512fd2..7ff48a5652 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -235,8 +235,7 @@ pub const ChildProcess = struct { }); defer poller.deinit(); - while (!poller.done()) { - try poller.poll(); + while (try poller.poll()) { if (poller.fifo(.stdout).count > max_output_bytes) return error.StdoutStreamTooLong; if (poller.fifo(.stderr).count > max_output_bytes) diff --git a/lib/std/io.zig b/lib/std/io.zig index ea48b2d121..0faba2b652 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -249,7 +249,7 @@ pub fn Poller(comptime StreamEnum: type) type { self.* = undefined; } - pub fn poll(self: *Self) !void { + pub fn poll(self: *Self) !bool { if (builtin.os.tag == .windows) { return pollWindows(self); } else { @@ -261,16 +261,7 @@ pub fn Poller(comptime StreamEnum: type) type { return &self.fifos[@enumToInt(which)]; } - pub fn done(self: Self) bool { - if (builtin.os.tag == .windows) - return self.windows.first_read_done and self.windows.active.count == 0; - - for (self.poll_fds) |poll_fd| { - if (poll_fd.fd != -1) return false; - } else return true; - } - - fn pollWindows(self: *Self) !void { + fn pollWindows(self: *Self) !bool { const bump_amt = 512; if (!self.windows.first_read_done) { @@ -295,7 +286,7 @@ pub fn Poller(comptime StreamEnum: type) type { } while (true) { - if (self.windows.active.count == 0) return; + if (self.windows.active.count == 0) return false; const status = os.windows.kernel32.WaitForMultipleObjects( self.windows.active.count, @@ -338,11 +329,11 @@ pub fn Poller(comptime StreamEnum: type) type { .pending => {}, .closed => self.windows.active.removeAt(active_idx), } - return; + return true; } } - fn pollPosix(self: *Self) !void { + fn pollPosix(self: *Self) !bool { // We ask for ensureUnusedCapacity with this much extra space. This // has more of an effect on small reads because once the reads // start to get larger the amount of space an ArrayList will @@ -352,8 +343,13 @@ pub fn Poller(comptime StreamEnum: type) type { const err_mask = os.POLL.ERR | os.POLL.NVAL | os.POLL.HUP; const events_len = try os.poll(&self.poll_fds, std.math.maxInt(i32)); - if (events_len == 0) return; + if (events_len == 0) { + for (self.poll_fds) |poll_fd| { + if (poll_fd.fd != -1) return true; + } else return false; + } + var keep_polling = false; inline for (&self.poll_fds, &self.fifos) |*poll_fd, *q| { // Try reading whatever is available before checking the error // conditions. @@ -366,12 +362,17 @@ pub fn Poller(comptime StreamEnum: type) type { if (amt == 0) { // Remove the fd when the EOF condition is met. poll_fd.fd = -1; + } else { + keep_polling = true; } } else if (poll_fd.revents & err_mask != 0) { // Exclude the fds that signaled an error. poll_fd.fd = -1; + } else if (poll_fd.fd != -1) { + keep_polling = true; } } + return keep_polling; } }; } From 25b83188d06d4cc760c723db0f6ec65db96373f5 Mon Sep 17 00:00:00 2001 From: Auguste Rame <19855629+SuperAuguste@users.noreply.github.com> Date: Wed, 1 Mar 2023 13:20:01 -0500 Subject: [PATCH 15/25] Add --build-runner `zig build` option (#14742) --- lib/build_runner.zig | 1 + src/main.zig | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/lib/build_runner.zig b/lib/build_runner.zig index ca78ce713f..64421d1031 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -362,6 +362,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory \\ --zig-lib-dir [arg] Override path to Zig lib directory + \\ --build-runner [file] Override path to build runner \\ --debug-log [scope] Enable debugging the compiler \\ --verbose-link Enable compiler debug output for linking \\ --verbose-air Enable compiler debug output for Zig AIR diff --git a/src/main.zig b/src/main.zig index d544940779..dedcaabfa2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4013,6 +4013,7 @@ pub const usage_build = \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory \\ --zig-lib-dir [arg] Override path to Zig lib directory + \\ --build-runner [file] Override path to build runner \\ --prominent-compile-errors Output compile errors formatted for a human to read \\ -h, --help Print this help and exit \\ @@ -4031,6 +4032,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi var override_lib_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LIB_DIR"); var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); var override_local_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LOCAL_CACHE_DIR"); + var override_build_runner: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_BUILD_RUNNER"); var child_argv = std.ArrayList([]const u8).init(arena); var reference_trace: ?u32 = null; var debug_compile_errors = false; @@ -4065,6 +4067,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi override_lib_dir = args[i]; try child_argv.appendSlice(&[_][]const u8{ arg, args[i] }); continue; + } else if (mem.eql(u8, arg, "--build-runner")) { + if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); + i += 1; + override_build_runner = args[i]; + continue; } else if (mem.eql(u8, arg, "--cache-dir")) { if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg}); i += 1; @@ -4197,10 +4204,29 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi try thread_pool.init(gpa); defer thread_pool.deinit(); - var main_pkg: Package = .{ - .root_src_directory = zig_lib_directory, - .root_src_path = "build_runner.zig", - }; + var cleanup_build_runner_dir: ?fs.Dir = null; + defer if (cleanup_build_runner_dir) |*dir| dir.close(); + + var main_pkg: Package = if (override_build_runner) |build_runner_path| + .{ + .root_src_directory = blk: { + if (std.fs.path.dirname(build_runner_path)) |dirname| { + const dir = fs.cwd().openDir(dirname, .{}) catch |err| { + fatal("unable to open directory to build runner from argument 'build-runner', '{s}': {s}", .{ dirname, @errorName(err) }); + }; + cleanup_build_runner_dir = dir; + break :blk .{ .path = dirname, .handle = dir }; + } + + break :blk .{ .path = null, .handle = fs.cwd() }; + }, + .root_src_path = std.fs.path.basename(build_runner_path), + } + else + .{ + .root_src_directory = zig_lib_directory, + .root_src_path = "build_runner.zig", + }; if (!build_options.omit_pkg_fetching_code) { var http_client: std.http.Client = .{ .allocator = gpa }; From db8217f9a080f7c645a6448640a9af65f3944818 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 1 Mar 2023 12:11:17 -0700 Subject: [PATCH 16/25] packages: avoid creating multiple modules with same build.zig When there is a diamond dependency, reuse a *Module instead of creating a redundant one using the same build.zig file. Otherwise, the compile error "file exists in multiple modules" would occur. --- src/Package.zig | 34 ++++++++++++++++++++++++++-------- src/main.zig | 4 ++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/Package.zig b/src/Package.zig index f0e389e7ef..2aa5e85294 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -225,6 +225,7 @@ pub fn fetchAndAddDependencies( build_roots_source: *std.ArrayList(u8), name_prefix: []const u8, color: main.Color, + all_modules: *AllModules, ) !void { const max_bytes = 10 * 1024 * 1024; const gpa = thread_pool.allocator; @@ -291,6 +292,7 @@ pub fn fetchAndAddDependencies( report, build_roots_source, fqn, + all_modules, ); try pkg.fetchAndAddDependencies( @@ -304,6 +306,7 @@ pub fn fetchAndAddDependencies( build_roots_source, sub_prefix, color, + all_modules, ); try add(pkg, gpa, fqn, sub_pkg); @@ -402,6 +405,11 @@ const Report = struct { } }; +const hex_multihash_len = 2 * Manifest.multihash_len; +const MultiHashHexDigest = [hex_multihash_len]u8; +/// This is to avoid creating multiple modules for the same build.zig file. +pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, *Package); + fn fetchAndUnpack( thread_pool: *ThreadPool, http_client: *std.http.Client, @@ -410,6 +418,7 @@ fn fetchAndUnpack( report: Report, build_roots_source: *std.ArrayList(u8), fqn: []const u8, + all_modules: *AllModules, ) !*Package { const gpa = http_client.allocator; const s = fs.path.sep_str; @@ -417,9 +426,24 @@ fn fetchAndUnpack( // Check if the expected_hash is already present in the global package // cache, and thereby avoid both fetching and unpacking. if (dep.hash) |h| cached: { - const hex_multihash_len = 2 * Manifest.multihash_len; const hex_digest = h[0..hex_multihash_len]; const pkg_dir_sub_path = "p" ++ s ++ hex_digest; + + const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); + errdefer gpa.free(build_root); + + try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{ + std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root), + }); + + // The compiler has a rule that a file must not be included in multiple modules, + // so we must detect if a module has been created for this package and reuse it. + const gop = try all_modules.getOrPut(gpa, hex_digest.*); + if (gop.found_existing) { + gpa.free(build_root); + return gop.value_ptr.*; + } + var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { error.FileNotFound => break :cached, else => |e| return e, @@ -432,13 +456,6 @@ fn fetchAndUnpack( const owned_src_path = try gpa.dupe(u8, build_zig_basename); errdefer gpa.free(owned_src_path); - const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); - errdefer gpa.free(build_root); - - try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{ - std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root), - }); - ptr.* = .{ .root_src_directory = .{ .path = build_root, @@ -448,6 +465,7 @@ fn fetchAndUnpack( .root_src_path = owned_src_path, }; + gop.value_ptr.* = ptr; return ptr; } diff --git a/src/main.zig b/src/main.zig index dedcaabfa2..fb02628c61 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4244,6 +4244,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi var build_roots_source = std.ArrayList(u8).init(gpa); defer build_roots_source.deinit(); + var all_modules: Package.AllModules = .{}; + defer all_modules.deinit(gpa); + // Here we borrow main package's table and will replace it with a fresh // one after this process completes. main_pkg.fetchAndAddDependencies( @@ -4257,6 +4260,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi &build_roots_source, "", color, + &all_modules, ) catch |err| switch (err) { error.PackageFetchFailed => process.exit(1), else => |e| return e, From 28364166e83ed52a7053029d5d7b33ad956d804d Mon Sep 17 00:00:00 2001 From: Frank Denis <124872+jedisct1@users.noreply.github.com> Date: Thu, 2 Mar 2023 07:13:40 +0100 Subject: [PATCH 17/25] crypto.hash.sha3: make permutation generic and public, add SHAKE (#14756) Make the Keccak permutation public, as it's useful for more than SHA-3 (kMAC, SHAKE, TurboSHAKE, TupleHash, etc). Our Keccak implementation was accepting f as a comptime parameter, but always used 64-bit words and 200 byte states, so it actually didn't work with anything besides f=1600. That has been fixed. The ability to use reduced-round versions was also added in order to support M14 and K12. The state was constantly converted back and forth between bytes and words, even though only a part of the state is actually used for absorbing and squeezing bytes. It was changed to something similar to the other permutations we have, so we can avoid extra copies, and eventually add vectorized implementations. In addition, the SHAKE extendable output function (XOF) was added (SHAKE128, SHAKE256). It is required by newer schemes, such as the Kyber post-quantum key exchange mechanism, whose implementation is currently blocked by SHAKE missing from our standard library. Breaking change: `Keccak_256` and `Keccak_512` were renamed to `Keccak256` and `Keccak512` for consistency with all other hash functions. --- lib/std/crypto.zig | 2 + lib/std/crypto/benchmark.zig | 2 + lib/std/crypto/keccak_p.zig | 251 ++++++++++++++++++++++++++++ lib/std/crypto/sha3.zig | 315 +++++++++++++++++++---------------- 4 files changed, 427 insertions(+), 143 deletions(-) create mode 100644 lib/std/crypto/keccak_p.zig diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 1602fb926c..f46e7b1022 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -47,6 +47,8 @@ pub const auth = struct { /// Core functions, that should rarely be used directly by applications. pub const core = struct { pub const aes = @import("crypto/aes.zig"); + pub const keccak = @import("crypto/keccak_p.zig"); + pub const Ascon = @import("crypto/ascon.zig").State; pub const Gimli = @import("crypto/gimli.zig").State; pub const Xoodoo = @import("crypto/xoodoo.zig").State; diff --git a/lib/std/crypto/benchmark.zig b/lib/std/crypto/benchmark.zig index 71c22f2b4c..e6e0e1fc39 100644 --- a/lib/std/crypto/benchmark.zig +++ b/lib/std/crypto/benchmark.zig @@ -25,6 +25,8 @@ const hashes = [_]Crypto{ Crypto{ .ty = crypto.hash.sha2.Sha512, .name = "sha512" }, Crypto{ .ty = crypto.hash.sha3.Sha3_256, .name = "sha3-256" }, Crypto{ .ty = crypto.hash.sha3.Sha3_512, .name = "sha3-512" }, + Crypto{ .ty = crypto.hash.sha3.Shake128, .name = "shake-128" }, + Crypto{ .ty = crypto.hash.sha3.Shake256, .name = "shake-256" }, Crypto{ .ty = crypto.hash.Gimli, .name = "gimli-hash" }, Crypto{ .ty = crypto.hash.blake2.Blake2s256, .name = "blake2s" }, Crypto{ .ty = crypto.hash.blake2.Blake2b512, .name = "blake2b" }, diff --git a/lib/std/crypto/keccak_p.zig b/lib/std/crypto/keccak_p.zig new file mode 100644 index 0000000000..5d13677bde --- /dev/null +++ b/lib/std/crypto/keccak_p.zig @@ -0,0 +1,251 @@ +const std = @import("std"); +const assert = std.debug.assert; +const math = std.math; +const mem = std.mem; + +/// The Keccak-f permutation. +pub fn KeccakF(comptime f: u11) type { + comptime assert(f > 200 and f <= 1600 and f % 200 == 0); // invalid bit size + const T = std.meta.Int(.unsigned, f / 25); + const Block = [25]T; + + const RC = [_]u64{ + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, 0x8000000080008000, + 0x000000000000808b, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009, + 0x000000000000008a, 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, 0x8000000000008003, + 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, + 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, + }; + + const RHO = [_]u6{ + 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44, + }; + + const PI = [_]u5{ + 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1, + }; + + return struct { + const Self = @This(); + + /// Number of bytes in the state. + pub const block_bytes = f / 8; + + st: Block = [_]T{0} ** 25, + + /// Initialize the state from a slice of bytes. + pub fn init(bytes: [block_bytes]u8) Self { + var self: Self = undefined; + inline for (&self.st, 0..) |*r, i| { + r.* = mem.readIntLittle(T, bytes[@sizeOf(T) * i ..][0..@sizeOf(T)]); + } + return self; + } + + /// A representation of the state as bytes. The byte order is architecture-dependent. + pub fn asBytes(self: *Self) *[block_bytes]u8 { + return mem.asBytes(&self.st); + } + + /// Byte-swap the entire state if the architecture doesn't match the required endianness. + pub fn endianSwap(self: *Self) void { + for (&self.st) |*w| { + w.* = mem.littleTooNative(T, w.*); + } + } + + /// Set bytes starting at the beginning of the state. + pub fn setBytes(self: *Self, bytes: []const u8) void { + var i: usize = 0; + while (i + @sizeOf(T) <= bytes.len) : (i += @sizeOf(T)) { + self.st[i / @sizeOf(T)] = mem.readIntLittle(T, bytes[i..][0..@sizeOf(T)]); + } + if (i < bytes.len) { + var padded = [_]u8{0} ** @sizeOf(T); + mem.copy(u8, padded[0 .. bytes.len - i], bytes[i..]); + self.st[i / @sizeOf(T)] = mem.readIntLittle(T, padded[0..]); + } + } + + /// XOR a byte into the state at a given offset. + pub fn addByte(self: *Self, byte: u8, offset: usize) void { + const z = @sizeOf(T) * @truncate(math.Log2Int(T), offset % @sizeOf(T)); + self.st[offset / @sizeOf(T)] ^= @as(T, byte) << z; + } + + /// XOR bytes into the beginning of the state. + pub fn addBytes(self: *Self, bytes: []const u8) void { + var i: usize = 0; + while (i + @sizeOf(T) <= bytes.len) : (i += @sizeOf(T)) { + self.st[i / @sizeOf(T)] ^= mem.readIntLittle(T, bytes[i..][0..@sizeOf(T)]); + } + if (i < bytes.len) { + var padded = [_]u8{0} ** @sizeOf(T); + mem.copy(u8, padded[0 .. bytes.len - i], bytes[i..]); + self.st[i / @sizeOf(T)] ^= mem.readIntLittle(T, padded[0..]); + } + } + + /// Extract the first bytes of the state. + pub fn extractBytes(self: *Self, out: []u8) void { + var i: usize = 0; + while (i + @sizeOf(T) <= out.len) : (i += @sizeOf(T)) { + mem.writeIntLittle(T, out[i..][0..@sizeOf(T)], self.st[i / @sizeOf(T)]); + } + if (i < out.len) { + var padded = [_]u8{0} ** @sizeOf(T); + mem.writeIntLittle(T, padded[0..], self.st[i / @sizeOf(T)]); + mem.copy(u8, out[i..], padded[0 .. out.len - i]); + } + } + + /// XOR the first bytes of the state into a slice of bytes. + pub fn xorBytes(self: *Self, out: []u8, in: []const u8) void { + assert(out.len == in.len); + + var i: usize = 0; + while (i + @sizeOf(T) <= in.len) : (i += @sizeOf(T)) { + const x = mem.readIntNative(T, in[i..][0..@sizeOf(T)]) ^ mem.nativeToLittle(T, self.st[i / @sizeOf(T)]); + mem.writeIntNative(T, out[i..][0..@sizeOf(T)], x); + } + if (i < in.len) { + var padded = [_]u8{0} ** @sizeOf(T); + mem.copy(u8, padded[0 .. in.len - i], in[i..]); + const x = mem.readIntNative(T, &padded) ^ mem.nativeToLittle(T, self.st[i / @sizeOf(T)]); + mem.writeIntNative(T, &padded, x); + mem.copy(u8, out[i..], padded[0 .. in.len - i]); + } + } + + /// Set the words storing the bytes of a given range to zero. + pub fn clear(self: *Self, from: usize, to: usize) void { + mem.set(T, self.st[from / @sizeOf(T) .. (to + @sizeOf(T) - 1) / @sizeOf(T)], 0); + } + + /// Clear the entire state, disabling compiler optimizations. + pub fn secureZero(self: *Self) void { + std.crypto.utils.secureZero(T, &self.st); + } + + inline fn round(self: *Self, rc: T) void { + const st = &self.st; + + // theta + var t = [_]T{0} ** 5; + inline for (0..5) |i| { + inline for (0..5) |j| { + t[i] ^= st[j * 5 + i]; + } + } + inline for (0..5) |i| { + inline for (0..5) |j| { + st[j * 5 + i] ^= t[(i + 4) % 5] ^ math.rotl(T, t[(i + 1) % 5], 1); + } + } + + // rho+pi + var last = st[1]; + inline for (0..24) |i| { + const x = PI[i]; + const tmp = st[x]; + st[x] = math.rotl(T, last, RHO[i]); + last = tmp; + } + inline for (0..5) |i| { + inline for (0..5) |j| { + t[j] = st[i * 5 + j]; + } + inline for (0..5) |j| { + st[i * 5 + j] = t[j] ^ (~t[(j + 1) % 5] & t[(j + 2) % 5]); + } + } + + // iota + st[0] ^= rc; + } + + /// Apply a (possibly) reduced-round permutation to the state. + pub fn permuteR(self: *Self, comptime rounds: u5) void { + var i = RC.len - rounds; + while (i < rounds - rounds % 3) : (i += 3) { + self.round(RC[i]); + self.round(RC[i + 1]); + self.round(RC[i + 2]); + } + while (i < rounds) : (i += 1) { + self.round(RC[i]); + } + } + + /// Apply a full-round permutation to the state. + pub fn permute(self: *Self) void { + self.permuteR(comptime 12 + 2 * math.log2(f / 25)); + } + }; +} + +/// A generic Keccak-P state. +pub fn State(comptime f: u11, comptime capacity: u11, comptime delim: u8, comptime rounds: u5) type { + comptime assert(f > 200 and f <= 1600 and f % 200 == 0); // invalid state size + comptime assert(capacity < f and capacity % 8 == 0); // invalid capacity size + + return struct { + const Self = @This(); + + /// The block length, or rate, in bytes. + pub const rate = KeccakF(f).block_bytes - capacity / 8; + /// Keccak does not have any options. + pub const Options = struct {}; + + offset: usize = 0, + buf: [rate]u8 = undefined, + + st: KeccakF(f) = .{}, + + /// Absorb a slice of bytes into the sponge. + pub fn absorb(self: *Self, bytes_: []const u8) void { + var bytes = bytes_; + if (self.offset > 0) { + const left = math.min(rate - self.offset, bytes.len); + mem.copy(u8, self.buf[self.offset..], bytes[0..left]); + self.offset += left; + if (self.offset == rate) { + self.offset = 0; + self.st.addBytes(self.buf[0..]); + self.st.permuteR(rounds); + } + if (left == bytes.len) return; + bytes = bytes[left..]; + } + while (bytes.len >= rate) { + self.st.addBytes(bytes[0..rate]); + self.st.permuteR(rounds); + bytes = bytes[rate..]; + } + if (bytes.len > 0) { + self.st.addBytes(bytes[0..]); + self.offset = bytes.len; + } + } + + /// Mark the end of the input. + pub fn pad(self: *Self) void { + self.st.addBytes(self.buf[0..self.offset]); + self.st.addByte(delim, self.offset); + self.st.addByte(0x80, rate - 1); + self.st.permuteR(rounds); + self.offset = 0; + } + + /// Squeeze a slice of bytes from the sponge. + pub fn squeeze(self: *Self, out: []u8) void { + var i: usize = 0; + while (i < out.len) : (i += rate) { + const left = math.min(rate, out.len - i); + self.st.extractBytes(out[i..][0..left]); + self.st.permuteR(rounds); + } + } + }; +} diff --git a/lib/std/crypto/sha3.zig b/lib/std/crypto/sha3.zig index c2801a4709..985027f3ee 100644 --- a/lib/std/crypto/sha3.zig +++ b/lib/std/crypto/sha3.zig @@ -1,84 +1,63 @@ -const std = @import("../std.zig"); -const mem = std.mem; +const std = @import("std"); +const assert = std.debug.assert; const math = std.math; -const debug = std.debug; -const htest = @import("test.zig"); +const mem = std.mem; -pub const Sha3_224 = Keccak(224, 0x06); -pub const Sha3_256 = Keccak(256, 0x06); -pub const Sha3_384 = Keccak(384, 0x06); -pub const Sha3_512 = Keccak(512, 0x06); -pub const Keccak_256 = Keccak(256, 0x01); -pub const Keccak_512 = Keccak(512, 0x01); +const KeccakState = std.crypto.core.keccak.State; + +pub const Sha3_224 = Keccak(1600, 224, 0x06, 24); +pub const Sha3_256 = Keccak(1600, 256, 0x06, 24); +pub const Sha3_384 = Keccak(1600, 384, 0x06, 24); +pub const Sha3_512 = Keccak(1600, 512, 0x06, 24); + +pub const Keccak256 = Keccak(1600, 256, 0x01, 24); +pub const Keccak512 = Keccak(1600, 512, 0x01, 24); +pub const Keccak_256 = @compileError("Deprecated: use `Keccak256` instead"); +pub const Keccak_512 = @compileError("Deprecated: use `Keccak512` instead"); + +pub const Shake128 = Shake(128); +pub const Shake256 = Shake(256); + +/// A generic Keccak hash function. +pub fn Keccak(comptime f: u11, comptime output_bits: u11, comptime delim: u8, comptime rounds: u5) type { + comptime assert(output_bits > 0 and output_bits * 2 < f and output_bits % 8 == 0); // invalid output length + + const State = KeccakState(f, output_bits * 2, delim, rounds); -fn Keccak(comptime bits: usize, comptime delim: u8) type { return struct { const Self = @This(); + + st: State = .{}, + /// The output length, in bytes. - pub const digest_length = bits / 8; + pub const digest_length = output_bits / 8; /// The block length, or rate, in bytes. - pub const block_length = 200 - bits / 4; + pub const block_length = State.rate; /// Keccak does not have any options. pub const Options = struct {}; - s: [200]u8, - offset: usize, - + /// Initialize a Keccak hash function. pub fn init(options: Options) Self { _ = options; - return Self{ .s = [_]u8{0} ** 200, .offset = 0 }; + return Self{}; } - pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { - var d = Self.init(options); - d.update(b); - d.final(out); + /// Hash a slice of bytes. + pub fn hash(bytes: []const u8, out: *[digest_length]u8, options: Options) void { + var st = Self.init(options); + st.update(bytes); + st.final(out); } - pub fn update(d: *Self, b: []const u8) void { - var ip: usize = 0; - var len = b.len; - var rate = block_length - d.offset; - var offset = d.offset; - - // absorb - while (len >= rate) { - for (d.s[offset .. offset + rate], 0..) |*r, i| - r.* ^= b[ip..][i]; - - keccakF(1600, &d.s); - - ip += rate; - len -= rate; - rate = block_length; - offset = 0; - } - - for (d.s[offset .. offset + len], 0..) |*r, i| - r.* ^= b[ip..][i]; - - d.offset = offset + len; + /// Absorb a slice of bytes into the state. + pub fn update(self: *Self, bytes: []const u8) void { + self.st.absorb(bytes); } - pub fn final(d: *Self, out: *[digest_length]u8) void { - // padding - d.s[d.offset] ^= delim; - d.s[block_length - 1] ^= 0x80; - - keccakF(1600, &d.s); - - // squeeze - var op: usize = 0; - var len: usize = bits / 8; - - while (len >= block_length) { - mem.copy(u8, out[op..], d.s[0..block_length]); - keccakF(1600, &d.s); - op += block_length; - len -= block_length; - } - - mem.copy(u8, out[op..], d.s[0..len]); + /// Return the hash of the absorbed bytes. + pub fn final(self: *Self, out: *[digest_length]u8) void { + self.st.pad(); + self.st.squeeze(out[0..]); } pub const Error = error{}; @@ -95,87 +74,101 @@ fn Keccak(comptime bits: usize, comptime delim: u8) type { }; } -const RC = [_]u64{ - 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, 0x8000000080008000, - 0x000000000000808b, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009, - 0x000000000000008a, 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, - 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, 0x8000000000008003, - 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, - 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, -}; +/// The SHAKE extendable output hash function. +pub fn Shake(comptime security_level: u11) type { + const f = 1600; + const rounds = 24; + const State = KeccakState(f, security_level * 2, 0x1f, rounds); -const ROTC = [_]usize{ - 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44, -}; + return struct { + const Self = @This(); -const PIL = [_]usize{ - 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1, -}; + st: State = .{}, + buf: [State.rate]u8 = undefined, + offset: usize = 0, + padded: bool = false, -const M5 = [_]usize{ - 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, -}; + /// The recommended output length, in bytes. + pub const digest_length = security_level / 2; + /// The block length, or rate, in bytes. + pub const block_length = State.rate; + /// Keccak does not have any options. + pub const Options = struct {}; -fn keccakF(comptime F: usize, d: *[F / 8]u8) void { - const B = F / 25; - const no_rounds = comptime x: { - break :x 12 + 2 * math.log2(B); + /// Initialize a SHAKE extensible hash function. + pub fn init(options: Options) Self { + _ = options; + return Self{}; + } + + /// Hash a slice of bytes. + /// `out` can be any length. + pub fn hash(bytes: []const u8, out: []u8, options: Options) void { + var st = Self.init(options); + st.update(bytes); + st.squeeze(out); + } + + /// Absorb a slice of bytes into the state. + pub fn update(self: *Self, bytes: []const u8) void { + self.st.absorb(bytes); + } + + /// Squeeze a slice of bytes from the state. + /// `out` can be any length, and the function can be called multiple times. + pub fn squeeze(self: *Self, out_: []u8) void { + if (!self.padded) { + self.st.pad(); + self.padded = true; + } + var out = out_; + if (self.offset > 0) { + const left = self.buf.len - self.offset; + if (left > 0) { + const n = math.min(left, out.len); + mem.copy(u8, out[0..n], self.buf[self.offset..][0..n]); + out = out[n..]; + self.offset += n; + if (out.len == 0) { + return; + } + } + } + const full_blocks = out[0 .. out.len - out.len % State.rate]; + if (full_blocks.len > 0) { + self.st.squeeze(full_blocks); + out = out[full_blocks.len..]; + } + if (out.len > 0) { + self.st.squeeze(self.buf[0..]); + mem.copy(u8, out[0..], self.buf[0..out.len]); + self.offset = out.len; + } + } + + /// Return the hash of the absorbed bytes. + /// `out` can be of any length, but the function must not be called multiple times (use `squeeze` for that purpose instead). + pub fn final(self: *Self, out: []u8) void { + self.squeeze(out); + self.st.st.clear(0, State.rate); + } + + pub const Error = error{}; + pub const Writer = std.io.Writer(*Self, Error, write); + + fn write(self: *Self, bytes: []const u8) Error!usize { + self.update(bytes); + return bytes.len; + } + + pub fn writer(self: *Self) Writer { + return .{ .context = self }; + } }; - - var s = [_]u64{0} ** 25; - var t = [_]u64{0} ** 1; - var c = [_]u64{0} ** 5; - - for (&s, 0..) |*r, i| { - r.* = mem.readIntLittle(u64, d[8 * i ..][0..8]); - } - - for (RC[0..no_rounds]) |round| { - // theta - comptime var x: usize = 0; - inline while (x < 5) : (x += 1) { - c[x] = s[x] ^ s[x + 5] ^ s[x + 10] ^ s[x + 15] ^ s[x + 20]; - } - x = 0; - inline while (x < 5) : (x += 1) { - t[0] = c[M5[x + 4]] ^ math.rotl(u64, c[M5[x + 1]], @as(usize, 1)); - comptime var y: usize = 0; - inline while (y < 5) : (y += 1) { - s[x + y * 5] ^= t[0]; - } - } - - // rho+pi - t[0] = s[1]; - x = 0; - inline while (x < 24) : (x += 1) { - c[0] = s[PIL[x]]; - s[PIL[x]] = math.rotl(u64, t[0], ROTC[x]); - t[0] = c[0]; - } - - // chi - comptime var y: usize = 0; - inline while (y < 5) : (y += 1) { - x = 0; - inline while (x < 5) : (x += 1) { - c[x] = s[x + y * 5]; - } - x = 0; - inline while (x < 5) : (x += 1) { - s[x + y * 5] = c[x] ^ (~c[M5[x + 1]] & c[M5[x + 2]]); - } - } - - // iota - s[0] ^= round; - } - - for (s, 0..) |r, i| { - mem.writeIntLittle(u64, d[8 * i ..][0..8], r); - } } +const htest = @import("test.zig"); + test "sha3-224 single" { try htest.assertEqualHash(Sha3_224, "6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7", ""); try htest.assertEqualHash(Sha3_224, "e642824c3f8cf24ad09234ee7d3c766fc9a3a5168d0c94ad73b46fdf", "abc"); @@ -309,13 +302,49 @@ test "sha3-512 aligned final" { } test "keccak-256 single" { - try htest.assertEqualHash(Keccak_256, "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ""); - try htest.assertEqualHash(Keccak_256, "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45", "abc"); - try htest.assertEqualHash(Keccak_256, "f519747ed599024f3882238e5ab43960132572b7345fbeb9a90769dafd21ad67", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); + try htest.assertEqualHash(Keccak256, "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ""); + try htest.assertEqualHash(Keccak256, "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45", "abc"); + try htest.assertEqualHash(Keccak256, "f519747ed599024f3882238e5ab43960132572b7345fbeb9a90769dafd21ad67", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); } test "keccak-512 single" { - try htest.assertEqualHash(Keccak_512, "0eab42de4c3ceb9235fc91acffe746b29c29a8c366b7c60e4e67c466f36a4304c00fa9caf9d87976ba469bcbe06713b435f091ef2769fb160cdab33d3670680e", ""); - try htest.assertEqualHash(Keccak_512, "18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", "abc"); - try htest.assertEqualHash(Keccak_512, "ac2fb35251825d3aa48468a9948c0a91b8256f6d97d8fa4160faff2dd9dfcc24f3f1db7a983dad13d53439ccac0b37e24037e7b95f80f59f37a2f683c4ba4682", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); + try htest.assertEqualHash(Keccak512, "0eab42de4c3ceb9235fc91acffe746b29c29a8c366b7c60e4e67c466f36a4304c00fa9caf9d87976ba469bcbe06713b435f091ef2769fb160cdab33d3670680e", ""); + try htest.assertEqualHash(Keccak512, "18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", "abc"); + try htest.assertEqualHash(Keccak512, "ac2fb35251825d3aa48468a9948c0a91b8256f6d97d8fa4160faff2dd9dfcc24f3f1db7a983dad13d53439ccac0b37e24037e7b95f80f59f37a2f683c4ba4682", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); +} + +test "SHAKE-128 single" { + var out: [10]u8 = undefined; + Shake128.hash("hello123", &out, .{}); + try htest.assertEqual("1b85861510bc4d8e467d", &out); +} + +test "SHAKE-128 multisqueeze" { + var out: [10]u8 = undefined; + var h = Shake128.init(.{}); + h.update("hello123"); + h.squeeze(out[0..4]); + h.squeeze(out[4..]); + try htest.assertEqual("1b85861510bc4d8e467d", &out); +} + +test "SHAKE-128 multisqueeze with multiple blocks" { + var out: [100]u8 = undefined; + var out2: [100]u8 = undefined; + + var h = Shake128.init(.{}); + h.update("hello123"); + h.squeeze(out[0..50]); + h.squeeze(out[50..]); + + var h2 = Shake128.init(.{}); + h2.update("hello123"); + h2.squeeze(&out2); + try std.testing.expectEqualSlices(u8, &out, &out2); +} + +test "SHAKE-256 single" { + var out: [10]u8 = undefined; + Shake256.hash("hello123", &out, .{}); + try htest.assertEqual("ade612ba265f92de4a37", &out); } From 4789cc0249f8dd6b76471abda0c6eaa16d75bd69 Mon Sep 17 00:00:00 2001 From: Frank Denis <124872+jedisct1@users.noreply.github.com> Date: Thu, 2 Mar 2023 20:14:41 +0100 Subject: [PATCH 18/25] crypto.KeccakF: compute rotations at comptime, add a test with f=800 (#14760) --- lib/std/crypto/keccak_p.zig | 56 +++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/lib/std/crypto/keccak_p.zig b/lib/std/crypto/keccak_p.zig index 5d13677bde..10367caffd 100644 --- a/lib/std/crypto/keccak_p.zig +++ b/lib/std/crypto/keccak_p.zig @@ -9,19 +9,6 @@ pub fn KeccakF(comptime f: u11) type { const T = std.meta.Int(.unsigned, f / 25); const Block = [25]T; - const RC = [_]u64{ - 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, 0x8000000080008000, - 0x000000000000808b, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009, - 0x000000000000008a, 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, - 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, 0x8000000000008003, - 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, - 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, - }; - - const RHO = [_]u6{ - 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44, - }; - const PI = [_]u5{ 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1, }; @@ -32,6 +19,24 @@ pub fn KeccakF(comptime f: u11) type { /// Number of bytes in the state. pub const block_bytes = f / 8; + /// Maximum number of rounds for the given f parameter. + pub const max_rounds = 12 + 2 * math.log2(f / 25); + + // Round constants + const RC = rc: { + const RC64 = [_]u64{ + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, 0x8000000080008000, + 0x000000000000808b, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009, + 0x000000000000008a, 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, 0x8000000000008003, + 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, + 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, + }; + var rc: [max_rounds]T = undefined; + for (&rc, RC64[0..max_rounds]) |*t, c| t.* = @truncate(T, c); + break :rc rc; + }; + st: Block = [_]T{0} ** 25, /// Initialize the state from a slice of bytes. @@ -146,10 +151,12 @@ pub fn KeccakF(comptime f: u11) type { // rho+pi var last = st[1]; + comptime var rotc = 0; inline for (0..24) |i| { const x = PI[i]; const tmp = st[x]; - st[x] = math.rotl(T, last, RHO[i]); + rotc = (rotc + i + 1) % @bitSizeOf(T); + st[x] = math.rotl(T, last, rotc); last = tmp; } inline for (0..5) |i| { @@ -180,7 +187,7 @@ pub fn KeccakF(comptime f: u11) type { /// Apply a full-round permutation to the state. pub fn permute(self: *Self) void { - self.permuteR(comptime 12 + 2 * math.log2(f / 25)); + self.permuteR(max_rounds); } }; } @@ -249,3 +256,22 @@ pub fn State(comptime f: u11, comptime capacity: u11, comptime delim: u8, compti } }; } + +test "Keccak-f800" { + var st: KeccakF(800) = .{ + .st = .{ + 0xE531D45D, 0xF404C6FB, 0x23A0BF99, 0xF1F8452F, 0x51FFD042, 0xE539F578, 0xF00B80A7, + 0xAF973664, 0xBF5AF34C, 0x227A2424, 0x88172715, 0x9F685884, 0xB15CD054, 0x1BF4FC0E, + 0x6166FA91, 0x1A9E599A, 0xA3970A1F, 0xAB659687, 0xAFAB8D68, 0xE74B1015, 0x34001A98, + 0x4119EFF3, 0x930A0E76, 0x87B28070, 0x11EFE996, + }, + }; + st.permute(); + const expected: [25]u32 = .{ + 0x75BF2D0D, 0x9B610E89, 0xC826AF40, 0x64CD84AB, 0xF905BDD6, 0xBC832835, 0x5F8001B9, + 0x15662CCE, 0x8E38C95E, 0x701FE543, 0x1B544380, 0x89ACDEFF, 0x51EDB5DE, 0x0E9702D9, + 0x6C19AA16, 0xA2913EEE, 0x60754E9A, 0x9819063C, 0xF4709254, 0xD09F9084, 0x772DA259, + 0x1DB35DF7, 0x5AA60162, 0x358825D5, 0xB3783BAB, + }; + try std.testing.expectEqualSlices(u32, &st.st, &expected); +} From 426c13dddfa9e19cf505fc26561d2a8637165d64 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Mar 2023 18:56:43 -0700 Subject: [PATCH 19/25] add doc comments to std.fs.File.default_mode --- lib/std/fs/file.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 1ba4bc18fd..bf93a61239 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -48,6 +48,12 @@ pub const File = struct { Unknown, }; + /// This is the default mode given to POSIX operating systems for creating + /// files. `0o666` is "-rw-rw-rw-" which is counter-intuitive at first, + /// since most people would expect "-rw-r--r--", for example, when using + /// the `touch` command, which would correspond to `0o644`. However, POSIX + /// libc implementations use `0o666` inside `fopen` and then rely on the + /// process-scoped "umask" setting to adjust this number for file creation. pub const default_mode = switch (builtin.os.tag) { .windows => 0, .wasi => 0, From bb5006d7287607e49b71d25d263ebcc3d5845c60 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Mar 2023 18:32:15 -0700 Subject: [PATCH 20/25] std: add fchmodat Also add `std.fs.has_executable_bit` for doing conditional compilation. This adds the linux syscalls for chmod and fchmodat, as well as the extern libc function declarations. Only `fchmodat` is added to `std.os`, and it is not yet added to std.fs. --- lib/std/c.zig | 2 ++ lib/std/fs.zig | 5 +++ lib/std/os.zig | 35 +++++++++++++++++++-- lib/std/os/linux.zig | 18 +++++++++++ lib/std/os/test.zig | 72 ++++++++++++++++++++++++++------------------ 5 files changed, 100 insertions(+), 32 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 1334b2f2c1..9fc3b1d57e 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -171,7 +171,9 @@ pub extern "c" fn dup(fd: c.fd_t) c_int; pub extern "c" fn dup2(old_fd: c.fd_t, new_fd: c.fd_t) c_int; pub extern "c" fn readlink(noalias path: [*:0]const u8, noalias buf: [*]u8, bufsize: usize) isize; pub extern "c" fn readlinkat(dirfd: c.fd_t, noalias path: [*:0]const u8, noalias buf: [*]u8, bufsize: usize) isize; +pub extern "c" fn chmod(path: [*:0]const u8, mode: c.mode_t) c_int; pub extern "c" fn fchmod(fd: c.fd_t, mode: c.mode_t) c_int; +pub extern "c" fn fchmodat(fd: c.fd_t, path: [*:0]const u8, mode: c.mode_t, flags: c_uint) c_int; pub extern "c" fn fchown(fd: c.fd_t, owner: c.uid_t, group: c.gid_t) c_int; pub extern "c" fn umask(mode: c.mode_t) c.mode_t; diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 52a93a498f..def4332700 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -11,6 +11,11 @@ const math = std.math; const is_darwin = builtin.os.tag.isDarwin(); +pub const has_executable_bit = switch (builtin.os.tag) { + .windows, .wasi => false, + else => true, +}; + pub const path = @import("fs/path.zig"); pub const File = @import("fs/file.zig").File; pub const wasi = @import("fs/wasi.zig"); diff --git a/lib/std/os.zig b/lib/std/os.zig index bd6719ec8f..fe664302a7 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -302,8 +302,7 @@ pub const FChmodError = error{ /// successfully, or must have the effective user ID matching the owner /// of the file. pub fn fchmod(fd: fd_t, mode: mode_t) FChmodError!void { - if (builtin.os.tag == .windows or builtin.os.tag == .wasi) - @compileError("Unsupported OS"); + if (!std.fs.has_executable_bit) @compileError("fchmod unsupported by target OS"); while (true) { const res = system.fchmod(fd, mode); @@ -311,8 +310,38 @@ pub fn fchmod(fd: fd_t, mode: mode_t) FChmodError!void { switch (system.getErrno(res)) { .SUCCESS => return, .INTR => continue, - .BADF => unreachable, // Can be reached if the fd refers to a non-iterable directory. + .BADF => unreachable, + .FAULT => unreachable, + .INVAL => unreachable, + .ACCES => return error.AccessDenied, + .IO => return error.InputOutput, + .LOOP => return error.SymLinkLoop, + .NOENT => return error.FileNotFound, + .NOMEM => return error.SystemResources, + .NOTDIR => return error.FileNotFound, + .PERM => return error.AccessDenied, + .ROFS => return error.ReadOnlyFileSystem, + else => |err| return unexpectedErrno(err), + } + } +} +const FChmodAtError = FChmodError || error{ + NameTooLong, +}; + +pub fn fchmodat(dirfd: fd_t, path: []const u8, mode: mode_t, flags: u32) FChmodAtError!void { + if (!std.fs.has_executable_bit) @compileError("fchmodat unsupported by target OS"); + + const path_c = try toPosixPath(path); + + while (true) { + const res = system.fchmodat(dirfd, &path_c, mode, flags); + + switch (system.getErrno(res)) { + .SUCCESS => return, + .INTR => continue, + .BADF => unreachable, .FAULT => unreachable, .INVAL => unreachable, .ACCES => return error.AccessDenied, diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index fe2f8404e2..b151e5f235 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -769,6 +769,20 @@ pub fn fchmod(fd: i32, mode: mode_t) usize { return syscall2(.fchmod, @bitCast(usize, @as(isize, fd)), mode); } +pub fn chmod(path: [*:0]const u8, mode: mode_t) usize { + if (@hasField(SYS, "chmod")) { + return syscall2(.chmod, @ptrToInt(path), mode); + } else { + return syscall4( + .fchmodat, + @bitCast(usize, @as(isize, AT.FDCWD)), + @ptrToInt(path), + mode, + 0, + ); + } +} + pub fn fchown(fd: i32, owner: uid_t, group: gid_t) usize { if (@hasField(SYS, "fchown32")) { return syscall3(.fchown32, @bitCast(usize, @as(isize, fd)), owner, group); @@ -777,6 +791,10 @@ pub fn fchown(fd: i32, owner: uid_t, group: gid_t) usize { } } +pub fn fchmodat(fd: i32, path: [*:0]const u8, mode: mode_t, flags: u32) usize { + return syscall4(.fchmodat, @bitCast(usize, @as(isize, fd)), @ptrToInt(path), mode, flags); +} + /// Can only be called on 32 bit systems. For 64 bit see `lseek`. pub fn llseek(fd: i32, offset: u64, result: ?*u64, whence: usize) usize { // NOTE: The offset parameter splitting is independent from the target diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 177e32f772..7a5fc0de33 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -531,17 +531,17 @@ test "memfd_create" { else => return error.SkipZigTest, } - const fd = std.os.memfd_create("test", 0) catch |err| switch (err) { + const fd = os.memfd_create("test", 0) catch |err| switch (err) { // Related: https://github.com/ziglang/zig/issues/4019 error.SystemOutdated => return error.SkipZigTest, else => |e| return e, }; - defer std.os.close(fd); - try expect((try std.os.write(fd, "test")) == 4); - try std.os.lseek_SET(fd, 0); + defer os.close(fd); + try expect((try os.write(fd, "test")) == 4); + try os.lseek_SET(fd, 0); var buf: [10]u8 = undefined; - const bytes_read = try std.os.read(fd, &buf); + const bytes_read = try os.read(fd, &buf); try expect(bytes_read == 4); try expect(mem.eql(u8, buf[0..4], "test")); } @@ -688,7 +688,7 @@ test "signalfd" { .linux, .solaris => {}, else => return error.SkipZigTest, } - _ = std.os.signalfd; + _ = os.signalfd; } test "sync" { @@ -757,11 +757,11 @@ test "shutdown socket" { if (native_os == .wasi) return error.SkipZigTest; if (native_os == .windows) { - _ = try std.os.windows.WSAStartup(2, 2); + _ = try os.windows.WSAStartup(2, 2); } defer { if (native_os == .windows) { - std.os.windows.WSACleanup() catch unreachable; + os.windows.WSACleanup() catch unreachable; } } const sock = try os.socket(os.AF.INET, os.SOCK.STREAM, 0); @@ -855,13 +855,13 @@ test "dup & dup2" { var file = try tmp.dir.createFile("os_dup_test", .{}); defer file.close(); - var duped = std.fs.File{ .handle = try std.os.dup(file.handle) }; + var duped = std.fs.File{ .handle = try os.dup(file.handle) }; defer duped.close(); try duped.writeAll("dup"); // Tests aren't run in parallel so using the next fd shouldn't be an issue. const new_fd = duped.handle + 1; - try std.os.dup2(file.handle, new_fd); + try os.dup2(file.handle, new_fd); var dup2ed = std.fs.File{ .handle = new_fd }; defer dup2ed.close(); try dup2ed.writeAll("dup2"); @@ -909,46 +909,46 @@ test "POSIX file locking with fcntl" { const fd = file.handle; // Place an exclusive lock on the first byte, and a shared lock on the second byte: - var struct_flock = std.mem.zeroInit(std.os.Flock, .{ .type = std.os.F.WRLCK }); - _ = try std.os.fcntl(fd, std.os.F.SETLK, @ptrToInt(&struct_flock)); + var struct_flock = std.mem.zeroInit(os.Flock, .{ .type = os.F.WRLCK }); + _ = try os.fcntl(fd, os.F.SETLK, @ptrToInt(&struct_flock)); struct_flock.start = 1; - struct_flock.type = std.os.F.RDLCK; - _ = try std.os.fcntl(fd, std.os.F.SETLK, @ptrToInt(&struct_flock)); + struct_flock.type = os.F.RDLCK; + _ = try os.fcntl(fd, os.F.SETLK, @ptrToInt(&struct_flock)); // Check the locks in a child process: - const pid = try std.os.fork(); + const pid = try os.fork(); if (pid == 0) { // child expects be denied the exclusive lock: struct_flock.start = 0; - struct_flock.type = std.os.F.WRLCK; - try expectError(error.Locked, std.os.fcntl(fd, std.os.F.SETLK, @ptrToInt(&struct_flock))); + struct_flock.type = os.F.WRLCK; + try expectError(error.Locked, os.fcntl(fd, os.F.SETLK, @ptrToInt(&struct_flock))); // child expects to get the shared lock: struct_flock.start = 1; - struct_flock.type = std.os.F.RDLCK; - _ = try std.os.fcntl(fd, std.os.F.SETLK, @ptrToInt(&struct_flock)); + struct_flock.type = os.F.RDLCK; + _ = try os.fcntl(fd, os.F.SETLK, @ptrToInt(&struct_flock)); // child waits for the exclusive lock in order to test deadlock: struct_flock.start = 0; - struct_flock.type = std.os.F.WRLCK; - _ = try std.os.fcntl(fd, std.os.F.SETLKW, @ptrToInt(&struct_flock)); + struct_flock.type = os.F.WRLCK; + _ = try os.fcntl(fd, os.F.SETLKW, @ptrToInt(&struct_flock)); // child exits without continuing: - std.os.exit(0); + os.exit(0); } else { // parent waits for child to get shared lock: std.time.sleep(1 * std.time.ns_per_ms); // parent expects deadlock when attempting to upgrade the shared lock to exclusive: struct_flock.start = 1; - struct_flock.type = std.os.F.WRLCK; - try expectError(error.DeadLock, std.os.fcntl(fd, std.os.F.SETLKW, @ptrToInt(&struct_flock))); + struct_flock.type = os.F.WRLCK; + try expectError(error.DeadLock, os.fcntl(fd, os.F.SETLKW, @ptrToInt(&struct_flock))); // parent releases exclusive lock: struct_flock.start = 0; - struct_flock.type = std.os.F.UNLCK; - _ = try std.os.fcntl(fd, std.os.F.SETLK, @ptrToInt(&struct_flock)); + struct_flock.type = os.F.UNLCK; + _ = try os.fcntl(fd, os.F.SETLK, @ptrToInt(&struct_flock)); // parent releases shared lock: struct_flock.start = 1; - struct_flock.type = std.os.F.UNLCK; - _ = try std.os.fcntl(fd, std.os.F.SETLK, @ptrToInt(&struct_flock)); + struct_flock.type = os.F.UNLCK; + _ = try os.fcntl(fd, os.F.SETLK, @ptrToInt(&struct_flock)); // parent waits for child: - const result = std.os.waitpid(pid, 0); + const result = os.waitpid(pid, 0); try expect(result.status == 0 * 256); } } @@ -1182,3 +1182,17 @@ test "pwrite with empty buffer" { _ = try os.pwrite(file.handle, bytes, 0); } + +test "fchmodat smoke test" { + if (!std.fs.has_executable_bit) return error.SkipZigTest; + + var tmp = tmpDir(.{}); + defer tmp.cleanup(); + + try expectError(error.FileNotFound, os.fchmodat(tmp.dir.fd, "foo.txt", 0o666, 0)); + const fd = try os.openat(tmp.dir.fd, "foo.txt", os.O.RDWR | os.O.CREAT | os.O.EXCL, 0o666); + os.close(fd); + try os.fchmodat(tmp.dir.fd, "foo.txt", 0o755, 0); + const st = try os.fstatat(tmp.dir.fd, "foo.txt", 0); + try expectEqual(@as(os.mode_t, 0o755), st.mode & 0b111_111_111); +} From aaaaab9ec2d5d6a4c1ed2b7171bd6f599f7a31ce Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Mar 2023 22:56:52 -0700 Subject: [PATCH 21/25] std.process.Child: remove pid and handle, add id Previously, this API had pid, to be used on POSIX systems, and handle, to be used on Windows. This commit unifies the API, defining an Id type that is either the pid or the HANDLE depending on the target OS. This commit also prepares for the future by allowing one to import via `std.process.Child` which is the fully qualified namespace that I intend to migrate to in the future. --- lib/std/child_process.zig | 55 +++++++++++++++++++++++++-------------- lib/std/process.zig | 1 + lib/std/std.zig | 1 + 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 7ff48a5652..dba92ab998 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -19,8 +19,15 @@ const maxInt = std.math.maxInt; const assert = std.debug.assert; pub const ChildProcess = struct { - pid: if (builtin.os.tag == .windows) void else i32, - handle: if (builtin.os.tag == .windows) windows.HANDLE else void, + pub const Id = switch (builtin.os.tag) { + .windows => windows.HANDLE, + else => os.pid_t, + }; + + /// Available after calling `spawn()`. This becomes `undefined` after calling `wait()`. + /// On Windows this is the hProcess. + /// On POSIX this is the pid. + id: Id, thread_handle: if (builtin.os.tag == .windows) windows.HANDLE else void, allocator: mem.Allocator, @@ -105,8 +112,7 @@ pub const ChildProcess = struct { return .{ .allocator = allocator, .argv = argv, - .pid = undefined, - .handle = undefined, + .id = undefined, .thread_handle = undefined, .err_pipe = null, .term = null, @@ -131,6 +137,7 @@ pub const ChildProcess = struct { } /// On success must call `kill` or `wait`. + /// After spawning the `id` is available. pub fn spawn(self: *ChildProcess) SpawnError!void { if (!std.process.can_spawn) { @compileError("the target operating system cannot spawn processes"); @@ -167,7 +174,7 @@ pub const ChildProcess = struct { return term; } - try windows.TerminateProcess(self.handle, exit_code); + try windows.TerminateProcess(self.id, exit_code); try self.waitUnwrappedWindows(); return self.term.?; } @@ -177,18 +184,21 @@ pub const ChildProcess = struct { self.cleanupStreams(); return term; } - try os.kill(self.pid, os.SIG.TERM); + try os.kill(self.id, os.SIG.TERM); try self.waitUnwrapped(); return self.term.?; } /// Blocks until child process terminates and then cleans up all resources. pub fn wait(self: *ChildProcess) !Term { - if (builtin.os.tag == .windows) { - return self.waitWindows(); - } else { - return self.waitPosix(); - } + const term = if (builtin.os.tag == .windows) + try self.waitWindows() + else + try self.waitPosix(); + + self.id = undefined; + + return term; } pub const ExecResult = struct { @@ -246,6 +256,11 @@ pub const ChildProcess = struct { stderr.* = fifoToOwnedArrayList(poller.fifo(.stderr)); } + pub const ExecError = os.GetCwdError || os.ReadError || SpawnError || os.PollError || error{ + StdoutStreamTooLong, + StderrStreamTooLong, + }; + /// Spawns a child process, waits for it, collecting stdout and stderr, and then returns. /// If it succeeds, the caller owns result.stdout and result.stderr memory. pub fn exec(args: struct { @@ -256,7 +271,7 @@ pub const ChildProcess = struct { env_map: ?*const EnvMap = null, max_output_bytes: usize = 50 * 1024, expand_arg0: Arg0Expand = .no_expand, - }) !ExecResult { + }) ExecError!ExecResult { var child = ChildProcess.init(args.argv, args.allocator); child.stdin_behavior = .Ignore; child.stdout_behavior = .Pipe; @@ -304,18 +319,18 @@ pub const ChildProcess = struct { } fn waitUnwrappedWindows(self: *ChildProcess) !void { - const result = windows.WaitForSingleObjectEx(self.handle, windows.INFINITE, false); + const result = windows.WaitForSingleObjectEx(self.id, windows.INFINITE, false); self.term = @as(SpawnError!Term, x: { var exit_code: windows.DWORD = undefined; - if (windows.kernel32.GetExitCodeProcess(self.handle, &exit_code) == 0) { + if (windows.kernel32.GetExitCodeProcess(self.id, &exit_code) == 0) { break :x Term{ .Unknown = 0 }; } else { break :x Term{ .Exited = @truncate(u8, exit_code) }; } }); - os.close(self.handle); + os.close(self.id); os.close(self.thread_handle); self.cleanupStreams(); return result; @@ -323,9 +338,9 @@ pub const ChildProcess = struct { fn waitUnwrapped(self: *ChildProcess) !void { const res: os.WaitPidResult = if (comptime builtin.target.isDarwin()) - try os.posix_spawn.waitpid(self.pid, 0) + try os.posix_spawn.waitpid(self.id, 0) else - os.waitpid(self.pid, 0); + os.waitpid(self.id, 0); const status = res.status; self.cleanupStreams(); self.handleWaitResult(status); @@ -483,7 +498,7 @@ pub const ChildProcess = struct { self.stderr = null; } - self.pid = pid; + self.id = pid; self.term = null; if (self.stdin_behavior == StdIo.Pipe) { @@ -657,7 +672,7 @@ pub const ChildProcess = struct { self.stderr = null; } - self.pid = pid; + self.id = pid; self.err_pipe = err_pipe; self.term = null; @@ -923,7 +938,7 @@ pub const ChildProcess = struct { self.stderr = null; } - self.handle = piProcInfo.hProcess; + self.id = piProcInfo.hProcess; self.thread_handle = piProcInfo.hThread; self.term = null; diff --git a/lib/std/process.zig b/lib/std/process.zig index 777bcbbab0..eff29e86fa 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -9,6 +9,7 @@ const assert = std.debug.assert; const testing = std.testing; const child_process = @import("child_process.zig"); +pub const Child = child_process.ChildProcess; pub const abort = os.abort; pub const exit = os.exit; pub const changeCurDir = os.chdir; diff --git a/lib/std/std.zig b/lib/std/std.zig index 4a6d330003..c1c682e224 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -12,6 +12,7 @@ pub const BoundedArray = @import("bounded_array.zig").BoundedArray; pub const Build = @import("Build.zig"); pub const BufMap = @import("buf_map.zig").BufMap; pub const BufSet = @import("buf_set.zig").BufSet; +/// Deprecated: use `process.Child`. pub const ChildProcess = @import("child_process.zig").ChildProcess; pub const ComptimeStringMap = @import("comptime_string_map.zig").ComptimeStringMap; pub const DynLib = @import("dynamic_library.zig").DynLib; From fdee558e45e4587aa5345b0de531a9b56a8682fd Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Thu, 2 Mar 2023 23:36:06 +0100 Subject: [PATCH 22/25] crypto.25519.field: de-inline _sq() May fix #14764 --- lib/std/crypto/25519/field.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/crypto/25519/field.zig b/lib/std/crypto/25519/field.zig index 1a786e0c32..66a50bee70 100644 --- a/lib/std/crypto/25519/field.zig +++ b/lib/std/crypto/25519/field.zig @@ -287,7 +287,7 @@ pub const Fe = struct { return _carry128(&r); } - inline fn _sq(a: Fe, comptime double: bool) Fe { + fn _sq(a: Fe, comptime double: bool) Fe { var ax: [5]u128 = undefined; var r: [5]u128 = undefined; comptime var i = 0; From 6be5946ed8026f2a7ae990aced9572f229acecf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eckhart=20K=C3=B6ppen?= Date: Wed, 1 Mar 2023 16:49:12 +0200 Subject: [PATCH 23/25] sema: Place functions on AVR in flash addrspace - Use .flash as the default address space for functions on AVR - Return .flash as the address space for function pointers on AVR without explicit address space --- src/Sema.zig | 4 ++-- src/target.zig | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Sema.zig b/src/Sema.zig index 46b47cd23d..f9a6f39867 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -17328,11 +17328,11 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air break :blk abi_align; } else 0; - const address_space = if (inst_data.flags.has_addrspace) blk: { + const address_space: std.builtin.AddressSpace = if (inst_data.flags.has_addrspace) blk: { const ref = @intToEnum(Zir.Inst.Ref, sema.code.extra[extra_i]); extra_i += 1; break :blk try sema.analyzeAddressSpace(block, addrspace_src, ref, .pointer); - } else .generic; + } else if (elem_ty.zigTypeTag() == .Fn and target.cpu.arch == .avr) .flash else .generic; const bit_offset = if (inst_data.flags.has_bit_range) blk: { const ref = @intToEnum(Zir.Inst.Ref, sema.code.extra[extra_i]); diff --git a/src/target.zig b/src/target.zig index 6d6933e9e7..f1c9a4056f 100644 --- a/src/target.zig +++ b/src/target.zig @@ -644,8 +644,9 @@ pub fn defaultAddressSpace( function, }, ) AddressSpace { - _ = target; - _ = context; + // The default address space for functions on AVR is .flash to produce + // correct fixups into progmem. + if (context == .function and target.cpu.arch == .avr) return .flash; return .generic; } From 2834b937f143aa80e3730706dac5ff44cea3a67d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 2 Mar 2023 19:04:55 -0700 Subject: [PATCH 24/25] link: give executable bit to wasm executables sometimes Give +x to the .wasm file if it is an executable and the OS is WASI. Some systems may be configured to execute such binaries directly. Even if that is not the case, it means we will get "exec format error" when trying to run it rather than "access denied", and then can react to that in the same way as trying to run an ELF file from a foreign CPU architecture. This is part of the strategy to unify RunStep and EmulatableRunStep. --- src/link/Wasm.zig | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 00a52177f7..74525138a1 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -345,7 +345,17 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option } // TODO: read the file and keep valid parts instead of truncating - const file = try options.emit.?.directory.handle.createFile(sub_path, .{ .truncate = true, .read = true }); + const file = try options.emit.?.directory.handle.createFile(sub_path, .{ + .truncate = true, + .read = true, + .mode = if (fs.has_executable_bit) + if (options.target.os.tag == .wasi and options.output_mode == .Exe) + fs.File.default_mode | 0b001_000_000 + else + fs.File.default_mode + else + 0, + }); wasm_bin.base.file = file; wasm_bin.name = sub_path; @@ -3750,10 +3760,7 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) ! if (wasm.base.options.import_symbols) { try argv.append("--allow-undefined"); } - try argv.appendSlice(&[_][]const u8{ - "-o", - full_out_path, - }); + try argv.appendSlice(&.{ "-o", full_out_path }); if (target.cpu.arch == .wasm64) { try argv.append("-mwasm64"); @@ -3889,6 +3896,21 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) ! } } } + + // Give +x to the .wasm file if it is an executable and the OS is WASI. + // Some systems may be configured to execute such binaries directly. Even if that + // is not the case, it means we will get "exec format error" when trying to run + // it, and then can react to that in the same way as trying to run an ELF file + // from a foreign CPU architecture. + if (fs.has_executable_bit and target.os.tag == .wasi and + wasm.base.options.output_mode == .Exe) + { + // TODO: what's our strategy for reporting linker errors from this function? + // report a nice error here with the file path if it fails instead of + // just returning the error code. + // chmod does not interact with umask, so we use a conservative -rwxr--r-- here. + try std.os.fchmodat(fs.cwd().fd, full_out_path, 0o744, 0); + } } if (!wasm.base.options.disable_lld_caching) { From 75ff34db9e93056482233f8476a06f78b4a2f3c2 Mon Sep 17 00:00:00 2001 From: Ali Chraghi Date: Fri, 3 Mar 2023 11:04:08 +0330 Subject: [PATCH 25/25] std.Build.Cache: remove 'test-filetimestamp.tmp' once timestamp returned --- lib/std/Build/Cache.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index c459fca633..d4dbe6ec14 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -956,11 +956,16 @@ fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void { // Create/Write a file, close it, then grab its stat.mtime timestamp. fn testGetCurrentFileTimestamp() !i128 { - var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{ + const test_out_file = "test-filetimestamp.tmp"; + + var file = try fs.cwd().createFile(test_out_file, .{ .read = true, .truncate = true, }); - defer file.close(); + defer { + file.close(); + fs.cwd().deleteFile(test_out_file) catch {}; + } return (try file.stat()).mtime; }