src/codegen.cpp - motiejus/zig - fork of https://codeberg.org/ziglang/zig

src/codegen.cpp (446608B) - Raw
      1 /*
      2  * Copyright (c) 2015 Andrew Kelley
      3  *
      4  * This file is part of zig, which is MIT licensed.
      5  * See http://opensource.org/licenses/MIT
      6  */
      7 
      8 #include "analyze.hpp"
      9 #include "ast_render.hpp"
     10 #include "codegen.hpp"
     11 #include "compiler.hpp"
     12 #include "config.h"
     13 #include "errmsg.hpp"
     14 #include "error.hpp"
     15 #include "hash_map.hpp"
     16 #include "ir.hpp"
     17 #include "os.hpp"
     18 #include "translate_c.hpp"
     19 #include "target.hpp"
     20 #include "util.hpp"
     21 #include "zig_llvm.h"
     22 #include "userland.h"
     23 
     24 #include <stdio.h>
     25 #include <errno.h>
     26 
     27 enum ResumeId {
     28     ResumeIdManual,
     29     ResumeIdReturn,
     30     ResumeIdCall,
     31 };
     32 
     33 static void init_darwin_native(CodeGen *g) {
     34     char *osx_target = getenv("MACOSX_DEPLOYMENT_TARGET");
     35     char *ios_target = getenv("IPHONEOS_DEPLOYMENT_TARGET");
     36 
     37     // Allow conflicts among OSX and iOS, but choose the default platform.
     38     if (osx_target && ios_target) {
     39         if (g->zig_target->arch == ZigLLVM_arm ||
     40             g->zig_target->arch == ZigLLVM_aarch64 ||
     41             g->zig_target->arch == ZigLLVM_thumb)
     42         {
     43             osx_target = nullptr;
     44         } else {
     45             ios_target = nullptr;
     46         }
     47     }
     48 
     49     if (osx_target) {
     50         g->mmacosx_version_min = buf_create_from_str(osx_target);
     51     } else if (ios_target) {
     52         g->mios_version_min = buf_create_from_str(ios_target);
     53     } else if (g->zig_target->os != OsIOS) {
     54         g->mmacosx_version_min = buf_create_from_str("10.14");
     55     }
     56 }
     57 
     58 static ZigPackage *new_package(const char *root_src_dir, const char *root_src_path, const char *pkg_path) {
     59     ZigPackage *entry = allocate<ZigPackage>(1);
     60     entry->package_table.init(4);
     61     buf_init_from_str(&entry->root_src_dir, root_src_dir);
     62     buf_init_from_str(&entry->root_src_path, root_src_path);
     63     buf_init_from_str(&entry->pkg_path, pkg_path);
     64     return entry;
     65 }
     66 
     67 ZigPackage *new_anonymous_package() {
     68     return new_package("", "", "");
     69 }
     70 
     71 static const char *symbols_that_llvm_depends_on[] = {
     72     "memcpy",
     73     "memset",
     74     "sqrt",
     75     "powi",
     76     "sin",
     77     "cos",
     78     "pow",
     79     "exp",
     80     "exp2",
     81     "log",
     82     "log10",
     83     "log2",
     84     "fma",
     85     "fabs",
     86     "minnum",
     87     "maxnum",
     88     "copysign",
     89     "floor",
     90     "ceil",
     91     "trunc",
     92     "rint",
     93     "nearbyint",
     94     "round",
     95     // TODO probably all of compiler-rt needs to go here
     96 };
     97 
     98 void codegen_set_clang_argv(CodeGen *g, const char **args, size_t len) {
     99     g->clang_argv = args;
    100     g->clang_argv_len = len;
    101 }
    102 
    103 void codegen_set_llvm_argv(CodeGen *g, const char **args, size_t len) {
    104     g->llvm_argv = args;
    105     g->llvm_argv_len = len;
    106 }
    107 
    108 void codegen_set_test_filter(CodeGen *g, Buf *filter) {
    109     g->test_filter = filter;
    110 }
    111 
    112 void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix) {
    113     g->test_name_prefix = prefix;
    114 }
    115 
    116 void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch) {
    117     g->version_major = major;
    118     g->version_minor = minor;
    119     g->version_patch = patch;
    120 }
    121 
    122 void codegen_set_emit_file_type(CodeGen *g, EmitFileType emit_file_type) {
    123     g->emit_file_type = emit_file_type;
    124 }
    125 
    126 void codegen_set_each_lib_rpath(CodeGen *g, bool each_lib_rpath) {
    127     g->each_lib_rpath = each_lib_rpath;
    128 }
    129 
    130 void codegen_set_errmsg_color(CodeGen *g, ErrColor err_color) {
    131     g->err_color = err_color;
    132 }
    133 
    134 void codegen_set_strip(CodeGen *g, bool strip) {
    135     g->strip_debug_symbols = strip;
    136     if (!target_has_debug_info(g->zig_target)) {
    137         g->strip_debug_symbols = true;
    138     }
    139 }
    140 
    141 void codegen_set_out_name(CodeGen *g, Buf *out_name) {
    142     g->root_out_name = out_name;
    143 }
    144 
    145 void codegen_add_lib_dir(CodeGen *g, const char *dir) {
    146     g->lib_dirs.append(dir);
    147 }
    148 
    149 void codegen_add_rpath(CodeGen *g, const char *name) {
    150     g->rpath_list.append(buf_create_from_str(name));
    151 }
    152 
    153 LinkLib *codegen_add_link_lib(CodeGen *g, Buf *name) {
    154     return add_link_lib(g, name);
    155 }
    156 
    157 void codegen_add_forbidden_lib(CodeGen *codegen, Buf *lib) {
    158     codegen->forbidden_libs.append(lib);
    159 }
    160 
    161 void codegen_add_framework(CodeGen *g, const char *framework) {
    162     g->darwin_frameworks.append(buf_create_from_str(framework));
    163 }
    164 
    165 void codegen_set_mmacosx_version_min(CodeGen *g, Buf *mmacosx_version_min) {
    166     g->mmacosx_version_min = mmacosx_version_min;
    167 }
    168 
    169 void codegen_set_mios_version_min(CodeGen *g, Buf *mios_version_min) {
    170     g->mios_version_min = mios_version_min;
    171 }
    172 
    173 void codegen_set_rdynamic(CodeGen *g, bool rdynamic) {
    174     g->linker_rdynamic = rdynamic;
    175 }
    176 
    177 void codegen_set_linker_script(CodeGen *g, const char *linker_script) {
    178     g->linker_script = linker_script;
    179 }
    180 
    181 
    182 static void render_const_val(CodeGen *g, ConstExprValue *const_val, const char *name);
    183 static void render_const_val_global(CodeGen *g, ConstExprValue *const_val, const char *name);
    184 static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const char *name);
    185 static void generate_error_name_table(CodeGen *g);
    186 static bool value_is_all_undef(ConstExprValue *const_val);
    187 static void gen_undef_init(CodeGen *g, uint32_t ptr_align_bytes, ZigType *value_type, LLVMValueRef ptr);
    188 static LLVMValueRef build_alloca(CodeGen *g, ZigType *type_entry, const char *name, uint32_t alignment);
    189 
    190 static void addLLVMAttr(LLVMValueRef val, LLVMAttributeIndex attr_index, const char *attr_name) {
    191     unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name, strlen(attr_name));
    192     assert(kind_id != 0);
    193     LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(LLVMGetGlobalContext(), kind_id, 0);
    194     LLVMAddAttributeAtIndex(val, attr_index, llvm_attr);
    195 }
    196 
    197 static void addLLVMAttrStr(LLVMValueRef val, LLVMAttributeIndex attr_index,
    198         const char *attr_name, const char *attr_val)
    199 {
    200     LLVMAttributeRef llvm_attr = LLVMCreateStringAttribute(LLVMGetGlobalContext(),
    201             attr_name, (unsigned)strlen(attr_name), attr_val, (unsigned)strlen(attr_val));
    202     LLVMAddAttributeAtIndex(val, attr_index, llvm_attr);
    203 }
    204 
    205 static void addLLVMAttrInt(LLVMValueRef val, LLVMAttributeIndex attr_index,
    206         const char *attr_name, uint64_t attr_val)
    207 {
    208     unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name, strlen(attr_name));
    209     assert(kind_id != 0);
    210     LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(LLVMGetGlobalContext(), kind_id, attr_val);
    211     LLVMAddAttributeAtIndex(val, attr_index, llvm_attr);
    212 }
    213 
    214 static void addLLVMFnAttr(LLVMValueRef fn_val, const char *attr_name) {
    215     return addLLVMAttr(fn_val, -1, attr_name);
    216 }
    217 
    218 static void addLLVMFnAttrStr(LLVMValueRef fn_val, const char *attr_name, const char *attr_val) {
    219     return addLLVMAttrStr(fn_val, -1, attr_name, attr_val);
    220 }
    221 
    222 static void addLLVMFnAttrInt(LLVMValueRef fn_val, const char *attr_name, uint64_t attr_val) {
    223     return addLLVMAttrInt(fn_val, -1, attr_name, attr_val);
    224 }
    225 
    226 static void addLLVMArgAttr(LLVMValueRef fn_val, unsigned param_index, const char *attr_name) {
    227     return addLLVMAttr(fn_val, param_index + 1, attr_name);
    228 }
    229 
    230 static void addLLVMArgAttrInt(LLVMValueRef fn_val, unsigned param_index, const char *attr_name, uint64_t attr_val) {
    231     return addLLVMAttrInt(fn_val, param_index + 1, attr_name, attr_val);
    232 }
    233 
    234 static bool is_symbol_available(CodeGen *g, Buf *name) {
    235     return g->exported_symbol_names.maybe_get(name) == nullptr && g->external_prototypes.maybe_get(name) == nullptr;
    236 }
    237 
    238 static Buf *get_mangled_name(CodeGen *g, Buf *original_name, bool external_linkage) {
    239     if (external_linkage || is_symbol_available(g, original_name)) {
    240         return original_name;
    241     }
    242 
    243     int n = 0;
    244     for (;; n += 1) {
    245         Buf *new_name = buf_sprintf("%s.%d", buf_ptr(original_name), n);
    246         if (is_symbol_available(g, new_name)) {
    247             return new_name;
    248         }
    249     }
    250 }
    251 
    252 static LLVMCallConv get_llvm_cc(CodeGen *g, CallingConvention cc) {
    253     switch (cc) {
    254         case CallingConventionUnspecified: return LLVMFastCallConv;
    255         case CallingConventionC: return LLVMCCallConv;
    256         case CallingConventionCold:
    257             // cold calling convention only works on x86.
    258             if (g->zig_target->arch == ZigLLVM_x86 ||
    259                 g->zig_target->arch == ZigLLVM_x86_64)
    260             {
    261                 // cold calling convention is not supported on windows
    262                 if (g->zig_target->os == OsWindows) {
    263                     return LLVMCCallConv;
    264                 } else {
    265                     return LLVMColdCallConv;
    266                 }
    267             } else {
    268                 return LLVMCCallConv;
    269             }
    270             break;
    271         case CallingConventionNaked:
    272             zig_unreachable();
    273         case CallingConventionStdcall:
    274             // stdcall calling convention only works on x86.
    275             if (g->zig_target->arch == ZigLLVM_x86) {
    276                 return LLVMX86StdcallCallConv;
    277             } else {
    278                 return LLVMCCallConv;
    279             }
    280         case CallingConventionAsync:
    281             return LLVMFastCallConv;
    282     }
    283     zig_unreachable();
    284 }
    285 
    286 static void add_uwtable_attr(CodeGen *g, LLVMValueRef fn_val) {
    287     if (g->zig_target->os == OsWindows) {
    288         addLLVMFnAttr(fn_val, "uwtable");
    289     }
    290 }
    291 
    292 static LLVMLinkage to_llvm_linkage(GlobalLinkageId id) {
    293     switch (id) {
    294         case GlobalLinkageIdInternal:
    295             return LLVMInternalLinkage;
    296         case GlobalLinkageIdStrong:
    297             return LLVMExternalLinkage;
    298         case GlobalLinkageIdWeak:
    299             return LLVMWeakODRLinkage;
    300         case GlobalLinkageIdLinkOnce:
    301             return LLVMLinkOnceODRLinkage;
    302     }
    303     zig_unreachable();
    304 }
    305 
    306 // label (grep this): [fn_frame_struct_layout]
    307 static uint32_t frame_index_trace_arg(CodeGen *g, ZigType *return_type) {
    308     // [0] *ReturnType (callee's)
    309     // [1] *ReturnType (awaiter's)
    310     // [2] ReturnType
    311     uint32_t return_field_count = type_has_bits(return_type) ? 3 : 0;
    312     return frame_ret_start + return_field_count;
    313 }
    314 
    315 // label (grep this): [fn_frame_struct_layout]
    316 static uint32_t frame_index_arg(CodeGen *g, ZigType *return_type) {
    317     bool have_stack_trace = codegen_fn_has_err_ret_tracing_arg(g, return_type);
    318     // [0] *StackTrace (callee's)
    319     // [1] *StackTrace (awaiter's)
    320     uint32_t trace_field_count = have_stack_trace ? 2 : 0;
    321     return frame_index_trace_arg(g, return_type) + trace_field_count;
    322 }
    323 
    324 // label (grep this): [fn_frame_struct_layout]
    325 static uint32_t frame_index_trace_stack(CodeGen *g, FnTypeId *fn_type_id) {
    326     uint32_t result = frame_index_arg(g, fn_type_id->return_type);
    327     for (size_t i = 0; i < fn_type_id->param_count; i += 1) {
    328         if (type_has_bits(fn_type_id->param_info->type)) {
    329             result += 1;
    330         }
    331     }
    332     return result;
    333 }
    334 
    335 
    336 static uint32_t get_err_ret_trace_arg_index(CodeGen *g, ZigFn *fn_table_entry) {
    337     if (!g->have_err_ret_tracing) {
    338         return UINT32_MAX;
    339     }
    340     if (fn_is_async(fn_table_entry)) {
    341         return UINT32_MAX;
    342     }
    343     ZigType *fn_type = fn_table_entry->type_entry;
    344     if (!fn_type_can_fail(&fn_type->data.fn.fn_type_id)) {
    345         return UINT32_MAX;
    346     }
    347     ZigType *return_type = fn_type->data.fn.fn_type_id.return_type;
    348     bool first_arg_ret = type_has_bits(return_type) && handle_is_ptr(return_type);
    349     return first_arg_ret ? 1 : 0;
    350 }
    351 
    352 static void maybe_export_dll(CodeGen *g, LLVMValueRef global_value, GlobalLinkageId linkage) {
    353     if (linkage != GlobalLinkageIdInternal && g->zig_target->os == OsWindows && g->is_dynamic) {
    354         LLVMSetDLLStorageClass(global_value, LLVMDLLExportStorageClass);
    355     }
    356 }
    357 
    358 static void maybe_import_dll(CodeGen *g, LLVMValueRef global_value, GlobalLinkageId linkage) {
    359     if (linkage != GlobalLinkageIdInternal && g->zig_target->os == OsWindows) {
    360         // TODO come up with a good explanation/understanding for why we never do
    361         // DLLImportStorageClass. Empirically it only causes problems. But let's have
    362         // this documented and then clean up the code accordingly.
    363         //LLVMSetDLLStorageClass(global_value, LLVMDLLImportStorageClass);
    364     }
    365 }
    366 
    367 static bool cc_want_sret_attr(CallingConvention cc) {
    368     switch (cc) {
    369         case CallingConventionNaked:
    370             zig_unreachable();
    371         case CallingConventionC:
    372         case CallingConventionCold:
    373         case CallingConventionStdcall:
    374             return true;
    375         case CallingConventionAsync:
    376         case CallingConventionUnspecified:
    377             return false;
    378     }
    379     zig_unreachable();
    380 }
    381 
    382 static bool codegen_have_frame_pointer(CodeGen *g) {
    383     return g->build_mode == BuildModeDebug;
    384 }
    385 
    386 static LLVMValueRef make_fn_llvm_value(CodeGen *g, ZigFn *fn) {
    387     Buf *unmangled_name = &fn->symbol_name;
    388     Buf *symbol_name;
    389     GlobalLinkageId linkage;
    390     if (fn->body_node == nullptr) {
    391         symbol_name = unmangled_name;
    392         linkage = GlobalLinkageIdStrong;
    393     } else if (fn->export_list.length == 0) {
    394         symbol_name = get_mangled_name(g, unmangled_name, false);
    395         linkage = GlobalLinkageIdInternal;
    396     } else {
    397         GlobalExport *fn_export = &fn->export_list.items[0];
    398         symbol_name = &fn_export->name;
    399         linkage = fn_export->linkage;
    400     }
    401 
    402     bool external_linkage = linkage != GlobalLinkageIdInternal;
    403     CallingConvention cc = fn->type_entry->data.fn.fn_type_id.cc;
    404     if (cc == CallingConventionStdcall && external_linkage &&
    405         g->zig_target->arch == ZigLLVM_x86)
    406     {
    407         // prevent llvm name mangling
    408         symbol_name = buf_sprintf("\x01_%s", buf_ptr(symbol_name));
    409     }
    410 
    411     bool is_async = fn_is_async(fn);
    412 
    413 
    414     ZigType *fn_type = fn->type_entry;
    415     // Make the raw_type_ref populated
    416     resolve_llvm_types_fn(g, fn);
    417     LLVMTypeRef fn_llvm_type = fn->raw_type_ref;
    418     LLVMValueRef llvm_fn = nullptr;
    419     if (fn->body_node == nullptr) {
    420         LLVMValueRef existing_llvm_fn = LLVMGetNamedFunction(g->module, buf_ptr(symbol_name));
    421         if (existing_llvm_fn) {
    422             return LLVMConstBitCast(existing_llvm_fn, LLVMPointerType(fn_llvm_type, 0));
    423         } else {
    424             auto entry = g->exported_symbol_names.maybe_get(symbol_name);
    425             if (entry == nullptr) {
    426                 llvm_fn = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type);
    427 
    428                 if (target_is_wasm(g->zig_target)) {
    429                     assert(fn->proto_node->type == NodeTypeFnProto);
    430                     AstNodeFnProto *fn_proto = &fn->proto_node->data.fn_proto;
    431                     if (fn_proto-> is_extern && fn_proto->lib_name != nullptr ) {
    432                         addLLVMFnAttrStr(llvm_fn, "wasm-import-module", buf_ptr(fn_proto->lib_name));
    433                     }
    434                 }
    435             } else {
    436                 assert(entry->value->id == TldIdFn);
    437                 TldFn *tld_fn = reinterpret_cast<TldFn *>(entry->value);
    438                 // Make the raw_type_ref populated
    439                 resolve_llvm_types_fn(g, tld_fn->fn_entry);
    440                 tld_fn->fn_entry->llvm_value = LLVMAddFunction(g->module, buf_ptr(symbol_name),
    441                         tld_fn->fn_entry->raw_type_ref);
    442                 llvm_fn = LLVMConstBitCast(tld_fn->fn_entry->llvm_value, LLVMPointerType(fn_llvm_type, 0));
    443                 return llvm_fn;
    444             }
    445         }
    446     } else {
    447         if (llvm_fn == nullptr) {
    448             llvm_fn = LLVMAddFunction(g->module, buf_ptr(symbol_name), fn_llvm_type);
    449         }
    450 
    451         for (size_t i = 1; i < fn->export_list.length; i += 1) {
    452             GlobalExport *fn_export = &fn->export_list.items[i];
    453             LLVMAddAlias(g->module, LLVMTypeOf(llvm_fn), llvm_fn, buf_ptr(&fn_export->name));
    454         }
    455     }
    456 
    457     switch (fn->fn_inline) {
    458         case FnInlineAlways:
    459             addLLVMFnAttr(llvm_fn, "alwaysinline");
    460             g->inline_fns.append(fn);
    461             break;
    462         case FnInlineNever:
    463             addLLVMFnAttr(llvm_fn, "noinline");
    464             break;
    465         case FnInlineAuto:
    466             if (fn->alignstack_value != 0) {
    467                 addLLVMFnAttr(llvm_fn, "noinline");
    468             }
    469             break;
    470     }
    471 
    472     if (cc == CallingConventionNaked) {
    473         addLLVMFnAttr(llvm_fn, "naked");
    474     } else {
    475         LLVMSetFunctionCallConv(llvm_fn, get_llvm_cc(g, fn_type->data.fn.fn_type_id.cc));
    476     }
    477 
    478     bool want_cold = fn->is_cold || cc == CallingConventionCold;
    479     if (want_cold) {
    480         ZigLLVMAddFunctionAttrCold(llvm_fn);
    481     }
    482 
    483 
    484     LLVMSetLinkage(llvm_fn, to_llvm_linkage(linkage));
    485 
    486     if (linkage == GlobalLinkageIdInternal) {
    487         LLVMSetUnnamedAddr(llvm_fn, true);
    488     }
    489 
    490     ZigType *return_type = fn_type->data.fn.fn_type_id.return_type;
    491     if (return_type->id == ZigTypeIdUnreachable) {
    492         addLLVMFnAttr(llvm_fn, "noreturn");
    493     }
    494 
    495     if (fn->body_node != nullptr) {
    496         maybe_export_dll(g, llvm_fn, linkage);
    497 
    498         bool want_fn_safety = g->build_mode != BuildModeFastRelease &&
    499                               g->build_mode != BuildModeSmallRelease &&
    500                               !fn->def_scope->safety_off;
    501         if (want_fn_safety) {
    502             if (g->libc_link_lib != nullptr) {
    503                 addLLVMFnAttr(llvm_fn, "sspstrong");
    504                 addLLVMFnAttrStr(llvm_fn, "stack-protector-buffer-size", "4");
    505             }
    506         }
    507         if (g->have_stack_probing && !fn->def_scope->safety_off) {
    508             addLLVMFnAttrStr(llvm_fn, "probe-stack", "__zig_probe_stack");
    509         }
    510     } else {
    511         maybe_import_dll(g, llvm_fn, linkage);
    512     }
    513 
    514     if (fn->alignstack_value != 0) {
    515         addLLVMFnAttrInt(llvm_fn, "alignstack", fn->alignstack_value);
    516     }
    517 
    518     addLLVMFnAttr(llvm_fn, "nounwind");
    519     add_uwtable_attr(g, llvm_fn);
    520     addLLVMFnAttr(llvm_fn, "nobuiltin");
    521     if (codegen_have_frame_pointer(g) && fn->fn_inline != FnInlineAlways) {
    522         ZigLLVMAddFunctionAttr(llvm_fn, "no-frame-pointer-elim", "true");
    523         ZigLLVMAddFunctionAttr(llvm_fn, "no-frame-pointer-elim-non-leaf", nullptr);
    524     }
    525     if (fn->section_name) {
    526         LLVMSetSection(llvm_fn, buf_ptr(fn->section_name));
    527     }
    528     if (fn->align_bytes > 0) {
    529         LLVMSetAlignment(llvm_fn, (unsigned)fn->align_bytes);
    530     } else {
    531         // We'd like to set the best alignment for the function here, but on Darwin LLVM gives
    532         // "Cannot getTypeInfo() on a type that is unsized!" assertion failure when calling
    533         // any of the functions for getting alignment. Not specifying the alignment should
    534         // use the ABI alignment, which is fine.
    535     }
    536 
    537     if (is_async) {
    538         addLLVMArgAttr(llvm_fn, 0, "nonnull");
    539     } else {
    540         unsigned init_gen_i = 0;
    541         if (!type_has_bits(return_type)) {
    542             // nothing to do
    543         } else if (type_is_nonnull_ptr(return_type)) {
    544             addLLVMAttr(llvm_fn, 0, "nonnull");
    545         } else if (want_first_arg_sret(g, &fn_type->data.fn.fn_type_id)) {
    546             // Sret pointers must not be address 0
    547             addLLVMArgAttr(llvm_fn, 0, "nonnull");
    548             addLLVMArgAttr(llvm_fn, 0, "sret");
    549             if (cc_want_sret_attr(cc)) {
    550                 addLLVMArgAttr(llvm_fn, 0, "noalias");
    551             }
    552             init_gen_i = 1;
    553         }
    554 
    555         // set parameter attributes
    556         FnWalk fn_walk = {};
    557         fn_walk.id = FnWalkIdAttrs;
    558         fn_walk.data.attrs.fn = fn;
    559         fn_walk.data.attrs.llvm_fn = llvm_fn;
    560         fn_walk.data.attrs.gen_i = init_gen_i;
    561         walk_function_params(g, fn_type, &fn_walk);
    562 
    563         uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn);
    564         if (err_ret_trace_arg_index != UINT32_MAX) {
    565             // Error return trace memory is in the stack, which is impossible to be at address 0
    566             // on any architecture.
    567             addLLVMArgAttr(llvm_fn, (unsigned)err_ret_trace_arg_index, "nonnull");
    568         }
    569     }
    570 
    571     return llvm_fn;
    572 }
    573 
    574 static LLVMValueRef fn_llvm_value(CodeGen *g, ZigFn *fn) {
    575     if (fn->llvm_value)
    576         return fn->llvm_value;
    577 
    578     fn->llvm_value = make_fn_llvm_value(g, fn);
    579     fn->llvm_name = strdup(LLVMGetValueName(fn->llvm_value));
    580     return fn->llvm_value;
    581 }
    582 
    583 static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) {
    584     if (scope->di_scope)
    585         return scope->di_scope;
    586 
    587     ZigType *import = get_scope_import(scope);
    588     switch (scope->id) {
    589         case ScopeIdCImport:
    590             zig_unreachable();
    591         case ScopeIdFnDef:
    592         {
    593             assert(scope->parent);
    594             ScopeFnDef *fn_scope = (ScopeFnDef *)scope;
    595             ZigFn *fn_table_entry = fn_scope->fn_entry;
    596             if (!fn_table_entry->proto_node)
    597                 return get_di_scope(g, scope->parent);
    598             unsigned line_number = (unsigned)(fn_table_entry->proto_node->line == 0) ?
    599                 0 : (fn_table_entry->proto_node->line + 1);
    600             unsigned scope_line = line_number;
    601             bool is_definition = fn_table_entry->body_node != nullptr;
    602             bool is_optimized = g->build_mode != BuildModeDebug;
    603             bool is_internal_linkage = (fn_table_entry->body_node != nullptr &&
    604                     fn_table_entry->export_list.length == 0);
    605             unsigned flags = ZigLLVM_DIFlags_StaticMember;
    606             ZigLLVMDIScope *fn_di_scope = get_di_scope(g, scope->parent);
    607             assert(fn_di_scope != nullptr);
    608             assert(fn_table_entry->raw_di_type != nullptr);
    609             ZigLLVMDISubprogram *subprogram = ZigLLVMCreateFunction(g->dbuilder,
    610                 fn_di_scope, buf_ptr(&fn_table_entry->symbol_name), "",
    611                 import->data.structure.root_struct->di_file, line_number,
    612                 fn_table_entry->raw_di_type, is_internal_linkage,
    613                 is_definition, scope_line, flags, is_optimized, nullptr);
    614 
    615             scope->di_scope = ZigLLVMSubprogramToScope(subprogram);
    616             if (!g->strip_debug_symbols) {
    617                 ZigLLVMFnSetSubprogram(fn_llvm_value(g, fn_table_entry), subprogram);
    618             }
    619             return scope->di_scope;
    620         }
    621         case ScopeIdDecls:
    622             if (scope->parent) {
    623                 ScopeDecls *decls_scope = (ScopeDecls *)scope;
    624                 assert(decls_scope->container_type);
    625                 scope->di_scope = ZigLLVMTypeToScope(get_llvm_di_type(g, decls_scope->container_type));
    626             } else {
    627                 scope->di_scope = ZigLLVMFileToScope(import->data.structure.root_struct->di_file);
    628             }
    629             return scope->di_scope;
    630         case ScopeIdBlock:
    631         case ScopeIdDefer:
    632         {
    633             assert(scope->parent);
    634             ZigLLVMDILexicalBlock *di_block = ZigLLVMCreateLexicalBlock(g->dbuilder,
    635                 get_di_scope(g, scope->parent),
    636                 import->data.structure.root_struct->di_file,
    637                 (unsigned)scope->source_node->line + 1,
    638                 (unsigned)scope->source_node->column + 1);
    639             scope->di_scope = ZigLLVMLexicalBlockToScope(di_block);
    640             return scope->di_scope;
    641         }
    642         case ScopeIdVarDecl:
    643         case ScopeIdDeferExpr:
    644         case ScopeIdLoop:
    645         case ScopeIdSuspend:
    646         case ScopeIdCompTime:
    647         case ScopeIdRuntime:
    648             return get_di_scope(g, scope->parent);
    649     }
    650     zig_unreachable();
    651 }
    652 
    653 static void clear_debug_source_node(CodeGen *g) {
    654     ZigLLVMClearCurrentDebugLocation(g->builder);
    655 }
    656 
    657 static LLVMValueRef get_arithmetic_overflow_fn(CodeGen *g, ZigType *operand_type,
    658         const char *signed_name, const char *unsigned_name)
    659 {
    660     ZigType *int_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;
    661     char fn_name[64];
    662 
    663     assert(int_type->id == ZigTypeIdInt);
    664     const char *signed_str = int_type->data.integral.is_signed ? signed_name : unsigned_name;
    665 
    666     LLVMTypeRef param_types[] = {
    667         get_llvm_type(g, operand_type),
    668         get_llvm_type(g, operand_type),
    669     };
    670 
    671     if (operand_type->id == ZigTypeIdVector) {
    672         sprintf(fn_name, "llvm.%s.with.overflow.v%" PRIu32 "i%" PRIu32, signed_str,
    673                 operand_type->data.vector.len, int_type->data.integral.bit_count);
    674 
    675         LLVMTypeRef return_elem_types[] = {
    676             get_llvm_type(g, operand_type),
    677             LLVMVectorType(LLVMInt1Type(), operand_type->data.vector.len),
    678         };
    679         LLVMTypeRef return_struct_type = LLVMStructType(return_elem_types, 2, false);
    680         LLVMTypeRef fn_type = LLVMFunctionType(return_struct_type, param_types, 2, false);
    681         LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);
    682         assert(LLVMGetIntrinsicID(fn_val));
    683         return fn_val;
    684     } else {
    685         sprintf(fn_name, "llvm.%s.with.overflow.i%" PRIu32, signed_str, int_type->data.integral.bit_count);
    686 
    687         LLVMTypeRef return_elem_types[] = {
    688             get_llvm_type(g, operand_type),
    689             LLVMInt1Type(),
    690         };
    691         LLVMTypeRef return_struct_type = LLVMStructType(return_elem_types, 2, false);
    692         LLVMTypeRef fn_type = LLVMFunctionType(return_struct_type, param_types, 2, false);
    693         LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);
    694         assert(LLVMGetIntrinsicID(fn_val));
    695         return fn_val;
    696     }
    697 }
    698 
    699 static LLVMValueRef get_int_overflow_fn(CodeGen *g, ZigType *operand_type, AddSubMul add_sub_mul) {
    700     ZigType *int_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;
    701     assert(int_type->id == ZigTypeIdInt);
    702 
    703     ZigLLVMFnKey key = {};
    704     key.id = ZigLLVMFnIdOverflowArithmetic;
    705     key.data.overflow_arithmetic.is_signed = int_type->data.integral.is_signed;
    706     key.data.overflow_arithmetic.add_sub_mul = add_sub_mul;
    707     key.data.overflow_arithmetic.bit_count = (uint32_t)int_type->data.integral.bit_count;
    708     key.data.overflow_arithmetic.vector_len = (operand_type->id == ZigTypeIdVector) ?
    709         operand_type->data.vector.len : 0;
    710 
    711     auto existing_entry = g->llvm_fn_table.maybe_get(key);
    712     if (existing_entry)
    713         return existing_entry->value;
    714 
    715     LLVMValueRef fn_val;
    716     switch (add_sub_mul) {
    717         case AddSubMulAdd:
    718             fn_val = get_arithmetic_overflow_fn(g, operand_type, "sadd", "uadd");
    719             break;
    720         case AddSubMulSub:
    721             fn_val = get_arithmetic_overflow_fn(g, operand_type, "ssub", "usub");
    722             break;
    723         case AddSubMulMul:
    724             fn_val = get_arithmetic_overflow_fn(g, operand_type, "smul", "umul");
    725             break;
    726     }
    727 
    728     g->llvm_fn_table.put(key, fn_val);
    729     return fn_val;
    730 }
    731 
    732 static LLVMValueRef get_float_fn(CodeGen *g, ZigType *type_entry, ZigLLVMFnId fn_id, BuiltinFnId op) {
    733     assert(type_entry->id == ZigTypeIdFloat ||
    734            type_entry->id == ZigTypeIdVector);
    735 
    736     bool is_vector = (type_entry->id == ZigTypeIdVector);
    737     ZigType *float_type = is_vector ? type_entry->data.vector.elem_type : type_entry;
    738 
    739     ZigLLVMFnKey key = {};
    740     key.id = fn_id;
    741     key.data.floating.bit_count = (uint32_t)float_type->data.floating.bit_count;
    742     key.data.floating.vector_len = is_vector ? (uint32_t)type_entry->data.vector.len : 0;
    743     key.data.floating.op = op;
    744 
    745     auto existing_entry = g->llvm_fn_table.maybe_get(key);
    746     if (existing_entry)
    747         return existing_entry->value;
    748 
    749     const char *name;
    750     uint32_t num_args;
    751     if (fn_id == ZigLLVMFnIdFMA) {
    752         name = "fma";
    753         num_args = 3;
    754     } else if (fn_id == ZigLLVMFnIdFloatOp) {
    755         name = float_op_to_name(op, true);
    756         num_args = 1;
    757     } else {
    758         zig_unreachable();
    759     }
    760 
    761     char fn_name[64];
    762     if (is_vector)
    763         sprintf(fn_name, "llvm.%s.v%" PRIu32 "f%" PRIu32, name, key.data.floating.vector_len, key.data.floating.bit_count);
    764     else
    765         sprintf(fn_name, "llvm.%s.f%" PRIu32, name, key.data.floating.bit_count);
    766     LLVMTypeRef float_type_ref = get_llvm_type(g, type_entry);
    767     LLVMTypeRef return_elem_types[3] = {
    768         float_type_ref,
    769         float_type_ref,
    770         float_type_ref,
    771     };
    772     LLVMTypeRef fn_type = LLVMFunctionType(float_type_ref, return_elem_types, num_args, false);
    773     LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);
    774     assert(LLVMGetIntrinsicID(fn_val));
    775 
    776     g->llvm_fn_table.put(key, fn_val);
    777     return fn_val;
    778 }
    779 
    780 static LLVMValueRef gen_store_untyped(CodeGen *g, LLVMValueRef value, LLVMValueRef ptr,
    781         uint32_t alignment, bool is_volatile)
    782 {
    783     LLVMValueRef instruction = LLVMBuildStore(g->builder, value, ptr);
    784     if (is_volatile) LLVMSetVolatile(instruction, true);
    785     if (alignment != 0) {
    786         LLVMSetAlignment(instruction, alignment);
    787     }
    788     return instruction;
    789 }
    790 
    791 static LLVMValueRef gen_store(CodeGen *g, LLVMValueRef value, LLVMValueRef ptr, ZigType *ptr_type) {
    792     assert(ptr_type->id == ZigTypeIdPointer);
    793     uint32_t alignment = get_ptr_align(g, ptr_type);
    794     return gen_store_untyped(g, value, ptr, alignment, ptr_type->data.pointer.is_volatile);
    795 }
    796 
    797 static LLVMValueRef gen_load_untyped(CodeGen *g, LLVMValueRef ptr, uint32_t alignment, bool is_volatile,
    798         const char *name)
    799 {
    800     LLVMValueRef result = LLVMBuildLoad(g->builder, ptr, name);
    801     if (is_volatile) LLVMSetVolatile(result, true);
    802     if (alignment == 0) {
    803         LLVMSetAlignment(result, LLVMABIAlignmentOfType(g->target_data_ref, LLVMGetElementType(LLVMTypeOf(ptr))));
    804     } else {
    805         LLVMSetAlignment(result, alignment);
    806     }
    807     return result;
    808 }
    809 
    810 static LLVMValueRef gen_load(CodeGen *g, LLVMValueRef ptr, ZigType *ptr_type, const char *name) {
    811     assert(ptr_type->id == ZigTypeIdPointer);
    812     uint32_t alignment = get_ptr_align(g, ptr_type);
    813     return gen_load_untyped(g, ptr, alignment, ptr_type->data.pointer.is_volatile, name);
    814 }
    815 
    816 static LLVMValueRef get_handle_value(CodeGen *g, LLVMValueRef ptr, ZigType *type, ZigType *ptr_type) {
    817     if (type_has_bits(type)) {
    818         if (handle_is_ptr(type)) {
    819             return ptr;
    820         } else {
    821             assert(ptr_type->id == ZigTypeIdPointer);
    822             return gen_load(g, ptr, ptr_type, "");
    823         }
    824     } else {
    825         return nullptr;
    826     }
    827 }
    828 
    829 static bool ir_want_fast_math(CodeGen *g, IrInstruction *instruction) {
    830     // TODO memoize
    831     Scope *scope = instruction->scope;
    832     while (scope) {
    833         if (scope->id == ScopeIdBlock) {
    834             ScopeBlock *block_scope = (ScopeBlock *)scope;
    835             if (block_scope->fast_math_set_node)
    836                 return block_scope->fast_math_on;
    837         } else if (scope->id == ScopeIdDecls) {
    838             ScopeDecls *decls_scope = (ScopeDecls *)scope;
    839             if (decls_scope->fast_math_set_node)
    840                 return decls_scope->fast_math_on;
    841         }
    842         scope = scope->parent;
    843     }
    844     return false;
    845 }
    846 
    847 static bool ir_want_runtime_safety_scope(CodeGen *g, Scope *scope) {
    848     // TODO memoize
    849     while (scope) {
    850         if (scope->id == ScopeIdBlock) {
    851             ScopeBlock *block_scope = (ScopeBlock *)scope;
    852             if (block_scope->safety_set_node)
    853                 return !block_scope->safety_off;
    854         } else if (scope->id == ScopeIdDecls) {
    855             ScopeDecls *decls_scope = (ScopeDecls *)scope;
    856             if (decls_scope->safety_set_node)
    857                 return !decls_scope->safety_off;
    858         }
    859         scope = scope->parent;
    860     }
    861 
    862     return (g->build_mode != BuildModeFastRelease &&
    863             g->build_mode != BuildModeSmallRelease);
    864 }
    865 
    866 static bool ir_want_runtime_safety(CodeGen *g, IrInstruction *instruction) {
    867     return ir_want_runtime_safety_scope(g, instruction->scope);
    868 }
    869 
    870 static Buf *panic_msg_buf(PanicMsgId msg_id) {
    871     switch (msg_id) {
    872         case PanicMsgIdCount:
    873             zig_unreachable();
    874         case PanicMsgIdBoundsCheckFailure:
    875             return buf_create_from_str("index out of bounds");
    876         case PanicMsgIdCastNegativeToUnsigned:
    877             return buf_create_from_str("attempt to cast negative value to unsigned integer");
    878         case PanicMsgIdCastTruncatedData:
    879             return buf_create_from_str("integer cast truncated bits");
    880         case PanicMsgIdIntegerOverflow:
    881             return buf_create_from_str("integer overflow");
    882         case PanicMsgIdShlOverflowedBits:
    883             return buf_create_from_str("left shift overflowed bits");
    884         case PanicMsgIdShrOverflowedBits:
    885             return buf_create_from_str("right shift overflowed bits");
    886         case PanicMsgIdDivisionByZero:
    887             return buf_create_from_str("division by zero");
    888         case PanicMsgIdRemainderDivisionByZero:
    889             return buf_create_from_str("remainder division by zero or negative value");
    890         case PanicMsgIdExactDivisionRemainder:
    891             return buf_create_from_str("exact division produced remainder");
    892         case PanicMsgIdSliceWidenRemainder:
    893             return buf_create_from_str("slice widening size mismatch");
    894         case PanicMsgIdUnwrapOptionalFail:
    895             return buf_create_from_str("attempt to unwrap null");
    896         case PanicMsgIdUnreachable:
    897             return buf_create_from_str("reached unreachable code");
    898         case PanicMsgIdInvalidErrorCode:
    899             return buf_create_from_str("invalid error code");
    900         case PanicMsgIdIncorrectAlignment:
    901             return buf_create_from_str("incorrect alignment");
    902         case PanicMsgIdBadUnionField:
    903             return buf_create_from_str("access of inactive union field");
    904         case PanicMsgIdBadEnumValue:
    905             return buf_create_from_str("invalid enum value");
    906         case PanicMsgIdFloatToInt:
    907             return buf_create_from_str("integer part of floating point value out of bounds");
    908         case PanicMsgIdPtrCastNull:
    909             return buf_create_from_str("cast causes pointer to be null");
    910         case PanicMsgIdBadResume:
    911             return buf_create_from_str("resumed an async function which already returned");
    912         case PanicMsgIdBadAwait:
    913             return buf_create_from_str("async function awaited twice");
    914         case PanicMsgIdBadReturn:
    915             return buf_create_from_str("async function returned twice");
    916         case PanicMsgIdResumedAnAwaitingFn:
    917             return buf_create_from_str("awaiting function resumed");
    918         case PanicMsgIdFrameTooSmall:
    919             return buf_create_from_str("frame too small");
    920         case PanicMsgIdResumedFnPendingAwait:
    921             return buf_create_from_str("resumed an async function which can only be awaited");
    922     }
    923     zig_unreachable();
    924 }
    925 
    926 static LLVMValueRef get_panic_msg_ptr_val(CodeGen *g, PanicMsgId msg_id) {
    927     ConstExprValue *val = &g->panic_msg_vals[msg_id];
    928     if (!val->global_refs->llvm_global) {
    929 
    930         Buf *buf_msg = panic_msg_buf(msg_id);
    931         ConstExprValue *array_val = create_const_str_lit(g, buf_msg);
    932         init_const_slice(g, val, array_val, 0, buf_len(buf_msg), true);
    933 
    934         render_const_val(g, val, "");
    935         render_const_val_global(g, val, "");
    936 
    937         assert(val->global_refs->llvm_global);
    938     }
    939 
    940     ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, true, false,
    941             PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false);
    942     ZigType *str_type = get_slice_type(g, u8_ptr_type);
    943     return LLVMConstBitCast(val->global_refs->llvm_global, LLVMPointerType(get_llvm_type(g, str_type), 0));
    944 }
    945 
    946 static ZigType *ptr_to_stack_trace_type(CodeGen *g) {
    947     return get_pointer_to_type(g, get_stack_trace_type(g), false);
    948 }
    949 
    950 static void gen_panic(CodeGen *g, LLVMValueRef msg_arg, LLVMValueRef stack_trace_arg) {
    951     assert(g->panic_fn != nullptr);
    952     LLVMValueRef fn_val = fn_llvm_value(g, g->panic_fn);
    953     LLVMCallConv llvm_cc = get_llvm_cc(g, g->panic_fn->type_entry->data.fn.fn_type_id.cc);
    954     if (stack_trace_arg == nullptr) {
    955         stack_trace_arg = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
    956     }
    957     LLVMValueRef args[] = {
    958         msg_arg,
    959         stack_trace_arg,
    960     };
    961     LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, fn_val, args, 2, llvm_cc, ZigLLVM_FnInlineAuto, "");
    962     LLVMSetTailCall(call_instruction, true);
    963     LLVMBuildUnreachable(g->builder);
    964 }
    965 
    966 // TODO update most callsites to call gen_assertion instead of this
    967 static void gen_safety_crash(CodeGen *g, PanicMsgId msg_id) {
    968     gen_panic(g, get_panic_msg_ptr_val(g, msg_id), nullptr);
    969 }
    970 
    971 static void gen_assertion_scope(CodeGen *g, PanicMsgId msg_id, Scope *source_scope) {
    972     if (ir_want_runtime_safety_scope(g, source_scope)) {
    973         gen_safety_crash(g, msg_id);
    974     } else {
    975         LLVMBuildUnreachable(g->builder);
    976     }
    977 }
    978 
    979 static void gen_assertion(CodeGen *g, PanicMsgId msg_id, IrInstruction *source_instruction) {
    980     return gen_assertion_scope(g, msg_id, source_instruction->scope);
    981 }
    982 
    983 static LLVMValueRef get_stacksave_fn_val(CodeGen *g) {
    984     if (g->stacksave_fn_val)
    985         return g->stacksave_fn_val;
    986 
    987     // declare i8* @llvm.stacksave()
    988 
    989     LLVMTypeRef fn_type = LLVMFunctionType(LLVMPointerType(LLVMInt8Type(), 0), nullptr, 0, false);
    990     g->stacksave_fn_val = LLVMAddFunction(g->module, "llvm.stacksave", fn_type);
    991     assert(LLVMGetIntrinsicID(g->stacksave_fn_val));
    992 
    993     return g->stacksave_fn_val;
    994 }
    995 
    996 static LLVMValueRef get_stackrestore_fn_val(CodeGen *g) {
    997     if (g->stackrestore_fn_val)
    998         return g->stackrestore_fn_val;
    999 
   1000     // declare void @llvm.stackrestore(i8* %ptr)
   1001 
   1002     LLVMTypeRef param_type = LLVMPointerType(LLVMInt8Type(), 0);
   1003     LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), &param_type, 1, false);
   1004     g->stackrestore_fn_val = LLVMAddFunction(g->module, "llvm.stackrestore", fn_type);
   1005     assert(LLVMGetIntrinsicID(g->stackrestore_fn_val));
   1006 
   1007     return g->stackrestore_fn_val;
   1008 }
   1009 
   1010 static LLVMValueRef get_write_register_fn_val(CodeGen *g) {
   1011     if (g->write_register_fn_val)
   1012         return g->write_register_fn_val;
   1013 
   1014     // declare void @llvm.write_register.i64(metadata, i64 @value)
   1015     // !0 = !{!"sp\00"}
   1016 
   1017     LLVMTypeRef param_types[] = {
   1018         LLVMMetadataTypeInContext(LLVMGetGlobalContext()),
   1019         LLVMIntType(g->pointer_size_bytes * 8),
   1020     };
   1021 
   1022     LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 2, false);
   1023     Buf *name = buf_sprintf("llvm.write_register.i%d", g->pointer_size_bytes * 8);
   1024     g->write_register_fn_val = LLVMAddFunction(g->module, buf_ptr(name), fn_type);
   1025     assert(LLVMGetIntrinsicID(g->write_register_fn_val));
   1026 
   1027     return g->write_register_fn_val;
   1028 }
   1029 
   1030 static LLVMValueRef get_return_address_fn_val(CodeGen *g) {
   1031     if (g->return_address_fn_val)
   1032         return g->return_address_fn_val;
   1033 
   1034     ZigType *return_type = get_pointer_to_type(g, g->builtin_types.entry_u8, true);
   1035 
   1036     LLVMTypeRef fn_type = LLVMFunctionType(get_llvm_type(g, return_type),
   1037             &g->builtin_types.entry_i32->llvm_type, 1, false);
   1038     g->return_address_fn_val = LLVMAddFunction(g->module, "llvm.returnaddress", fn_type);
   1039     assert(LLVMGetIntrinsicID(g->return_address_fn_val));
   1040 
   1041     return g->return_address_fn_val;
   1042 }
   1043 
   1044 static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) {
   1045     if (g->add_error_return_trace_addr_fn_val != nullptr)
   1046         return g->add_error_return_trace_addr_fn_val;
   1047 
   1048     LLVMTypeRef arg_types[] = {
   1049         get_llvm_type(g, ptr_to_stack_trace_type(g)),
   1050         g->builtin_types.entry_usize->llvm_type,
   1051     };
   1052     LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false);
   1053 
   1054     Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_add_err_ret_trace_addr"), false);
   1055     LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
   1056     addLLVMFnAttr(fn_val, "alwaysinline");
   1057     LLVMSetLinkage(fn_val, LLVMInternalLinkage);
   1058     LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
   1059     addLLVMFnAttr(fn_val, "nounwind");
   1060     add_uwtable_attr(g, fn_val);
   1061     // Error return trace memory is in the stack, which is impossible to be at address 0
   1062     // on any architecture.
   1063     addLLVMArgAttr(fn_val, (unsigned)0, "nonnull");
   1064     if (codegen_have_frame_pointer(g)) {
   1065         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
   1066         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
   1067     }
   1068 
   1069     LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
   1070     LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
   1071     LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
   1072     LLVMPositionBuilderAtEnd(g->builder, entry_block);
   1073     ZigLLVMClearCurrentDebugLocation(g->builder);
   1074 
   1075     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   1076 
   1077     // stack_trace.instruction_addresses[stack_trace.index & (stack_trace.instruction_addresses.len - 1)] = return_address;
   1078 
   1079     LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0);
   1080     LLVMValueRef address_value = LLVMGetParam(fn_val, 1);
   1081 
   1082     size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
   1083     LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)index_field_index, "");
   1084     size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
   1085     LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)addresses_field_index, "");
   1086 
   1087     ZigType *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry;
   1088     size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
   1089     LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, "");
   1090     size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index;
   1091     LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, "");
   1092 
   1093     LLVMValueRef len_value = gen_load_untyped(g, len_field_ptr, 0, false, "");
   1094     LLVMValueRef index_val = gen_load_untyped(g, index_field_ptr, 0, false, "");
   1095     LLVMValueRef len_val_minus_one = LLVMBuildSub(g->builder, len_value, LLVMConstInt(usize_type_ref, 1, false), "");
   1096     LLVMValueRef masked_val = LLVMBuildAnd(g->builder, index_val, len_val_minus_one, "");
   1097     LLVMValueRef address_indices[] = {
   1098         masked_val,
   1099     };
   1100 
   1101     LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, "");
   1102     LLVMValueRef address_slot = LLVMBuildInBoundsGEP(g->builder, ptr_value, address_indices, 1, "");
   1103 
   1104     gen_store_untyped(g, address_value, address_slot, 0, false);
   1105 
   1106     // stack_trace.index += 1;
   1107     LLVMValueRef index_plus_one_val = LLVMBuildNUWAdd(g->builder, index_val, LLVMConstInt(usize_type_ref, 1, false), "");
   1108     gen_store_untyped(g, index_plus_one_val, index_field_ptr, 0, false);
   1109 
   1110     // return;
   1111     LLVMBuildRetVoid(g->builder);
   1112 
   1113     LLVMPositionBuilderAtEnd(g->builder, prev_block);
   1114     if (!g->strip_debug_symbols) {
   1115         LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
   1116     }
   1117 
   1118     g->add_error_return_trace_addr_fn_val = fn_val;
   1119     return fn_val;
   1120 }
   1121 
   1122 static LLVMValueRef get_return_err_fn(CodeGen *g) {
   1123     if (g->return_err_fn != nullptr)
   1124         return g->return_err_fn;
   1125 
   1126     assert(g->err_tag_type != nullptr);
   1127 
   1128     LLVMTypeRef arg_types[] = {
   1129         // error return trace pointer
   1130         get_llvm_type(g, ptr_to_stack_trace_type(g)),
   1131     };
   1132     LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 1, false);
   1133 
   1134     Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_return_error"), false);
   1135     LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
   1136     addLLVMFnAttr(fn_val, "noinline"); // so that we can look at return address
   1137     addLLVMFnAttr(fn_val, "cold");
   1138     LLVMSetLinkage(fn_val, LLVMInternalLinkage);
   1139     LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
   1140     addLLVMFnAttr(fn_val, "nounwind");
   1141     add_uwtable_attr(g, fn_val);
   1142     if (codegen_have_frame_pointer(g)) {
   1143         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
   1144         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
   1145     }
   1146 
   1147     // this is above the ZigLLVMClearCurrentDebugLocation
   1148     LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g);
   1149 
   1150     LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
   1151     LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
   1152     LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
   1153     LLVMPositionBuilderAtEnd(g->builder, entry_block);
   1154     ZigLLVMClearCurrentDebugLocation(g->builder);
   1155 
   1156     LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0);
   1157 
   1158     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   1159     LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, g->builtin_types.entry_i32));
   1160     LLVMValueRef return_address_ptr = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, "");
   1161     LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, "");
   1162 
   1163     LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return");
   1164     LLVMBasicBlockRef dest_non_null_block = LLVMAppendBasicBlock(fn_val, "DestNonNull");
   1165 
   1166     LLVMValueRef null_dest_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, err_ret_trace_ptr,
   1167             LLVMConstNull(LLVMTypeOf(err_ret_trace_ptr)), "");
   1168     LLVMBuildCondBr(g->builder, null_dest_bit, return_block, dest_non_null_block);
   1169 
   1170     LLVMPositionBuilderAtEnd(g->builder, return_block);
   1171     LLVMBuildRetVoid(g->builder);
   1172 
   1173     LLVMPositionBuilderAtEnd(g->builder, dest_non_null_block);
   1174     LLVMValueRef args[] = { err_ret_trace_ptr, return_address };
   1175     ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, "");
   1176     LLVMBuildRetVoid(g->builder);
   1177 
   1178     LLVMPositionBuilderAtEnd(g->builder, prev_block);
   1179     if (!g->strip_debug_symbols) {
   1180         LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
   1181     }
   1182 
   1183     g->return_err_fn = fn_val;
   1184     return fn_val;
   1185 }
   1186 
   1187 static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) {
   1188     if (g->safety_crash_err_fn != nullptr)
   1189         return g->safety_crash_err_fn;
   1190 
   1191     static const char *unwrap_err_msg_text = "attempt to unwrap error: ";
   1192 
   1193     g->generate_error_name_table = true;
   1194     generate_error_name_table(g);
   1195     assert(g->err_name_table != nullptr);
   1196 
   1197     // Generate the constant part of the error message
   1198     LLVMValueRef msg_prefix_init = LLVMConstString(unwrap_err_msg_text, strlen(unwrap_err_msg_text), 1);
   1199     LLVMValueRef msg_prefix = LLVMAddGlobal(g->module, LLVMTypeOf(msg_prefix_init), "");
   1200     LLVMSetInitializer(msg_prefix, msg_prefix_init);
   1201     LLVMSetLinkage(msg_prefix, LLVMInternalLinkage);
   1202     LLVMSetGlobalConstant(msg_prefix, true);
   1203 
   1204     Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_fail_unwrap"), false);
   1205     LLVMTypeRef fn_type_ref;
   1206     if (g->have_err_ret_tracing) {
   1207         LLVMTypeRef arg_types[] = {
   1208             get_llvm_type(g, get_pointer_to_type(g, get_stack_trace_type(g), false)),
   1209             get_llvm_type(g, g->err_tag_type),
   1210         };
   1211         fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false);
   1212     } else {
   1213         LLVMTypeRef arg_types[] = {
   1214             get_llvm_type(g, g->err_tag_type),
   1215         };
   1216         fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 1, false);
   1217     }
   1218     LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
   1219     addLLVMFnAttr(fn_val, "noreturn");
   1220     addLLVMFnAttr(fn_val, "cold");
   1221     LLVMSetLinkage(fn_val, LLVMInternalLinkage);
   1222     LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
   1223     addLLVMFnAttr(fn_val, "nounwind");
   1224     add_uwtable_attr(g, fn_val);
   1225     if (codegen_have_frame_pointer(g)) {
   1226         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
   1227         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
   1228     }
   1229     // Not setting alignment here. See the comment above about
   1230     // "Cannot getTypeInfo() on a type that is unsized!"
   1231     // assertion failure on Darwin.
   1232 
   1233     LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
   1234     LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
   1235     LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
   1236     LLVMPositionBuilderAtEnd(g->builder, entry_block);
   1237     ZigLLVMClearCurrentDebugLocation(g->builder);
   1238 
   1239     ZigType *usize_ty = g->builtin_types.entry_usize;
   1240     ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, true, false,
   1241             PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false);
   1242     ZigType *str_type = get_slice_type(g, u8_ptr_type);
   1243 
   1244     // Allocate a buffer to hold the fully-formatted error message
   1245     const size_t err_buf_len = strlen(unwrap_err_msg_text) + g->largest_err_name_len;
   1246     LLVMValueRef max_msg_len = LLVMConstInt(usize_ty->llvm_type, err_buf_len, 0);
   1247     LLVMValueRef msg_buffer = LLVMBuildArrayAlloca(g->builder, LLVMInt8Type(), max_msg_len, "msg_buffer");
   1248 
   1249     // Allocate a []u8 slice for the message
   1250     LLVMValueRef msg_slice = build_alloca(g, str_type, "msg_slice", 0);
   1251 
   1252     LLVMValueRef err_ret_trace_arg;
   1253     LLVMValueRef err_val;
   1254     if (g->have_err_ret_tracing) {
   1255         err_ret_trace_arg = LLVMGetParam(fn_val, 0);
   1256         err_val = LLVMGetParam(fn_val, 1);
   1257     } else {
   1258         err_ret_trace_arg = nullptr;
   1259         err_val = LLVMGetParam(fn_val, 0);
   1260     }
   1261 
   1262     // Fetch the error name from the global table
   1263     LLVMValueRef err_table_indices[] = {
   1264         LLVMConstNull(usize_ty->llvm_type),
   1265         err_val,
   1266     };
   1267     LLVMValueRef err_name_val = LLVMBuildInBoundsGEP(g->builder, g->err_name_table, err_table_indices, 2, "");
   1268 
   1269     LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, err_name_val, slice_ptr_index, "");
   1270     LLVMValueRef err_name_ptr = gen_load_untyped(g, ptr_field_ptr, 0, false, "");
   1271 
   1272     LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, err_name_val, slice_len_index, "");
   1273     LLVMValueRef err_name_len = gen_load_untyped(g, len_field_ptr, 0, false, "");
   1274 
   1275     LLVMValueRef msg_prefix_len = LLVMConstInt(usize_ty->llvm_type, strlen(unwrap_err_msg_text), false);
   1276     // Points to the beginning of msg_buffer
   1277     LLVMValueRef msg_buffer_ptr_indices[] = {
   1278         LLVMConstNull(usize_ty->llvm_type),
   1279     };
   1280     LLVMValueRef msg_buffer_ptr = LLVMBuildInBoundsGEP(g->builder, msg_buffer, msg_buffer_ptr_indices, 1, "");
   1281     // Points to the beginning of the constant prefix message
   1282     LLVMValueRef msg_prefix_ptr_indices[] = {
   1283         LLVMConstNull(usize_ty->llvm_type),
   1284     };
   1285     LLVMValueRef msg_prefix_ptr = LLVMConstInBoundsGEP(msg_prefix, msg_prefix_ptr_indices, 1);
   1286 
   1287     // Build the message using the prefix...
   1288     ZigLLVMBuildMemCpy(g->builder, msg_buffer_ptr, 1, msg_prefix_ptr, 1, msg_prefix_len, false);
   1289     // ..and append the error name
   1290     LLVMValueRef msg_buffer_ptr_after_indices[] = {
   1291         msg_prefix_len,
   1292     };
   1293     LLVMValueRef msg_buffer_ptr_after = LLVMBuildInBoundsGEP(g->builder, msg_buffer, msg_buffer_ptr_after_indices, 1, "");
   1294     ZigLLVMBuildMemCpy(g->builder, msg_buffer_ptr_after, 1, err_name_ptr, 1, err_name_len, false);
   1295 
   1296     // Set the slice pointer
   1297     LLVMValueRef msg_slice_ptr_field_ptr = LLVMBuildStructGEP(g->builder, msg_slice, slice_ptr_index, "");
   1298     gen_store_untyped(g, msg_buffer_ptr, msg_slice_ptr_field_ptr, 0, false);
   1299 
   1300     // Set the slice length
   1301     LLVMValueRef slice_len = LLVMBuildNUWAdd(g->builder, msg_prefix_len, err_name_len, "");
   1302     LLVMValueRef msg_slice_len_field_ptr = LLVMBuildStructGEP(g->builder, msg_slice, slice_len_index, "");
   1303     gen_store_untyped(g, slice_len, msg_slice_len_field_ptr, 0, false);
   1304 
   1305     // Call panic()
   1306     gen_panic(g, msg_slice, err_ret_trace_arg);
   1307 
   1308     LLVMPositionBuilderAtEnd(g->builder, prev_block);
   1309     if (!g->strip_debug_symbols) {
   1310         LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
   1311     }
   1312 
   1313     g->safety_crash_err_fn = fn_val;
   1314     return fn_val;
   1315 }
   1316 
   1317 static LLVMValueRef get_cur_err_ret_trace_val(CodeGen *g, Scope *scope) {
   1318     if (!g->have_err_ret_tracing) {
   1319         return nullptr;
   1320     }
   1321     if (g->cur_err_ret_trace_val_stack != nullptr) {
   1322         return g->cur_err_ret_trace_val_stack;
   1323     }
   1324     return g->cur_err_ret_trace_val_arg;
   1325 }
   1326 
   1327 static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val, Scope *scope) {
   1328     LLVMValueRef safety_crash_err_fn = get_safety_crash_err_fn(g);
   1329     LLVMValueRef call_instruction;
   1330     if (g->have_err_ret_tracing) {
   1331         LLVMValueRef err_ret_trace_val = get_cur_err_ret_trace_val(g, scope);
   1332         if (err_ret_trace_val == nullptr) {
   1333             err_ret_trace_val = LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
   1334         }
   1335         LLVMValueRef args[] = {
   1336             err_ret_trace_val,
   1337             err_val,
   1338         };
   1339         call_instruction = ZigLLVMBuildCall(g->builder, safety_crash_err_fn, args, 2,
   1340                 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
   1341     } else {
   1342         LLVMValueRef args[] = {
   1343             err_val,
   1344         };
   1345         call_instruction = ZigLLVMBuildCall(g->builder, safety_crash_err_fn, args, 1,
   1346                 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
   1347     }
   1348     LLVMSetTailCall(call_instruction, true);
   1349     LLVMBuildUnreachable(g->builder);
   1350 }
   1351 
   1352 static void add_bounds_check(CodeGen *g, LLVMValueRef target_val,
   1353         LLVMIntPredicate lower_pred, LLVMValueRef lower_value,
   1354         LLVMIntPredicate upper_pred, LLVMValueRef upper_value)
   1355 {
   1356     if (!lower_value && !upper_value) {
   1357         return;
   1358     }
   1359     if (upper_value && !lower_value) {
   1360         lower_value = upper_value;
   1361         lower_pred = upper_pred;
   1362         upper_value = nullptr;
   1363     }
   1364 
   1365     LLVMBasicBlockRef bounds_check_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "BoundsCheckFail");
   1366     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "BoundsCheckOk");
   1367     LLVMBasicBlockRef lower_ok_block = upper_value ?
   1368         LLVMAppendBasicBlock(g->cur_fn_val, "FirstBoundsCheckOk") : ok_block;
   1369 
   1370     LLVMValueRef lower_ok_val = LLVMBuildICmp(g->builder, lower_pred, target_val, lower_value, "");
   1371     LLVMBuildCondBr(g->builder, lower_ok_val, lower_ok_block, bounds_check_fail_block);
   1372 
   1373     LLVMPositionBuilderAtEnd(g->builder, bounds_check_fail_block);
   1374     gen_safety_crash(g, PanicMsgIdBoundsCheckFailure);
   1375 
   1376     if (upper_value) {
   1377         LLVMPositionBuilderAtEnd(g->builder, lower_ok_block);
   1378         LLVMValueRef upper_ok_val = LLVMBuildICmp(g->builder, upper_pred, target_val, upper_value, "");
   1379         LLVMBuildCondBr(g->builder, upper_ok_val, ok_block, bounds_check_fail_block);
   1380     }
   1381 
   1382     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   1383 }
   1384 
   1385 static LLVMValueRef gen_assert_zero(CodeGen *g, LLVMValueRef expr_val, ZigType *int_type) {
   1386     LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, int_type));
   1387     LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, expr_val, zero, "");
   1388     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "CastShortenOk");
   1389     LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "CastShortenFail");
   1390     LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   1391 
   1392     LLVMPositionBuilderAtEnd(g->builder, fail_block);
   1393     gen_safety_crash(g, PanicMsgIdCastTruncatedData);
   1394 
   1395     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   1396     return nullptr;
   1397 }
   1398 
   1399 static LLVMValueRef gen_widen_or_shorten(CodeGen *g, bool want_runtime_safety, ZigType *actual_type,
   1400         ZigType *wanted_type, LLVMValueRef expr_val)
   1401 {
   1402     assert(actual_type->id == wanted_type->id);
   1403     assert(expr_val != nullptr);
   1404 
   1405     uint64_t actual_bits;
   1406     uint64_t wanted_bits;
   1407     if (actual_type->id == ZigTypeIdFloat) {
   1408         actual_bits = actual_type->data.floating.bit_count;
   1409         wanted_bits = wanted_type->data.floating.bit_count;
   1410     } else if (actual_type->id == ZigTypeIdInt) {
   1411         actual_bits = actual_type->data.integral.bit_count;
   1412         wanted_bits = wanted_type->data.integral.bit_count;
   1413     } else {
   1414         zig_unreachable();
   1415     }
   1416 
   1417     if (actual_type->id == ZigTypeIdInt &&
   1418         !wanted_type->data.integral.is_signed && actual_type->data.integral.is_signed &&
   1419         want_runtime_safety)
   1420     {
   1421         LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, actual_type));
   1422         LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntSGE, expr_val, zero, "");
   1423 
   1424         LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "SignCastOk");
   1425         LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "SignCastFail");
   1426         LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   1427 
   1428         LLVMPositionBuilderAtEnd(g->builder, fail_block);
   1429         gen_safety_crash(g, PanicMsgIdCastNegativeToUnsigned);
   1430 
   1431         LLVMPositionBuilderAtEnd(g->builder, ok_block);
   1432     }
   1433 
   1434     if (actual_bits == wanted_bits) {
   1435         return expr_val;
   1436     } else if (actual_bits < wanted_bits) {
   1437         if (actual_type->id == ZigTypeIdFloat) {
   1438             return LLVMBuildFPExt(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   1439         } else if (actual_type->id == ZigTypeIdInt) {
   1440             if (actual_type->data.integral.is_signed) {
   1441                 return LLVMBuildSExt(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   1442             } else {
   1443                 return LLVMBuildZExt(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   1444             }
   1445         } else {
   1446             zig_unreachable();
   1447         }
   1448     } else if (actual_bits > wanted_bits) {
   1449         if (actual_type->id == ZigTypeIdFloat) {
   1450             return LLVMBuildFPTrunc(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   1451         } else if (actual_type->id == ZigTypeIdInt) {
   1452             if (wanted_bits == 0) {
   1453                 if (!want_runtime_safety)
   1454                     return nullptr;
   1455 
   1456                 return gen_assert_zero(g, expr_val, actual_type);
   1457             }
   1458             LLVMValueRef trunc_val = LLVMBuildTrunc(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   1459             if (!want_runtime_safety) {
   1460                 return trunc_val;
   1461             }
   1462             LLVMValueRef orig_val;
   1463             if (wanted_type->data.integral.is_signed) {
   1464                 orig_val = LLVMBuildSExt(g->builder, trunc_val, get_llvm_type(g, actual_type), "");
   1465             } else {
   1466                 orig_val = LLVMBuildZExt(g->builder, trunc_val, get_llvm_type(g, actual_type), "");
   1467             }
   1468             LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, expr_val, orig_val, "");
   1469             LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "CastShortenOk");
   1470             LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "CastShortenFail");
   1471             LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   1472 
   1473             LLVMPositionBuilderAtEnd(g->builder, fail_block);
   1474             gen_safety_crash(g, PanicMsgIdCastTruncatedData);
   1475 
   1476             LLVMPositionBuilderAtEnd(g->builder, ok_block);
   1477             return trunc_val;
   1478         } else {
   1479             zig_unreachable();
   1480         }
   1481     } else {
   1482         zig_unreachable();
   1483     }
   1484 }
   1485 
   1486 typedef LLVMValueRef (*BuildBinOpFunc)(LLVMBuilderRef, LLVMValueRef, LLVMValueRef, const char *);
   1487 // These are lookup table using the AddSubMul enum as the lookup.
   1488 // If AddSubMul ever changes, then these tables will be out of
   1489 // date.
   1490 static const BuildBinOpFunc float_op[3] = { LLVMBuildFAdd, LLVMBuildFSub, LLVMBuildFMul };
   1491 static const BuildBinOpFunc wrap_op[3] = { LLVMBuildAdd, LLVMBuildSub, LLVMBuildMul };
   1492 static const BuildBinOpFunc signed_op[3] = { LLVMBuildNSWAdd, LLVMBuildNSWSub, LLVMBuildNSWMul };
   1493 static const BuildBinOpFunc unsigned_op[3] = { LLVMBuildNUWAdd, LLVMBuildNUWSub, LLVMBuildNUWMul };
   1494 
   1495 static LLVMValueRef gen_overflow_op(CodeGen *g, ZigType *operand_type, AddSubMul op,
   1496         LLVMValueRef val1, LLVMValueRef val2)
   1497 {
   1498     LLVMValueRef overflow_bit;
   1499     LLVMValueRef result;
   1500 
   1501     if (operand_type->id == ZigTypeIdVector) {
   1502         ZigType *int_type = operand_type->data.vector.elem_type;
   1503         assert(int_type->id == ZigTypeIdInt);
   1504         LLVMTypeRef one_more_bit_int = LLVMIntType(int_type->data.integral.bit_count + 1);
   1505         LLVMTypeRef one_more_bit_int_vector = LLVMVectorType(one_more_bit_int, operand_type->data.vector.len);
   1506         const auto buildExtFn = int_type->data.integral.is_signed ? LLVMBuildSExt : LLVMBuildZExt;
   1507         LLVMValueRef extended1 = buildExtFn(g->builder, val1, one_more_bit_int_vector, "");
   1508         LLVMValueRef extended2 = buildExtFn(g->builder, val2, one_more_bit_int_vector, "");
   1509         LLVMValueRef extended_result = wrap_op[op](g->builder, extended1, extended2, "");
   1510         result = LLVMBuildTrunc(g->builder, extended_result, get_llvm_type(g, operand_type), "");
   1511 
   1512         LLVMValueRef re_extended_result = buildExtFn(g->builder, result, one_more_bit_int_vector, "");
   1513         LLVMValueRef overflow_vector = LLVMBuildICmp(g->builder, LLVMIntNE, extended_result, re_extended_result, "");
   1514         LLVMTypeRef bitcast_int_type = LLVMIntType(operand_type->data.vector.len);
   1515         LLVMValueRef bitcasted_overflow = LLVMBuildBitCast(g->builder, overflow_vector, bitcast_int_type, "");
   1516         LLVMValueRef zero = LLVMConstNull(bitcast_int_type);
   1517         overflow_bit = LLVMBuildICmp(g->builder, LLVMIntNE, bitcasted_overflow, zero, "");
   1518     } else {
   1519         LLVMValueRef fn_val = get_int_overflow_fn(g, operand_type, op);
   1520         LLVMValueRef params[] = {
   1521             val1,
   1522             val2,
   1523         };
   1524         LLVMValueRef result_struct = LLVMBuildCall(g->builder, fn_val, params, 2, "");
   1525         result = LLVMBuildExtractValue(g->builder, result_struct, 0, "");
   1526         overflow_bit = LLVMBuildExtractValue(g->builder, result_struct, 1, "");
   1527     }
   1528 
   1529     LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowFail");
   1530     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowOk");
   1531     LLVMBuildCondBr(g->builder, overflow_bit, fail_block, ok_block);
   1532 
   1533     LLVMPositionBuilderAtEnd(g->builder, fail_block);
   1534     gen_safety_crash(g, PanicMsgIdIntegerOverflow);
   1535 
   1536     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   1537     return result;
   1538 }
   1539 
   1540 static LLVMIntPredicate cmp_op_to_int_predicate(IrBinOp cmp_op, bool is_signed) {
   1541     switch (cmp_op) {
   1542         case IrBinOpCmpEq:
   1543             return LLVMIntEQ;
   1544         case IrBinOpCmpNotEq:
   1545             return LLVMIntNE;
   1546         case IrBinOpCmpLessThan:
   1547             return is_signed ? LLVMIntSLT : LLVMIntULT;
   1548         case IrBinOpCmpGreaterThan:
   1549             return is_signed ? LLVMIntSGT : LLVMIntUGT;
   1550         case IrBinOpCmpLessOrEq:
   1551             return is_signed ? LLVMIntSLE : LLVMIntULE;
   1552         case IrBinOpCmpGreaterOrEq:
   1553             return is_signed ? LLVMIntSGE : LLVMIntUGE;
   1554         default:
   1555             zig_unreachable();
   1556     }
   1557 }
   1558 
   1559 static LLVMRealPredicate cmp_op_to_real_predicate(IrBinOp cmp_op) {
   1560     switch (cmp_op) {
   1561         case IrBinOpCmpEq:
   1562             return LLVMRealOEQ;
   1563         case IrBinOpCmpNotEq:
   1564             return LLVMRealUNE;
   1565         case IrBinOpCmpLessThan:
   1566             return LLVMRealOLT;
   1567         case IrBinOpCmpGreaterThan:
   1568             return LLVMRealOGT;
   1569         case IrBinOpCmpLessOrEq:
   1570             return LLVMRealOLE;
   1571         case IrBinOpCmpGreaterOrEq:
   1572             return LLVMRealOGE;
   1573         default:
   1574             zig_unreachable();
   1575     }
   1576 }
   1577 
   1578 static void gen_assign_raw(CodeGen *g, LLVMValueRef ptr, ZigType *ptr_type,
   1579         LLVMValueRef value)
   1580 {
   1581     assert(ptr_type->id == ZigTypeIdPointer);
   1582     ZigType *child_type = ptr_type->data.pointer.child_type;
   1583 
   1584     if (!type_has_bits(child_type))
   1585         return;
   1586 
   1587     if (handle_is_ptr(child_type)) {
   1588         assert(LLVMGetTypeKind(LLVMTypeOf(value)) == LLVMPointerTypeKind);
   1589         assert(LLVMGetTypeKind(LLVMTypeOf(ptr)) == LLVMPointerTypeKind);
   1590 
   1591         LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
   1592 
   1593         LLVMValueRef src_ptr = LLVMBuildBitCast(g->builder, value, ptr_u8, "");
   1594         LLVMValueRef dest_ptr = LLVMBuildBitCast(g->builder, ptr, ptr_u8, "");
   1595 
   1596         ZigType *usize = g->builtin_types.entry_usize;
   1597         uint64_t size_bytes = LLVMStoreSizeOfType(g->target_data_ref, get_llvm_type(g, child_type));
   1598         uint64_t align_bytes = get_ptr_align(g, ptr_type);
   1599         assert(size_bytes > 0);
   1600         assert(align_bytes > 0);
   1601 
   1602         ZigLLVMBuildMemCpy(g->builder, dest_ptr, align_bytes, src_ptr, align_bytes,
   1603                 LLVMConstInt(usize->llvm_type, size_bytes, false),
   1604                 ptr_type->data.pointer.is_volatile);
   1605         return;
   1606     }
   1607 
   1608     uint32_t host_int_bytes = ptr_type->data.pointer.host_int_bytes;
   1609     if (host_int_bytes == 0) {
   1610         gen_store(g, value, ptr, ptr_type);
   1611         return;
   1612     }
   1613 
   1614     bool big_endian = g->is_big_endian;
   1615 
   1616     LLVMValueRef containing_int = gen_load(g, ptr, ptr_type, "");
   1617     uint32_t host_bit_count = LLVMGetIntTypeWidth(LLVMTypeOf(containing_int));
   1618     assert(host_bit_count == host_int_bytes * 8);
   1619     uint32_t size_in_bits = type_size_bits(g, child_type);
   1620 
   1621     uint32_t bit_offset = ptr_type->data.pointer.bit_offset_in_host;
   1622     uint32_t shift_amt = big_endian ? host_bit_count - bit_offset - size_in_bits : bit_offset;
   1623     LLVMValueRef shift_amt_val = LLVMConstInt(LLVMTypeOf(containing_int), shift_amt, false);
   1624 
   1625     // Convert to equally-sized integer type in order to perform the bit
   1626     // operations on the value to store
   1627     LLVMTypeRef value_bits_type = LLVMIntType(size_in_bits);
   1628     LLVMValueRef value_bits = LLVMBuildBitCast(g->builder, value, value_bits_type, "");
   1629 
   1630     LLVMValueRef mask_val = LLVMConstAllOnes(value_bits_type);
   1631     mask_val = LLVMConstZExt(mask_val, LLVMTypeOf(containing_int));
   1632     mask_val = LLVMConstShl(mask_val, shift_amt_val);
   1633     mask_val = LLVMConstNot(mask_val);
   1634 
   1635     LLVMValueRef anded_containing_int = LLVMBuildAnd(g->builder, containing_int, mask_val, "");
   1636     LLVMValueRef extended_value = LLVMBuildZExt(g->builder, value_bits, LLVMTypeOf(containing_int), "");
   1637     LLVMValueRef shifted_value = LLVMBuildShl(g->builder, extended_value, shift_amt_val, "");
   1638     LLVMValueRef ored_value = LLVMBuildOr(g->builder, shifted_value, anded_containing_int, "");
   1639 
   1640     gen_store(g, ored_value, ptr, ptr_type);
   1641     return;
   1642 }
   1643 
   1644 static void gen_var_debug_decl(CodeGen *g, ZigVar *var) {
   1645     if (g->strip_debug_symbols) return;
   1646     assert(var->di_loc_var != nullptr);
   1647     AstNode *source_node = var->decl_node;
   1648     ZigLLVMDILocation *debug_loc = ZigLLVMGetDebugLoc((unsigned)source_node->line + 1,
   1649             (unsigned)source_node->column + 1, get_di_scope(g, var->parent_scope));
   1650     ZigLLVMInsertDeclareAtEnd(g->dbuilder, var->value_ref, var->di_loc_var, debug_loc,
   1651             LLVMGetInsertBlock(g->builder));
   1652 }
   1653 
   1654 static LLVMValueRef ir_llvm_value(CodeGen *g, IrInstruction *instruction) {
   1655     if (!type_has_bits(instruction->value.type))
   1656         return nullptr;
   1657     if (!instruction->llvm_value) {
   1658         src_assert(instruction->value.special != ConstValSpecialRuntime, instruction->source_node);
   1659         assert(instruction->value.type);
   1660         render_const_val(g, &instruction->value, "");
   1661         // we might have to do some pointer casting here due to the way union
   1662         // values are rendered with a type other than the one we expect
   1663         if (handle_is_ptr(instruction->value.type)) {
   1664             render_const_val_global(g, &instruction->value, "");
   1665             ZigType *ptr_type = get_pointer_to_type(g, instruction->value.type, true);
   1666             instruction->llvm_value = LLVMBuildBitCast(g->builder, instruction->value.global_refs->llvm_global, get_llvm_type(g, ptr_type), "");
   1667         } else {
   1668             instruction->llvm_value = LLVMBuildBitCast(g->builder, instruction->value.global_refs->llvm_value,
   1669                     get_llvm_type(g, instruction->value.type), "");
   1670         }
   1671         assert(instruction->llvm_value);
   1672     }
   1673     return instruction->llvm_value;
   1674 }
   1675 
   1676 ATTRIBUTE_NORETURN
   1677 static void report_errors_and_exit(CodeGen *g) {
   1678     assert(g->errors.length != 0);
   1679     for (size_t i = 0; i < g->errors.length; i += 1) {
   1680         ErrorMsg *err = g->errors.at(i);
   1681         print_err_msg(err, g->err_color);
   1682     }
   1683     exit(1);
   1684 }
   1685 
   1686 static void report_errors_and_maybe_exit(CodeGen *g) {
   1687     if (g->errors.length != 0) {
   1688         report_errors_and_exit(g);
   1689     }
   1690 }
   1691 
   1692 ATTRIBUTE_NORETURN
   1693 static void give_up_with_c_abi_error(CodeGen *g, AstNode *source_node) {
   1694     ErrorMsg *msg = add_node_error(g, source_node,
   1695             buf_sprintf("TODO: support C ABI for more targets. https://github.com/ziglang/zig/issues/1481"));
   1696     add_error_note(g, msg, source_node,
   1697         buf_sprintf("pointers, integers, floats, bools, and enums work on all targets"));
   1698     report_errors_and_exit(g);
   1699 }
   1700 
   1701 static LLVMValueRef build_alloca(CodeGen *g, ZigType *type_entry, const char *name, uint32_t alignment) {
   1702     LLVMValueRef result = LLVMBuildAlloca(g->builder, get_llvm_type(g, type_entry), name);
   1703     LLVMSetAlignment(result, (alignment == 0) ? get_abi_alignment(g, type_entry) : alignment);
   1704     return result;
   1705 }
   1706 
   1707 static bool iter_function_params_c_abi(CodeGen *g, ZigType *fn_type, FnWalk *fn_walk, size_t src_i) {
   1708     // Initialized from the type for some walks, but because of C var args,
   1709     // initialized based on callsite instructions for that one.
   1710     FnTypeParamInfo *param_info = nullptr;
   1711     ZigType *ty;
   1712     ZigType *dest_ty = nullptr;
   1713     AstNode *source_node = nullptr;
   1714     LLVMValueRef val;
   1715     LLVMValueRef llvm_fn;
   1716     unsigned di_arg_index;
   1717     ZigVar *var;
   1718     switch (fn_walk->id) {
   1719         case FnWalkIdAttrs:
   1720             if (src_i >= fn_type->data.fn.fn_type_id.param_count)
   1721                 return false;
   1722             param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
   1723             ty = param_info->type;
   1724             source_node = fn_walk->data.attrs.fn->proto_node;
   1725             llvm_fn = fn_walk->data.attrs.llvm_fn;
   1726             break;
   1727         case FnWalkIdCall: {
   1728             if (src_i >= fn_walk->data.call.inst->arg_count)
   1729                 return false;
   1730             IrInstruction *arg = fn_walk->data.call.inst->args[src_i];
   1731             ty = arg->value.type;
   1732             source_node = arg->source_node;
   1733             val = ir_llvm_value(g, arg);
   1734             break;
   1735         }
   1736         case FnWalkIdTypes:
   1737             if (src_i >= fn_type->data.fn.fn_type_id.param_count)
   1738                 return false;
   1739             param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
   1740             ty = param_info->type;
   1741             break;
   1742         case FnWalkIdVars:
   1743             assert(src_i < fn_type->data.fn.fn_type_id.param_count);
   1744             param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
   1745             ty = param_info->type;
   1746             var = fn_walk->data.vars.var;
   1747             source_node = var->decl_node;
   1748             llvm_fn = fn_walk->data.vars.llvm_fn;
   1749             break;
   1750         case FnWalkIdInits:
   1751             if (src_i >= fn_type->data.fn.fn_type_id.param_count)
   1752                 return false;
   1753             param_info = &fn_type->data.fn.fn_type_id.param_info[src_i];
   1754             ty = param_info->type;
   1755             var = fn_walk->data.inits.fn->variable_list.at(src_i);
   1756             source_node = fn_walk->data.inits.fn->proto_node;
   1757             llvm_fn = fn_walk->data.inits.llvm_fn;
   1758             break;
   1759     }
   1760 
   1761     if (type_is_c_abi_int(g, ty) || ty->id == ZigTypeIdFloat || ty->id == ZigTypeIdVector ||
   1762         ty->id == ZigTypeIdInt // TODO investigate if we need to change this
   1763     ) {
   1764         switch (fn_walk->id) {
   1765             case FnWalkIdAttrs: {
   1766                 ZigType *ptr_type = get_codegen_ptr_type(ty);
   1767                 if (ptr_type != nullptr) {
   1768                     if (type_is_nonnull_ptr(ty)) {
   1769                         addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "nonnull");
   1770                     }
   1771                     if (ptr_type->data.pointer.is_const) {
   1772                         addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "readonly");
   1773                     }
   1774                     if (param_info->is_noalias) {
   1775                         addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "noalias");
   1776                     }
   1777                 }
   1778                 fn_walk->data.attrs.gen_i += 1;
   1779                 break;
   1780             }
   1781             case FnWalkIdCall:
   1782                 fn_walk->data.call.gen_param_values->append(val);
   1783                 break;
   1784             case FnWalkIdTypes:
   1785                 fn_walk->data.types.gen_param_types->append(get_llvm_type(g, ty));
   1786                 fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, ty));
   1787                 break;
   1788             case FnWalkIdVars: {
   1789                 var->value_ref = build_alloca(g, ty, buf_ptr(&var->name), var->align_bytes);
   1790                 di_arg_index = fn_walk->data.vars.gen_i;
   1791                 fn_walk->data.vars.gen_i += 1;
   1792                 dest_ty = ty;
   1793                 goto var_ok;
   1794             }
   1795             case FnWalkIdInits:
   1796                 clear_debug_source_node(g);
   1797                 gen_store_untyped(g, LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i), var->value_ref, var->align_bytes, false);
   1798                 if (var->decl_node) {
   1799                     gen_var_debug_decl(g, var);
   1800                 }
   1801                 fn_walk->data.inits.gen_i += 1;
   1802                 break;
   1803         }
   1804         return true;
   1805     }
   1806 
   1807     // Arrays are just pointers
   1808     if (ty->id == ZigTypeIdArray) {
   1809         assert(handle_is_ptr(ty));
   1810         switch (fn_walk->id) {
   1811             case FnWalkIdAttrs:
   1812                 // arrays passed to C ABI functions may not be at address 0
   1813                 addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "nonnull");
   1814                 addLLVMArgAttrInt(llvm_fn, fn_walk->data.attrs.gen_i, "align", get_abi_alignment(g, ty));
   1815                 fn_walk->data.attrs.gen_i += 1;
   1816                 break;
   1817             case FnWalkIdCall:
   1818                 fn_walk->data.call.gen_param_values->append(val);
   1819                 break;
   1820             case FnWalkIdTypes: {
   1821                 ZigType *gen_type = get_pointer_to_type(g, ty, true);
   1822                 fn_walk->data.types.gen_param_types->append(get_llvm_type(g, gen_type));
   1823                 fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, gen_type));
   1824                 break;
   1825             }
   1826             case FnWalkIdVars: {
   1827                 var->value_ref = LLVMGetParam(llvm_fn,  fn_walk->data.vars.gen_i);
   1828                 di_arg_index = fn_walk->data.vars.gen_i;
   1829                 dest_ty = get_pointer_to_type(g, ty, false);
   1830                 fn_walk->data.vars.gen_i += 1;
   1831                 goto var_ok;
   1832             }
   1833             case FnWalkIdInits:
   1834                 if (var->decl_node) {
   1835                     gen_var_debug_decl(g, var);
   1836                 }
   1837                 fn_walk->data.inits.gen_i += 1;
   1838                 break;
   1839         }
   1840         return true;
   1841     }
   1842 
   1843     if (g->zig_target->arch == ZigLLVM_x86_64) {
   1844         X64CABIClass abi_class = type_c_abi_x86_64_class(g, ty);
   1845         size_t ty_size = type_size(g, ty);
   1846         if (abi_class == X64CABIClass_MEMORY) {
   1847             assert(handle_is_ptr(ty));
   1848             switch (fn_walk->id) {
   1849                 case FnWalkIdAttrs:
   1850                     ZigLLVMAddByValAttr(llvm_fn, fn_walk->data.attrs.gen_i + 1, get_llvm_type(g, ty));
   1851                     addLLVMArgAttrInt(llvm_fn, fn_walk->data.attrs.gen_i, "align", get_abi_alignment(g, ty));
   1852                     // Byvalue parameters must not have address 0
   1853                     addLLVMArgAttr(llvm_fn, fn_walk->data.attrs.gen_i, "nonnull");
   1854                     fn_walk->data.attrs.gen_i += 1;
   1855                     break;
   1856                 case FnWalkIdCall:
   1857                     fn_walk->data.call.gen_param_values->append(val);
   1858                     break;
   1859                 case FnWalkIdTypes: {
   1860                     ZigType *gen_type = get_pointer_to_type(g, ty, true);
   1861                     fn_walk->data.types.gen_param_types->append(get_llvm_type(g, gen_type));
   1862                     fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, gen_type));
   1863                     break;
   1864                 }
   1865                 case FnWalkIdVars: {
   1866                     di_arg_index = fn_walk->data.vars.gen_i;
   1867                     var->value_ref = LLVMGetParam(llvm_fn,  fn_walk->data.vars.gen_i);
   1868                     dest_ty = get_pointer_to_type(g, ty, false);
   1869                     fn_walk->data.vars.gen_i += 1;
   1870                     goto var_ok;
   1871                 }
   1872                 case FnWalkIdInits:
   1873                     if (var->decl_node) {
   1874                         gen_var_debug_decl(g, var);
   1875                     }
   1876                     fn_walk->data.inits.gen_i += 1;
   1877                     break;
   1878             }
   1879             return true;
   1880         } else if (abi_class == X64CABIClass_INTEGER) {
   1881             switch (fn_walk->id) {
   1882                 case FnWalkIdAttrs:
   1883                     fn_walk->data.attrs.gen_i += 1;
   1884                     break;
   1885                 case FnWalkIdCall: {
   1886                     LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(LLVMIntType((unsigned)ty_size * 8), 0);
   1887                     LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, val, ptr_to_int_type_ref, "");
   1888                     LLVMValueRef loaded = LLVMBuildLoad(g->builder, bitcasted, "");
   1889                     fn_walk->data.call.gen_param_values->append(loaded);
   1890                     break;
   1891                 }
   1892                 case FnWalkIdTypes: {
   1893                     ZigType *gen_type = get_int_type(g, false, ty_size * 8);
   1894                     fn_walk->data.types.gen_param_types->append(get_llvm_type(g, gen_type));
   1895                     fn_walk->data.types.param_di_types->append(get_llvm_di_type(g, gen_type));
   1896                     break;
   1897                 }
   1898                 case FnWalkIdVars: {
   1899                     di_arg_index = fn_walk->data.vars.gen_i;
   1900                     var->value_ref = build_alloca(g, ty, buf_ptr(&var->name), var->align_bytes);
   1901                     fn_walk->data.vars.gen_i += 1;
   1902                     dest_ty = ty;
   1903                     goto var_ok;
   1904                 }
   1905                 case FnWalkIdInits: {
   1906                     clear_debug_source_node(g);
   1907                     if (!fn_is_async(fn_walk->data.inits.fn)) {
   1908                         LLVMValueRef arg = LLVMGetParam(llvm_fn, fn_walk->data.inits.gen_i);
   1909                         LLVMTypeRef ptr_to_int_type_ref = LLVMPointerType(LLVMIntType((unsigned)ty_size * 8), 0);
   1910                         LLVMValueRef bitcasted = LLVMBuildBitCast(g->builder, var->value_ref, ptr_to_int_type_ref, "");
   1911                         gen_store_untyped(g, arg, bitcasted, var->align_bytes, false);
   1912                     }
   1913                     if (var->decl_node) {
   1914                         gen_var_debug_decl(g, var);
   1915                     }
   1916                     fn_walk->data.inits.gen_i += 1;
   1917                     break;
   1918                 }
   1919             }
   1920             return true;
   1921         }
   1922     }
   1923     if (source_node != nullptr) {
   1924         give_up_with_c_abi_error(g, source_node);
   1925     }
   1926     // otherwise allow codegen code to report a compile error
   1927     return false;
   1928 
   1929 var_ok:
   1930     if (dest_ty != nullptr && var->decl_node) {
   1931         // arg index + 1 because the 0 index is return value
   1932         var->di_loc_var = ZigLLVMCreateParameterVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
   1933                 buf_ptr(&var->name), fn_walk->data.vars.import->data.structure.root_struct->di_file,
   1934                 (unsigned)(var->decl_node->line + 1),
   1935                 get_llvm_di_type(g, dest_ty), !g->strip_debug_symbols, 0, di_arg_index + 1);
   1936     }
   1937     return true;
   1938 }
   1939 
   1940 void walk_function_params(CodeGen *g, ZigType *fn_type, FnWalk *fn_walk) {
   1941     CallingConvention cc = fn_type->data.fn.fn_type_id.cc;
   1942     if (cc == CallingConventionC) {
   1943         size_t src_i = 0;
   1944         for (;;) {
   1945             if (!iter_function_params_c_abi(g, fn_type, fn_walk, src_i))
   1946                 break;
   1947             src_i += 1;
   1948         }
   1949         return;
   1950     }
   1951     if (fn_walk->id == FnWalkIdCall) {
   1952         IrInstructionCallGen *instruction = fn_walk->data.call.inst;
   1953         bool is_var_args = fn_walk->data.call.is_var_args;
   1954         for (size_t call_i = 0; call_i < instruction->arg_count; call_i += 1) {
   1955             IrInstruction *param_instruction = instruction->args[call_i];
   1956             ZigType *param_type = param_instruction->value.type;
   1957             if (is_var_args || type_has_bits(param_type)) {
   1958                 LLVMValueRef param_value = ir_llvm_value(g, param_instruction);
   1959                 assert(param_value);
   1960                 fn_walk->data.call.gen_param_values->append(param_value);
   1961                 fn_walk->data.call.gen_param_types->append(param_type);
   1962             }
   1963         }
   1964         return;
   1965     }
   1966     size_t next_var_i = 0;
   1967     for (size_t param_i = 0; param_i < fn_type->data.fn.fn_type_id.param_count; param_i += 1) {
   1968         FnGenParamInfo *gen_info = &fn_type->data.fn.gen_param_info[param_i];
   1969         size_t gen_index = gen_info->gen_index;
   1970 
   1971         if (gen_index == SIZE_MAX) {
   1972             continue;
   1973         }
   1974 
   1975         switch (fn_walk->id) {
   1976             case FnWalkIdAttrs: {
   1977                 LLVMValueRef llvm_fn = fn_walk->data.attrs.llvm_fn;
   1978                 bool is_byval = gen_info->is_byval;
   1979                 FnTypeParamInfo *param_info = &fn_type->data.fn.fn_type_id.param_info[param_i];
   1980 
   1981                 ZigType *param_type = gen_info->type;
   1982                 if (param_info->is_noalias) {
   1983                     addLLVMArgAttr(llvm_fn, (unsigned)gen_index, "noalias");
   1984                 }
   1985                 if ((param_type->id == ZigTypeIdPointer && param_type->data.pointer.is_const) || is_byval) {
   1986                     addLLVMArgAttr(llvm_fn, (unsigned)gen_index, "readonly");
   1987                 }
   1988                 if (get_codegen_ptr_type(param_type) != nullptr) {
   1989                     addLLVMArgAttrInt(llvm_fn, (unsigned)gen_index, "align", get_ptr_align(g, param_type));
   1990                 }
   1991                 if (type_is_nonnull_ptr(param_type)) {
   1992                     addLLVMArgAttr(llvm_fn, (unsigned)gen_index, "nonnull");
   1993                 }
   1994                 break;
   1995             }
   1996             case FnWalkIdInits: {
   1997                 ZigFn *fn_table_entry = fn_walk->data.inits.fn;
   1998                 LLVMValueRef llvm_fn = fn_table_entry->llvm_value;
   1999                 ZigVar *variable = fn_table_entry->variable_list.at(next_var_i);
   2000                 assert(variable->src_arg_index != SIZE_MAX);
   2001                 next_var_i += 1;
   2002 
   2003                 assert(variable);
   2004                 assert(variable->value_ref);
   2005 
   2006                 if (!handle_is_ptr(variable->var_type) && !fn_is_async(fn_walk->data.inits.fn)) {
   2007                     clear_debug_source_node(g);
   2008                     ZigType *fn_type = fn_table_entry->type_entry;
   2009                     unsigned gen_arg_index = fn_type->data.fn.gen_param_info[variable->src_arg_index].gen_index;
   2010                     gen_store_untyped(g, LLVMGetParam(llvm_fn, gen_arg_index),
   2011                             variable->value_ref, variable->align_bytes, false);
   2012                 }
   2013 
   2014                 if (variable->decl_node) {
   2015                     gen_var_debug_decl(g, variable);
   2016                 }
   2017                 break;
   2018             }
   2019             case FnWalkIdCall:
   2020                 // handled before for loop
   2021                 zig_unreachable();
   2022             case FnWalkIdTypes:
   2023                 // Not called for non-c-abi
   2024                 zig_unreachable();
   2025             case FnWalkIdVars:
   2026                 // iter_function_params_c_abi is called directly for this one
   2027                 zig_unreachable();
   2028         }
   2029     }
   2030 }
   2031 
   2032 static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) {
   2033     if (g->merge_err_ret_traces_fn_val)
   2034         return g->merge_err_ret_traces_fn_val;
   2035 
   2036     assert(g->stack_trace_type != nullptr);
   2037 
   2038     LLVMTypeRef param_types[] = {
   2039         get_llvm_type(g, ptr_to_stack_trace_type(g)),
   2040         get_llvm_type(g, ptr_to_stack_trace_type(g)),
   2041     };
   2042     LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false);
   2043 
   2044     Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_merge_error_return_traces"), false);
   2045     LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
   2046     LLVMSetLinkage(fn_val, LLVMInternalLinkage);
   2047     LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
   2048     addLLVMFnAttr(fn_val, "nounwind");
   2049     add_uwtable_attr(g, fn_val);
   2050     addLLVMArgAttr(fn_val, (unsigned)0, "noalias");
   2051     addLLVMArgAttr(fn_val, (unsigned)0, "writeonly");
   2052 
   2053     addLLVMArgAttr(fn_val, (unsigned)1, "noalias");
   2054     addLLVMArgAttr(fn_val, (unsigned)1, "readonly");
   2055     if (g->build_mode == BuildModeDebug) {
   2056         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
   2057         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
   2058     }
   2059 
   2060     // this is above the ZigLLVMClearCurrentDebugLocation
   2061     LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g);
   2062 
   2063     LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
   2064     LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
   2065     LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
   2066     LLVMPositionBuilderAtEnd(g->builder, entry_block);
   2067     ZigLLVMClearCurrentDebugLocation(g->builder);
   2068 
   2069     // if (dest_stack_trace == null or src_stack_trace == null) return;
   2070     // var frame_index: usize = undefined;
   2071     // var frames_left: usize = undefined;
   2072     // if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) {
   2073     //     frame_index = 0;
   2074     //     frames_left = src_stack_trace.index;
   2075     //     if (frames_left == 0) return;
   2076     // } else {
   2077     //     frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len;
   2078     //     frames_left = src_stack_trace.instruction_addresses.len;
   2079     // }
   2080     // while (true) {
   2081     //     __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]);
   2082     //     frames_left -= 1;
   2083     //     if (frames_left == 0) return;
   2084     //     frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len;
   2085     // }
   2086     LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return");
   2087     LLVMBasicBlockRef non_null_block = LLVMAppendBasicBlock(fn_val, "NonNull");
   2088 
   2089     LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frame_index");
   2090     LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->llvm_type, "frames_left");
   2091 
   2092     LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0);
   2093     LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1);
   2094 
   2095     LLVMValueRef null_dest_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, dest_stack_trace_ptr,
   2096             LLVMConstNull(LLVMTypeOf(dest_stack_trace_ptr)), "");
   2097     LLVMValueRef null_src_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_stack_trace_ptr,
   2098             LLVMConstNull(LLVMTypeOf(src_stack_trace_ptr)), "");
   2099     LLVMValueRef null_bit = LLVMBuildOr(g->builder, null_dest_bit, null_src_bit, "");
   2100     LLVMBuildCondBr(g->builder, null_bit, return_block, non_null_block);
   2101 
   2102     LLVMPositionBuilderAtEnd(g->builder, non_null_block);
   2103     size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
   2104     size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
   2105     LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
   2106             (unsigned)src_index_field_index, "");
   2107     LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr,
   2108             (unsigned)src_addresses_field_index, "");
   2109     ZigType *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry;
   2110     size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
   2111     LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, "");
   2112     size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index;
   2113     LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, "");
   2114     LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, "");
   2115     LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, "");
   2116     LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, "");
   2117     LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, "");
   2118     LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap");
   2119     LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap");
   2120     LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop");
   2121     LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block);
   2122 
   2123     LLVMPositionBuilderAtEnd(g->builder, no_wrap_block);
   2124     LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->llvm_type);
   2125     LLVMBuildStore(g->builder, usize_zero, frame_index_ptr);
   2126     LLVMBuildStore(g->builder, src_index_val, frames_left_ptr);
   2127     LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, "");
   2128     LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block);
   2129 
   2130     LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block);
   2131     LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 1, false);
   2132     LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, "");
   2133     LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, "");
   2134     LLVMBuildStore(g->builder, mod_len, frame_index_ptr);
   2135     LLVMBuildStore(g->builder, src_len_val, frames_left_ptr);
   2136     LLVMBuildBr(g->builder, loop_block);
   2137 
   2138     LLVMPositionBuilderAtEnd(g->builder, loop_block);
   2139     LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
   2140     LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, "");
   2141     LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, "");
   2142     LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val};
   2143     ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, "");
   2144     LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, "");
   2145     LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, "");
   2146     LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, "");
   2147     LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue");
   2148     LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block);
   2149 
   2150     LLVMPositionBuilderAtEnd(g->builder, return_block);
   2151     LLVMBuildRetVoid(g->builder);
   2152 
   2153     LLVMPositionBuilderAtEnd(g->builder, continue_block);
   2154     LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr);
   2155     LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, "");
   2156     LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, "");
   2157     LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, "");
   2158     LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr);
   2159     LLVMBuildBr(g->builder, loop_block);
   2160 
   2161     LLVMPositionBuilderAtEnd(g->builder, prev_block);
   2162     if (!g->strip_debug_symbols) {
   2163         LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
   2164     }
   2165 
   2166     g->merge_err_ret_traces_fn_val = fn_val;
   2167     return fn_val;
   2168 
   2169 }
   2170 static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *executable,
   2171         IrInstructionSaveErrRetAddr *save_err_ret_addr_instruction)
   2172 {
   2173     assert(g->have_err_ret_tracing);
   2174 
   2175     LLVMValueRef return_err_fn = get_return_err_fn(g);
   2176     LLVMValueRef my_err_trace_val = get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope);
   2177     ZigLLVMBuildCall(g->builder, return_err_fn, &my_err_trace_val, 1,
   2178             get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
   2179 
   2180     ZigType *ret_type = g->cur_fn->type_entry->data.fn.fn_type_id.return_type;
   2181     if (fn_is_async(g->cur_fn) && codegen_fn_has_err_ret_tracing_arg(g, ret_type)) {
   2182         LLVMValueRef trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
   2183                 frame_index_trace_arg(g, ret_type), "");
   2184         LLVMBuildStore(g->builder, my_err_trace_val, trace_ptr_ptr);
   2185     }
   2186 
   2187     return nullptr;
   2188 }
   2189 
   2190 static void gen_assert_resume_id(CodeGen *g, IrInstruction *source_instr, ResumeId resume_id, PanicMsgId msg_id,
   2191         LLVMBasicBlockRef end_bb)
   2192 {
   2193     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   2194     LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
   2195     if (end_bb == nullptr) end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume");
   2196     LLVMValueRef expected_value = LLVMConstSub(LLVMConstAllOnes(usize_type_ref),
   2197             LLVMConstInt(usize_type_ref, resume_id, false));
   2198     LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, LLVMGetParam(g->cur_fn_val, 1), expected_value, "");
   2199     LLVMBuildCondBr(g->builder, ok_bit, end_bb, bad_resume_block);
   2200 
   2201     LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
   2202     gen_assertion(g, msg_id, source_instr);
   2203 
   2204     LLVMPositionBuilderAtEnd(g->builder, end_bb);
   2205 }
   2206 
   2207 static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef target_frame_ptr, ResumeId resume_id) {
   2208     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   2209     if (fn_val == nullptr) {
   2210         LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_fn_ptr_index, "");
   2211         fn_val = LLVMBuildLoad(g->builder, fn_ptr_ptr, "");
   2212     }
   2213     LLVMValueRef arg_val = LLVMConstSub(LLVMConstAllOnes(usize_type_ref),
   2214             LLVMConstInt(usize_type_ref, resume_id, false));
   2215     LLVMValueRef args[] = {target_frame_ptr, arg_val};
   2216     return ZigLLVMBuildCall(g->builder, fn_val, args, 2, LLVMFastCallConv, ZigLLVM_FnInlineAuto, "");
   2217 }
   2218 
   2219 static LLVMBasicBlockRef gen_suspend_begin(CodeGen *g, const char *name_hint) {
   2220     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   2221     LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, name_hint);
   2222     size_t new_block_index = g->cur_resume_block_count;
   2223     g->cur_resume_block_count += 1;
   2224     LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false);
   2225     LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb);
   2226     LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr);
   2227     return resume_bb;
   2228 }
   2229 
   2230 static void set_tail_call_if_appropriate(CodeGen *g, LLVMValueRef call_inst) {
   2231     LLVMSetTailCall(call_inst, true);
   2232 }
   2233 
   2234 static LLVMValueRef gen_maybe_atomic_op(CodeGen *g, LLVMAtomicRMWBinOp op, LLVMValueRef ptr, LLVMValueRef val,
   2235         LLVMAtomicOrdering order)
   2236 {
   2237     if (g->is_single_threaded) {
   2238         LLVMValueRef loaded = LLVMBuildLoad(g->builder, ptr, "");
   2239         LLVMValueRef modified;
   2240         switch (op) {
   2241             case LLVMAtomicRMWBinOpXchg:
   2242                 modified = val;
   2243                 break;
   2244             case LLVMAtomicRMWBinOpXor:
   2245                 modified = LLVMBuildXor(g->builder, loaded, val, "");
   2246                 break;
   2247             default:
   2248                 zig_unreachable();
   2249         }
   2250         LLVMBuildStore(g->builder, modified, ptr);
   2251         return loaded;
   2252     } else {
   2253         return LLVMBuildAtomicRMW(g->builder, op, ptr, val, order, false);
   2254     }
   2255 }
   2256 
   2257 static void gen_async_return(CodeGen *g, IrInstructionReturn *instruction) {
   2258     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   2259 
   2260     ZigType *operand_type = (instruction->operand != nullptr) ? instruction->operand->value.type : nullptr;
   2261     bool operand_has_bits = (operand_type != nullptr) && type_has_bits(operand_type);
   2262     ZigType *ret_type = g->cur_fn->type_entry->data.fn.fn_type_id.return_type;
   2263     bool ret_type_has_bits = type_has_bits(ret_type);
   2264 
   2265     if (operand_has_bits && instruction->operand != nullptr) {
   2266         bool need_store = instruction->operand->value.special != ConstValSpecialRuntime || !handle_is_ptr(ret_type);
   2267         if (need_store) {
   2268             // It didn't get written to the result ptr. We do that now.
   2269             ZigType *ret_ptr_type = get_pointer_to_type(g, ret_type, true);
   2270             gen_assign_raw(g, g->cur_ret_ptr, ret_ptr_type, ir_llvm_value(g, instruction->operand));
   2271         }
   2272     }
   2273 
   2274     // Whether we tail resume the awaiter, or do an early return, we are done and will not be resumed.
   2275     if (ir_want_runtime_safety(g, &instruction->base)) {
   2276         LLVMValueRef new_resume_index = LLVMConstAllOnes(usize_type_ref);
   2277         LLVMBuildStore(g->builder, new_resume_index, g->cur_async_resume_index_ptr);
   2278     }
   2279 
   2280     LLVMValueRef zero = LLVMConstNull(usize_type_ref);
   2281     LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
   2282 
   2283     LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXor, g->cur_async_awaiter_ptr,
   2284             all_ones, LLVMAtomicOrderingAcquire);
   2285 
   2286     LLVMBasicBlockRef bad_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadReturn");
   2287     LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
   2288     LLVMBasicBlockRef resume_them_block = LLVMAppendBasicBlock(g->cur_fn_val, "ResumeThem");
   2289 
   2290     LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, resume_them_block, 2);
   2291 
   2292     LLVMAddCase(switch_instr, zero, early_return_block);
   2293     LLVMAddCase(switch_instr, all_ones, bad_return_block);
   2294 
   2295     // Something has gone horribly wrong, and this is an invalid second return.
   2296     LLVMPositionBuilderAtEnd(g->builder, bad_return_block);
   2297     gen_assertion(g, PanicMsgIdBadReturn, &instruction->base);
   2298 
   2299     // There is no awaiter yet, but we're completely done.
   2300     LLVMPositionBuilderAtEnd(g->builder, early_return_block);
   2301     LLVMBuildRetVoid(g->builder);
   2302 
   2303     // We need to resume the caller by tail calling them,
   2304     // but first write through the result pointer and possibly
   2305     // error return trace pointer.
   2306     LLVMPositionBuilderAtEnd(g->builder, resume_them_block);
   2307 
   2308     if (ret_type_has_bits) {
   2309         // If the awaiter result pointer is non-null, we need to copy the result to there.
   2310         LLVMBasicBlockRef copy_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResult");
   2311         LLVMBasicBlockRef copy_end_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResultEnd");
   2312         LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_ret_start + 1, "");
   2313         LLVMValueRef awaiter_ret_ptr = LLVMBuildLoad(g->builder, awaiter_ret_ptr_ptr, "");
   2314         LLVMValueRef zero_ptr = LLVMConstNull(LLVMTypeOf(awaiter_ret_ptr));
   2315         LLVMValueRef need_copy_bit = LLVMBuildICmp(g->builder, LLVMIntNE, awaiter_ret_ptr, zero_ptr, "");
   2316         LLVMBuildCondBr(g->builder, need_copy_bit, copy_block, copy_end_block);
   2317 
   2318         LLVMPositionBuilderAtEnd(g->builder, copy_block);
   2319         LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
   2320         LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, awaiter_ret_ptr, ptr_u8, "");
   2321         LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, g->cur_ret_ptr, ptr_u8, "");
   2322         bool is_volatile = false;
   2323         uint32_t abi_align = get_abi_alignment(g, ret_type);
   2324         LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, ret_type), false);
   2325         ZigLLVMBuildMemCpy(g->builder,
   2326                 dest_ptr_casted, abi_align,
   2327                 src_ptr_casted, abi_align, byte_count_val, is_volatile);
   2328         LLVMBuildBr(g->builder, copy_end_block);
   2329 
   2330         LLVMPositionBuilderAtEnd(g->builder, copy_end_block);
   2331         if (codegen_fn_has_err_ret_tracing_arg(g, ret_type)) {
   2332             LLVMValueRef awaiter_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
   2333                     frame_index_trace_arg(g, ret_type) + 1, "");
   2334             LLVMValueRef dest_trace_ptr = LLVMBuildLoad(g->builder, awaiter_trace_ptr_ptr, "");
   2335             LLVMValueRef my_err_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
   2336             LLVMValueRef args[] = { dest_trace_ptr, my_err_trace_val };
   2337             ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2,
   2338                     get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
   2339         }
   2340     }
   2341 
   2342     // Resume the caller by tail calling them.
   2343     ZigType *any_frame_type = get_any_frame_type(g, ret_type);
   2344     LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, prev_val, get_llvm_type(g, any_frame_type), "");
   2345     LLVMValueRef call_inst = gen_resume(g, nullptr, their_frame_ptr, ResumeIdReturn);
   2346     set_tail_call_if_appropriate(g, call_inst);
   2347     LLVMBuildRetVoid(g->builder);
   2348 }
   2349 
   2350 static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrInstructionReturn *instruction) {
   2351     if (fn_is_async(g->cur_fn)) {
   2352         gen_async_return(g, instruction);
   2353         return nullptr;
   2354     }
   2355 
   2356     if (want_first_arg_sret(g, &g->cur_fn->type_entry->data.fn.fn_type_id)) {
   2357         if (instruction->operand == nullptr) {
   2358             LLVMBuildRetVoid(g->builder);
   2359             return nullptr;
   2360         }
   2361         assert(g->cur_ret_ptr);
   2362         src_assert(instruction->operand->value.special != ConstValSpecialRuntime,
   2363                 instruction->base.source_node);
   2364         LLVMValueRef value = ir_llvm_value(g, instruction->operand);
   2365         ZigType *return_type = instruction->operand->value.type;
   2366         gen_assign_raw(g, g->cur_ret_ptr, get_pointer_to_type(g, return_type, false), value);
   2367         LLVMBuildRetVoid(g->builder);
   2368     } else if (g->cur_fn->type_entry->data.fn.fn_type_id.cc != CallingConventionAsync &&
   2369             handle_is_ptr(g->cur_fn->type_entry->data.fn.fn_type_id.return_type))
   2370     {
   2371         if (instruction->operand == nullptr) {
   2372             LLVMValueRef by_val_value = gen_load_untyped(g, g->cur_ret_ptr, 0, false, "");
   2373             LLVMBuildRet(g->builder, by_val_value);
   2374         } else {
   2375             LLVMValueRef value = ir_llvm_value(g, instruction->operand);
   2376             LLVMValueRef by_val_value = gen_load_untyped(g, value, 0, false, "");
   2377             LLVMBuildRet(g->builder, by_val_value);
   2378         }
   2379     } else if (instruction->operand == nullptr) {
   2380         LLVMBuildRetVoid(g->builder);
   2381     } else {
   2382         LLVMValueRef value = ir_llvm_value(g, instruction->operand);
   2383         LLVMBuildRet(g->builder, value);
   2384     }
   2385     return nullptr;
   2386 }
   2387 
   2388 static LLVMValueRef gen_overflow_shl_op(CodeGen *g, ZigType *type_entry,
   2389         LLVMValueRef val1, LLVMValueRef val2)
   2390 {
   2391     // for unsigned left shifting, we do the lossy shift, then logically shift
   2392     // right the same number of bits
   2393     // if the values don't match, we have an overflow
   2394     // for signed left shifting we do the same except arithmetic shift right
   2395 
   2396     assert(type_entry->id == ZigTypeIdInt);
   2397 
   2398     LLVMValueRef result = LLVMBuildShl(g->builder, val1, val2, "");
   2399     LLVMValueRef orig_val;
   2400     if (type_entry->data.integral.is_signed) {
   2401         orig_val = LLVMBuildAShr(g->builder, result, val2, "");
   2402     } else {
   2403         orig_val = LLVMBuildLShr(g->builder, result, val2, "");
   2404     }
   2405     LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, val1, orig_val, "");
   2406 
   2407     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowOk");
   2408     LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowFail");
   2409     LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   2410 
   2411     LLVMPositionBuilderAtEnd(g->builder, fail_block);
   2412     gen_safety_crash(g, PanicMsgIdShlOverflowedBits);
   2413 
   2414     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   2415     return result;
   2416 }
   2417 
   2418 static LLVMValueRef gen_overflow_shr_op(CodeGen *g, ZigType *type_entry,
   2419         LLVMValueRef val1, LLVMValueRef val2)
   2420 {
   2421     assert(type_entry->id == ZigTypeIdInt);
   2422 
   2423     LLVMValueRef result;
   2424     if (type_entry->data.integral.is_signed) {
   2425         result = LLVMBuildAShr(g->builder, val1, val2, "");
   2426     } else {
   2427         result = LLVMBuildLShr(g->builder, val1, val2, "");
   2428     }
   2429     LLVMValueRef orig_val = LLVMBuildShl(g->builder, result, val2, "");
   2430     LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, val1, orig_val, "");
   2431 
   2432     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowOk");
   2433     LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowFail");
   2434     LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   2435 
   2436     LLVMPositionBuilderAtEnd(g->builder, fail_block);
   2437     gen_safety_crash(g, PanicMsgIdShrOverflowedBits);
   2438 
   2439     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   2440     return result;
   2441 }
   2442 
   2443 static LLVMValueRef gen_float_op(CodeGen *g, LLVMValueRef val, ZigType *type_entry, BuiltinFnId op) {
   2444     if ((op == BuiltinFnIdCeil ||
   2445          op == BuiltinFnIdFloor) &&
   2446         type_entry->id == ZigTypeIdInt)
   2447         return val;
   2448     assert(type_entry->id == ZigTypeIdFloat);
   2449 
   2450     LLVMValueRef floor_fn = get_float_fn(g, type_entry, ZigLLVMFnIdFloatOp, op);
   2451     return LLVMBuildCall(g->builder, floor_fn, &val, 1, "");
   2452 }
   2453 
   2454 enum DivKind {
   2455     DivKindFloat,
   2456     DivKindTrunc,
   2457     DivKindFloor,
   2458     DivKindExact,
   2459 };
   2460 
   2461 static LLVMValueRef bigint_to_llvm_const(LLVMTypeRef type_ref, BigInt *bigint) {
   2462     if (bigint->digit_count == 0) {
   2463         return LLVMConstNull(type_ref);
   2464     }
   2465     LLVMValueRef unsigned_val;
   2466     if (bigint->digit_count == 1) {
   2467         unsigned_val = LLVMConstInt(type_ref, bigint_ptr(bigint)[0], false);
   2468     } else {
   2469         unsigned_val = LLVMConstIntOfArbitraryPrecision(type_ref, bigint->digit_count, bigint_ptr(bigint));
   2470     }
   2471     if (bigint->is_negative) {
   2472         return LLVMConstNeg(unsigned_val);
   2473     } else {
   2474         return unsigned_val;
   2475     }
   2476 }
   2477 
   2478 static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast_math,
   2479         LLVMValueRef val1, LLVMValueRef val2,
   2480         ZigType *type_entry, DivKind div_kind)
   2481 {
   2482     ZigLLVMSetFastMath(g->builder, want_fast_math);
   2483 
   2484     LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, type_entry));
   2485     if (want_runtime_safety && (want_fast_math || type_entry->id != ZigTypeIdFloat)) {
   2486         LLVMValueRef is_zero_bit;
   2487         if (type_entry->id == ZigTypeIdInt) {
   2488             is_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, val2, zero, "");
   2489         } else if (type_entry->id == ZigTypeIdFloat) {
   2490             is_zero_bit = LLVMBuildFCmp(g->builder, LLVMRealOEQ, val2, zero, "");
   2491         } else {
   2492             zig_unreachable();
   2493         }
   2494         LLVMBasicBlockRef div_zero_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivZeroFail");
   2495         LLVMBasicBlockRef div_zero_ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivZeroOk");
   2496         LLVMBuildCondBr(g->builder, is_zero_bit, div_zero_fail_block, div_zero_ok_block);
   2497 
   2498         LLVMPositionBuilderAtEnd(g->builder, div_zero_fail_block);
   2499         gen_safety_crash(g, PanicMsgIdDivisionByZero);
   2500 
   2501         LLVMPositionBuilderAtEnd(g->builder, div_zero_ok_block);
   2502 
   2503         if (type_entry->id == ZigTypeIdInt && type_entry->data.integral.is_signed) {
   2504             LLVMValueRef neg_1_value = LLVMConstInt(get_llvm_type(g, type_entry), -1, true);
   2505             BigInt int_min_bi = {0};
   2506             eval_min_max_value_int(g, type_entry, &int_min_bi, false);
   2507             LLVMValueRef int_min_value = bigint_to_llvm_const(get_llvm_type(g, type_entry), &int_min_bi);
   2508             LLVMBasicBlockRef overflow_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivOverflowFail");
   2509             LLVMBasicBlockRef overflow_ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivOverflowOk");
   2510             LLVMValueRef num_is_int_min = LLVMBuildICmp(g->builder, LLVMIntEQ, val1, int_min_value, "");
   2511             LLVMValueRef den_is_neg_1 = LLVMBuildICmp(g->builder, LLVMIntEQ, val2, neg_1_value, "");
   2512             LLVMValueRef overflow_fail_bit = LLVMBuildAnd(g->builder, num_is_int_min, den_is_neg_1, "");
   2513             LLVMBuildCondBr(g->builder, overflow_fail_bit, overflow_fail_block, overflow_ok_block);
   2514 
   2515             LLVMPositionBuilderAtEnd(g->builder, overflow_fail_block);
   2516             gen_safety_crash(g, PanicMsgIdIntegerOverflow);
   2517 
   2518             LLVMPositionBuilderAtEnd(g->builder, overflow_ok_block);
   2519         }
   2520     }
   2521 
   2522     if (type_entry->id == ZigTypeIdFloat) {
   2523         LLVMValueRef result = LLVMBuildFDiv(g->builder, val1, val2, "");
   2524         switch (div_kind) {
   2525             case DivKindFloat:
   2526                 return result;
   2527             case DivKindExact:
   2528                 if (want_runtime_safety) {
   2529                     LLVMValueRef floored = gen_float_op(g, result, type_entry, BuiltinFnIdFloor);
   2530                     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactOk");
   2531                     LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
   2532                     LLVMValueRef ok_bit = LLVMBuildFCmp(g->builder, LLVMRealOEQ, floored, result, "");
   2533 
   2534                     LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   2535 
   2536                     LLVMPositionBuilderAtEnd(g->builder, fail_block);
   2537                     gen_safety_crash(g, PanicMsgIdExactDivisionRemainder);
   2538 
   2539                     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   2540                 }
   2541                 return result;
   2542             case DivKindTrunc:
   2543                 {
   2544                     LLVMBasicBlockRef ltz_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivTruncLTZero");
   2545                     LLVMBasicBlockRef gez_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivTruncGEZero");
   2546                     LLVMBasicBlockRef end_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivTruncEnd");
   2547                     LLVMValueRef ltz = LLVMBuildFCmp(g->builder, LLVMRealOLT, val1, zero, "");
   2548                     LLVMBuildCondBr(g->builder, ltz, ltz_block, gez_block);
   2549 
   2550                     LLVMPositionBuilderAtEnd(g->builder, ltz_block);
   2551                     LLVMValueRef ceiled = gen_float_op(g, result, type_entry, BuiltinFnIdCeil);
   2552                     LLVMBasicBlockRef ceiled_end_block = LLVMGetInsertBlock(g->builder);
   2553                     LLVMBuildBr(g->builder, end_block);
   2554 
   2555                     LLVMPositionBuilderAtEnd(g->builder, gez_block);
   2556                     LLVMValueRef floored = gen_float_op(g, result, type_entry, BuiltinFnIdFloor);
   2557                     LLVMBasicBlockRef floored_end_block = LLVMGetInsertBlock(g->builder);
   2558                     LLVMBuildBr(g->builder, end_block);
   2559 
   2560                     LLVMPositionBuilderAtEnd(g->builder, end_block);
   2561                     LLVMValueRef phi = LLVMBuildPhi(g->builder, get_llvm_type(g, type_entry), "");
   2562                     LLVMValueRef incoming_values[] = { ceiled, floored };
   2563                     LLVMBasicBlockRef incoming_blocks[] = { ceiled_end_block, floored_end_block };
   2564                     LLVMAddIncoming(phi, incoming_values, incoming_blocks, 2);
   2565                     return phi;
   2566                 }
   2567             case DivKindFloor:
   2568                 return gen_float_op(g, result, type_entry, BuiltinFnIdFloor);
   2569         }
   2570         zig_unreachable();
   2571     }
   2572 
   2573     assert(type_entry->id == ZigTypeIdInt);
   2574 
   2575     switch (div_kind) {
   2576         case DivKindFloat:
   2577             zig_unreachable();
   2578         case DivKindTrunc:
   2579             if (type_entry->data.integral.is_signed) {
   2580                 return LLVMBuildSDiv(g->builder, val1, val2, "");
   2581             } else {
   2582                 return LLVMBuildUDiv(g->builder, val1, val2, "");
   2583             }
   2584         case DivKindExact:
   2585             if (want_runtime_safety) {
   2586                 LLVMValueRef remainder_val;
   2587                 if (type_entry->data.integral.is_signed) {
   2588                     remainder_val = LLVMBuildSRem(g->builder, val1, val2, "");
   2589                 } else {
   2590                     remainder_val = LLVMBuildURem(g->builder, val1, val2, "");
   2591                 }
   2592                 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, remainder_val, zero, "");
   2593 
   2594                 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactOk");
   2595                 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
   2596                 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   2597 
   2598                 LLVMPositionBuilderAtEnd(g->builder, fail_block);
   2599                 gen_safety_crash(g, PanicMsgIdExactDivisionRemainder);
   2600 
   2601                 LLVMPositionBuilderAtEnd(g->builder, ok_block);
   2602             }
   2603             if (type_entry->data.integral.is_signed) {
   2604                 return LLVMBuildExactSDiv(g->builder, val1, val2, "");
   2605             } else {
   2606                 return LLVMBuildExactUDiv(g->builder, val1, val2, "");
   2607             }
   2608         case DivKindFloor:
   2609             {
   2610                 if (!type_entry->data.integral.is_signed) {
   2611                     return LLVMBuildUDiv(g->builder, val1, val2, "");
   2612                 }
   2613                 // const d = @divTrunc(a, b);
   2614                 // const r = @rem(a, b);
   2615                 // return if (r == 0) d else d - ((a < 0) ^ (b < 0));
   2616 
   2617                 LLVMValueRef div_trunc = LLVMBuildSDiv(g->builder, val1, val2, "");
   2618                 LLVMValueRef rem = LLVMBuildSRem(g->builder, val1, val2, "");
   2619                 LLVMValueRef rem_eq_0 = LLVMBuildICmp(g->builder, LLVMIntEQ, rem, zero, "");
   2620                 LLVMValueRef a_lt_0 = LLVMBuildICmp(g->builder, LLVMIntSLT, val1, zero, "");
   2621                 LLVMValueRef b_lt_0 = LLVMBuildICmp(g->builder, LLVMIntSLT, val2, zero, "");
   2622                 LLVMValueRef a_b_xor = LLVMBuildXor(g->builder, a_lt_0, b_lt_0, "");
   2623                 LLVMValueRef a_b_xor_ext = LLVMBuildZExt(g->builder, a_b_xor, LLVMTypeOf(div_trunc), "");
   2624                 LLVMValueRef d_sub_xor = LLVMBuildSub(g->builder, div_trunc, a_b_xor_ext, "");
   2625                 return LLVMBuildSelect(g->builder, rem_eq_0, div_trunc, d_sub_xor, "");
   2626             }
   2627     }
   2628     zig_unreachable();
   2629 }
   2630 
   2631 enum RemKind {
   2632     RemKindRem,
   2633     RemKindMod,
   2634 };
   2635 
   2636 static LLVMValueRef gen_rem(CodeGen *g, bool want_runtime_safety, bool want_fast_math,
   2637         LLVMValueRef val1, LLVMValueRef val2,
   2638         ZigType *type_entry, RemKind rem_kind)
   2639 {
   2640     ZigLLVMSetFastMath(g->builder, want_fast_math);
   2641 
   2642     LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, type_entry));
   2643     if (want_runtime_safety) {
   2644         LLVMValueRef is_zero_bit;
   2645         if (type_entry->id == ZigTypeIdInt) {
   2646             LLVMIntPredicate pred = type_entry->data.integral.is_signed ? LLVMIntSLE : LLVMIntEQ;
   2647             is_zero_bit = LLVMBuildICmp(g->builder, pred, val2, zero, "");
   2648         } else if (type_entry->id == ZigTypeIdFloat) {
   2649             is_zero_bit = LLVMBuildFCmp(g->builder, LLVMRealOEQ, val2, zero, "");
   2650         } else {
   2651             zig_unreachable();
   2652         }
   2653         LLVMBasicBlockRef rem_zero_ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "RemZeroOk");
   2654         LLVMBasicBlockRef rem_zero_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "RemZeroFail");
   2655         LLVMBuildCondBr(g->builder, is_zero_bit, rem_zero_fail_block, rem_zero_ok_block);
   2656 
   2657         LLVMPositionBuilderAtEnd(g->builder, rem_zero_fail_block);
   2658         gen_safety_crash(g, PanicMsgIdRemainderDivisionByZero);
   2659 
   2660         LLVMPositionBuilderAtEnd(g->builder, rem_zero_ok_block);
   2661     }
   2662 
   2663     if (type_entry->id == ZigTypeIdFloat) {
   2664         if (rem_kind == RemKindRem) {
   2665             return LLVMBuildFRem(g->builder, val1, val2, "");
   2666         } else {
   2667             LLVMValueRef a = LLVMBuildFRem(g->builder, val1, val2, "");
   2668             LLVMValueRef b = LLVMBuildFAdd(g->builder, a, val2, "");
   2669             LLVMValueRef c = LLVMBuildFRem(g->builder, b, val2, "");
   2670             LLVMValueRef ltz = LLVMBuildFCmp(g->builder, LLVMRealOLT, val1, zero, "");
   2671             return LLVMBuildSelect(g->builder, ltz, c, a, "");
   2672         }
   2673     } else {
   2674         assert(type_entry->id == ZigTypeIdInt);
   2675         if (type_entry->data.integral.is_signed) {
   2676             if (rem_kind == RemKindRem) {
   2677                 return LLVMBuildSRem(g->builder, val1, val2, "");
   2678             } else {
   2679                 LLVMValueRef a = LLVMBuildSRem(g->builder, val1, val2, "");
   2680                 LLVMValueRef b = LLVMBuildNSWAdd(g->builder, a, val2, "");
   2681                 LLVMValueRef c = LLVMBuildSRem(g->builder, b, val2, "");
   2682                 LLVMValueRef ltz = LLVMBuildICmp(g->builder, LLVMIntSLT, val1, zero, "");
   2683                 return LLVMBuildSelect(g->builder, ltz, c, a, "");
   2684             }
   2685         } else {
   2686             return LLVMBuildURem(g->builder, val1, val2, "");
   2687         }
   2688     }
   2689 
   2690 }
   2691 
   2692 static LLVMValueRef ir_render_bin_op(CodeGen *g, IrExecutable *executable,
   2693         IrInstructionBinOp *bin_op_instruction)
   2694 {
   2695     IrBinOp op_id = bin_op_instruction->op_id;
   2696     IrInstruction *op1 = bin_op_instruction->op1;
   2697     IrInstruction *op2 = bin_op_instruction->op2;
   2698 
   2699     assert(op1->value.type == op2->value.type || op_id == IrBinOpBitShiftLeftLossy ||
   2700         op_id == IrBinOpBitShiftLeftExact || op_id == IrBinOpBitShiftRightLossy ||
   2701         op_id == IrBinOpBitShiftRightExact ||
   2702         (op1->value.type->id == ZigTypeIdErrorSet && op2->value.type->id == ZigTypeIdErrorSet) ||
   2703         (op1->value.type->id == ZigTypeIdPointer &&
   2704             (op_id == IrBinOpAdd || op_id == IrBinOpSub) &&
   2705             op1->value.type->data.pointer.ptr_len != PtrLenSingle)
   2706     );
   2707     ZigType *operand_type = op1->value.type;
   2708     ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;
   2709 
   2710     bool want_runtime_safety = bin_op_instruction->safety_check_on &&
   2711         ir_want_runtime_safety(g, &bin_op_instruction->base);
   2712 
   2713     LLVMValueRef op1_value = ir_llvm_value(g, op1);
   2714     LLVMValueRef op2_value = ir_llvm_value(g, op2);
   2715 
   2716 
   2717     switch (op_id) {
   2718         case IrBinOpInvalid:
   2719         case IrBinOpArrayCat:
   2720         case IrBinOpArrayMult:
   2721         case IrBinOpRemUnspecified:
   2722         case IrBinOpMergeErrorSets:
   2723             zig_unreachable();
   2724         case IrBinOpBoolOr:
   2725             return LLVMBuildOr(g->builder, op1_value, op2_value, "");
   2726         case IrBinOpBoolAnd:
   2727             return LLVMBuildAnd(g->builder, op1_value, op2_value, "");
   2728         case IrBinOpCmpEq:
   2729         case IrBinOpCmpNotEq:
   2730         case IrBinOpCmpLessThan:
   2731         case IrBinOpCmpGreaterThan:
   2732         case IrBinOpCmpLessOrEq:
   2733         case IrBinOpCmpGreaterOrEq:
   2734             if (scalar_type->id == ZigTypeIdFloat) {
   2735                 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, &bin_op_instruction->base));
   2736                 LLVMRealPredicate pred = cmp_op_to_real_predicate(op_id);
   2737                 return LLVMBuildFCmp(g->builder, pred, op1_value, op2_value, "");
   2738             } else if (scalar_type->id == ZigTypeIdInt) {
   2739                 LLVMIntPredicate pred = cmp_op_to_int_predicate(op_id, scalar_type->data.integral.is_signed);
   2740                 return LLVMBuildICmp(g->builder, pred, op1_value, op2_value, "");
   2741             } else if (scalar_type->id == ZigTypeIdEnum ||
   2742                     scalar_type->id == ZigTypeIdErrorSet ||
   2743                     scalar_type->id == ZigTypeIdBool ||
   2744                     get_codegen_ptr_type(scalar_type) != nullptr)
   2745             {
   2746                 LLVMIntPredicate pred = cmp_op_to_int_predicate(op_id, false);
   2747                 return LLVMBuildICmp(g->builder, pred, op1_value, op2_value, "");
   2748             } else {
   2749                 zig_unreachable();
   2750             }
   2751         case IrBinOpMult:
   2752         case IrBinOpMultWrap:
   2753         case IrBinOpAdd:
   2754         case IrBinOpAddWrap:
   2755         case IrBinOpSub:
   2756         case IrBinOpSubWrap: {
   2757             bool is_wrapping = (op_id == IrBinOpSubWrap || op_id == IrBinOpAddWrap || op_id == IrBinOpMultWrap);
   2758             AddSubMul add_sub_mul =
   2759                 op_id == IrBinOpAdd || op_id == IrBinOpAddWrap ? AddSubMulAdd :
   2760                 op_id == IrBinOpSub || op_id == IrBinOpSubWrap ? AddSubMulSub :
   2761                 AddSubMulMul;
   2762 
   2763             if (scalar_type->id == ZigTypeIdPointer) {
   2764                 assert(scalar_type->data.pointer.ptr_len != PtrLenSingle);
   2765                 LLVMValueRef subscript_value;
   2766                 if (operand_type->id == ZigTypeIdVector)
   2767                     zig_panic("TODO: Implement vector operations on pointers.");
   2768 
   2769                 switch (add_sub_mul) {
   2770                     case AddSubMulAdd:
   2771                         subscript_value = op2_value;
   2772                         break;
   2773                     case AddSubMulSub:
   2774                         subscript_value = LLVMBuildNeg(g->builder, op2_value, "");
   2775                         break;
   2776                     case AddSubMulMul:
   2777                         zig_unreachable();
   2778                 }
   2779 
   2780                 // TODO runtime safety
   2781                 return LLVMBuildInBoundsGEP(g->builder, op1_value, &subscript_value, 1, "");
   2782             } else if (scalar_type->id == ZigTypeIdFloat) {
   2783                 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, &bin_op_instruction->base));
   2784                 return float_op[add_sub_mul](g->builder, op1_value, op2_value, "");
   2785             } else if (scalar_type->id == ZigTypeIdInt) {
   2786                 if (is_wrapping) {
   2787                     return wrap_op[add_sub_mul](g->builder, op1_value, op2_value, "");
   2788                 } else if (want_runtime_safety) {
   2789                     return gen_overflow_op(g, operand_type, add_sub_mul, op1_value, op2_value);
   2790                 } else if (scalar_type->data.integral.is_signed) {
   2791                     return signed_op[add_sub_mul](g->builder, op1_value, op2_value, "");
   2792                 } else {
   2793                     return unsigned_op[add_sub_mul](g->builder, op1_value, op2_value, "");
   2794                 }
   2795             } else {
   2796                 zig_unreachable();
   2797             }
   2798         }
   2799         case IrBinOpBinOr:
   2800             return LLVMBuildOr(g->builder, op1_value, op2_value, "");
   2801         case IrBinOpBinXor:
   2802             return LLVMBuildXor(g->builder, op1_value, op2_value, "");
   2803         case IrBinOpBinAnd:
   2804             return LLVMBuildAnd(g->builder, op1_value, op2_value, "");
   2805         case IrBinOpBitShiftLeftLossy:
   2806         case IrBinOpBitShiftLeftExact:
   2807             {
   2808                 assert(scalar_type->id == ZigTypeIdInt);
   2809                 LLVMValueRef op2_casted = gen_widen_or_shorten(g, false, op2->value.type, scalar_type, op2_value);
   2810                 bool is_sloppy = (op_id == IrBinOpBitShiftLeftLossy);
   2811                 if (is_sloppy) {
   2812                     return LLVMBuildShl(g->builder, op1_value, op2_casted, "");
   2813                 } else if (want_runtime_safety) {
   2814                     return gen_overflow_shl_op(g, scalar_type, op1_value, op2_casted);
   2815                 } else if (scalar_type->data.integral.is_signed) {
   2816                     return ZigLLVMBuildNSWShl(g->builder, op1_value, op2_casted, "");
   2817                 } else {
   2818                     return ZigLLVMBuildNUWShl(g->builder, op1_value, op2_casted, "");
   2819                 }
   2820             }
   2821         case IrBinOpBitShiftRightLossy:
   2822         case IrBinOpBitShiftRightExact:
   2823             {
   2824                 assert(scalar_type->id == ZigTypeIdInt);
   2825                 LLVMValueRef op2_casted = gen_widen_or_shorten(g, false, op2->value.type, scalar_type, op2_value);
   2826                 bool is_sloppy = (op_id == IrBinOpBitShiftRightLossy);
   2827                 if (is_sloppy) {
   2828                     if (scalar_type->data.integral.is_signed) {
   2829                         return LLVMBuildAShr(g->builder, op1_value, op2_casted, "");
   2830                     } else {
   2831                         return LLVMBuildLShr(g->builder, op1_value, op2_casted, "");
   2832                     }
   2833                 } else if (want_runtime_safety) {
   2834                     return gen_overflow_shr_op(g, scalar_type, op1_value, op2_casted);
   2835                 } else if (scalar_type->data.integral.is_signed) {
   2836                     return ZigLLVMBuildAShrExact(g->builder, op1_value, op2_casted, "");
   2837                 } else {
   2838                     return ZigLLVMBuildLShrExact(g->builder, op1_value, op2_casted, "");
   2839                 }
   2840             }
   2841         case IrBinOpDivUnspecified:
   2842             return gen_div(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
   2843                     op1_value, op2_value, scalar_type, DivKindFloat);
   2844         case IrBinOpDivExact:
   2845             return gen_div(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
   2846                     op1_value, op2_value, scalar_type, DivKindExact);
   2847         case IrBinOpDivTrunc:
   2848             return gen_div(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
   2849                     op1_value, op2_value, scalar_type, DivKindTrunc);
   2850         case IrBinOpDivFloor:
   2851             return gen_div(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
   2852                     op1_value, op2_value, scalar_type, DivKindFloor);
   2853         case IrBinOpRemRem:
   2854             return gen_rem(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
   2855                     op1_value, op2_value, scalar_type, RemKindRem);
   2856         case IrBinOpRemMod:
   2857             return gen_rem(g, want_runtime_safety, ir_want_fast_math(g, &bin_op_instruction->base),
   2858                     op1_value, op2_value, scalar_type, RemKindMod);
   2859     }
   2860     zig_unreachable();
   2861 }
   2862 
   2863 static void add_error_range_check(CodeGen *g, ZigType *err_set_type, ZigType *int_type, LLVMValueRef target_val) {
   2864     assert(err_set_type->id == ZigTypeIdErrorSet);
   2865 
   2866     if (type_is_global_error_set(err_set_type)) {
   2867         LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, int_type));
   2868         LLVMValueRef neq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntNE, target_val, zero, "");
   2869         LLVMValueRef ok_bit;
   2870 
   2871         BigInt biggest_possible_err_val = {0};
   2872         eval_min_max_value_int(g, int_type, &biggest_possible_err_val, true);
   2873 
   2874         if (bigint_fits_in_bits(&biggest_possible_err_val, 64, false) &&
   2875             bigint_as_unsigned(&biggest_possible_err_val) < g->errors_by_index.length)
   2876         {
   2877             ok_bit = neq_zero_bit;
   2878         } else {
   2879             LLVMValueRef error_value_count = LLVMConstInt(get_llvm_type(g, int_type), g->errors_by_index.length, false);
   2880             LLVMValueRef in_bounds_bit = LLVMBuildICmp(g->builder, LLVMIntULT, target_val, error_value_count, "");
   2881             ok_bit = LLVMBuildAnd(g->builder, neq_zero_bit, in_bounds_bit, "");
   2882         }
   2883 
   2884         LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "IntToErrOk");
   2885         LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "IntToErrFail");
   2886 
   2887         LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   2888 
   2889         LLVMPositionBuilderAtEnd(g->builder, fail_block);
   2890         gen_safety_crash(g, PanicMsgIdInvalidErrorCode);
   2891 
   2892         LLVMPositionBuilderAtEnd(g->builder, ok_block);
   2893     } else {
   2894         LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "IntToErrOk");
   2895         LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "IntToErrFail");
   2896 
   2897         uint32_t err_count = err_set_type->data.error_set.err_count;
   2898         LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, target_val, fail_block, err_count);
   2899         for (uint32_t i = 0; i < err_count; i += 1) {
   2900             LLVMValueRef case_value = LLVMConstInt(get_llvm_type(g, g->err_tag_type),
   2901                     err_set_type->data.error_set.errors[i]->value, false);
   2902             LLVMAddCase(switch_instr, case_value, ok_block);
   2903         }
   2904 
   2905         LLVMPositionBuilderAtEnd(g->builder, fail_block);
   2906         gen_safety_crash(g, PanicMsgIdInvalidErrorCode);
   2907 
   2908         LLVMPositionBuilderAtEnd(g->builder, ok_block);
   2909     }
   2910 }
   2911 
   2912 static LLVMValueRef ir_render_resize_slice(CodeGen *g, IrExecutable *executable,
   2913         IrInstructionResizeSlice *instruction)
   2914 {
   2915     ZigType *actual_type = instruction->operand->value.type;
   2916     ZigType *wanted_type = instruction->base.value.type;
   2917     LLVMValueRef expr_val = ir_llvm_value(g, instruction->operand);
   2918     assert(expr_val);
   2919 
   2920     LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   2921     assert(wanted_type->id == ZigTypeIdStruct);
   2922     assert(wanted_type->data.structure.is_slice);
   2923     assert(actual_type->id == ZigTypeIdStruct);
   2924     assert(actual_type->data.structure.is_slice);
   2925 
   2926     ZigType *actual_pointer_type = actual_type->data.structure.fields[0].type_entry;
   2927     ZigType *actual_child_type = actual_pointer_type->data.pointer.child_type;
   2928     ZigType *wanted_pointer_type = wanted_type->data.structure.fields[0].type_entry;
   2929     ZigType *wanted_child_type = wanted_pointer_type->data.pointer.child_type;
   2930 
   2931 
   2932     size_t actual_ptr_index = actual_type->data.structure.fields[slice_ptr_index].gen_index;
   2933     size_t actual_len_index = actual_type->data.structure.fields[slice_len_index].gen_index;
   2934     size_t wanted_ptr_index = wanted_type->data.structure.fields[slice_ptr_index].gen_index;
   2935     size_t wanted_len_index = wanted_type->data.structure.fields[slice_len_index].gen_index;
   2936 
   2937     LLVMValueRef src_ptr_ptr = LLVMBuildStructGEP(g->builder, expr_val, (unsigned)actual_ptr_index, "");
   2938     LLVMValueRef src_ptr = gen_load_untyped(g, src_ptr_ptr, 0, false, "");
   2939     LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, src_ptr,
   2940             get_llvm_type(g, wanted_type->data.structure.fields[0].type_entry), "");
   2941     LLVMValueRef dest_ptr_ptr = LLVMBuildStructGEP(g->builder, result_loc,
   2942             (unsigned)wanted_ptr_index, "");
   2943     gen_store_untyped(g, src_ptr_casted, dest_ptr_ptr, 0, false);
   2944 
   2945     LLVMValueRef src_len_ptr = LLVMBuildStructGEP(g->builder, expr_val, (unsigned)actual_len_index, "");
   2946     LLVMValueRef src_len = gen_load_untyped(g, src_len_ptr, 0, false, "");
   2947     uint64_t src_size = type_size(g, actual_child_type);
   2948     uint64_t dest_size = type_size(g, wanted_child_type);
   2949 
   2950     LLVMValueRef new_len;
   2951     if (dest_size == 1) {
   2952         LLVMValueRef src_size_val = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, src_size, false);
   2953         new_len = LLVMBuildMul(g->builder, src_len, src_size_val, "");
   2954     } else if (src_size == 1) {
   2955         LLVMValueRef dest_size_val = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, dest_size, false);
   2956         if (ir_want_runtime_safety(g, &instruction->base)) {
   2957             LLVMValueRef remainder_val = LLVMBuildURem(g->builder, src_len, dest_size_val, "");
   2958             LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_usize->llvm_type);
   2959             LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, remainder_val, zero, "");
   2960             LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "SliceWidenOk");
   2961             LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "SliceWidenFail");
   2962             LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   2963 
   2964             LLVMPositionBuilderAtEnd(g->builder, fail_block);
   2965             gen_safety_crash(g, PanicMsgIdSliceWidenRemainder);
   2966 
   2967             LLVMPositionBuilderAtEnd(g->builder, ok_block);
   2968         }
   2969         new_len = LLVMBuildExactUDiv(g->builder, src_len, dest_size_val, "");
   2970     } else {
   2971         zig_unreachable();
   2972     }
   2973 
   2974     LLVMValueRef dest_len_ptr = LLVMBuildStructGEP(g->builder, result_loc, (unsigned)wanted_len_index, "");
   2975     gen_store_untyped(g, new_len, dest_len_ptr, 0, false);
   2976 
   2977     return result_loc;
   2978 }
   2979 
   2980 static LLVMValueRef ir_render_cast(CodeGen *g, IrExecutable *executable,
   2981         IrInstructionCast *cast_instruction)
   2982 {
   2983     ZigType *actual_type = cast_instruction->value->value.type;
   2984     ZigType *wanted_type = cast_instruction->base.value.type;
   2985     LLVMValueRef expr_val = ir_llvm_value(g, cast_instruction->value);
   2986     assert(expr_val);
   2987 
   2988     switch (cast_instruction->cast_op) {
   2989         case CastOpNoCast:
   2990         case CastOpNumLitToConcrete:
   2991             zig_unreachable();
   2992         case CastOpNoop:
   2993             return expr_val;
   2994         case CastOpIntToFloat:
   2995             assert(actual_type->id == ZigTypeIdInt);
   2996             if (actual_type->data.integral.is_signed) {
   2997                 return LLVMBuildSIToFP(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   2998             } else {
   2999                 return LLVMBuildUIToFP(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   3000             }
   3001         case CastOpFloatToInt: {
   3002             assert(wanted_type->id == ZigTypeIdInt);
   3003             ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, &cast_instruction->base));
   3004 
   3005             bool want_safety = ir_want_runtime_safety(g, &cast_instruction->base);
   3006 
   3007             LLVMValueRef result;
   3008             if (wanted_type->data.integral.is_signed) {
   3009                 result = LLVMBuildFPToSI(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   3010             } else {
   3011                 result = LLVMBuildFPToUI(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   3012             }
   3013 
   3014             if (want_safety) {
   3015                 LLVMValueRef back_to_float;
   3016                 if (wanted_type->data.integral.is_signed) {
   3017                     back_to_float = LLVMBuildSIToFP(g->builder, result, LLVMTypeOf(expr_val), "");
   3018                 } else {
   3019                     back_to_float = LLVMBuildUIToFP(g->builder, result, LLVMTypeOf(expr_val), "");
   3020                 }
   3021                 LLVMValueRef difference = LLVMBuildFSub(g->builder, expr_val, back_to_float, "");
   3022                 LLVMValueRef one_pos = LLVMConstReal(LLVMTypeOf(expr_val), 1.0f);
   3023                 LLVMValueRef one_neg = LLVMConstReal(LLVMTypeOf(expr_val), -1.0f);
   3024                 LLVMValueRef ok_bit_pos = LLVMBuildFCmp(g->builder, LLVMRealOLT, difference, one_pos, "");
   3025                 LLVMValueRef ok_bit_neg = LLVMBuildFCmp(g->builder, LLVMRealOGT, difference, one_neg, "");
   3026                 LLVMValueRef ok_bit = LLVMBuildAnd(g->builder, ok_bit_pos, ok_bit_neg, "");
   3027                 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "FloatCheckOk");
   3028                 LLVMBasicBlockRef bad_block = LLVMAppendBasicBlock(g->cur_fn_val, "FloatCheckFail");
   3029                 LLVMBuildCondBr(g->builder, ok_bit, ok_block, bad_block);
   3030                 LLVMPositionBuilderAtEnd(g->builder, bad_block);
   3031                 gen_safety_crash(g, PanicMsgIdFloatToInt);
   3032                 LLVMPositionBuilderAtEnd(g->builder, ok_block);
   3033             }
   3034             return result;
   3035         }
   3036         case CastOpBoolToInt:
   3037             assert(wanted_type->id == ZigTypeIdInt);
   3038             assert(actual_type->id == ZigTypeIdBool);
   3039             return LLVMBuildZExt(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   3040         case CastOpErrSet:
   3041             if (ir_want_runtime_safety(g, &cast_instruction->base)) {
   3042                 add_error_range_check(g, wanted_type, g->err_tag_type, expr_val);
   3043             }
   3044             return expr_val;
   3045         case CastOpBitCast:
   3046             return LLVMBuildBitCast(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
   3047     }
   3048     zig_unreachable();
   3049 }
   3050 
   3051 static LLVMValueRef ir_render_ptr_of_array_to_slice(CodeGen *g, IrExecutable *executable,
   3052         IrInstructionPtrOfArrayToSlice *instruction)
   3053 {
   3054     ZigType *actual_type = instruction->operand->value.type;
   3055     LLVMValueRef expr_val = ir_llvm_value(g, instruction->operand);
   3056     assert(expr_val);
   3057 
   3058     LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   3059 
   3060     assert(actual_type->id == ZigTypeIdPointer);
   3061     ZigType *array_type = actual_type->data.pointer.child_type;
   3062     assert(array_type->id == ZigTypeIdArray);
   3063 
   3064     LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, result_loc, slice_ptr_index, "");
   3065     LLVMValueRef indices[] = {
   3066         LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
   3067         LLVMConstInt(g->builtin_types.entry_usize->llvm_type, 0, false),
   3068     };
   3069     LLVMValueRef slice_start_ptr = LLVMBuildInBoundsGEP(g->builder, expr_val, indices, 2, "");
   3070     gen_store_untyped(g, slice_start_ptr, ptr_field_ptr, 0, false);
   3071 
   3072     LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, result_loc, slice_len_index, "");
   3073     LLVMValueRef len_value = LLVMConstInt(g->builtin_types.entry_usize->llvm_type,
   3074             array_type->data.array.len, false);
   3075     gen_store_untyped(g, len_value, len_field_ptr, 0, false);
   3076 
   3077     return result_loc;
   3078 }
   3079 
   3080 static LLVMValueRef ir_render_ptr_cast(CodeGen *g, IrExecutable *executable,
   3081         IrInstructionPtrCastGen *instruction)
   3082 {
   3083     ZigType *wanted_type = instruction->base.value.type;
   3084     if (!type_has_bits(wanted_type)) {
   3085         return nullptr;
   3086     }
   3087     LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
   3088     LLVMValueRef result_ptr = LLVMBuildBitCast(g->builder, ptr, get_llvm_type(g, wanted_type), "");
   3089     bool want_safety_check = instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base);
   3090     if (!want_safety_check || ptr_allows_addr_zero(wanted_type))
   3091         return result_ptr;
   3092 
   3093     LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(result_ptr));
   3094     LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntNE, result_ptr, zero, "");
   3095     LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrCastFail");
   3096     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrCastOk");
   3097     LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   3098 
   3099     LLVMPositionBuilderAtEnd(g->builder, fail_block);
   3100     gen_safety_crash(g, PanicMsgIdPtrCastNull);
   3101 
   3102     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   3103     return result_ptr;
   3104 }
   3105 
   3106 static LLVMValueRef ir_render_bit_cast(CodeGen *g, IrExecutable *executable,
   3107         IrInstructionBitCastGen *instruction)
   3108 {
   3109     ZigType *wanted_type = instruction->base.value.type;
   3110     ZigType *actual_type = instruction->operand->value.type;
   3111     LLVMValueRef value = ir_llvm_value(g, instruction->operand);
   3112 
   3113     bool wanted_is_ptr = handle_is_ptr(wanted_type);
   3114     bool actual_is_ptr = handle_is_ptr(actual_type);
   3115     if (wanted_is_ptr == actual_is_ptr) {
   3116         // We either bitcast the value directly or bitcast the pointer which does a pointer cast
   3117         LLVMTypeRef wanted_type_ref = wanted_is_ptr ?
   3118             LLVMPointerType(get_llvm_type(g, wanted_type), 0) : get_llvm_type(g, wanted_type);
   3119         return LLVMBuildBitCast(g->builder, value, wanted_type_ref, "");
   3120     } else if (actual_is_ptr) {
   3121         LLVMTypeRef wanted_ptr_type_ref = LLVMPointerType(get_llvm_type(g, wanted_type), 0);
   3122         LLVMValueRef bitcasted_ptr = LLVMBuildBitCast(g->builder, value, wanted_ptr_type_ref, "");
   3123         uint32_t alignment = get_abi_alignment(g, actual_type);
   3124         return gen_load_untyped(g, bitcasted_ptr, alignment, false, "");
   3125     } else {
   3126         zig_unreachable();
   3127     }
   3128 }
   3129 
   3130 static LLVMValueRef ir_render_widen_or_shorten(CodeGen *g, IrExecutable *executable,
   3131         IrInstructionWidenOrShorten *instruction)
   3132 {
   3133     ZigType *actual_type = instruction->target->value.type;
   3134     // TODO instead of this logic, use the Noop instruction to change the type from
   3135     // enum_tag to the underlying int type
   3136     ZigType *int_type;
   3137     if (actual_type->id == ZigTypeIdEnum) {
   3138         int_type = actual_type->data.enumeration.tag_int_type;
   3139     } else {
   3140         int_type = actual_type;
   3141     }
   3142     LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   3143     return gen_widen_or_shorten(g, ir_want_runtime_safety(g, &instruction->base), int_type,
   3144             instruction->base.value.type, target_val);
   3145 }
   3146 
   3147 static LLVMValueRef ir_render_int_to_ptr(CodeGen *g, IrExecutable *executable, IrInstructionIntToPtr *instruction) {
   3148     ZigType *wanted_type = instruction->base.value.type;
   3149     LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   3150     if (!ptr_allows_addr_zero(wanted_type) && ir_want_runtime_safety(g, &instruction->base)) {
   3151         LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(target_val));
   3152         LLVMValueRef is_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, target_val, zero, "");
   3153         LLVMBasicBlockRef bad_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrToIntBad");
   3154         LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "PtrToIntOk");
   3155         LLVMBuildCondBr(g->builder, is_zero_bit, bad_block, ok_block);
   3156 
   3157         LLVMPositionBuilderAtEnd(g->builder, bad_block);
   3158         gen_safety_crash(g, PanicMsgIdPtrCastNull);
   3159 
   3160         LLVMPositionBuilderAtEnd(g->builder, ok_block);
   3161     }
   3162     return LLVMBuildIntToPtr(g->builder, target_val, get_llvm_type(g, wanted_type), "");
   3163 }
   3164 
   3165 static LLVMValueRef ir_render_ptr_to_int(CodeGen *g, IrExecutable *executable, IrInstructionPtrToInt *instruction) {
   3166     ZigType *wanted_type = instruction->base.value.type;
   3167     LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   3168     return LLVMBuildPtrToInt(g->builder, target_val, get_llvm_type(g, wanted_type), "");
   3169 }
   3170 
   3171 static LLVMValueRef ir_render_int_to_enum(CodeGen *g, IrExecutable *executable, IrInstructionIntToEnum *instruction) {
   3172     ZigType *wanted_type = instruction->base.value.type;
   3173     assert(wanted_type->id == ZigTypeIdEnum);
   3174     ZigType *tag_int_type = wanted_type->data.enumeration.tag_int_type;
   3175 
   3176     LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   3177     LLVMValueRef tag_int_value = gen_widen_or_shorten(g, ir_want_runtime_safety(g, &instruction->base),
   3178             instruction->target->value.type, tag_int_type, target_val);
   3179 
   3180     if (ir_want_runtime_safety(g, &instruction->base)) {
   3181         LLVMBasicBlockRef bad_value_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadValue");
   3182         LLVMBasicBlockRef ok_value_block = LLVMAppendBasicBlock(g->cur_fn_val, "OkValue");
   3183         size_t field_count = wanted_type->data.enumeration.src_field_count;
   3184         LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, tag_int_value, bad_value_block, field_count);
   3185         for (size_t field_i = 0; field_i < field_count; field_i += 1) {
   3186             LLVMValueRef this_tag_int_value = bigint_to_llvm_const(get_llvm_type(g, tag_int_type),
   3187                     &wanted_type->data.enumeration.fields[field_i].value);
   3188             LLVMAddCase(switch_instr, this_tag_int_value, ok_value_block);
   3189         }
   3190         LLVMPositionBuilderAtEnd(g->builder, bad_value_block);
   3191         gen_safety_crash(g, PanicMsgIdBadEnumValue);
   3192 
   3193         LLVMPositionBuilderAtEnd(g->builder, ok_value_block);
   3194     }
   3195     return tag_int_value;
   3196 }
   3197 
   3198 static LLVMValueRef ir_render_int_to_err(CodeGen *g, IrExecutable *executable, IrInstructionIntToErr *instruction) {
   3199     ZigType *wanted_type = instruction->base.value.type;
   3200     assert(wanted_type->id == ZigTypeIdErrorSet);
   3201 
   3202     ZigType *actual_type = instruction->target->value.type;
   3203     assert(actual_type->id == ZigTypeIdInt);
   3204     assert(!actual_type->data.integral.is_signed);
   3205 
   3206     LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   3207 
   3208     if (ir_want_runtime_safety(g, &instruction->base)) {
   3209         add_error_range_check(g, wanted_type, actual_type, target_val);
   3210     }
   3211 
   3212     return gen_widen_or_shorten(g, false, actual_type, g->err_tag_type, target_val);
   3213 }
   3214 
   3215 static LLVMValueRef ir_render_err_to_int(CodeGen *g, IrExecutable *executable, IrInstructionErrToInt *instruction) {
   3216     ZigType *wanted_type = instruction->base.value.type;
   3217     assert(wanted_type->id == ZigTypeIdInt);
   3218     assert(!wanted_type->data.integral.is_signed);
   3219 
   3220     ZigType *actual_type = instruction->target->value.type;
   3221     LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   3222 
   3223     if (actual_type->id == ZigTypeIdErrorSet) {
   3224         return gen_widen_or_shorten(g, ir_want_runtime_safety(g, &instruction->base),
   3225             g->err_tag_type, wanted_type, target_val);
   3226     } else if (actual_type->id == ZigTypeIdErrorUnion) {
   3227         // this should have been a compile time constant
   3228         assert(type_has_bits(actual_type->data.error_union.err_set_type));
   3229 
   3230         if (!type_has_bits(actual_type->data.error_union.payload_type)) {
   3231             return gen_widen_or_shorten(g, ir_want_runtime_safety(g, &instruction->base),
   3232                 g->err_tag_type, wanted_type, target_val);
   3233         } else {
   3234             zig_panic("TODO err to int when error union payload type not void");
   3235         }
   3236     } else {
   3237         zig_unreachable();
   3238     }
   3239 }
   3240 
   3241 static LLVMValueRef ir_render_unreachable(CodeGen *g, IrExecutable *executable,
   3242         IrInstructionUnreachable *unreachable_instruction)
   3243 {
   3244     if (ir_want_runtime_safety(g, &unreachable_instruction->base)) {
   3245         gen_safety_crash(g, PanicMsgIdUnreachable);
   3246     } else {
   3247         LLVMBuildUnreachable(g->builder);
   3248     }
   3249     return nullptr;
   3250 }
   3251 
   3252 static LLVMValueRef ir_render_cond_br(CodeGen *g, IrExecutable *executable,
   3253         IrInstructionCondBr *cond_br_instruction)
   3254 {
   3255     LLVMBuildCondBr(g->builder,
   3256             ir_llvm_value(g, cond_br_instruction->condition),
   3257             cond_br_instruction->then_block->llvm_block,
   3258             cond_br_instruction->else_block->llvm_block);
   3259     return nullptr;
   3260 }
   3261 
   3262 static LLVMValueRef ir_render_br(CodeGen *g, IrExecutable *executable, IrInstructionBr *br_instruction) {
   3263     LLVMBuildBr(g->builder, br_instruction->dest_block->llvm_block);
   3264     return nullptr;
   3265 }
   3266 
   3267 static LLVMValueRef ir_render_un_op(CodeGen *g, IrExecutable *executable, IrInstructionUnOp *un_op_instruction) {
   3268     IrUnOp op_id = un_op_instruction->op_id;
   3269     LLVMValueRef expr = ir_llvm_value(g, un_op_instruction->value);
   3270     ZigType *operand_type = un_op_instruction->value->value.type;
   3271     ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;
   3272 
   3273     switch (op_id) {
   3274         case IrUnOpInvalid:
   3275         case IrUnOpOptional:
   3276         case IrUnOpDereference:
   3277             zig_unreachable();
   3278         case IrUnOpNegation:
   3279         case IrUnOpNegationWrap:
   3280             {
   3281                 if (scalar_type->id == ZigTypeIdFloat) {
   3282                     ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, &un_op_instruction->base));
   3283                     return LLVMBuildFNeg(g->builder, expr, "");
   3284                 } else if (scalar_type->id == ZigTypeIdInt) {
   3285                     if (op_id == IrUnOpNegationWrap) {
   3286                         return LLVMBuildNeg(g->builder, expr, "");
   3287                     } else if (ir_want_runtime_safety(g, &un_op_instruction->base)) {
   3288                         LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(expr));
   3289                         return gen_overflow_op(g, operand_type, AddSubMulSub, zero, expr);
   3290                     } else if (scalar_type->data.integral.is_signed) {
   3291                         return LLVMBuildNSWNeg(g->builder, expr, "");
   3292                     } else {
   3293                         return LLVMBuildNUWNeg(g->builder, expr, "");
   3294                     }
   3295                 } else {
   3296                     zig_unreachable();
   3297                 }
   3298             }
   3299         case IrUnOpBinNot:
   3300             return LLVMBuildNot(g->builder, expr, "");
   3301     }
   3302 
   3303     zig_unreachable();
   3304 }
   3305 
   3306 static LLVMValueRef ir_render_bool_not(CodeGen *g, IrExecutable *executable, IrInstructionBoolNot *instruction) {
   3307     LLVMValueRef value = ir_llvm_value(g, instruction->value);
   3308     LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(value));
   3309     return LLVMBuildICmp(g->builder, LLVMIntEQ, value, zero, "");
   3310 }
   3311 
   3312 static void render_decl_var(CodeGen *g, ZigVar *var) {
   3313     if (!type_has_bits(var->var_type))
   3314         return;
   3315 
   3316     var->value_ref = ir_llvm_value(g, var->ptr_instruction);
   3317     gen_var_debug_decl(g, var);
   3318 }
   3319 
   3320 static LLVMValueRef ir_render_decl_var(CodeGen *g, IrExecutable *executable, IrInstructionDeclVarGen *instruction) {
   3321     instruction->var->ptr_instruction = instruction->var_ptr;
   3322     render_decl_var(g, instruction->var);
   3323     return nullptr;
   3324 }
   3325 
   3326 static LLVMValueRef ir_render_load_ptr(CodeGen *g, IrExecutable *executable, IrInstructionLoadPtrGen *instruction) {
   3327     ZigType *child_type = instruction->base.value.type;
   3328     if (!type_has_bits(child_type))
   3329         return nullptr;
   3330 
   3331     LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
   3332     ZigType *ptr_type = instruction->ptr->value.type;
   3333     assert(ptr_type->id == ZigTypeIdPointer);
   3334 
   3335     uint32_t host_int_bytes = ptr_type->data.pointer.host_int_bytes;
   3336     if (host_int_bytes == 0)
   3337         return get_handle_value(g, ptr, child_type, ptr_type);
   3338 
   3339     bool big_endian = g->is_big_endian;
   3340 
   3341     LLVMValueRef containing_int = gen_load(g, ptr, ptr_type, "");
   3342     uint32_t host_bit_count = LLVMGetIntTypeWidth(LLVMTypeOf(containing_int));
   3343     assert(host_bit_count == host_int_bytes * 8);
   3344     uint32_t size_in_bits = type_size_bits(g, child_type);
   3345 
   3346     uint32_t bit_offset = ptr_type->data.pointer.bit_offset_in_host;
   3347     uint32_t shift_amt = big_endian ? host_bit_count - bit_offset - size_in_bits : bit_offset;
   3348 
   3349     LLVMValueRef shift_amt_val = LLVMConstInt(LLVMTypeOf(containing_int), shift_amt, false);
   3350     LLVMValueRef shifted_value = LLVMBuildLShr(g->builder, containing_int, shift_amt_val, "");
   3351 
   3352     if (handle_is_ptr(child_type)) {
   3353         LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   3354         LLVMTypeRef same_size_int = LLVMIntType(size_in_bits);
   3355         LLVMValueRef truncated_int = LLVMBuildTrunc(g->builder, shifted_value, same_size_int, "");
   3356         LLVMValueRef bitcasted_ptr = LLVMBuildBitCast(g->builder, result_loc,
   3357                                                       LLVMPointerType(same_size_int, 0), "");
   3358         LLVMBuildStore(g->builder, truncated_int, bitcasted_ptr);
   3359         return result_loc;
   3360     }
   3361 
   3362     if (child_type->id == ZigTypeIdFloat) {
   3363         LLVMTypeRef same_size_int = LLVMIntType(size_in_bits);
   3364         LLVMValueRef truncated_int = LLVMBuildTrunc(g->builder, shifted_value, same_size_int, "");
   3365         return LLVMBuildBitCast(g->builder, truncated_int, get_llvm_type(g, child_type), "");
   3366     }
   3367 
   3368     return LLVMBuildTrunc(g->builder, shifted_value, get_llvm_type(g, child_type), "");
   3369 }
   3370 
   3371 static bool value_is_all_undef_array(ConstExprValue *const_val, size_t len) {
   3372     switch (const_val->data.x_array.special) {
   3373         case ConstArraySpecialUndef:
   3374             return true;
   3375         case ConstArraySpecialBuf:
   3376             return false;
   3377         case ConstArraySpecialNone:
   3378             for (size_t i = 0; i < len; i += 1) {
   3379                 if (!value_is_all_undef(&const_val->data.x_array.data.s_none.elements[i]))
   3380                     return false;
   3381             }
   3382             return true;
   3383     }
   3384     zig_unreachable();
   3385 }
   3386 
   3387 static bool value_is_all_undef(ConstExprValue *const_val) {
   3388     switch (const_val->special) {
   3389         case ConstValSpecialRuntime:
   3390             return false;
   3391         case ConstValSpecialUndef:
   3392             return true;
   3393         case ConstValSpecialStatic:
   3394             if (const_val->type->id == ZigTypeIdStruct) {
   3395                 for (size_t i = 0; i < const_val->type->data.structure.src_field_count; i += 1) {
   3396                     if (!value_is_all_undef(&const_val->data.x_struct.fields[i]))
   3397                         return false;
   3398                 }
   3399                 return true;
   3400             } else if (const_val->type->id == ZigTypeIdArray) {
   3401                 return value_is_all_undef_array(const_val, const_val->type->data.array.len);
   3402             } else if (const_val->type->id == ZigTypeIdVector) {
   3403                 return value_is_all_undef_array(const_val, const_val->type->data.vector.len);
   3404             } else {
   3405                 return false;
   3406             }
   3407     }
   3408     zig_unreachable();
   3409 }
   3410 
   3411 static LLVMValueRef gen_valgrind_client_request(CodeGen *g, LLVMValueRef default_value, LLVMValueRef request,
   3412         LLVMValueRef a1, LLVMValueRef a2, LLVMValueRef a3, LLVMValueRef a4, LLVMValueRef a5)
   3413 {
   3414     if (!target_has_valgrind_support(g->zig_target)) {
   3415         return default_value;
   3416     }
   3417     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   3418     bool asm_has_side_effects = true;
   3419     bool asm_is_alignstack = false;
   3420     if (g->zig_target->arch == ZigLLVM_x86_64) {
   3421         if (g->zig_target->os == OsLinux || target_os_is_darwin(g->zig_target->os) || g->zig_target->os == OsSolaris ||
   3422             (g->zig_target->os == OsWindows && g->zig_target->abi != ZigLLVM_MSVC))
   3423         {
   3424             if (g->cur_fn->valgrind_client_request_array == nullptr) {
   3425                 LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
   3426                 LLVMBasicBlockRef entry_block = LLVMGetEntryBasicBlock(g->cur_fn->llvm_value);
   3427                 LLVMValueRef first_inst = LLVMGetFirstInstruction(entry_block);
   3428                 LLVMPositionBuilderBefore(g->builder, first_inst);
   3429                 LLVMTypeRef array_type_ref = LLVMArrayType(usize_type_ref, 6);
   3430                 g->cur_fn->valgrind_client_request_array = LLVMBuildAlloca(g->builder, array_type_ref, "");
   3431                 LLVMPositionBuilderAtEnd(g->builder, prev_block);
   3432             }
   3433             LLVMValueRef array_ptr = g->cur_fn->valgrind_client_request_array;
   3434             LLVMValueRef array_elements[] = {request, a1, a2, a3, a4, a5};
   3435             LLVMValueRef zero = LLVMConstInt(usize_type_ref, 0, false);
   3436             for (unsigned i = 0; i < 6; i += 1) {
   3437                 LLVMValueRef indexes[] = {
   3438                     zero,
   3439                     LLVMConstInt(usize_type_ref, i, false),
   3440                 };
   3441                 LLVMValueRef elem_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, indexes, 2, "");
   3442                 LLVMBuildStore(g->builder, array_elements[i], elem_ptr);
   3443             }
   3444 
   3445             Buf *asm_template = buf_create_from_str(
   3446                 "rolq $$3,  %rdi ; rolq $$13, %rdi\n"
   3447                 "rolq $$61, %rdi ; rolq $$51, %rdi\n"
   3448                 "xchgq %rbx,%rbx\n"
   3449             );
   3450             Buf *asm_constraints = buf_create_from_str(
   3451                 "={rdx},{rax},0,~{cc},~{memory}"
   3452             );
   3453             unsigned input_and_output_count = 2;
   3454             LLVMValueRef array_ptr_as_usize = LLVMBuildPtrToInt(g->builder, array_ptr, usize_type_ref, "");
   3455             LLVMValueRef param_values[] = { array_ptr_as_usize, default_value };
   3456             LLVMTypeRef param_types[] = {usize_type_ref, usize_type_ref};
   3457             LLVMTypeRef function_type = LLVMFunctionType(usize_type_ref, param_types,
   3458                     input_and_output_count, false);
   3459             LLVMValueRef asm_fn = LLVMGetInlineAsm(function_type, buf_ptr(asm_template), buf_len(asm_template),
   3460                     buf_ptr(asm_constraints), buf_len(asm_constraints), asm_has_side_effects, asm_is_alignstack,
   3461                     LLVMInlineAsmDialectATT);
   3462             return LLVMBuildCall(g->builder, asm_fn, param_values, input_and_output_count, "");
   3463         }
   3464     }
   3465     zig_unreachable();
   3466 }
   3467 
   3468 static bool want_valgrind_support(CodeGen *g) {
   3469     if (!target_has_valgrind_support(g->zig_target))
   3470         return false;
   3471     switch (g->valgrind_support) {
   3472         case ValgrindSupportDisabled:
   3473             return false;
   3474         case ValgrindSupportEnabled:
   3475             return true;
   3476         case ValgrindSupportAuto:
   3477             return g->build_mode == BuildModeDebug;
   3478     }
   3479     zig_unreachable();
   3480 }
   3481 
   3482 static void gen_valgrind_undef(CodeGen *g, LLVMValueRef dest_ptr, LLVMValueRef byte_count) {
   3483     static const uint32_t VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
   3484     ZigType *usize = g->builtin_types.entry_usize;
   3485     LLVMValueRef zero = LLVMConstInt(usize->llvm_type, 0, false);
   3486     LLVMValueRef req = LLVMConstInt(usize->llvm_type, VG_USERREQ__MAKE_MEM_UNDEFINED, false);
   3487     LLVMValueRef ptr_as_usize = LLVMBuildPtrToInt(g->builder, dest_ptr, usize->llvm_type, "");
   3488     gen_valgrind_client_request(g, zero, req, ptr_as_usize, byte_count, zero, zero, zero);
   3489 }
   3490 
   3491 static void gen_undef_init(CodeGen *g, uint32_t ptr_align_bytes, ZigType *value_type, LLVMValueRef ptr) {
   3492     assert(type_has_bits(value_type));
   3493     uint64_t size_bytes = LLVMStoreSizeOfType(g->target_data_ref, get_llvm_type(g, value_type));
   3494     assert(size_bytes > 0);
   3495     assert(ptr_align_bytes > 0);
   3496     // memset uninitialized memory to 0xaa
   3497     LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
   3498     LLVMValueRef fill_char = LLVMConstInt(LLVMInt8Type(), 0xaa, false);
   3499     LLVMValueRef dest_ptr = LLVMBuildBitCast(g->builder, ptr, ptr_u8, "");
   3500     ZigType *usize = g->builtin_types.entry_usize;
   3501     LLVMValueRef byte_count = LLVMConstInt(usize->llvm_type, size_bytes, false);
   3502     ZigLLVMBuildMemSet(g->builder, dest_ptr, fill_char, byte_count, ptr_align_bytes, false);
   3503     // then tell valgrind that the memory is undefined even though we just memset it
   3504     if (want_valgrind_support(g)) {
   3505         gen_valgrind_undef(g, dest_ptr, byte_count);
   3506     }
   3507 }
   3508 
   3509 static LLVMValueRef ir_render_store_ptr(CodeGen *g, IrExecutable *executable, IrInstructionStorePtr *instruction) {
   3510     ZigType *ptr_type = instruction->ptr->value.type;
   3511     assert(ptr_type->id == ZigTypeIdPointer);
   3512     if (!type_has_bits(ptr_type))
   3513         return nullptr;
   3514 
   3515     bool have_init_expr = !value_is_all_undef(&instruction->value->value);
   3516     if (have_init_expr) {
   3517         LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
   3518         LLVMValueRef value = ir_llvm_value(g, instruction->value);
   3519         gen_assign_raw(g, ptr, ptr_type, value);
   3520     } else if (ir_want_runtime_safety(g, &instruction->base)) {
   3521         gen_undef_init(g, get_ptr_align(g, ptr_type), instruction->value->value.type,
   3522             ir_llvm_value(g, instruction->ptr));
   3523     }
   3524     return nullptr;
   3525 }
   3526 
   3527 static LLVMValueRef ir_render_var_ptr(CodeGen *g, IrExecutable *executable, IrInstructionVarPtr *instruction) {
   3528     ZigVar *var = instruction->var;
   3529     if (type_has_bits(var->var_type)) {
   3530         assert(var->value_ref);
   3531         return var->value_ref;
   3532     } else {
   3533         return nullptr;
   3534     }
   3535 }
   3536 
   3537 static LLVMValueRef ir_render_return_ptr(CodeGen *g, IrExecutable *executable,
   3538         IrInstructionReturnPtr *instruction)
   3539 {
   3540     if (!type_has_bits(instruction->base.value.type))
   3541         return nullptr;
   3542     src_assert(g->cur_ret_ptr != nullptr, instruction->base.source_node);
   3543     return g->cur_ret_ptr;
   3544 }
   3545 
   3546 static LLVMValueRef ir_render_elem_ptr(CodeGen *g, IrExecutable *executable, IrInstructionElemPtr *instruction) {
   3547     LLVMValueRef array_ptr_ptr = ir_llvm_value(g, instruction->array_ptr);
   3548     ZigType *array_ptr_type = instruction->array_ptr->value.type;
   3549     assert(array_ptr_type->id == ZigTypeIdPointer);
   3550     ZigType *array_type = array_ptr_type->data.pointer.child_type;
   3551     LLVMValueRef array_ptr = get_handle_value(g, array_ptr_ptr, array_type, array_ptr_type);
   3552     LLVMValueRef subscript_value = ir_llvm_value(g, instruction->elem_index);
   3553     assert(subscript_value);
   3554 
   3555     if (!type_has_bits(array_type))
   3556         return nullptr;
   3557 
   3558     bool safety_check_on = ir_want_runtime_safety(g, &instruction->base) && instruction->safety_check_on;
   3559 
   3560     if (array_type->id == ZigTypeIdArray ||
   3561         (array_type->id == ZigTypeIdPointer && array_type->data.pointer.ptr_len == PtrLenSingle))
   3562     {
   3563         if (array_type->id == ZigTypeIdPointer) {
   3564             assert(array_type->data.pointer.child_type->id == ZigTypeIdArray);
   3565             array_type = array_type->data.pointer.child_type;
   3566         }
   3567         if (safety_check_on) {
   3568             LLVMValueRef end = LLVMConstInt(g->builtin_types.entry_usize->llvm_type,
   3569                     array_type->data.array.len, false);
   3570             add_bounds_check(g, subscript_value, LLVMIntEQ, nullptr, LLVMIntULT, end);
   3571         }
   3572         if (array_ptr_type->data.pointer.host_int_bytes != 0) {
   3573             return array_ptr_ptr;
   3574         }
   3575         ZigType *child_type = array_type->data.array.child_type;
   3576         if (child_type->id == ZigTypeIdStruct &&
   3577             child_type->data.structure.layout == ContainerLayoutPacked)
   3578         {
   3579             ZigType *ptr_type = instruction->base.value.type;
   3580             size_t host_int_bytes = ptr_type->data.pointer.host_int_bytes;
   3581             if (host_int_bytes != 0) {
   3582                 uint32_t size_in_bits = type_size_bits(g, ptr_type->data.pointer.child_type);
   3583                 LLVMTypeRef ptr_u8_type_ref = LLVMPointerType(LLVMInt8Type(), 0);
   3584                 LLVMValueRef u8_array_ptr = LLVMBuildBitCast(g->builder, array_ptr, ptr_u8_type_ref, "");
   3585                 assert(size_in_bits % 8 == 0);
   3586                 LLVMValueRef elem_size_bytes = LLVMConstInt(g->builtin_types.entry_usize->llvm_type,
   3587                         size_in_bits / 8, false);
   3588                 LLVMValueRef byte_offset = LLVMBuildNUWMul(g->builder, subscript_value, elem_size_bytes, "");
   3589                 LLVMValueRef indices[] = {
   3590                     byte_offset
   3591                 };
   3592                 LLVMValueRef elem_byte_ptr = LLVMBuildInBoundsGEP(g->builder, u8_array_ptr, indices, 1, "");
   3593                 return LLVMBuildBitCast(g->builder, elem_byte_ptr, LLVMPointerType(get_llvm_type(g, child_type), 0), "");
   3594             }
   3595         }
   3596         LLVMValueRef indices[] = {
   3597             LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
   3598             subscript_value
   3599         };
   3600         return LLVMBuildInBoundsGEP(g->builder, array_ptr, indices, 2, "");
   3601     } else if (array_type->id == ZigTypeIdPointer) {
   3602         assert(LLVMGetTypeKind(LLVMTypeOf(array_ptr)) == LLVMPointerTypeKind);
   3603         LLVMValueRef indices[] = {
   3604             subscript_value
   3605         };
   3606         return LLVMBuildInBoundsGEP(g->builder, array_ptr, indices, 1, "");
   3607     } else if (array_type->id == ZigTypeIdStruct) {
   3608         assert(array_type->data.structure.is_slice);
   3609 
   3610         ZigType *ptr_type = instruction->base.value.type;
   3611         if (!type_has_bits(ptr_type)) {
   3612             if (safety_check_on) {
   3613                 assert(LLVMGetTypeKind(LLVMTypeOf(array_ptr)) == LLVMIntegerTypeKind);
   3614                 add_bounds_check(g, subscript_value, LLVMIntEQ, nullptr, LLVMIntULT, array_ptr);
   3615             }
   3616             return nullptr;
   3617         }
   3618 
   3619         assert(LLVMGetTypeKind(LLVMTypeOf(array_ptr)) == LLVMPointerTypeKind);
   3620         assert(LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(array_ptr))) == LLVMStructTypeKind);
   3621 
   3622         if (safety_check_on) {
   3623             size_t len_index = array_type->data.structure.fields[slice_len_index].gen_index;
   3624             assert(len_index != SIZE_MAX);
   3625             LLVMValueRef len_ptr = LLVMBuildStructGEP(g->builder, array_ptr, (unsigned)len_index, "");
   3626             LLVMValueRef len = gen_load_untyped(g, len_ptr, 0, false, "");
   3627             add_bounds_check(g, subscript_value, LLVMIntEQ, nullptr, LLVMIntULT, len);
   3628         }
   3629 
   3630         size_t ptr_index = array_type->data.structure.fields[slice_ptr_index].gen_index;
   3631         assert(ptr_index != SIZE_MAX);
   3632         LLVMValueRef ptr_ptr = LLVMBuildStructGEP(g->builder, array_ptr, (unsigned)ptr_index, "");
   3633         LLVMValueRef ptr = gen_load_untyped(g, ptr_ptr, 0, false, "");
   3634         return LLVMBuildInBoundsGEP(g->builder, ptr, &subscript_value, 1, "");
   3635     } else {
   3636         zig_unreachable();
   3637     }
   3638 }
   3639 
   3640 static LLVMValueRef get_new_stack_addr(CodeGen *g, LLVMValueRef new_stack) {
   3641     LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, new_stack, (unsigned)slice_ptr_index, "");
   3642     LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, new_stack, (unsigned)slice_len_index, "");
   3643 
   3644     LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, "");
   3645     LLVMValueRef len_value = gen_load_untyped(g, len_field_ptr, 0, false, "");
   3646 
   3647     LLVMValueRef ptr_addr = LLVMBuildPtrToInt(g->builder, ptr_value, LLVMTypeOf(len_value), "");
   3648     LLVMValueRef end_addr = LLVMBuildNUWAdd(g->builder, ptr_addr, len_value, "");
   3649     LLVMValueRef align_amt = LLVMConstInt(LLVMTypeOf(end_addr), get_abi_alignment(g, g->builtin_types.entry_usize), false);
   3650     LLVMValueRef align_adj = LLVMBuildURem(g->builder, end_addr, align_amt, "");
   3651     return LLVMBuildNUWSub(g->builder, end_addr, align_adj, "");
   3652 }
   3653 
   3654 static void gen_set_stack_pointer(CodeGen *g, LLVMValueRef aligned_end_addr) {
   3655     LLVMValueRef write_register_fn_val = get_write_register_fn_val(g);
   3656 
   3657     if (g->sp_md_node == nullptr) {
   3658         Buf *sp_reg_name = buf_create_from_str(arch_stack_pointer_register_name(g->zig_target->arch));
   3659         LLVMValueRef str_node = LLVMMDString(buf_ptr(sp_reg_name), buf_len(sp_reg_name) + 1);
   3660         g->sp_md_node = LLVMMDNode(&str_node, 1);
   3661     }
   3662 
   3663     LLVMValueRef params[] = {
   3664         g->sp_md_node,
   3665         aligned_end_addr,
   3666     };
   3667 
   3668     LLVMBuildCall(g->builder, write_register_fn_val, params, 2, "");
   3669 }
   3670 
   3671 static void set_call_instr_sret(CodeGen *g, LLVMValueRef call_instr) {
   3672     unsigned attr_kind_id = LLVMGetEnumAttributeKindForName("sret", 4);
   3673     LLVMAttributeRef sret_attr = LLVMCreateEnumAttribute(LLVMGetGlobalContext(), attr_kind_id, 0);
   3674     LLVMAddCallSiteAttribute(call_instr, 1, sret_attr);
   3675 }
   3676 
   3677 static void render_async_spills(CodeGen *g) {
   3678     ZigType *fn_type = g->cur_fn->type_entry;
   3679     ZigType *import = get_scope_import(&g->cur_fn->fndef_scope->base);
   3680     uint32_t async_var_index = frame_index_arg(g, fn_type->data.fn.fn_type_id.return_type);
   3681     for (size_t var_i = 0; var_i < g->cur_fn->variable_list.length; var_i += 1) {
   3682         ZigVar *var = g->cur_fn->variable_list.at(var_i);
   3683 
   3684         if (!type_has_bits(var->var_type)) {
   3685             continue;
   3686         }
   3687         if (ir_get_var_is_comptime(var))
   3688             continue;
   3689         switch (type_requires_comptime(g, var->var_type)) {
   3690             case ReqCompTimeInvalid:
   3691                 zig_unreachable();
   3692             case ReqCompTimeYes:
   3693                 continue;
   3694             case ReqCompTimeNo:
   3695                 break;
   3696         }
   3697         if (var->src_arg_index == SIZE_MAX) {
   3698             continue;
   3699         }
   3700 
   3701         var->value_ref = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, async_var_index,
   3702                 buf_ptr(&var->name));
   3703         async_var_index += 1;
   3704         if (var->decl_node) {
   3705             var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
   3706                 buf_ptr(&var->name), import->data.structure.root_struct->di_file,
   3707                 (unsigned)(var->decl_node->line + 1),
   3708                 get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
   3709             gen_var_debug_decl(g, var);
   3710         }
   3711     }
   3712 
   3713     ZigType *frame_type = g->cur_fn->frame_type->data.frame.locals_struct;
   3714 
   3715     for (size_t alloca_i = 0; alloca_i < g->cur_fn->alloca_gen_list.length; alloca_i += 1) {
   3716         IrInstructionAllocaGen *instruction = g->cur_fn->alloca_gen_list.at(alloca_i);
   3717         if (instruction->field_index == SIZE_MAX)
   3718             continue;
   3719 
   3720         size_t gen_index = frame_type->data.structure.fields[instruction->field_index].gen_index;
   3721         instruction->base.llvm_value = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, gen_index,
   3722                 instruction->name_hint);
   3723     }
   3724 }
   3725 
   3726 static void render_async_var_decls(CodeGen *g, Scope *scope) {
   3727     for (;;) {
   3728         switch (scope->id) {
   3729             case ScopeIdCImport:
   3730                 zig_unreachable();
   3731             case ScopeIdFnDef:
   3732                 return;
   3733             case ScopeIdVarDecl: {
   3734                 ZigVar *var = reinterpret_cast<ScopeVarDecl *>(scope)->var;
   3735                 if (var->ptr_instruction != nullptr) {
   3736                     render_decl_var(g, var);
   3737                 }
   3738                 // fallthrough
   3739             }
   3740             case ScopeIdDecls:
   3741             case ScopeIdBlock:
   3742             case ScopeIdDefer:
   3743             case ScopeIdDeferExpr:
   3744             case ScopeIdLoop:
   3745             case ScopeIdSuspend:
   3746             case ScopeIdCompTime:
   3747             case ScopeIdRuntime:
   3748                 scope = scope->parent;
   3749                 continue;
   3750         }
   3751     }
   3752 }
   3753 
   3754 static LLVMValueRef gen_frame_size(CodeGen *g, LLVMValueRef fn_val) {
   3755     LLVMTypeRef usize_llvm_type = g->builtin_types.entry_usize->llvm_type;
   3756     LLVMTypeRef ptr_usize_llvm_type = LLVMPointerType(usize_llvm_type, 0);
   3757     LLVMValueRef casted_fn_val = LLVMBuildBitCast(g->builder, fn_val, ptr_usize_llvm_type, "");
   3758     LLVMValueRef negative_one = LLVMConstInt(LLVMInt32Type(), -1, true);
   3759     LLVMValueRef prefix_ptr = LLVMBuildInBoundsGEP(g->builder, casted_fn_val, &negative_one, 1, "");
   3760     return LLVMBuildLoad(g->builder, prefix_ptr, "");
   3761 }
   3762 
   3763 static void gen_init_stack_trace(CodeGen *g, LLVMValueRef trace_field_ptr, LLVMValueRef addrs_field_ptr) {
   3764     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   3765     LLVMValueRef zero = LLVMConstNull(usize_type_ref);
   3766 
   3767     LLVMValueRef index_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 0, "");
   3768     LLVMBuildStore(g->builder, zero, index_ptr);
   3769 
   3770     LLVMValueRef addrs_slice_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 1, "");
   3771     LLVMValueRef addrs_ptr_ptr = LLVMBuildStructGEP(g->builder, addrs_slice_ptr, slice_ptr_index, "");
   3772     LLVMValueRef indices[] = { LLVMConstNull(usize_type_ref), LLVMConstNull(usize_type_ref) };
   3773     LLVMValueRef trace_field_addrs_as_ptr = LLVMBuildInBoundsGEP(g->builder, addrs_field_ptr, indices, 2, "");
   3774     LLVMBuildStore(g->builder, trace_field_addrs_as_ptr, addrs_ptr_ptr);
   3775 
   3776     LLVMValueRef addrs_len_ptr = LLVMBuildStructGEP(g->builder, addrs_slice_ptr, slice_len_index, "");
   3777     LLVMBuildStore(g->builder, LLVMConstInt(usize_type_ref, stack_trace_ptr_count, false), addrs_len_ptr);
   3778 }
   3779 
   3780 static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstructionCallGen *instruction) {
   3781     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   3782 
   3783     LLVMValueRef fn_val;
   3784     ZigType *fn_type;
   3785     bool callee_is_async;
   3786     if (instruction->fn_entry) {
   3787         fn_val = fn_llvm_value(g, instruction->fn_entry);
   3788         fn_type = instruction->fn_entry->type_entry;
   3789         callee_is_async = fn_is_async(instruction->fn_entry);
   3790     } else {
   3791         assert(instruction->fn_ref);
   3792         fn_val = ir_llvm_value(g, instruction->fn_ref);
   3793         fn_type = instruction->fn_ref->value.type;
   3794         callee_is_async = fn_type->data.fn.fn_type_id.cc == CallingConventionAsync;
   3795     }
   3796 
   3797     FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id;
   3798 
   3799     ZigType *src_return_type = fn_type_id->return_type;
   3800     bool ret_has_bits = type_has_bits(src_return_type);
   3801 
   3802     CallingConvention cc = fn_type->data.fn.fn_type_id.cc;
   3803 
   3804     bool first_arg_ret = ret_has_bits && want_first_arg_sret(g, fn_type_id);
   3805     bool prefix_arg_err_ret_stack = codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type);
   3806     bool is_var_args = fn_type_id->is_var_args;
   3807     ZigList<LLVMValueRef> gen_param_values = {};
   3808     ZigList<ZigType *> gen_param_types = {};
   3809     LLVMValueRef result_loc = instruction->result_loc ? ir_llvm_value(g, instruction->result_loc) : nullptr;
   3810     LLVMValueRef zero = LLVMConstNull(usize_type_ref);
   3811     LLVMValueRef frame_result_loc;
   3812     LLVMValueRef awaiter_init_val;
   3813     LLVMValueRef ret_ptr;
   3814     if (callee_is_async) {
   3815         if (instruction->is_async) {
   3816             if (instruction->new_stack == nullptr) {
   3817                 awaiter_init_val = zero;
   3818                 frame_result_loc = result_loc;
   3819 
   3820                 if (ret_has_bits) {
   3821                     // Use the result location which is inside the frame if this is an async call.
   3822                     ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
   3823                 }
   3824             } else if (cc == CallingConventionAsync) {
   3825                 awaiter_init_val = zero;
   3826                 LLVMValueRef frame_slice_ptr = ir_llvm_value(g, instruction->new_stack);
   3827                 if (ir_want_runtime_safety(g, &instruction->base)) {
   3828                     LLVMValueRef given_len_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_len_index, "");
   3829                     LLVMValueRef given_frame_len = LLVMBuildLoad(g->builder, given_len_ptr, "");
   3830                     LLVMValueRef actual_frame_len = gen_frame_size(g, fn_val);
   3831 
   3832                     LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "FrameSizeCheckFail");
   3833                     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "FrameSizeCheckOk");
   3834 
   3835                     LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntUGE, given_frame_len, actual_frame_len, "");
   3836                     LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   3837 
   3838                     LLVMPositionBuilderAtEnd(g->builder, fail_block);
   3839                     gen_safety_crash(g, PanicMsgIdFrameTooSmall);
   3840 
   3841                     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   3842                 }
   3843                 LLVMValueRef frame_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_slice_ptr, slice_ptr_index, "");
   3844                 LLVMValueRef frame_ptr = LLVMBuildLoad(g->builder, frame_ptr_ptr, "");
   3845                 frame_result_loc = LLVMBuildBitCast(g->builder, frame_ptr,
   3846                         get_llvm_type(g, instruction->base.value.type), "");
   3847 
   3848                 if (ret_has_bits) {
   3849                     // Use the result location provided to the @asyncCall builtin
   3850                     ret_ptr = result_loc;
   3851                 }
   3852             } else {
   3853                 zig_unreachable();
   3854             }
   3855 
   3856             // even if prefix_arg_err_ret_stack is true, let the async function do its own
   3857             // initialization.
   3858         } else {
   3859             // async function called as a normal function
   3860 
   3861             frame_result_loc = ir_llvm_value(g, instruction->frame_result_loc);
   3862             awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, ""); // caller's own frame pointer
   3863             if (ret_has_bits) {
   3864                 if (result_loc == nullptr) {
   3865                     // return type is a scalar, but we still need a pointer to it. Use the async fn frame.
   3866                     ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
   3867                 } else {
   3868                     // Use the call instruction's result location.
   3869                     ret_ptr = result_loc;
   3870                 }
   3871 
   3872                 // Store a zero in the awaiter's result ptr to indicate we do not need a copy made.
   3873                 LLVMValueRef awaiter_ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 1, "");
   3874                 LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr)));
   3875                 LLVMBuildStore(g->builder, zero_ptr, awaiter_ret_ptr);
   3876             }
   3877 
   3878             if (prefix_arg_err_ret_stack) {
   3879                 LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
   3880                         frame_index_trace_arg(g, src_return_type) + 1, "");
   3881                 LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
   3882                 LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr);
   3883             }
   3884         }
   3885 
   3886         assert(frame_result_loc != nullptr);
   3887 
   3888         LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_fn_ptr_index, "");
   3889         LLVMValueRef bitcasted_fn_val = LLVMBuildBitCast(g->builder, fn_val,
   3890                 LLVMGetElementType(LLVMTypeOf(fn_ptr_ptr)), "");
   3891         LLVMBuildStore(g->builder, bitcasted_fn_val, fn_ptr_ptr);
   3892 
   3893         LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_resume_index, "");
   3894         LLVMBuildStore(g->builder, zero, resume_index_ptr);
   3895 
   3896         LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_awaiter_index, "");
   3897         LLVMBuildStore(g->builder, awaiter_init_val, awaiter_ptr);
   3898 
   3899         if (ret_has_bits) {
   3900             LLVMValueRef ret_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start, "");
   3901             LLVMBuildStore(g->builder, ret_ptr, ret_ptr_ptr);
   3902         }
   3903     } else if (instruction->is_async) {
   3904         // Async call of blocking function
   3905         if (instruction->new_stack != nullptr) {
   3906             zig_panic("TODO @asyncCall of non-async function");
   3907         }
   3908         frame_result_loc = result_loc;
   3909         awaiter_init_val = LLVMConstAllOnes(usize_type_ref);
   3910 
   3911         LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_awaiter_index, "");
   3912         LLVMBuildStore(g->builder, awaiter_init_val, awaiter_ptr);
   3913 
   3914         if (ret_has_bits) {
   3915             LLVMValueRef ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
   3916             LLVMValueRef ret_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start, "");
   3917             LLVMBuildStore(g->builder, ret_ptr, ret_ptr_ptr);
   3918 
   3919             if (first_arg_ret) {
   3920                 gen_param_values.append(ret_ptr);
   3921             }
   3922             if (prefix_arg_err_ret_stack) {
   3923                 // Set up the callee stack trace pointer pointing into the frame.
   3924                 // Then we have to wire up the StackTrace pointers.
   3925                 // Await is responsible for merging error return traces.
   3926                 uint32_t trace_field_index_start = frame_index_trace_arg(g, src_return_type);
   3927                 LLVMValueRef callee_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
   3928                         trace_field_index_start, "");
   3929                 LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
   3930                         trace_field_index_start + 2, "");
   3931                 LLVMValueRef addrs_field_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
   3932                         trace_field_index_start + 3, "");
   3933 
   3934                 LLVMBuildStore(g->builder, trace_field_ptr, callee_trace_ptr_ptr);
   3935 
   3936                 gen_init_stack_trace(g, trace_field_ptr, addrs_field_ptr);
   3937 
   3938                 gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope));
   3939             }
   3940         }
   3941     } else {
   3942         if (first_arg_ret) {
   3943             gen_param_values.append(result_loc);
   3944         }
   3945         if (prefix_arg_err_ret_stack) {
   3946             gen_param_values.append(get_cur_err_ret_trace_val(g, instruction->base.scope));
   3947         }
   3948     }
   3949     FnWalk fn_walk = {};
   3950     fn_walk.id = FnWalkIdCall;
   3951     fn_walk.data.call.inst = instruction;
   3952     fn_walk.data.call.is_var_args = is_var_args;
   3953     fn_walk.data.call.gen_param_values = &gen_param_values;
   3954     fn_walk.data.call.gen_param_types = &gen_param_types;
   3955     walk_function_params(g, fn_type, &fn_walk);
   3956 
   3957     ZigLLVM_FnInline fn_inline;
   3958     switch (instruction->fn_inline) {
   3959         case FnInlineAuto:
   3960             fn_inline = ZigLLVM_FnInlineAuto;
   3961             break;
   3962         case FnInlineAlways:
   3963             fn_inline = (instruction->fn_entry == nullptr) ? ZigLLVM_FnInlineAuto : ZigLLVM_FnInlineAlways;
   3964             break;
   3965         case FnInlineNever:
   3966             fn_inline = ZigLLVM_FnInlineNever;
   3967             break;
   3968     }
   3969 
   3970     LLVMCallConv llvm_cc = get_llvm_cc(g, cc);
   3971     LLVMValueRef result;
   3972 
   3973     if (callee_is_async) {
   3974         uint32_t arg_start_i = frame_index_arg(g, fn_type->data.fn.fn_type_id.return_type);
   3975 
   3976         LLVMValueRef casted_frame;
   3977         if (instruction->new_stack != nullptr) {
   3978             // We need the frame type to be a pointer to a struct that includes the args
   3979             size_t field_count = arg_start_i + gen_param_values.length;
   3980             LLVMTypeRef *field_types = allocate_nonzero<LLVMTypeRef>(field_count);
   3981             LLVMGetStructElementTypes(LLVMGetElementType(LLVMTypeOf(frame_result_loc)), field_types);
   3982             assert(LLVMCountStructElementTypes(LLVMGetElementType(LLVMTypeOf(frame_result_loc))) == arg_start_i);
   3983             for (size_t arg_i = 0; arg_i < gen_param_values.length; arg_i += 1) {
   3984                 field_types[arg_start_i + arg_i] = LLVMTypeOf(gen_param_values.at(arg_i));
   3985             }
   3986             LLVMTypeRef frame_with_args_type = LLVMStructType(field_types, field_count, false);
   3987             LLVMTypeRef ptr_frame_with_args_type = LLVMPointerType(frame_with_args_type, 0);
   3988 
   3989             casted_frame = LLVMBuildBitCast(g->builder, frame_result_loc, ptr_frame_with_args_type, "");
   3990         } else {
   3991             casted_frame = frame_result_loc;
   3992         }
   3993 
   3994         for (size_t arg_i = 0; arg_i < gen_param_values.length; arg_i += 1) {
   3995             LLVMValueRef arg_ptr = LLVMBuildStructGEP(g->builder, casted_frame, arg_start_i + arg_i, "");
   3996             gen_assign_raw(g, arg_ptr, get_pointer_to_type(g, gen_param_types.at(arg_i), true),
   3997                     gen_param_values.at(arg_i));
   3998         }
   3999 
   4000         if (instruction->is_async) {
   4001             gen_resume(g, fn_val, frame_result_loc, ResumeIdCall);
   4002             if (instruction->new_stack != nullptr) {
   4003                 return frame_result_loc;
   4004             }
   4005             return nullptr;
   4006         } else {
   4007             ZigType *ptr_result_type = get_pointer_to_type(g, src_return_type, true);
   4008 
   4009             LLVMBasicBlockRef call_bb = gen_suspend_begin(g, "CallResume");
   4010 
   4011             LLVMValueRef call_inst = gen_resume(g, fn_val, frame_result_loc, ResumeIdCall);
   4012             set_tail_call_if_appropriate(g, call_inst);
   4013             LLVMBuildRetVoid(g->builder);
   4014 
   4015             LLVMPositionBuilderAtEnd(g->builder, call_bb);
   4016             gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr);
   4017             render_async_var_decls(g, instruction->base.scope);
   4018 
   4019             if (!type_has_bits(src_return_type))
   4020                 return nullptr;
   4021 
   4022             if (result_loc != nullptr) 
   4023                 return get_handle_value(g, result_loc, src_return_type, ptr_result_type);
   4024 
   4025             LLVMValueRef result_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, frame_ret_start + 2, "");
   4026             return LLVMBuildLoad(g->builder, result_ptr, "");
   4027         }
   4028     }
   4029 
   4030     if (instruction->new_stack == nullptr) {
   4031         result = ZigLLVMBuildCall(g->builder, fn_val,
   4032                 gen_param_values.items, (unsigned)gen_param_values.length, llvm_cc, fn_inline, "");
   4033     } else if (instruction->is_async) {
   4034         zig_panic("TODO @asyncCall of non-async function");
   4035     } else {
   4036         LLVMValueRef stacksave_fn_val = get_stacksave_fn_val(g);
   4037         LLVMValueRef stackrestore_fn_val = get_stackrestore_fn_val(g);
   4038 
   4039         LLVMValueRef new_stack_addr = get_new_stack_addr(g, ir_llvm_value(g, instruction->new_stack));
   4040         LLVMValueRef old_stack_ref = LLVMBuildCall(g->builder, stacksave_fn_val, nullptr, 0, "");
   4041         gen_set_stack_pointer(g, new_stack_addr);
   4042         result = ZigLLVMBuildCall(g->builder, fn_val,
   4043                 gen_param_values.items, (unsigned)gen_param_values.length, llvm_cc, fn_inline, "");
   4044         LLVMBuildCall(g->builder, stackrestore_fn_val, &old_stack_ref, 1, "");
   4045     }
   4046 
   4047     if (src_return_type->id == ZigTypeIdUnreachable) {
   4048         return LLVMBuildUnreachable(g->builder);
   4049     } else if (!ret_has_bits) {
   4050         return nullptr;
   4051     } else if (first_arg_ret) {
   4052         set_call_instr_sret(g, result);
   4053         return result_loc;
   4054     } else if (handle_is_ptr(src_return_type)) {
   4055         LLVMValueRef store_instr = LLVMBuildStore(g->builder, result, result_loc);
   4056         LLVMSetAlignment(store_instr, get_ptr_align(g, instruction->result_loc->value.type));
   4057         return result_loc;
   4058     } else {
   4059         return result;
   4060     }
   4061 }
   4062 
   4063 static LLVMValueRef ir_render_struct_field_ptr(CodeGen *g, IrExecutable *executable,
   4064     IrInstructionStructFieldPtr *instruction)
   4065 {
   4066     if (instruction->base.value.special != ConstValSpecialRuntime)
   4067         return nullptr;
   4068 
   4069     LLVMValueRef struct_ptr = ir_llvm_value(g, instruction->struct_ptr);
   4070     // not necessarily a pointer. could be ZigTypeIdStruct
   4071     ZigType *struct_ptr_type = instruction->struct_ptr->value.type;
   4072     TypeStructField *field = instruction->field;
   4073 
   4074     if (!type_has_bits(field->type_entry))
   4075         return nullptr;
   4076 
   4077     if (struct_ptr_type->id == ZigTypeIdPointer &&
   4078         struct_ptr_type->data.pointer.host_int_bytes != 0)
   4079     {
   4080         return struct_ptr;
   4081     }
   4082 
   4083     assert(field->gen_index != SIZE_MAX);
   4084     return LLVMBuildStructGEP(g->builder, struct_ptr, (unsigned)field->gen_index, "");
   4085 }
   4086 
   4087 static LLVMValueRef ir_render_union_field_ptr(CodeGen *g, IrExecutable *executable,
   4088     IrInstructionUnionFieldPtr *instruction)
   4089 {
   4090     if (instruction->base.value.special != ConstValSpecialRuntime)
   4091         return nullptr;
   4092 
   4093     ZigType *union_ptr_type = instruction->union_ptr->value.type;
   4094     assert(union_ptr_type->id == ZigTypeIdPointer);
   4095     ZigType *union_type = union_ptr_type->data.pointer.child_type;
   4096     assert(union_type->id == ZigTypeIdUnion);
   4097 
   4098     TypeUnionField *field = instruction->field;
   4099 
   4100     if (!type_has_bits(field->type_entry)) {
   4101         if (union_type->data.unionation.gen_tag_index == SIZE_MAX) {
   4102             return nullptr;
   4103         }
   4104         if (instruction->initializing) {
   4105             LLVMValueRef union_ptr = ir_llvm_value(g, instruction->union_ptr);
   4106             LLVMValueRef tag_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr,
   4107                     union_type->data.unionation.gen_tag_index, "");
   4108             LLVMValueRef tag_value = bigint_to_llvm_const(get_llvm_type(g, union_type->data.unionation.tag_type),
   4109                     &field->enum_field->value);
   4110             gen_store_untyped(g, tag_value, tag_field_ptr, 0, false);
   4111         }
   4112         return nullptr;
   4113     }
   4114 
   4115     LLVMValueRef union_ptr = ir_llvm_value(g, instruction->union_ptr);
   4116     LLVMTypeRef field_type_ref = LLVMPointerType(get_llvm_type(g, field->type_entry), 0);
   4117 
   4118     if (union_type->data.unionation.gen_tag_index == SIZE_MAX) {
   4119         LLVMValueRef union_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr, 0, "");
   4120         LLVMValueRef bitcasted_union_field_ptr = LLVMBuildBitCast(g->builder, union_field_ptr, field_type_ref, "");
   4121         return bitcasted_union_field_ptr;
   4122     }
   4123 
   4124     if (instruction->initializing) {
   4125         LLVMValueRef tag_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr, union_type->data.unionation.gen_tag_index, "");
   4126         LLVMValueRef tag_value = bigint_to_llvm_const(get_llvm_type(g, union_type->data.unionation.tag_type),
   4127                 &field->enum_field->value);
   4128         gen_store_untyped(g, tag_value, tag_field_ptr, 0, false);
   4129     } else if (instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base)) {
   4130         LLVMValueRef tag_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr, union_type->data.unionation.gen_tag_index, "");
   4131         LLVMValueRef tag_value = gen_load_untyped(g, tag_field_ptr, 0, false, "");
   4132 
   4133 
   4134         LLVMValueRef expected_tag_value = bigint_to_llvm_const(get_llvm_type(g, union_type->data.unionation.tag_type),
   4135                 &field->enum_field->value);
   4136         LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnionCheckOk");
   4137         LLVMBasicBlockRef bad_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnionCheckFail");
   4138         LLVMValueRef ok_val = LLVMBuildICmp(g->builder, LLVMIntEQ, tag_value, expected_tag_value, "");
   4139         LLVMBuildCondBr(g->builder, ok_val, ok_block, bad_block);
   4140 
   4141         LLVMPositionBuilderAtEnd(g->builder, bad_block);
   4142         gen_safety_crash(g, PanicMsgIdBadUnionField);
   4143 
   4144         LLVMPositionBuilderAtEnd(g->builder, ok_block);
   4145     }
   4146 
   4147     LLVMValueRef union_field_ptr = LLVMBuildStructGEP(g->builder, union_ptr,
   4148             union_type->data.unionation.gen_union_index, "");
   4149     LLVMValueRef bitcasted_union_field_ptr = LLVMBuildBitCast(g->builder, union_field_ptr, field_type_ref, "");
   4150     return bitcasted_union_field_ptr;
   4151 }
   4152 
   4153 static size_t find_asm_index(CodeGen *g, AstNode *node, AsmToken *tok, Buf *src_template) {
   4154     const char *ptr = buf_ptr(src_template) + tok->start + 2;
   4155     size_t len = tok->end - tok->start - 2;
   4156     size_t result = 0;
   4157     for (size_t i = 0; i < node->data.asm_expr.output_list.length; i += 1, result += 1) {
   4158         AsmOutput *asm_output = node->data.asm_expr.output_list.at(i);
   4159         if (buf_eql_mem(asm_output->asm_symbolic_name, ptr, len)) {
   4160             return result;
   4161         }
   4162     }
   4163     for (size_t i = 0; i < node->data.asm_expr.input_list.length; i += 1, result += 1) {
   4164         AsmInput *asm_input = node->data.asm_expr.input_list.at(i);
   4165         if (buf_eql_mem(asm_input->asm_symbolic_name, ptr, len)) {
   4166             return result;
   4167         }
   4168     }
   4169     return SIZE_MAX;
   4170 }
   4171 
   4172 static LLVMValueRef ir_render_asm(CodeGen *g, IrExecutable *executable, IrInstructionAsm *instruction) {
   4173     AstNode *asm_node = instruction->base.source_node;
   4174     assert(asm_node->type == NodeTypeAsmExpr);
   4175     AstNodeAsmExpr *asm_expr = &asm_node->data.asm_expr;
   4176 
   4177     Buf *src_template = instruction->asm_template;
   4178 
   4179     Buf llvm_template = BUF_INIT;
   4180     buf_resize(&llvm_template, 0);
   4181 
   4182     for (size_t token_i = 0; token_i < instruction->token_list_len; token_i += 1) {
   4183         AsmToken *asm_token = &instruction->token_list[token_i];
   4184         switch (asm_token->id) {
   4185             case AsmTokenIdTemplate:
   4186                 for (size_t offset = asm_token->start; offset < asm_token->end; offset += 1) {
   4187                     uint8_t c = *((uint8_t*)(buf_ptr(src_template) + offset));
   4188                     if (c == '$') {
   4189                         buf_append_str(&llvm_template, "$$");
   4190                     } else {
   4191                         buf_append_char(&llvm_template, c);
   4192                     }
   4193                 }
   4194                 break;
   4195             case AsmTokenIdPercent:
   4196                 buf_append_char(&llvm_template, '%');
   4197                 break;
   4198             case AsmTokenIdVar:
   4199                 {
   4200                     size_t index = find_asm_index(g, asm_node, asm_token, src_template);
   4201                     assert(index < SIZE_MAX);
   4202                     buf_appendf(&llvm_template, "$%" ZIG_PRI_usize "", index);
   4203                     break;
   4204                 }
   4205             case AsmTokenIdUniqueId:
   4206                 buf_append_str(&llvm_template, "${:uid}");
   4207                 break;
   4208         }
   4209     }
   4210 
   4211     Buf constraint_buf = BUF_INIT;
   4212     buf_resize(&constraint_buf, 0);
   4213 
   4214     assert(instruction->return_count == 0 || instruction->return_count == 1);
   4215 
   4216     size_t total_constraint_count = asm_expr->output_list.length +
   4217                                  asm_expr->input_list.length +
   4218                                  asm_expr->clobber_list.length;
   4219     size_t input_and_output_count = asm_expr->output_list.length +
   4220                                  asm_expr->input_list.length -
   4221                                  instruction->return_count;
   4222     size_t total_index = 0;
   4223     size_t param_index = 0;
   4224     LLVMTypeRef *param_types = allocate<LLVMTypeRef>(input_and_output_count);
   4225     LLVMValueRef *param_values = allocate<LLVMValueRef>(input_and_output_count);
   4226     for (size_t i = 0; i < asm_expr->output_list.length; i += 1, total_index += 1) {
   4227         AsmOutput *asm_output = asm_expr->output_list.at(i);
   4228         bool is_return = (asm_output->return_type != nullptr);
   4229         assert(*buf_ptr(asm_output->constraint) == '=');
   4230         // LLVM uses commas internally to separate different constraints,
   4231         // alternative constraints are achieved with pipes.
   4232         // We still allow the user to use commas in a way that is similar
   4233         // to GCC's inline assembly.
   4234         // http://llvm.org/docs/LangRef.html#constraint-codes
   4235         buf_replace(asm_output->constraint, ',', '|');
   4236 
   4237         if (is_return) {
   4238             buf_appendf(&constraint_buf, "=%s", buf_ptr(asm_output->constraint) + 1);
   4239         } else {
   4240             buf_appendf(&constraint_buf, "=*%s", buf_ptr(asm_output->constraint) + 1);
   4241         }
   4242         if (total_index + 1 < total_constraint_count) {
   4243             buf_append_char(&constraint_buf, ',');
   4244         }
   4245 
   4246         if (!is_return) {
   4247             ZigVar *variable = instruction->output_vars[i];
   4248             assert(variable);
   4249             param_types[param_index] = LLVMTypeOf(variable->value_ref);
   4250             param_values[param_index] = variable->value_ref;
   4251             param_index += 1;
   4252         }
   4253     }
   4254     for (size_t i = 0; i < asm_expr->input_list.length; i += 1, total_index += 1, param_index += 1) {
   4255         AsmInput *asm_input = asm_expr->input_list.at(i);
   4256         buf_replace(asm_input->constraint, ',', '|');
   4257         IrInstruction *ir_input = instruction->input_list[i];
   4258         buf_append_buf(&constraint_buf, asm_input->constraint);
   4259         if (total_index + 1 < total_constraint_count) {
   4260             buf_append_char(&constraint_buf, ',');
   4261         }
   4262 
   4263         ZigType *const type = ir_input->value.type;
   4264         LLVMTypeRef type_ref = get_llvm_type(g, type);
   4265         LLVMValueRef value_ref = ir_llvm_value(g, ir_input);
   4266         // Handle integers of non pot bitsize by widening them.
   4267         if (type->id == ZigTypeIdInt) {
   4268             const size_t bitsize = type->data.integral.bit_count;
   4269             if (bitsize < 8 || !is_power_of_2(bitsize)) {
   4270                 const bool is_signed = type->data.integral.is_signed;
   4271                 const size_t wider_bitsize = bitsize < 8 ? 8 : round_to_next_power_of_2(bitsize);
   4272                 ZigType *const wider_type = get_int_type(g, is_signed, wider_bitsize);
   4273                 type_ref = get_llvm_type(g, wider_type);
   4274                 value_ref = gen_widen_or_shorten(g, false, type, wider_type, value_ref);
   4275             }
   4276         }
   4277 
   4278         param_types[param_index] = type_ref;
   4279         param_values[param_index] = value_ref;
   4280     }
   4281     for (size_t i = 0; i < asm_expr->clobber_list.length; i += 1, total_index += 1) {
   4282         Buf *clobber_buf = asm_expr->clobber_list.at(i);
   4283         buf_appendf(&constraint_buf, "~{%s}", buf_ptr(clobber_buf));
   4284         if (total_index + 1 < total_constraint_count) {
   4285             buf_append_char(&constraint_buf, ',');
   4286         }
   4287     }
   4288 
   4289     LLVMTypeRef ret_type;
   4290     if (instruction->return_count == 0) {
   4291         ret_type = LLVMVoidType();
   4292     } else {
   4293         ret_type = get_llvm_type(g, instruction->base.value.type);
   4294     }
   4295     LLVMTypeRef function_type = LLVMFunctionType(ret_type, param_types, (unsigned)input_and_output_count, false);
   4296 
   4297     bool is_volatile = instruction->has_side_effects || (asm_expr->output_list.length == 0);
   4298     LLVMValueRef asm_fn = LLVMGetInlineAsm(function_type, buf_ptr(&llvm_template), buf_len(&llvm_template),
   4299             buf_ptr(&constraint_buf), buf_len(&constraint_buf), is_volatile, false, LLVMInlineAsmDialectATT);
   4300 
   4301     return LLVMBuildCall(g->builder, asm_fn, param_values, (unsigned)input_and_output_count, "");
   4302 }
   4303 
   4304 static LLVMValueRef gen_non_null_bit(CodeGen *g, ZigType *maybe_type, LLVMValueRef maybe_handle) {
   4305     assert(maybe_type->id == ZigTypeIdOptional ||
   4306             (maybe_type->id == ZigTypeIdPointer && maybe_type->data.pointer.allow_zero));
   4307 
   4308     ZigType *child_type = maybe_type->data.maybe.child_type;
   4309     if (!type_has_bits(child_type))
   4310         return maybe_handle;
   4311 
   4312     bool is_scalar = !handle_is_ptr(maybe_type);
   4313     if (is_scalar)
   4314         return LLVMBuildICmp(g->builder, LLVMIntNE, maybe_handle, LLVMConstNull(get_llvm_type(g, maybe_type)), "");
   4315 
   4316     LLVMValueRef maybe_field_ptr = LLVMBuildStructGEP(g->builder, maybe_handle, maybe_null_index, "");
   4317     return gen_load_untyped(g, maybe_field_ptr, 0, false, "");
   4318 }
   4319 
   4320 static LLVMValueRef ir_render_test_non_null(CodeGen *g, IrExecutable *executable,
   4321     IrInstructionTestNonNull *instruction)
   4322 {
   4323     return gen_non_null_bit(g, instruction->value->value.type, ir_llvm_value(g, instruction->value));
   4324 }
   4325 
   4326 static LLVMValueRef ir_render_optional_unwrap_ptr(CodeGen *g, IrExecutable *executable,
   4327         IrInstructionOptionalUnwrapPtr *instruction)
   4328 {
   4329     if (instruction->base.value.special != ConstValSpecialRuntime)
   4330         return nullptr;
   4331 
   4332     ZigType *ptr_type = instruction->base_ptr->value.type;
   4333     assert(ptr_type->id == ZigTypeIdPointer);
   4334     ZigType *maybe_type = ptr_type->data.pointer.child_type;
   4335     assert(maybe_type->id == ZigTypeIdOptional);
   4336     ZigType *child_type = maybe_type->data.maybe.child_type;
   4337     LLVMValueRef base_ptr = ir_llvm_value(g, instruction->base_ptr);
   4338     if (instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base)) {
   4339         LLVMValueRef maybe_handle = get_handle_value(g, base_ptr, maybe_type, ptr_type);
   4340         LLVMValueRef non_null_bit = gen_non_null_bit(g, maybe_type, maybe_handle);
   4341         LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnwrapOptionalFail");
   4342         LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnwrapOptionalOk");
   4343         LLVMBuildCondBr(g->builder, non_null_bit, ok_block, fail_block);
   4344 
   4345         LLVMPositionBuilderAtEnd(g->builder, fail_block);
   4346         gen_safety_crash(g, PanicMsgIdUnwrapOptionalFail);
   4347 
   4348         LLVMPositionBuilderAtEnd(g->builder, ok_block);
   4349     }
   4350     if (!type_has_bits(child_type)) {
   4351         return nullptr;
   4352     } else {
   4353         bool is_scalar = !handle_is_ptr(maybe_type);
   4354         if (is_scalar) {
   4355             return base_ptr;
   4356         } else {
   4357             LLVMValueRef optional_struct_ref = get_handle_value(g, base_ptr, maybe_type, ptr_type);
   4358             if (instruction->initializing) {
   4359                 LLVMValueRef non_null_bit_ptr = LLVMBuildStructGEP(g->builder, optional_struct_ref,
   4360                         maybe_null_index, "");
   4361                 LLVMValueRef non_null_bit = LLVMConstInt(LLVMInt1Type(), 1, false);
   4362                 gen_store_untyped(g, non_null_bit, non_null_bit_ptr, 0, false);
   4363             }
   4364             return LLVMBuildStructGEP(g->builder, optional_struct_ref, maybe_child_index, "");
   4365         }
   4366     }
   4367 }
   4368 
   4369 static LLVMValueRef get_int_builtin_fn(CodeGen *g, ZigType *int_type, BuiltinFnId fn_id) {
   4370     ZigLLVMFnKey key = {};
   4371     const char *fn_name;
   4372     uint32_t n_args;
   4373     if (fn_id == BuiltinFnIdCtz) {
   4374         fn_name = "cttz";
   4375         n_args = 2;
   4376         key.id = ZigLLVMFnIdCtz;
   4377         key.data.ctz.bit_count = (uint32_t)int_type->data.integral.bit_count;
   4378     } else if (fn_id == BuiltinFnIdClz) {
   4379         fn_name = "ctlz";
   4380         n_args = 2;
   4381         key.id = ZigLLVMFnIdClz;
   4382         key.data.clz.bit_count = (uint32_t)int_type->data.integral.bit_count;
   4383     } else if (fn_id == BuiltinFnIdPopCount) {
   4384         fn_name = "ctpop";
   4385         n_args = 1;
   4386         key.id = ZigLLVMFnIdPopCount;
   4387         key.data.pop_count.bit_count = (uint32_t)int_type->data.integral.bit_count;
   4388     } else if (fn_id == BuiltinFnIdBswap) {
   4389         fn_name = "bswap";
   4390         n_args = 1;
   4391         key.id = ZigLLVMFnIdBswap;
   4392         key.data.bswap.bit_count = (uint32_t)int_type->data.integral.bit_count;
   4393     } else if (fn_id == BuiltinFnIdBitReverse) {
   4394         fn_name = "bitreverse";
   4395         n_args = 1;
   4396         key.id = ZigLLVMFnIdBitReverse;
   4397         key.data.bit_reverse.bit_count = (uint32_t)int_type->data.integral.bit_count;
   4398     } else {
   4399         zig_unreachable();
   4400     }
   4401 
   4402     auto existing_entry = g->llvm_fn_table.maybe_get(key);
   4403     if (existing_entry)
   4404         return existing_entry->value;
   4405 
   4406     char llvm_name[64];
   4407     sprintf(llvm_name, "llvm.%s.i%" PRIu32, fn_name, int_type->data.integral.bit_count);
   4408     LLVMTypeRef param_types[] = {
   4409         get_llvm_type(g, int_type),
   4410         LLVMInt1Type(),
   4411     };
   4412     LLVMTypeRef fn_type = LLVMFunctionType(get_llvm_type(g, int_type), param_types, n_args, false);
   4413     LLVMValueRef fn_val = LLVMAddFunction(g->module, llvm_name, fn_type);
   4414     assert(LLVMGetIntrinsicID(fn_val));
   4415 
   4416     g->llvm_fn_table.put(key, fn_val);
   4417 
   4418     return fn_val;
   4419 }
   4420 
   4421 static LLVMValueRef ir_render_clz(CodeGen *g, IrExecutable *executable, IrInstructionClz *instruction) {
   4422     ZigType *int_type = instruction->op->value.type;
   4423     LLVMValueRef fn_val = get_int_builtin_fn(g, int_type, BuiltinFnIdClz);
   4424     LLVMValueRef operand = ir_llvm_value(g, instruction->op);
   4425     LLVMValueRef params[] {
   4426         operand,
   4427         LLVMConstNull(LLVMInt1Type()),
   4428     };
   4429     LLVMValueRef wrong_size_int = LLVMBuildCall(g->builder, fn_val, params, 2, "");
   4430     return gen_widen_or_shorten(g, false, int_type, instruction->base.value.type, wrong_size_int);
   4431 }
   4432 
   4433 static LLVMValueRef ir_render_ctz(CodeGen *g, IrExecutable *executable, IrInstructionCtz *instruction) {
   4434     ZigType *int_type = instruction->op->value.type;
   4435     LLVMValueRef fn_val = get_int_builtin_fn(g, int_type, BuiltinFnIdCtz);
   4436     LLVMValueRef operand = ir_llvm_value(g, instruction->op);
   4437     LLVMValueRef params[] {
   4438         operand,
   4439         LLVMConstNull(LLVMInt1Type()),
   4440     };
   4441     LLVMValueRef wrong_size_int = LLVMBuildCall(g->builder, fn_val, params, 2, "");
   4442     return gen_widen_or_shorten(g, false, int_type, instruction->base.value.type, wrong_size_int);
   4443 }
   4444 
   4445 static LLVMValueRef ir_render_pop_count(CodeGen *g, IrExecutable *executable, IrInstructionPopCount *instruction) {
   4446     ZigType *int_type = instruction->op->value.type;
   4447     LLVMValueRef fn_val = get_int_builtin_fn(g, int_type, BuiltinFnIdPopCount);
   4448     LLVMValueRef operand = ir_llvm_value(g, instruction->op);
   4449     LLVMValueRef wrong_size_int = LLVMBuildCall(g->builder, fn_val, &operand, 1, "");
   4450     return gen_widen_or_shorten(g, false, int_type, instruction->base.value.type, wrong_size_int);
   4451 }
   4452 
   4453 static LLVMValueRef ir_render_switch_br(CodeGen *g, IrExecutable *executable, IrInstructionSwitchBr *instruction) {
   4454     LLVMValueRef target_value = ir_llvm_value(g, instruction->target_value);
   4455     LLVMBasicBlockRef else_block = instruction->else_block->llvm_block;
   4456     LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, target_value, else_block,
   4457             (unsigned)instruction->case_count);
   4458     for (size_t i = 0; i < instruction->case_count; i += 1) {
   4459         IrInstructionSwitchBrCase *this_case = &instruction->cases[i];
   4460         LLVMAddCase(switch_instr, ir_llvm_value(g, this_case->value), this_case->block->llvm_block);
   4461     }
   4462     return nullptr;
   4463 }
   4464 
   4465 static LLVMValueRef ir_render_phi(CodeGen *g, IrExecutable *executable, IrInstructionPhi *instruction) {
   4466     if (!type_has_bits(instruction->base.value.type))
   4467         return nullptr;
   4468 
   4469     LLVMTypeRef phi_type;
   4470     if (handle_is_ptr(instruction->base.value.type)) {
   4471         phi_type = LLVMPointerType(get_llvm_type(g,instruction->base.value.type), 0);
   4472     } else {
   4473         phi_type = get_llvm_type(g, instruction->base.value.type);
   4474     }
   4475 
   4476     LLVMValueRef phi = LLVMBuildPhi(g->builder, phi_type, "");
   4477     LLVMValueRef *incoming_values = allocate<LLVMValueRef>(instruction->incoming_count);
   4478     LLVMBasicBlockRef *incoming_blocks = allocate<LLVMBasicBlockRef>(instruction->incoming_count);
   4479     for (size_t i = 0; i < instruction->incoming_count; i += 1) {
   4480         incoming_values[i] = ir_llvm_value(g, instruction->incoming_values[i]);
   4481         incoming_blocks[i] = instruction->incoming_blocks[i]->llvm_exit_block;
   4482     }
   4483     LLVMAddIncoming(phi, incoming_values, incoming_blocks, (unsigned)instruction->incoming_count);
   4484     return phi;
   4485 }
   4486 
   4487 static LLVMValueRef ir_render_ref(CodeGen *g, IrExecutable *executable, IrInstructionRefGen *instruction) {
   4488     if (!type_has_bits(instruction->base.value.type)) {
   4489         return nullptr;
   4490     }
   4491     LLVMValueRef value = ir_llvm_value(g, instruction->operand);
   4492     if (handle_is_ptr(instruction->operand->value.type)) {
   4493         return value;
   4494     } else {
   4495         LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   4496         gen_store_untyped(g, value, result_loc, 0, false);
   4497         return result_loc;
   4498     }
   4499 }
   4500 
   4501 static LLVMValueRef ir_render_err_name(CodeGen *g, IrExecutable *executable, IrInstructionErrName *instruction) {
   4502     assert(g->generate_error_name_table);
   4503 
   4504     if (g->errors_by_index.length == 1) {
   4505         LLVMBuildUnreachable(g->builder);
   4506         return nullptr;
   4507     }
   4508 
   4509     LLVMValueRef err_val = ir_llvm_value(g, instruction->value);
   4510     if (ir_want_runtime_safety(g, &instruction->base)) {
   4511         LLVMValueRef zero = LLVMConstNull(LLVMTypeOf(err_val));
   4512         LLVMValueRef end_val = LLVMConstInt(LLVMTypeOf(err_val), g->errors_by_index.length, false);
   4513         add_bounds_check(g, err_val, LLVMIntNE, zero, LLVMIntULT, end_val);
   4514     }
   4515 
   4516     LLVMValueRef indices[] = {
   4517         LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
   4518         err_val,
   4519     };
   4520     return LLVMBuildInBoundsGEP(g->builder, g->err_name_table, indices, 2, "");
   4521 }
   4522 
   4523 static LLVMValueRef get_enum_tag_name_function(CodeGen *g, ZigType *enum_type) {
   4524     assert(enum_type->id == ZigTypeIdEnum);
   4525     if (enum_type->data.enumeration.name_function)
   4526         return enum_type->data.enumeration.name_function;
   4527 
   4528     ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, false, false,
   4529             PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false);
   4530     ZigType *u8_slice_type = get_slice_type(g, u8_ptr_type);
   4531     ZigType *tag_int_type = enum_type->data.enumeration.tag_int_type;
   4532 
   4533     LLVMTypeRef tag_int_llvm_type = get_llvm_type(g, tag_int_type);
   4534     LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMPointerType(get_llvm_type(g, u8_slice_type), 0),
   4535             &tag_int_llvm_type, 1, false);
   4536 
   4537     Buf *fn_name = get_mangled_name(g, buf_sprintf("__zig_tag_name_%s", buf_ptr(&enum_type->name)), false);
   4538     LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref);
   4539     LLVMSetLinkage(fn_val, LLVMInternalLinkage);
   4540     LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified));
   4541     addLLVMFnAttr(fn_val, "nounwind");
   4542     add_uwtable_attr(g, fn_val);
   4543     if (codegen_have_frame_pointer(g)) {
   4544         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true");
   4545         ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr);
   4546     }
   4547 
   4548     LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder);
   4549     LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder);
   4550     ZigFn *prev_cur_fn = g->cur_fn;
   4551     LLVMValueRef prev_cur_fn_val = g->cur_fn_val;
   4552 
   4553     LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry");
   4554     LLVMPositionBuilderAtEnd(g->builder, entry_block);
   4555     ZigLLVMClearCurrentDebugLocation(g->builder);
   4556     g->cur_fn = nullptr;
   4557     g->cur_fn_val = fn_val;
   4558 
   4559     size_t field_count = enum_type->data.enumeration.src_field_count;
   4560     LLVMBasicBlockRef bad_value_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadValue");
   4561     LLVMValueRef tag_int_value = LLVMGetParam(fn_val, 0);
   4562     LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, tag_int_value, bad_value_block, field_count);
   4563 
   4564 
   4565     ZigType *usize = g->builtin_types.entry_usize;
   4566     LLVMValueRef array_ptr_indices[] = {
   4567         LLVMConstNull(usize->llvm_type),
   4568         LLVMConstNull(usize->llvm_type),
   4569     };
   4570 
   4571     for (size_t field_i = 0; field_i < field_count; field_i += 1) {
   4572         Buf *name = enum_type->data.enumeration.fields[field_i].name;
   4573         LLVMValueRef str_init = LLVMConstString(buf_ptr(name), (unsigned)buf_len(name), true);
   4574         LLVMValueRef str_global = LLVMAddGlobal(g->module, LLVMTypeOf(str_init), "");
   4575         LLVMSetInitializer(str_global, str_init);
   4576         LLVMSetLinkage(str_global, LLVMPrivateLinkage);
   4577         LLVMSetGlobalConstant(str_global, true);
   4578         LLVMSetUnnamedAddr(str_global, true);
   4579         LLVMSetAlignment(str_global, LLVMABIAlignmentOfType(g->target_data_ref, LLVMTypeOf(str_init)));
   4580 
   4581         LLVMValueRef fields[] = {
   4582             LLVMConstGEP(str_global, array_ptr_indices, 2),
   4583             LLVMConstInt(g->builtin_types.entry_usize->llvm_type, buf_len(name), false),
   4584         };
   4585         LLVMValueRef slice_init_value = LLVMConstNamedStruct(get_llvm_type(g, u8_slice_type), fields, 2);
   4586 
   4587         LLVMValueRef slice_global = LLVMAddGlobal(g->module, LLVMTypeOf(slice_init_value), "");
   4588         LLVMSetInitializer(slice_global, slice_init_value);
   4589         LLVMSetLinkage(slice_global, LLVMPrivateLinkage);
   4590         LLVMSetGlobalConstant(slice_global, true);
   4591         LLVMSetUnnamedAddr(slice_global, true);
   4592         LLVMSetAlignment(slice_global, LLVMABIAlignmentOfType(g->target_data_ref, LLVMTypeOf(slice_init_value)));
   4593 
   4594         LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(g->cur_fn_val, "Name");
   4595         LLVMValueRef this_tag_int_value = bigint_to_llvm_const(get_llvm_type(g, tag_int_type),
   4596                 &enum_type->data.enumeration.fields[field_i].value);
   4597         LLVMAddCase(switch_instr, this_tag_int_value, return_block);
   4598 
   4599         LLVMPositionBuilderAtEnd(g->builder, return_block);
   4600         LLVMBuildRet(g->builder, slice_global);
   4601     }
   4602 
   4603     LLVMPositionBuilderAtEnd(g->builder, bad_value_block);
   4604     if (g->build_mode == BuildModeDebug || g->build_mode == BuildModeSafeRelease) {
   4605         gen_safety_crash(g, PanicMsgIdBadEnumValue);
   4606     } else {
   4607         LLVMBuildUnreachable(g->builder);
   4608     }
   4609 
   4610     g->cur_fn = prev_cur_fn;
   4611     g->cur_fn_val = prev_cur_fn_val;
   4612     LLVMPositionBuilderAtEnd(g->builder, prev_block);
   4613     if (!g->strip_debug_symbols) {
   4614         LLVMSetCurrentDebugLocation(g->builder, prev_debug_location);
   4615     }
   4616 
   4617     enum_type->data.enumeration.name_function = fn_val;
   4618     return fn_val;
   4619 }
   4620 
   4621 static LLVMValueRef ir_render_enum_tag_name(CodeGen *g, IrExecutable *executable,
   4622         IrInstructionTagName *instruction)
   4623 {
   4624     ZigType *enum_type = instruction->target->value.type;
   4625     assert(enum_type->id == ZigTypeIdEnum);
   4626 
   4627     LLVMValueRef enum_name_function = get_enum_tag_name_function(g, enum_type);
   4628 
   4629     LLVMValueRef enum_tag_value = ir_llvm_value(g, instruction->target);
   4630     return ZigLLVMBuildCall(g->builder, enum_name_function, &enum_tag_value, 1,
   4631             get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
   4632 }
   4633 
   4634 static LLVMValueRef ir_render_field_parent_ptr(CodeGen *g, IrExecutable *executable,
   4635         IrInstructionFieldParentPtr *instruction)
   4636 {
   4637     ZigType *container_ptr_type = instruction->base.value.type;
   4638     assert(container_ptr_type->id == ZigTypeIdPointer);
   4639 
   4640     ZigType *container_type = container_ptr_type->data.pointer.child_type;
   4641 
   4642     size_t byte_offset = LLVMOffsetOfElement(g->target_data_ref,
   4643             get_llvm_type(g, container_type), instruction->field->gen_index);
   4644 
   4645     LLVMValueRef field_ptr_val = ir_llvm_value(g, instruction->field_ptr);
   4646 
   4647     if (byte_offset == 0) {
   4648         return LLVMBuildBitCast(g->builder, field_ptr_val, get_llvm_type(g, container_ptr_type), "");
   4649     } else {
   4650         ZigType *usize = g->builtin_types.entry_usize;
   4651 
   4652         LLVMValueRef field_ptr_int = LLVMBuildPtrToInt(g->builder, field_ptr_val, usize->llvm_type, "");
   4653 
   4654         LLVMValueRef base_ptr_int = LLVMBuildNUWSub(g->builder, field_ptr_int,
   4655                 LLVMConstInt(usize->llvm_type, byte_offset, false), "");
   4656 
   4657         return LLVMBuildIntToPtr(g->builder, base_ptr_int, get_llvm_type(g, container_ptr_type), "");
   4658     }
   4659 }
   4660 
   4661 static LLVMValueRef ir_render_align_cast(CodeGen *g, IrExecutable *executable, IrInstructionAlignCast *instruction) {
   4662     LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   4663     assert(target_val);
   4664 
   4665     bool want_runtime_safety = ir_want_runtime_safety(g, &instruction->base);
   4666     if (!want_runtime_safety) {
   4667         return target_val;
   4668     }
   4669 
   4670     ZigType *target_type = instruction->base.value.type;
   4671     uint32_t align_bytes;
   4672     LLVMValueRef ptr_val;
   4673 
   4674     if (target_type->id == ZigTypeIdPointer) {
   4675         align_bytes = get_ptr_align(g, target_type);
   4676         ptr_val = target_val;
   4677     } else if (target_type->id == ZigTypeIdFn) {
   4678         align_bytes = target_type->data.fn.fn_type_id.alignment;
   4679         ptr_val = target_val;
   4680     } else if (target_type->id == ZigTypeIdOptional &&
   4681             target_type->data.maybe.child_type->id == ZigTypeIdPointer)
   4682     {
   4683         align_bytes = get_ptr_align(g, target_type->data.maybe.child_type);
   4684         ptr_val = target_val;
   4685     } else if (target_type->id == ZigTypeIdOptional &&
   4686             target_type->data.maybe.child_type->id == ZigTypeIdFn)
   4687     {
   4688         align_bytes = target_type->data.maybe.child_type->data.fn.fn_type_id.alignment;
   4689         ptr_val = target_val;
   4690     } else if (target_type->id == ZigTypeIdStruct && target_type->data.structure.is_slice) {
   4691         ZigType *slice_ptr_type = target_type->data.structure.fields[slice_ptr_index].type_entry;
   4692         align_bytes = get_ptr_align(g, slice_ptr_type);
   4693 
   4694         size_t ptr_index = target_type->data.structure.fields[slice_ptr_index].gen_index;
   4695         LLVMValueRef ptr_val_ptr = LLVMBuildStructGEP(g->builder, target_val, (unsigned)ptr_index, "");
   4696         ptr_val = gen_load_untyped(g, ptr_val_ptr, 0, false, "");
   4697     } else {
   4698         zig_unreachable();
   4699     }
   4700 
   4701     assert(align_bytes != 1);
   4702 
   4703     ZigType *usize = g->builtin_types.entry_usize;
   4704     LLVMValueRef ptr_as_int_val = LLVMBuildPtrToInt(g->builder, ptr_val, usize->llvm_type, "");
   4705     LLVMValueRef alignment_minus_1 = LLVMConstInt(usize->llvm_type, align_bytes - 1, false);
   4706     LLVMValueRef anded_val = LLVMBuildAnd(g->builder, ptr_as_int_val, alignment_minus_1, "");
   4707     LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, anded_val, LLVMConstNull(usize->llvm_type), "");
   4708 
   4709     LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "AlignCastOk");
   4710     LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "AlignCastFail");
   4711 
   4712     LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
   4713 
   4714     LLVMPositionBuilderAtEnd(g->builder, fail_block);
   4715     gen_safety_crash(g, PanicMsgIdIncorrectAlignment);
   4716 
   4717     LLVMPositionBuilderAtEnd(g->builder, ok_block);
   4718 
   4719     return target_val;
   4720 }
   4721 
   4722 static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *executable,
   4723         IrInstructionErrorReturnTrace *instruction)
   4724 {
   4725     LLVMValueRef cur_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
   4726     if (cur_err_ret_trace_val == nullptr) {
   4727         return LLVMConstNull(get_llvm_type(g, ptr_to_stack_trace_type(g)));
   4728     }
   4729     return cur_err_ret_trace_val;
   4730 }
   4731 
   4732 static LLVMAtomicOrdering to_LLVMAtomicOrdering(AtomicOrder atomic_order) {
   4733     switch (atomic_order) {
   4734         case AtomicOrderUnordered: return LLVMAtomicOrderingUnordered;
   4735         case AtomicOrderMonotonic: return LLVMAtomicOrderingMonotonic;
   4736         case AtomicOrderAcquire: return LLVMAtomicOrderingAcquire;
   4737         case AtomicOrderRelease: return LLVMAtomicOrderingRelease;
   4738         case AtomicOrderAcqRel: return LLVMAtomicOrderingAcquireRelease;
   4739         case AtomicOrderSeqCst: return LLVMAtomicOrderingSequentiallyConsistent;
   4740     }
   4741     zig_unreachable();
   4742 }
   4743 
   4744 static LLVMAtomicRMWBinOp to_LLVMAtomicRMWBinOp(AtomicRmwOp op, bool is_signed) {
   4745     switch (op) {
   4746         case AtomicRmwOp_xchg: return LLVMAtomicRMWBinOpXchg;
   4747         case AtomicRmwOp_add: return LLVMAtomicRMWBinOpAdd;
   4748         case AtomicRmwOp_sub: return LLVMAtomicRMWBinOpSub;
   4749         case AtomicRmwOp_and: return LLVMAtomicRMWBinOpAnd;
   4750         case AtomicRmwOp_nand: return LLVMAtomicRMWBinOpNand;
   4751         case AtomicRmwOp_or: return LLVMAtomicRMWBinOpOr;
   4752         case AtomicRmwOp_xor: return LLVMAtomicRMWBinOpXor;
   4753         case AtomicRmwOp_max:
   4754             return is_signed ? LLVMAtomicRMWBinOpMax : LLVMAtomicRMWBinOpUMax;
   4755         case AtomicRmwOp_min:
   4756             return is_signed ? LLVMAtomicRMWBinOpMin : LLVMAtomicRMWBinOpUMin;
   4757     }
   4758     zig_unreachable();
   4759 }
   4760 
   4761 static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutable *executable, IrInstructionCmpxchgGen *instruction) {
   4762     LLVMValueRef ptr_val = ir_llvm_value(g, instruction->ptr);
   4763     LLVMValueRef cmp_val = ir_llvm_value(g, instruction->cmp_value);
   4764     LLVMValueRef new_val = ir_llvm_value(g, instruction->new_value);
   4765 
   4766     LLVMAtomicOrdering success_order = to_LLVMAtomicOrdering(instruction->success_order);
   4767     LLVMAtomicOrdering failure_order = to_LLVMAtomicOrdering(instruction->failure_order);
   4768 
   4769     LLVMValueRef result_val = ZigLLVMBuildCmpXchg(g->builder, ptr_val, cmp_val, new_val,
   4770             success_order, failure_order, instruction->is_weak);
   4771 
   4772     ZigType *optional_type = instruction->base.value.type;
   4773     assert(optional_type->id == ZigTypeIdOptional);
   4774     ZigType *child_type = optional_type->data.maybe.child_type;
   4775 
   4776     if (!handle_is_ptr(optional_type)) {
   4777         LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");
   4778         LLVMValueRef success_bit = LLVMBuildExtractValue(g->builder, result_val, 1, "");
   4779         return LLVMBuildSelect(g->builder, success_bit, LLVMConstNull(get_llvm_type(g, child_type)), payload_val, "");
   4780     }
   4781 
   4782     // When the cmpxchg is discarded, the result location will have no bits.
   4783     if (!type_has_bits(instruction->result_loc->value.type)) {
   4784         return nullptr;
   4785     }
   4786 
   4787     LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   4788     src_assert(result_loc != nullptr, instruction->base.source_node);
   4789     src_assert(type_has_bits(child_type), instruction->base.source_node);
   4790 
   4791     LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");
   4792     LLVMValueRef val_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_child_index, "");
   4793     gen_assign_raw(g, val_ptr, get_pointer_to_type(g, child_type, false), payload_val);
   4794 
   4795     LLVMValueRef success_bit = LLVMBuildExtractValue(g->builder, result_val, 1, "");
   4796     LLVMValueRef nonnull_bit = LLVMBuildNot(g->builder, success_bit, "");
   4797     LLVMValueRef maybe_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_null_index, "");
   4798     gen_store_untyped(g, nonnull_bit, maybe_ptr, 0, false);
   4799     return result_loc;
   4800 }
   4801 
   4802 static LLVMValueRef ir_render_fence(CodeGen *g, IrExecutable *executable, IrInstructionFence *instruction) {
   4803     LLVMAtomicOrdering atomic_order = to_LLVMAtomicOrdering(instruction->order);
   4804     LLVMBuildFence(g->builder, atomic_order, false, "");
   4805     return nullptr;
   4806 }
   4807 
   4808 static LLVMValueRef ir_render_truncate(CodeGen *g, IrExecutable *executable, IrInstructionTruncate *instruction) {
   4809     LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   4810     ZigType *dest_type = instruction->base.value.type;
   4811     ZigType *src_type = instruction->target->value.type;
   4812     if (dest_type == src_type) {
   4813         // no-op
   4814         return target_val;
   4815     } if (src_type->data.integral.bit_count == dest_type->data.integral.bit_count) {
   4816         return LLVMBuildBitCast(g->builder, target_val, get_llvm_type(g, dest_type), "");
   4817     } else {
   4818         LLVMValueRef target_val = ir_llvm_value(g, instruction->target);
   4819         return LLVMBuildTrunc(g->builder, target_val, get_llvm_type(g, dest_type), "");
   4820     }
   4821 }
   4822 
   4823 static LLVMValueRef ir_render_memset(CodeGen *g, IrExecutable *executable, IrInstructionMemset *instruction) {
   4824     LLVMValueRef dest_ptr = ir_llvm_value(g, instruction->dest_ptr);
   4825     LLVMValueRef len_val = ir_llvm_value(g, instruction->count);
   4826 
   4827     LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
   4828     LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, dest_ptr, ptr_u8, "");
   4829 
   4830     ZigType *ptr_type = instruction->dest_ptr->value.type;
   4831     assert(ptr_type->id == ZigTypeIdPointer);
   4832 
   4833     bool val_is_undef = value_is_all_undef(&instruction->byte->value);
   4834     LLVMValueRef fill_char;
   4835     if (val_is_undef) {
   4836         fill_char = LLVMConstInt(LLVMInt8Type(), 0xaa, false);
   4837     } else {
   4838         fill_char = ir_llvm_value(g, instruction->byte);
   4839     }
   4840     ZigLLVMBuildMemSet(g->builder, dest_ptr_casted, fill_char, len_val, get_ptr_align(g, ptr_type),
   4841             ptr_type->data.pointer.is_volatile);
   4842 
   4843     if (val_is_undef && want_valgrind_support(g)) {
   4844         gen_valgrind_undef(g, dest_ptr_casted, len_val);
   4845     }
   4846     return nullptr;
   4847 }
   4848 
   4849 static LLVMValueRef ir_render_memcpy(CodeGen *g, IrExecutable *executable, IrInstructionMemcpy *instruction) {
   4850     LLVMValueRef dest_ptr = ir_llvm_value(g, instruction->dest_ptr);
   4851     LLVMValueRef src_ptr = ir_llvm_value(g, instruction->src_ptr);
   4852     LLVMValueRef len_val = ir_llvm_value(g, instruction->count);
   4853 
   4854     LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
   4855 
   4856     LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, dest_ptr, ptr_u8, "");
   4857     LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, src_ptr, ptr_u8, "");
   4858 
   4859     ZigType *dest_ptr_type = instruction->dest_ptr->value.type;
   4860     ZigType *src_ptr_type = instruction->src_ptr->value.type;
   4861 
   4862     assert(dest_ptr_type->id == ZigTypeIdPointer);
   4863     assert(src_ptr_type->id == ZigTypeIdPointer);
   4864 
   4865     bool is_volatile = (dest_ptr_type->data.pointer.is_volatile || src_ptr_type->data.pointer.is_volatile);
   4866     ZigLLVMBuildMemCpy(g->builder, dest_ptr_casted, get_ptr_align(g, dest_ptr_type),
   4867             src_ptr_casted, get_ptr_align(g, src_ptr_type), len_val, is_volatile);
   4868     return nullptr;
   4869 }
   4870 
   4871 static LLVMValueRef ir_render_slice(CodeGen *g, IrExecutable *executable, IrInstructionSliceGen *instruction) {
   4872     LLVMValueRef array_ptr_ptr = ir_llvm_value(g, instruction->ptr);
   4873     ZigType *array_ptr_type = instruction->ptr->value.type;
   4874     assert(array_ptr_type->id == ZigTypeIdPointer);
   4875     ZigType *array_type = array_ptr_type->data.pointer.child_type;
   4876     LLVMValueRef array_ptr = get_handle_value(g, array_ptr_ptr, array_type, array_ptr_type);
   4877 
   4878     LLVMValueRef tmp_struct_ptr = ir_llvm_value(g, instruction->result_loc);
   4879 
   4880     bool want_runtime_safety = instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base);
   4881 
   4882     if (array_type->id == ZigTypeIdArray ||
   4883         (array_type->id == ZigTypeIdPointer && array_type->data.pointer.ptr_len == PtrLenSingle))
   4884     {
   4885         if (array_type->id == ZigTypeIdPointer) {
   4886             array_type = array_type->data.pointer.child_type;
   4887         }
   4888         LLVMValueRef start_val = ir_llvm_value(g, instruction->start);
   4889         LLVMValueRef end_val;
   4890         if (instruction->end) {
   4891             end_val = ir_llvm_value(g, instruction->end);
   4892         } else {
   4893             end_val = LLVMConstInt(g->builtin_types.entry_usize->llvm_type, array_type->data.array.len, false);
   4894         }
   4895         if (want_runtime_safety) {
   4896             if (instruction->start->value.special == ConstValSpecialRuntime || instruction->end) {
   4897                 add_bounds_check(g, start_val, LLVMIntEQ, nullptr, LLVMIntULE, end_val);
   4898             }
   4899             if (instruction->end) {
   4900                 LLVMValueRef array_end = LLVMConstInt(g->builtin_types.entry_usize->llvm_type,
   4901                         array_type->data.array.len, false);
   4902                 add_bounds_check(g, end_val, LLVMIntEQ, nullptr, LLVMIntULE, array_end);
   4903             }
   4904         }
   4905         if (!type_has_bits(array_type)) {
   4906             LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, slice_len_index, "");
   4907 
   4908             // TODO if runtime safety is on, store 0xaaaaaaa in ptr field
   4909             LLVMValueRef len_value = LLVMBuildNSWSub(g->builder, end_val, start_val, "");
   4910             gen_store_untyped(g, len_value, len_field_ptr, 0, false);
   4911             return tmp_struct_ptr;
   4912         }
   4913 
   4914 
   4915         LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, slice_ptr_index, "");
   4916         LLVMValueRef indices[] = {
   4917             LLVMConstNull(g->builtin_types.entry_usize->llvm_type),
   4918             start_val,
   4919         };
   4920         LLVMValueRef slice_start_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, indices, 2, "");
   4921         gen_store_untyped(g, slice_start_ptr, ptr_field_ptr, 0, false);
   4922 
   4923         LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, slice_len_index, "");
   4924         LLVMValueRef len_value = LLVMBuildNSWSub(g->builder, end_val, start_val, "");
   4925         gen_store_untyped(g, len_value, len_field_ptr, 0, false);
   4926 
   4927         return tmp_struct_ptr;
   4928     } else if (array_type->id == ZigTypeIdPointer) {
   4929         assert(array_type->data.pointer.ptr_len != PtrLenSingle);
   4930         LLVMValueRef start_val = ir_llvm_value(g, instruction->start);
   4931         LLVMValueRef end_val = ir_llvm_value(g, instruction->end);
   4932 
   4933         if (want_runtime_safety) {
   4934             add_bounds_check(g, start_val, LLVMIntEQ, nullptr, LLVMIntULE, end_val);
   4935         }
   4936 
   4937         if (type_has_bits(array_type)) {
   4938             size_t gen_ptr_index = instruction->base.value.type->data.structure.fields[slice_ptr_index].gen_index;
   4939             LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, gen_ptr_index, "");
   4940             LLVMValueRef slice_start_ptr = LLVMBuildInBoundsGEP(g->builder, array_ptr, &start_val, 1, "");
   4941             gen_store_untyped(g, slice_start_ptr, ptr_field_ptr, 0, false);
   4942         }
   4943 
   4944         size_t gen_len_index = instruction->base.value.type->data.structure.fields[slice_len_index].gen_index;
   4945         LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, gen_len_index, "");
   4946         LLVMValueRef len_value = LLVMBuildNSWSub(g->builder, end_val, start_val, "");
   4947         gen_store_untyped(g, len_value, len_field_ptr, 0, false);
   4948 
   4949         return tmp_struct_ptr;
   4950     } else if (array_type->id == ZigTypeIdStruct) {
   4951         assert(array_type->data.structure.is_slice);
   4952         assert(LLVMGetTypeKind(LLVMTypeOf(array_ptr)) == LLVMPointerTypeKind);
   4953         assert(LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(array_ptr))) == LLVMStructTypeKind);
   4954         assert(LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(tmp_struct_ptr))) == LLVMStructTypeKind);
   4955 
   4956         size_t ptr_index = array_type->data.structure.fields[slice_ptr_index].gen_index;
   4957         assert(ptr_index != SIZE_MAX);
   4958         size_t len_index = array_type->data.structure.fields[slice_len_index].gen_index;
   4959         assert(len_index != SIZE_MAX);
   4960 
   4961         LLVMValueRef prev_end = nullptr;
   4962         if (!instruction->end || want_runtime_safety) {
   4963             LLVMValueRef src_len_ptr = LLVMBuildStructGEP(g->builder, array_ptr, (unsigned)len_index, "");
   4964             prev_end = gen_load_untyped(g, src_len_ptr, 0, false, "");
   4965         }
   4966 
   4967         LLVMValueRef start_val = ir_llvm_value(g, instruction->start);
   4968         LLVMValueRef end_val;
   4969         if (instruction->end) {
   4970             end_val = ir_llvm_value(g, instruction->end);
   4971         } else {
   4972             end_val = prev_end;
   4973         }
   4974 
   4975         if (want_runtime_safety) {
   4976             assert(prev_end);
   4977             add_bounds_check(g, start_val, LLVMIntEQ, nullptr, LLVMIntULE, end_val);
   4978             if (instruction->end) {
   4979                 add_bounds_check(g, end_val, LLVMIntEQ, nullptr, LLVMIntULE, prev_end);
   4980             }
   4981         }
   4982 
   4983         LLVMValueRef src_ptr_ptr = LLVMBuildStructGEP(g->builder, array_ptr, (unsigned)ptr_index, "");
   4984         LLVMValueRef src_ptr = gen_load_untyped(g, src_ptr_ptr, 0, false, "");
   4985         LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, (unsigned)ptr_index, "");
   4986         LLVMValueRef slice_start_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr, &start_val, (unsigned)len_index, "");
   4987         gen_store_untyped(g, slice_start_ptr, ptr_field_ptr, 0, false);
   4988 
   4989         LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, tmp_struct_ptr, (unsigned)len_index, "");
   4990         LLVMValueRef len_value = LLVMBuildNSWSub(g->builder, end_val, start_val, "");
   4991         gen_store_untyped(g, len_value, len_field_ptr, 0, false);
   4992 
   4993         return tmp_struct_ptr;
   4994     } else {
   4995         zig_unreachable();
   4996     }
   4997 }
   4998 
   4999 static LLVMValueRef get_trap_fn_val(CodeGen *g) {
   5000     if (g->trap_fn_val)
   5001         return g->trap_fn_val;
   5002 
   5003     LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), nullptr, 0, false);
   5004     g->trap_fn_val = LLVMAddFunction(g->module, "llvm.debugtrap", fn_type);
   5005     assert(LLVMGetIntrinsicID(g->trap_fn_val));
   5006 
   5007     return g->trap_fn_val;
   5008 }
   5009 
   5010 
   5011 static LLVMValueRef ir_render_breakpoint(CodeGen *g, IrExecutable *executable, IrInstructionBreakpoint *instruction) {
   5012     LLVMBuildCall(g->builder, get_trap_fn_val(g), nullptr, 0, "");
   5013     return nullptr;
   5014 }
   5015 
   5016 static LLVMValueRef ir_render_return_address(CodeGen *g, IrExecutable *executable,
   5017         IrInstructionReturnAddress *instruction)
   5018 {
   5019     LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->llvm_type);
   5020     LLVMValueRef ptr_val = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, "");
   5021     return LLVMBuildPtrToInt(g->builder, ptr_val, g->builtin_types.entry_usize->llvm_type, "");
   5022 }
   5023 
   5024 static LLVMValueRef get_frame_address_fn_val(CodeGen *g) {
   5025     if (g->frame_address_fn_val)
   5026         return g->frame_address_fn_val;
   5027 
   5028     ZigType *return_type = get_pointer_to_type(g, g->builtin_types.entry_u8, true);
   5029 
   5030     LLVMTypeRef fn_type = LLVMFunctionType(get_llvm_type(g, return_type),
   5031             &g->builtin_types.entry_i32->llvm_type, 1, false);
   5032     g->frame_address_fn_val = LLVMAddFunction(g->module, "llvm.frameaddress", fn_type);
   5033     assert(LLVMGetIntrinsicID(g->frame_address_fn_val));
   5034 
   5035     return g->frame_address_fn_val;
   5036 }
   5037 
   5038 static LLVMValueRef ir_render_frame_address(CodeGen *g, IrExecutable *executable,
   5039         IrInstructionFrameAddress *instruction)
   5040 {
   5041     LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->llvm_type);
   5042     LLVMValueRef ptr_val = LLVMBuildCall(g->builder, get_frame_address_fn_val(g), &zero, 1, "");
   5043     return LLVMBuildPtrToInt(g->builder, ptr_val, g->builtin_types.entry_usize->llvm_type, "");
   5044 }
   5045 
   5046 static LLVMValueRef ir_render_handle(CodeGen *g, IrExecutable *executable, IrInstructionFrameHandle *instruction) {
   5047     return g->cur_frame_ptr;
   5048 }
   5049 
   5050 static LLVMValueRef render_shl_with_overflow(CodeGen *g, IrInstructionOverflowOp *instruction) {
   5051     ZigType *int_type = instruction->result_ptr_type;
   5052     assert(int_type->id == ZigTypeIdInt);
   5053 
   5054     LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
   5055     LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);
   5056     LLVMValueRef ptr_result = ir_llvm_value(g, instruction->result_ptr);
   5057 
   5058     LLVMValueRef op2_casted = gen_widen_or_shorten(g, false, instruction->op2->value.type,
   5059             instruction->op1->value.type, op2);
   5060 
   5061     LLVMValueRef result = LLVMBuildShl(g->builder, op1, op2_casted, "");
   5062     LLVMValueRef orig_val;
   5063     if (int_type->data.integral.is_signed) {
   5064         orig_val = LLVMBuildAShr(g->builder, result, op2_casted, "");
   5065     } else {
   5066         orig_val = LLVMBuildLShr(g->builder, result, op2_casted, "");
   5067     }
   5068     LLVMValueRef overflow_bit = LLVMBuildICmp(g->builder, LLVMIntNE, op1, orig_val, "");
   5069 
   5070     gen_store(g, result, ptr_result, instruction->result_ptr->value.type);
   5071 
   5072     return overflow_bit;
   5073 }
   5074 
   5075 static LLVMValueRef ir_render_overflow_op(CodeGen *g, IrExecutable *executable, IrInstructionOverflowOp *instruction) {
   5076     AddSubMul add_sub_mul;
   5077     switch (instruction->op) {
   5078         case IrOverflowOpAdd:
   5079             add_sub_mul = AddSubMulAdd;
   5080             break;
   5081         case IrOverflowOpSub:
   5082             add_sub_mul = AddSubMulSub;
   5083             break;
   5084         case IrOverflowOpMul:
   5085             add_sub_mul = AddSubMulMul;
   5086             break;
   5087         case IrOverflowOpShl:
   5088             return render_shl_with_overflow(g, instruction);
   5089     }
   5090 
   5091     ZigType *int_type = instruction->result_ptr_type;
   5092     assert(int_type->id == ZigTypeIdInt);
   5093 
   5094     LLVMValueRef fn_val = get_int_overflow_fn(g, int_type, add_sub_mul);
   5095 
   5096     LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
   5097     LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);
   5098     LLVMValueRef ptr_result = ir_llvm_value(g, instruction->result_ptr);
   5099 
   5100     LLVMValueRef params[] = {
   5101         op1,
   5102         op2,
   5103     };
   5104 
   5105     LLVMValueRef result_struct = LLVMBuildCall(g->builder, fn_val, params, 2, "");
   5106     LLVMValueRef result = LLVMBuildExtractValue(g->builder, result_struct, 0, "");
   5107     LLVMValueRef overflow_bit = LLVMBuildExtractValue(g->builder, result_struct, 1, "");
   5108     gen_store(g, result, ptr_result, instruction->result_ptr->value.type);
   5109 
   5110     return overflow_bit;
   5111 }
   5112 
   5113 static LLVMValueRef ir_render_test_err(CodeGen *g, IrExecutable *executable, IrInstructionTestErrGen *instruction) {
   5114     ZigType *err_union_type = instruction->err_union->value.type;
   5115     ZigType *payload_type = err_union_type->data.error_union.payload_type;
   5116     LLVMValueRef err_union_handle = ir_llvm_value(g, instruction->err_union);
   5117 
   5118     LLVMValueRef err_val;
   5119     if (type_has_bits(payload_type)) {
   5120         LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, err_union_handle, err_union_err_index, "");
   5121         err_val = gen_load_untyped(g, err_val_ptr, 0, false, "");
   5122     } else {
   5123         err_val = err_union_handle;
   5124     }
   5125 
   5126     LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
   5127     return LLVMBuildICmp(g->builder, LLVMIntNE, err_val, zero, "");
   5128 }
   5129 
   5130 static LLVMValueRef ir_render_unwrap_err_code(CodeGen *g, IrExecutable *executable,
   5131         IrInstructionUnwrapErrCode *instruction)
   5132 {
   5133     if (instruction->base.value.special != ConstValSpecialRuntime)
   5134         return nullptr;
   5135 
   5136     ZigType *ptr_type = instruction->err_union_ptr->value.type;
   5137     assert(ptr_type->id == ZigTypeIdPointer);
   5138     ZigType *err_union_type = ptr_type->data.pointer.child_type;
   5139     ZigType *payload_type = err_union_type->data.error_union.payload_type;
   5140     LLVMValueRef err_union_ptr = ir_llvm_value(g, instruction->err_union_ptr);
   5141     if (!type_has_bits(payload_type)) {
   5142         return err_union_ptr;
   5143     } else {
   5144         // TODO assign undef to the payload
   5145         LLVMValueRef err_union_handle = get_handle_value(g, err_union_ptr, err_union_type, ptr_type);
   5146         return LLVMBuildStructGEP(g->builder, err_union_handle, err_union_err_index, "");
   5147     }
   5148 }
   5149 
   5150 static LLVMValueRef ir_render_unwrap_err_payload(CodeGen *g, IrExecutable *executable,
   5151         IrInstructionUnwrapErrPayload *instruction)
   5152 {
   5153     if (instruction->base.value.special != ConstValSpecialRuntime)
   5154         return nullptr;
   5155 
   5156     bool want_safety = instruction->safety_check_on && ir_want_runtime_safety(g, &instruction->base) &&
   5157         g->errors_by_index.length > 1;
   5158     if (!want_safety && !type_has_bits(instruction->base.value.type))
   5159         return nullptr;
   5160     ZigType *ptr_type = instruction->value->value.type;
   5161     assert(ptr_type->id == ZigTypeIdPointer);
   5162     ZigType *err_union_type = ptr_type->data.pointer.child_type;
   5163     ZigType *payload_type = err_union_type->data.error_union.payload_type;
   5164     LLVMValueRef err_union_ptr = ir_llvm_value(g, instruction->value);
   5165     LLVMValueRef err_union_handle = get_handle_value(g, err_union_ptr, err_union_type, ptr_type);
   5166 
   5167     if (!type_has_bits(err_union_type->data.error_union.err_set_type)) {
   5168         return err_union_handle;
   5169     }
   5170 
   5171     if (want_safety) {
   5172         LLVMValueRef err_val;
   5173         if (type_has_bits(payload_type)) {
   5174             LLVMValueRef err_val_ptr = LLVMBuildStructGEP(g->builder, err_union_handle, err_union_err_index, "");
   5175             err_val = gen_load_untyped(g, err_val_ptr, 0, false, "");
   5176         } else {
   5177             err_val = err_union_handle;
   5178         }
   5179         LLVMValueRef zero = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
   5180         LLVMValueRef cond_val = LLVMBuildICmp(g->builder, LLVMIntEQ, err_val, zero, "");
   5181         LLVMBasicBlockRef err_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnwrapErrError");
   5182         LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "UnwrapErrOk");
   5183         LLVMBuildCondBr(g->builder, cond_val, ok_block, err_block);
   5184 
   5185         LLVMPositionBuilderAtEnd(g->builder, err_block);
   5186         gen_safety_crash_for_err(g, err_val, instruction->base.scope);
   5187 
   5188         LLVMPositionBuilderAtEnd(g->builder, ok_block);
   5189     }
   5190 
   5191     if (type_has_bits(payload_type)) {
   5192         if (instruction->initializing) {
   5193             LLVMValueRef err_tag_ptr = LLVMBuildStructGEP(g->builder, err_union_handle, err_union_err_index, "");
   5194             LLVMValueRef ok_err_val = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
   5195             gen_store_untyped(g, ok_err_val, err_tag_ptr, 0, false);
   5196         }
   5197         return LLVMBuildStructGEP(g->builder, err_union_handle, err_union_payload_index, "");
   5198     } else {
   5199         return nullptr;
   5200     }
   5201 }
   5202 
   5203 static LLVMValueRef ir_render_optional_wrap(CodeGen *g, IrExecutable *executable, IrInstructionOptionalWrap *instruction) {
   5204     ZigType *wanted_type = instruction->base.value.type;
   5205 
   5206     assert(wanted_type->id == ZigTypeIdOptional);
   5207 
   5208     ZigType *child_type = wanted_type->data.maybe.child_type;
   5209 
   5210     if (!type_has_bits(child_type)) {
   5211         LLVMValueRef result = LLVMConstAllOnes(LLVMInt1Type());
   5212         if (instruction->result_loc != nullptr) {
   5213             LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   5214             gen_store_untyped(g, result, result_loc, 0, false);
   5215         }
   5216         return result;
   5217     }
   5218 
   5219     LLVMValueRef payload_val = ir_llvm_value(g, instruction->operand);
   5220     if (!handle_is_ptr(wanted_type)) {
   5221         if (instruction->result_loc != nullptr) {
   5222             LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   5223             gen_store_untyped(g, payload_val, result_loc, 0, false);
   5224         }
   5225         return payload_val;
   5226     }
   5227 
   5228     LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   5229 
   5230     LLVMValueRef val_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_child_index, "");
   5231     // child_type and instruction->value->value.type may differ by constness
   5232     gen_assign_raw(g, val_ptr, get_pointer_to_type(g, child_type, false), payload_val);
   5233     LLVMValueRef maybe_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_null_index, "");
   5234     gen_store_untyped(g, LLVMConstAllOnes(LLVMInt1Type()), maybe_ptr, 0, false);
   5235 
   5236     return result_loc;
   5237 }
   5238 
   5239 static LLVMValueRef ir_render_err_wrap_code(CodeGen *g, IrExecutable *executable, IrInstructionErrWrapCode *instruction) {
   5240     ZigType *wanted_type = instruction->base.value.type;
   5241 
   5242     assert(wanted_type->id == ZigTypeIdErrorUnion);
   5243 
   5244     LLVMValueRef err_val = ir_llvm_value(g, instruction->operand);
   5245 
   5246     if (!handle_is_ptr(wanted_type))
   5247         return err_val;
   5248 
   5249     LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   5250 
   5251     LLVMValueRef err_tag_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_err_index, "");
   5252     gen_store_untyped(g, err_val, err_tag_ptr, 0, false);
   5253 
   5254     // TODO store undef to the payload
   5255 
   5256     return result_loc;
   5257 }
   5258 
   5259 static LLVMValueRef ir_render_err_wrap_payload(CodeGen *g, IrExecutable *executable, IrInstructionErrWrapPayload *instruction) {
   5260     ZigType *wanted_type = instruction->base.value.type;
   5261 
   5262     assert(wanted_type->id == ZigTypeIdErrorUnion);
   5263 
   5264     ZigType *payload_type = wanted_type->data.error_union.payload_type;
   5265     ZigType *err_set_type = wanted_type->data.error_union.err_set_type;
   5266 
   5267     if (!type_has_bits(err_set_type)) {
   5268         return ir_llvm_value(g, instruction->operand);
   5269     }
   5270 
   5271     LLVMValueRef ok_err_val = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
   5272 
   5273     if (!type_has_bits(payload_type))
   5274         return ok_err_val;
   5275 
   5276 
   5277     LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   5278 
   5279     LLVMValueRef payload_val = ir_llvm_value(g, instruction->operand);
   5280 
   5281     LLVMValueRef err_tag_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_err_index, "");
   5282     gen_store_untyped(g, ok_err_val, err_tag_ptr, 0, false);
   5283 
   5284     LLVMValueRef payload_ptr = LLVMBuildStructGEP(g->builder, result_loc, err_union_payload_index, "");
   5285     gen_assign_raw(g, payload_ptr, get_pointer_to_type(g, payload_type, false), payload_val);
   5286 
   5287     return result_loc;
   5288 }
   5289 
   5290 static LLVMValueRef ir_render_union_tag(CodeGen *g, IrExecutable *executable, IrInstructionUnionTag *instruction) {
   5291     ZigType *union_type = instruction->value->value.type;
   5292 
   5293     ZigType *tag_type = union_type->data.unionation.tag_type;
   5294     if (!type_has_bits(tag_type))
   5295         return nullptr;
   5296 
   5297     LLVMValueRef union_val = ir_llvm_value(g, instruction->value);
   5298     if (union_type->data.unionation.gen_field_count == 0)
   5299         return union_val;
   5300 
   5301     assert(union_type->data.unionation.gen_tag_index != SIZE_MAX);
   5302     LLVMValueRef tag_field_ptr = LLVMBuildStructGEP(g->builder, union_val,
   5303             union_type->data.unionation.gen_tag_index, "");
   5304     ZigType *ptr_type = get_pointer_to_type(g, tag_type, false);
   5305     return get_handle_value(g, tag_field_ptr, tag_type, ptr_type);
   5306 }
   5307 
   5308 static LLVMValueRef ir_render_panic(CodeGen *g, IrExecutable *executable, IrInstructionPanic *instruction) {
   5309     gen_panic(g, ir_llvm_value(g, instruction->msg), get_cur_err_ret_trace_val(g, instruction->base.scope));
   5310     return nullptr;
   5311 }
   5312 
   5313 static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable,
   5314         IrInstructionAtomicRmw *instruction)
   5315 {
   5316     bool is_signed;
   5317     ZigType *operand_type = instruction->operand->value.type;
   5318     if (operand_type->id == ZigTypeIdInt) {
   5319         is_signed = operand_type->data.integral.is_signed;
   5320     } else {
   5321         is_signed = false;
   5322     }
   5323     LLVMAtomicRMWBinOp op = to_LLVMAtomicRMWBinOp(instruction->resolved_op, is_signed);
   5324     LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->resolved_ordering);
   5325     LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
   5326     LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
   5327 
   5328     if (get_codegen_ptr_type(operand_type) == nullptr) {
   5329         return LLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);
   5330     }
   5331 
   5332     // it's a pointer but we need to treat it as an int
   5333     LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,
   5334         LLVMPointerType(g->builtin_types.entry_usize->llvm_type, 0), "");
   5335     LLVMValueRef casted_operand = LLVMBuildPtrToInt(g->builder, operand, g->builtin_types.entry_usize->llvm_type, "");
   5336     LLVMValueRef uncasted_result = LLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
   5337             g->is_single_threaded);
   5338     return LLVMBuildIntToPtr(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
   5339 }
   5340 
   5341 static LLVMValueRef ir_render_atomic_load(CodeGen *g, IrExecutable *executable,
   5342         IrInstructionAtomicLoad *instruction)
   5343 {
   5344     LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->resolved_ordering);
   5345     LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
   5346     LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value.type, "");
   5347     LLVMSetOrdering(load_inst, ordering);
   5348     return load_inst;
   5349 }
   5350 
   5351 static LLVMValueRef ir_render_float_op(CodeGen *g, IrExecutable *executable, IrInstructionFloatOp *instruction) {
   5352     LLVMValueRef op = ir_llvm_value(g, instruction->op1);
   5353     assert(instruction->base.value.type->id == ZigTypeIdFloat);
   5354     LLVMValueRef fn_val = get_float_fn(g, instruction->base.value.type, ZigLLVMFnIdFloatOp, instruction->op);
   5355     return LLVMBuildCall(g->builder, fn_val, &op, 1, "");
   5356 }
   5357 
   5358 static LLVMValueRef ir_render_mul_add(CodeGen *g, IrExecutable *executable, IrInstructionMulAdd *instruction) {
   5359     LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
   5360     LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);
   5361     LLVMValueRef op3 = ir_llvm_value(g, instruction->op3);
   5362     assert(instruction->base.value.type->id == ZigTypeIdFloat ||
   5363            instruction->base.value.type->id == ZigTypeIdVector);
   5364     LLVMValueRef fn_val = get_float_fn(g, instruction->base.value.type, ZigLLVMFnIdFMA, BuiltinFnIdMulAdd);
   5365     LLVMValueRef args[3] = {
   5366         op1,
   5367         op2,
   5368         op3,
   5369     };
   5370     return LLVMBuildCall(g->builder, fn_val, args, 3, "");
   5371 }
   5372 
   5373 static LLVMValueRef ir_render_bswap(CodeGen *g, IrExecutable *executable, IrInstructionBswap *instruction) {
   5374     LLVMValueRef op = ir_llvm_value(g, instruction->op);
   5375     ZigType *int_type = instruction->base.value.type;
   5376     assert(int_type->id == ZigTypeIdInt);
   5377     if (int_type->data.integral.bit_count % 16 == 0) {
   5378         LLVMValueRef fn_val = get_int_builtin_fn(g, instruction->base.value.type, BuiltinFnIdBswap);
   5379         return LLVMBuildCall(g->builder, fn_val, &op, 1, "");
   5380     }
   5381     // Not an even number of bytes, so we zext 1 byte, then bswap, shift right 1 byte, truncate
   5382     ZigType *extended_type = get_int_type(g, int_type->data.integral.is_signed,
   5383             int_type->data.integral.bit_count + 8);
   5384     // aabbcc
   5385     LLVMValueRef extended = LLVMBuildZExt(g->builder, op, get_llvm_type(g, extended_type), "");
   5386     // 00aabbcc
   5387     LLVMValueRef fn_val = get_int_builtin_fn(g, extended_type, BuiltinFnIdBswap);
   5388     LLVMValueRef swapped = LLVMBuildCall(g->builder, fn_val, &extended, 1, "");
   5389     // ccbbaa00
   5390     LLVMValueRef shifted = ZigLLVMBuildLShrExact(g->builder, swapped,
   5391             LLVMConstInt(get_llvm_type(g, extended_type), 8, false), "");
   5392     // 00ccbbaa
   5393     return LLVMBuildTrunc(g->builder, shifted, get_llvm_type(g, int_type), "");
   5394 }
   5395 
   5396 static LLVMValueRef ir_render_bit_reverse(CodeGen *g, IrExecutable *executable, IrInstructionBitReverse *instruction) {
   5397     LLVMValueRef op = ir_llvm_value(g, instruction->op);
   5398     ZigType *int_type = instruction->base.value.type;
   5399     assert(int_type->id == ZigTypeIdInt);
   5400     LLVMValueRef fn_val = get_int_builtin_fn(g, instruction->base.value.type, BuiltinFnIdBitReverse);
   5401     return LLVMBuildCall(g->builder, fn_val, &op, 1, "");
   5402 }
   5403 
   5404 static LLVMValueRef ir_render_vector_to_array(CodeGen *g, IrExecutable *executable,
   5405         IrInstructionVectorToArray *instruction)
   5406 {
   5407     ZigType *array_type = instruction->base.value.type;
   5408     assert(array_type->id == ZigTypeIdArray);
   5409     assert(handle_is_ptr(array_type));
   5410     LLVMValueRef result_loc = ir_llvm_value(g, instruction->result_loc);
   5411     LLVMValueRef vector = ir_llvm_value(g, instruction->vector);
   5412     LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, result_loc,
   5413             LLVMPointerType(get_llvm_type(g, instruction->vector->value.type), 0), "");
   5414     uint32_t alignment = get_ptr_align(g, instruction->result_loc->value.type);
   5415     gen_store_untyped(g, vector, casted_ptr, alignment, false);
   5416     return result_loc;
   5417 }
   5418 
   5419 static LLVMValueRef ir_render_array_to_vector(CodeGen *g, IrExecutable *executable,
   5420         IrInstructionArrayToVector *instruction)
   5421 {
   5422     ZigType *vector_type = instruction->base.value.type;
   5423     assert(vector_type->id == ZigTypeIdVector);
   5424     assert(!handle_is_ptr(vector_type));
   5425     LLVMValueRef array_ptr = ir_llvm_value(g, instruction->array);
   5426     LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, array_ptr,
   5427             LLVMPointerType(get_llvm_type(g, vector_type), 0), "");
   5428     ZigType *array_type = instruction->array->value.type;
   5429     assert(array_type->id == ZigTypeIdArray);
   5430     uint32_t alignment = get_abi_alignment(g, array_type->data.array.child_type);
   5431     return gen_load_untyped(g, casted_ptr, alignment, false, "");
   5432 }
   5433 
   5434 static LLVMValueRef ir_render_assert_zero(CodeGen *g, IrExecutable *executable,
   5435         IrInstructionAssertZero *instruction)
   5436 {
   5437     LLVMValueRef target = ir_llvm_value(g, instruction->target);
   5438     ZigType *int_type = instruction->target->value.type;
   5439     if (ir_want_runtime_safety(g, &instruction->base)) {
   5440         return gen_assert_zero(g, target, int_type);
   5441     }
   5442     return nullptr;
   5443 }
   5444 
   5445 static LLVMValueRef ir_render_assert_non_null(CodeGen *g, IrExecutable *executable,
   5446         IrInstructionAssertNonNull *instruction)
   5447 {
   5448     LLVMValueRef target = ir_llvm_value(g, instruction->target);
   5449     ZigType *target_type = instruction->target->value.type;
   5450 
   5451     if (target_type->id == ZigTypeIdPointer) {
   5452         assert(target_type->data.pointer.ptr_len == PtrLenC);
   5453         LLVMValueRef non_null_bit = LLVMBuildICmp(g->builder, LLVMIntNE, target,
   5454                 LLVMConstNull(get_llvm_type(g, target_type)), "");
   5455 
   5456         LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "AssertNonNullFail");
   5457         LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "AssertNonNullOk");
   5458         LLVMBuildCondBr(g->builder, non_null_bit, ok_block, fail_block);
   5459 
   5460         LLVMPositionBuilderAtEnd(g->builder, fail_block);
   5461         gen_assertion(g, PanicMsgIdUnwrapOptionalFail, &instruction->base);
   5462 
   5463         LLVMPositionBuilderAtEnd(g->builder, ok_block);
   5464     } else {
   5465         zig_unreachable();
   5466     }
   5467     return nullptr;
   5468 }
   5469 
   5470 static LLVMValueRef ir_render_suspend_begin(CodeGen *g, IrExecutable *executable,
   5471         IrInstructionSuspendBegin *instruction)
   5472 {
   5473     if (fn_is_async(g->cur_fn)) {
   5474         instruction->resume_bb = gen_suspend_begin(g, "SuspendResume");
   5475     }
   5476     return nullptr;
   5477 }
   5478 
   5479 static LLVMValueRef ir_render_suspend_finish(CodeGen *g, IrExecutable *executable,
   5480         IrInstructionSuspendFinish *instruction)
   5481 {
   5482     LLVMBuildRetVoid(g->builder);
   5483 
   5484     LLVMPositionBuilderAtEnd(g->builder, instruction->begin->resume_bb);
   5485     render_async_var_decls(g, instruction->base.scope);
   5486     return nullptr;
   5487 }
   5488 
   5489 static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwaitGen *instruction) {
   5490     LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   5491     LLVMValueRef zero = LLVMConstNull(usize_type_ref);
   5492     LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame);
   5493     ZigType *result_type = instruction->base.value.type;
   5494     ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true);
   5495 
   5496     // Prepare to be suspended
   5497     LLVMBasicBlockRef resume_bb = gen_suspend_begin(g, "AwaitResume");
   5498     LLVMBasicBlockRef end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "AwaitEnd");
   5499 
   5500     // At this point resuming the function will continue from resume_bb.
   5501     // This code is as if it is running inside the suspend block.
   5502 
   5503     // supply the awaiter return pointer
   5504     LLVMValueRef result_loc = (instruction->result_loc == nullptr) ?
   5505         nullptr : ir_llvm_value(g, instruction->result_loc);
   5506     if (type_has_bits(result_type)) {
   5507         LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_ret_start + 1, "");
   5508         if (result_loc == nullptr) {
   5509             // no copy needed
   5510             LLVMBuildStore(g->builder, LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr_ptr))),
   5511                     awaiter_ret_ptr_ptr);
   5512         } else {
   5513             LLVMBuildStore(g->builder, result_loc, awaiter_ret_ptr_ptr);
   5514         }
   5515     }
   5516 
   5517     // supply the error return trace pointer
   5518     if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
   5519         LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
   5520         assert(my_err_ret_trace_val != nullptr);
   5521         LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr,
   5522                 frame_index_trace_arg(g, result_type) + 1, "");
   5523         LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr);
   5524     }
   5525 
   5526     // caller's own frame pointer
   5527     LLVMValueRef awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_frame_ptr, usize_type_ref, "");
   5528     LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_awaiter_index, "");
   5529     LLVMValueRef prev_val = gen_maybe_atomic_op(g, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_init_val,
   5530             LLVMAtomicOrderingRelease);
   5531 
   5532     LLVMBasicBlockRef bad_await_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadAwait");
   5533     LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CompleteSuspend");
   5534     LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
   5535 
   5536     LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
   5537     LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, bad_await_block, 2);
   5538 
   5539     LLVMAddCase(switch_instr, zero, complete_suspend_block);
   5540     LLVMAddCase(switch_instr, all_ones, early_return_block);
   5541 
   5542     // We discovered that another awaiter was already here.
   5543     LLVMPositionBuilderAtEnd(g->builder, bad_await_block);
   5544     gen_assertion(g, PanicMsgIdBadAwait, &instruction->base);
   5545 
   5546     // Rely on the target to resume us from suspension.
   5547     LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block);
   5548     LLVMBuildRetVoid(g->builder);
   5549 
   5550     // Early return: The async function has already completed. We must copy the result and
   5551     // the error return trace if applicable.
   5552     LLVMPositionBuilderAtEnd(g->builder, early_return_block);
   5553     if (type_has_bits(result_type) && result_loc != nullptr) {
   5554         LLVMValueRef their_result_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, frame_ret_start, "");
   5555         LLVMValueRef their_result_ptr = LLVMBuildLoad(g->builder, their_result_ptr_ptr, "");
   5556         LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
   5557         LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, result_loc, ptr_u8, "");
   5558         LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, their_result_ptr, ptr_u8, "");
   5559         bool is_volatile = false;
   5560         uint32_t abi_align = get_abi_alignment(g, result_type);
   5561         LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, result_type), false);
   5562         ZigLLVMBuildMemCpy(g->builder,
   5563                 dest_ptr_casted, abi_align,
   5564                 src_ptr_casted, abi_align, byte_count_val, is_volatile);
   5565     }
   5566     if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
   5567         LLVMValueRef their_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr,
   5568                 frame_index_trace_arg(g, result_type), "");
   5569         LLVMValueRef src_trace_ptr = LLVMBuildLoad(g->builder, their_trace_ptr_ptr, "");
   5570         LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope);
   5571         LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr };
   5572         ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2,
   5573                 get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, "");
   5574     }
   5575     LLVMBuildBr(g->builder, end_bb);
   5576 
   5577     LLVMPositionBuilderAtEnd(g->builder, resume_bb);
   5578     gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr);
   5579     LLVMBuildBr(g->builder, end_bb);
   5580 
   5581     LLVMPositionBuilderAtEnd(g->builder, end_bb);
   5582     if (type_has_bits(result_type) && result_loc != nullptr) {
   5583         return get_handle_value(g, result_loc, result_type, ptr_result_type);
   5584     }
   5585     return nullptr;
   5586 }
   5587 
   5588 static LLVMValueRef ir_render_resume(CodeGen *g, IrExecutable *executable, IrInstructionResume *instruction) {
   5589     LLVMValueRef frame = ir_llvm_value(g, instruction->frame);
   5590     ZigType *frame_type = instruction->frame->value.type;
   5591     assert(frame_type->id == ZigTypeIdAnyFrame);
   5592 
   5593     gen_resume(g, nullptr, frame, ResumeIdManual);
   5594     return nullptr;
   5595 }
   5596 
   5597 static LLVMValueRef ir_render_frame_size(CodeGen *g, IrExecutable *executable,
   5598         IrInstructionFrameSizeGen *instruction)
   5599 {
   5600     LLVMValueRef fn_val = ir_llvm_value(g, instruction->fn);
   5601     return gen_frame_size(g, fn_val);
   5602 }
   5603 
   5604 static LLVMValueRef ir_render_spill_begin(CodeGen *g, IrExecutable *executable,
   5605         IrInstructionSpillBegin *instruction)
   5606 {
   5607     if (!fn_is_async(g->cur_fn))
   5608         return nullptr;
   5609 
   5610     switch (instruction->spill_id) {
   5611         case SpillIdInvalid:
   5612             zig_unreachable();
   5613         case SpillIdRetErrCode: {
   5614             LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
   5615             LLVMValueRef ptr = ir_llvm_value(g, g->cur_fn->err_code_spill);
   5616             LLVMBuildStore(g->builder, operand, ptr);
   5617             return nullptr;
   5618         }
   5619 
   5620     }
   5621     zig_unreachable();
   5622 }
   5623 
   5624 static LLVMValueRef ir_render_spill_end(CodeGen *g, IrExecutable *executable, IrInstructionSpillEnd *instruction) {
   5625     if (!fn_is_async(g->cur_fn))
   5626         return ir_llvm_value(g, instruction->begin->operand);
   5627 
   5628     switch (instruction->begin->spill_id) {
   5629         case SpillIdInvalid:
   5630             zig_unreachable();
   5631         case SpillIdRetErrCode: {
   5632             LLVMValueRef ptr = ir_llvm_value(g, g->cur_fn->err_code_spill);
   5633             return LLVMBuildLoad(g->builder, ptr, "");
   5634         }
   5635 
   5636     }
   5637     zig_unreachable();
   5638 }
   5639 
   5640 static void set_debug_location(CodeGen *g, IrInstruction *instruction) {
   5641     AstNode *source_node = instruction->source_node;
   5642     Scope *scope = instruction->scope;
   5643 
   5644     assert(source_node);
   5645     assert(scope);
   5646 
   5647     ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1,
   5648             (int)source_node->column + 1, get_di_scope(g, scope));
   5649 }
   5650 
   5651 static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, IrInstruction *instruction) {
   5652     switch (instruction->id) {
   5653         case IrInstructionIdInvalid:
   5654         case IrInstructionIdConst:
   5655         case IrInstructionIdTypeOf:
   5656         case IrInstructionIdFieldPtr:
   5657         case IrInstructionIdSetCold:
   5658         case IrInstructionIdSetRuntimeSafety:
   5659         case IrInstructionIdSetFloatMode:
   5660         case IrInstructionIdArrayType:
   5661         case IrInstructionIdAnyFrameType:
   5662         case IrInstructionIdSliceType:
   5663         case IrInstructionIdSizeOf:
   5664         case IrInstructionIdSwitchTarget:
   5665         case IrInstructionIdContainerInitFields:
   5666         case IrInstructionIdCompileErr:
   5667         case IrInstructionIdCompileLog:
   5668         case IrInstructionIdImport:
   5669         case IrInstructionIdCImport:
   5670         case IrInstructionIdCInclude:
   5671         case IrInstructionIdCDefine:
   5672         case IrInstructionIdCUndef:
   5673         case IrInstructionIdEmbedFile:
   5674         case IrInstructionIdIntType:
   5675         case IrInstructionIdVectorType:
   5676         case IrInstructionIdMemberCount:
   5677         case IrInstructionIdMemberType:
   5678         case IrInstructionIdMemberName:
   5679         case IrInstructionIdAlignOf:
   5680         case IrInstructionIdFnProto:
   5681         case IrInstructionIdTestComptime:
   5682         case IrInstructionIdCheckSwitchProngs:
   5683         case IrInstructionIdCheckStatementIsVoid:
   5684         case IrInstructionIdTypeName:
   5685         case IrInstructionIdDeclRef:
   5686         case IrInstructionIdSwitchVar:
   5687         case IrInstructionIdSwitchElseVar:
   5688         case IrInstructionIdByteOffsetOf:
   5689         case IrInstructionIdBitOffsetOf:
   5690         case IrInstructionIdTypeInfo:
   5691         case IrInstructionIdHasField:
   5692         case IrInstructionIdTypeId:
   5693         case IrInstructionIdSetEvalBranchQuota:
   5694         case IrInstructionIdPtrType:
   5695         case IrInstructionIdOpaqueType:
   5696         case IrInstructionIdSetAlignStack:
   5697         case IrInstructionIdArgType:
   5698         case IrInstructionIdTagType:
   5699         case IrInstructionIdExport:
   5700         case IrInstructionIdErrorUnion:
   5701         case IrInstructionIdAddImplicitReturnType:
   5702         case IrInstructionIdIntCast:
   5703         case IrInstructionIdFloatCast:
   5704         case IrInstructionIdIntToFloat:
   5705         case IrInstructionIdFloatToInt:
   5706         case IrInstructionIdBoolToInt:
   5707         case IrInstructionIdErrSetCast:
   5708         case IrInstructionIdFromBytes:
   5709         case IrInstructionIdToBytes:
   5710         case IrInstructionIdEnumToInt:
   5711         case IrInstructionIdCheckRuntimeScope:
   5712         case IrInstructionIdDeclVarSrc:
   5713         case IrInstructionIdPtrCastSrc:
   5714         case IrInstructionIdCmpxchgSrc:
   5715         case IrInstructionIdLoadPtr:
   5716         case IrInstructionIdGlobalAsm:
   5717         case IrInstructionIdHasDecl:
   5718         case IrInstructionIdUndeclaredIdent:
   5719         case IrInstructionIdCallSrc:
   5720         case IrInstructionIdAllocaSrc:
   5721         case IrInstructionIdEndExpr:
   5722         case IrInstructionIdImplicitCast:
   5723         case IrInstructionIdResolveResult:
   5724         case IrInstructionIdResetResult:
   5725         case IrInstructionIdContainerInitList:
   5726         case IrInstructionIdSliceSrc:
   5727         case IrInstructionIdRef:
   5728         case IrInstructionIdBitCastSrc:
   5729         case IrInstructionIdTestErrSrc:
   5730         case IrInstructionIdUnionInitNamedField:
   5731         case IrInstructionIdFrameType:
   5732         case IrInstructionIdFrameSizeSrc:
   5733         case IrInstructionIdAllocaGen:
   5734         case IrInstructionIdAwaitSrc:
   5735             zig_unreachable();
   5736 
   5737         case IrInstructionIdDeclVarGen:
   5738             return ir_render_decl_var(g, executable, (IrInstructionDeclVarGen *)instruction);
   5739         case IrInstructionIdReturn:
   5740             return ir_render_return(g, executable, (IrInstructionReturn *)instruction);
   5741         case IrInstructionIdBinOp:
   5742             return ir_render_bin_op(g, executable, (IrInstructionBinOp *)instruction);
   5743         case IrInstructionIdCast:
   5744             return ir_render_cast(g, executable, (IrInstructionCast *)instruction);
   5745         case IrInstructionIdUnreachable:
   5746             return ir_render_unreachable(g, executable, (IrInstructionUnreachable *)instruction);
   5747         case IrInstructionIdCondBr:
   5748             return ir_render_cond_br(g, executable, (IrInstructionCondBr *)instruction);
   5749         case IrInstructionIdBr:
   5750             return ir_render_br(g, executable, (IrInstructionBr *)instruction);
   5751         case IrInstructionIdUnOp:
   5752             return ir_render_un_op(g, executable, (IrInstructionUnOp *)instruction);
   5753         case IrInstructionIdLoadPtrGen:
   5754             return ir_render_load_ptr(g, executable, (IrInstructionLoadPtrGen *)instruction);
   5755         case IrInstructionIdStorePtr:
   5756             return ir_render_store_ptr(g, executable, (IrInstructionStorePtr *)instruction);
   5757         case IrInstructionIdVarPtr:
   5758             return ir_render_var_ptr(g, executable, (IrInstructionVarPtr *)instruction);
   5759         case IrInstructionIdReturnPtr:
   5760             return ir_render_return_ptr(g, executable, (IrInstructionReturnPtr *)instruction);
   5761         case IrInstructionIdElemPtr:
   5762             return ir_render_elem_ptr(g, executable, (IrInstructionElemPtr *)instruction);
   5763         case IrInstructionIdCallGen:
   5764             return ir_render_call(g, executable, (IrInstructionCallGen *)instruction);
   5765         case IrInstructionIdStructFieldPtr:
   5766             return ir_render_struct_field_ptr(g, executable, (IrInstructionStructFieldPtr *)instruction);
   5767         case IrInstructionIdUnionFieldPtr:
   5768             return ir_render_union_field_ptr(g, executable, (IrInstructionUnionFieldPtr *)instruction);
   5769         case IrInstructionIdAsm:
   5770             return ir_render_asm(g, executable, (IrInstructionAsm *)instruction);
   5771         case IrInstructionIdTestNonNull:
   5772             return ir_render_test_non_null(g, executable, (IrInstructionTestNonNull *)instruction);
   5773         case IrInstructionIdOptionalUnwrapPtr:
   5774             return ir_render_optional_unwrap_ptr(g, executable, (IrInstructionOptionalUnwrapPtr *)instruction);
   5775         case IrInstructionIdClz:
   5776             return ir_render_clz(g, executable, (IrInstructionClz *)instruction);
   5777         case IrInstructionIdCtz:
   5778             return ir_render_ctz(g, executable, (IrInstructionCtz *)instruction);
   5779         case IrInstructionIdPopCount:
   5780             return ir_render_pop_count(g, executable, (IrInstructionPopCount *)instruction);
   5781         case IrInstructionIdSwitchBr:
   5782             return ir_render_switch_br(g, executable, (IrInstructionSwitchBr *)instruction);
   5783         case IrInstructionIdBswap:
   5784             return ir_render_bswap(g, executable, (IrInstructionBswap *)instruction);
   5785         case IrInstructionIdBitReverse:
   5786             return ir_render_bit_reverse(g, executable, (IrInstructionBitReverse *)instruction);
   5787         case IrInstructionIdPhi:
   5788             return ir_render_phi(g, executable, (IrInstructionPhi *)instruction);
   5789         case IrInstructionIdRefGen:
   5790             return ir_render_ref(g, executable, (IrInstructionRefGen *)instruction);
   5791         case IrInstructionIdErrName:
   5792             return ir_render_err_name(g, executable, (IrInstructionErrName *)instruction);
   5793         case IrInstructionIdCmpxchgGen:
   5794             return ir_render_cmpxchg(g, executable, (IrInstructionCmpxchgGen *)instruction);
   5795         case IrInstructionIdFence:
   5796             return ir_render_fence(g, executable, (IrInstructionFence *)instruction);
   5797         case IrInstructionIdTruncate:
   5798             return ir_render_truncate(g, executable, (IrInstructionTruncate *)instruction);
   5799         case IrInstructionIdBoolNot:
   5800             return ir_render_bool_not(g, executable, (IrInstructionBoolNot *)instruction);
   5801         case IrInstructionIdMemset:
   5802             return ir_render_memset(g, executable, (IrInstructionMemset *)instruction);
   5803         case IrInstructionIdMemcpy:
   5804             return ir_render_memcpy(g, executable, (IrInstructionMemcpy *)instruction);
   5805         case IrInstructionIdSliceGen:
   5806             return ir_render_slice(g, executable, (IrInstructionSliceGen *)instruction);
   5807         case IrInstructionIdBreakpoint:
   5808             return ir_render_breakpoint(g, executable, (IrInstructionBreakpoint *)instruction);
   5809         case IrInstructionIdReturnAddress:
   5810             return ir_render_return_address(g, executable, (IrInstructionReturnAddress *)instruction);
   5811         case IrInstructionIdFrameAddress:
   5812             return ir_render_frame_address(g, executable, (IrInstructionFrameAddress *)instruction);
   5813         case IrInstructionIdFrameHandle:
   5814             return ir_render_handle(g, executable, (IrInstructionFrameHandle *)instruction);
   5815         case IrInstructionIdOverflowOp:
   5816             return ir_render_overflow_op(g, executable, (IrInstructionOverflowOp *)instruction);
   5817         case IrInstructionIdTestErrGen:
   5818             return ir_render_test_err(g, executable, (IrInstructionTestErrGen *)instruction);
   5819         case IrInstructionIdUnwrapErrCode:
   5820             return ir_render_unwrap_err_code(g, executable, (IrInstructionUnwrapErrCode *)instruction);
   5821         case IrInstructionIdUnwrapErrPayload:
   5822             return ir_render_unwrap_err_payload(g, executable, (IrInstructionUnwrapErrPayload *)instruction);
   5823         case IrInstructionIdOptionalWrap:
   5824             return ir_render_optional_wrap(g, executable, (IrInstructionOptionalWrap *)instruction);
   5825         case IrInstructionIdErrWrapCode:
   5826             return ir_render_err_wrap_code(g, executable, (IrInstructionErrWrapCode *)instruction);
   5827         case IrInstructionIdErrWrapPayload:
   5828             return ir_render_err_wrap_payload(g, executable, (IrInstructionErrWrapPayload *)instruction);
   5829         case IrInstructionIdUnionTag:
   5830             return ir_render_union_tag(g, executable, (IrInstructionUnionTag *)instruction);
   5831         case IrInstructionIdPtrCastGen:
   5832             return ir_render_ptr_cast(g, executable, (IrInstructionPtrCastGen *)instruction);
   5833         case IrInstructionIdBitCastGen:
   5834             return ir_render_bit_cast(g, executable, (IrInstructionBitCastGen *)instruction);
   5835         case IrInstructionIdWidenOrShorten:
   5836             return ir_render_widen_or_shorten(g, executable, (IrInstructionWidenOrShorten *)instruction);
   5837         case IrInstructionIdPtrToInt:
   5838             return ir_render_ptr_to_int(g, executable, (IrInstructionPtrToInt *)instruction);
   5839         case IrInstructionIdIntToPtr:
   5840             return ir_render_int_to_ptr(g, executable, (IrInstructionIntToPtr *)instruction);
   5841         case IrInstructionIdIntToEnum:
   5842             return ir_render_int_to_enum(g, executable, (IrInstructionIntToEnum *)instruction);
   5843         case IrInstructionIdIntToErr:
   5844             return ir_render_int_to_err(g, executable, (IrInstructionIntToErr *)instruction);
   5845         case IrInstructionIdErrToInt:
   5846             return ir_render_err_to_int(g, executable, (IrInstructionErrToInt *)instruction);
   5847         case IrInstructionIdPanic:
   5848             return ir_render_panic(g, executable, (IrInstructionPanic *)instruction);
   5849         case IrInstructionIdTagName:
   5850             return ir_render_enum_tag_name(g, executable, (IrInstructionTagName *)instruction);
   5851         case IrInstructionIdFieldParentPtr:
   5852             return ir_render_field_parent_ptr(g, executable, (IrInstructionFieldParentPtr *)instruction);
   5853         case IrInstructionIdAlignCast:
   5854             return ir_render_align_cast(g, executable, (IrInstructionAlignCast *)instruction);
   5855         case IrInstructionIdErrorReturnTrace:
   5856             return ir_render_error_return_trace(g, executable, (IrInstructionErrorReturnTrace *)instruction);
   5857         case IrInstructionIdAtomicRmw:
   5858             return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction);
   5859         case IrInstructionIdAtomicLoad:
   5860             return ir_render_atomic_load(g, executable, (IrInstructionAtomicLoad *)instruction);
   5861         case IrInstructionIdSaveErrRetAddr:
   5862             return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction);
   5863         case IrInstructionIdFloatOp:
   5864             return ir_render_float_op(g, executable, (IrInstructionFloatOp *)instruction);
   5865         case IrInstructionIdMulAdd:
   5866             return ir_render_mul_add(g, executable, (IrInstructionMulAdd *)instruction);
   5867         case IrInstructionIdArrayToVector:
   5868             return ir_render_array_to_vector(g, executable, (IrInstructionArrayToVector *)instruction);
   5869         case IrInstructionIdVectorToArray:
   5870             return ir_render_vector_to_array(g, executable, (IrInstructionVectorToArray *)instruction);
   5871         case IrInstructionIdAssertZero:
   5872             return ir_render_assert_zero(g, executable, (IrInstructionAssertZero *)instruction);
   5873         case IrInstructionIdAssertNonNull:
   5874             return ir_render_assert_non_null(g, executable, (IrInstructionAssertNonNull *)instruction);
   5875         case IrInstructionIdResizeSlice:
   5876             return ir_render_resize_slice(g, executable, (IrInstructionResizeSlice *)instruction);
   5877         case IrInstructionIdPtrOfArrayToSlice:
   5878             return ir_render_ptr_of_array_to_slice(g, executable, (IrInstructionPtrOfArrayToSlice *)instruction);
   5879         case IrInstructionIdSuspendBegin:
   5880             return ir_render_suspend_begin(g, executable, (IrInstructionSuspendBegin *)instruction);
   5881         case IrInstructionIdSuspendFinish:
   5882             return ir_render_suspend_finish(g, executable, (IrInstructionSuspendFinish *)instruction);
   5883         case IrInstructionIdResume:
   5884             return ir_render_resume(g, executable, (IrInstructionResume *)instruction);
   5885         case IrInstructionIdFrameSizeGen:
   5886             return ir_render_frame_size(g, executable, (IrInstructionFrameSizeGen *)instruction);
   5887         case IrInstructionIdAwaitGen:
   5888             return ir_render_await(g, executable, (IrInstructionAwaitGen *)instruction);
   5889         case IrInstructionIdSpillBegin:
   5890             return ir_render_spill_begin(g, executable, (IrInstructionSpillBegin *)instruction);
   5891         case IrInstructionIdSpillEnd:
   5892             return ir_render_spill_end(g, executable, (IrInstructionSpillEnd *)instruction);
   5893     }
   5894     zig_unreachable();
   5895 }
   5896 
   5897 static void ir_render(CodeGen *g, ZigFn *fn_entry) {
   5898     assert(fn_entry);
   5899 
   5900     IrExecutable *executable = &fn_entry->analyzed_executable;
   5901     assert(executable->basic_block_list.length > 0);
   5902 
   5903     for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {
   5904         IrBasicBlock *current_block = executable->basic_block_list.at(block_i);
   5905         assert(current_block->llvm_block);
   5906         LLVMPositionBuilderAtEnd(g->builder, current_block->llvm_block);
   5907         for (size_t instr_i = 0; instr_i < current_block->instruction_list.length; instr_i += 1) {
   5908             IrInstruction *instruction = current_block->instruction_list.at(instr_i);
   5909             if (instruction->ref_count == 0 && !ir_has_side_effects(instruction))
   5910                 continue;
   5911 
   5912             if (!g->strip_debug_symbols) {
   5913                 set_debug_location(g, instruction);
   5914             }
   5915             instruction->llvm_value = ir_render_instruction(g, executable, instruction);
   5916         }
   5917         current_block->llvm_exit_block = LLVMGetInsertBlock(g->builder);
   5918     }
   5919 }
   5920 
   5921 static LLVMValueRef gen_const_ptr_struct_recursive(CodeGen *g, ConstExprValue *struct_const_val, size_t field_index);
   5922 static LLVMValueRef gen_const_ptr_array_recursive(CodeGen *g, ConstExprValue *array_const_val, size_t index);
   5923 static LLVMValueRef gen_const_ptr_union_recursive(CodeGen *g, ConstExprValue *union_const_val);
   5924 static LLVMValueRef gen_const_ptr_err_union_code_recursive(CodeGen *g, ConstExprValue *err_union_const_val);
   5925 static LLVMValueRef gen_const_ptr_err_union_payload_recursive(CodeGen *g, ConstExprValue *err_union_const_val);
   5926 static LLVMValueRef gen_const_ptr_optional_payload_recursive(CodeGen *g, ConstExprValue *optional_const_val);
   5927 
   5928 static LLVMValueRef gen_parent_ptr(CodeGen *g, ConstExprValue *val, ConstParent *parent) {
   5929     switch (parent->id) {
   5930         case ConstParentIdNone:
   5931             render_const_val(g, val, "");
   5932             render_const_val_global(g, val, "");
   5933             return val->global_refs->llvm_global;
   5934         case ConstParentIdStruct:
   5935             return gen_const_ptr_struct_recursive(g, parent->data.p_struct.struct_val,
   5936                     parent->data.p_struct.field_index);
   5937         case ConstParentIdErrUnionCode:
   5938             return gen_const_ptr_err_union_code_recursive(g, parent->data.p_err_union_code.err_union_val);
   5939         case ConstParentIdErrUnionPayload:
   5940             return gen_const_ptr_err_union_payload_recursive(g, parent->data.p_err_union_payload.err_union_val);
   5941         case ConstParentIdOptionalPayload:
   5942             return gen_const_ptr_optional_payload_recursive(g, parent->data.p_optional_payload.optional_val);
   5943         case ConstParentIdArray:
   5944             return gen_const_ptr_array_recursive(g, parent->data.p_array.array_val,
   5945                     parent->data.p_array.elem_index);
   5946         case ConstParentIdUnion:
   5947             return gen_const_ptr_union_recursive(g, parent->data.p_union.union_val);
   5948         case ConstParentIdScalar:
   5949             render_const_val(g, parent->data.p_scalar.scalar_val, "");
   5950             render_const_val_global(g, parent->data.p_scalar.scalar_val, "");
   5951             return parent->data.p_scalar.scalar_val->global_refs->llvm_global;
   5952     }
   5953     zig_unreachable();
   5954 }
   5955 
   5956 static LLVMValueRef gen_const_ptr_array_recursive(CodeGen *g, ConstExprValue *array_const_val, size_t index) {
   5957     expand_undef_array(g, array_const_val);
   5958     ConstParent *parent = &array_const_val->parent;
   5959     LLVMValueRef base_ptr = gen_parent_ptr(g, array_const_val, parent);
   5960 
   5961     LLVMTypeKind el_type = LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(base_ptr)));
   5962     if (el_type == LLVMArrayTypeKind) {
   5963         ZigType *usize = g->builtin_types.entry_usize;
   5964         LLVMValueRef indices[] = {
   5965             LLVMConstNull(usize->llvm_type),
   5966             LLVMConstInt(usize->llvm_type, index, false),
   5967         };
   5968         return LLVMConstInBoundsGEP(base_ptr, indices, 2);
   5969     } else if (el_type == LLVMStructTypeKind) {
   5970         ZigType *u32 = g->builtin_types.entry_u32;
   5971         LLVMValueRef indices[] = {
   5972             LLVMConstNull(get_llvm_type(g, u32)),
   5973             LLVMConstInt(get_llvm_type(g, u32), index, false),
   5974         };
   5975         return LLVMConstInBoundsGEP(base_ptr, indices, 2);
   5976     } else {
   5977         assert(parent->id == ConstParentIdScalar);
   5978         return base_ptr;
   5979     }
   5980 }
   5981 
   5982 static LLVMValueRef gen_const_ptr_struct_recursive(CodeGen *g, ConstExprValue *struct_const_val, size_t field_index) {
   5983     ConstParent *parent = &struct_const_val->parent;
   5984     LLVMValueRef base_ptr = gen_parent_ptr(g, struct_const_val, parent);
   5985 
   5986     ZigType *u32 = g->builtin_types.entry_u32;
   5987     LLVMValueRef indices[] = {
   5988         LLVMConstNull(get_llvm_type(g, u32)),
   5989         LLVMConstInt(get_llvm_type(g, u32), field_index, false),
   5990     };
   5991     return LLVMConstInBoundsGEP(base_ptr, indices, 2);
   5992 }
   5993 
   5994 static LLVMValueRef gen_const_ptr_err_union_code_recursive(CodeGen *g, ConstExprValue *err_union_const_val) {
   5995     ConstParent *parent = &err_union_const_val->parent;
   5996     LLVMValueRef base_ptr = gen_parent_ptr(g, err_union_const_val, parent);
   5997 
   5998     ZigType *u32 = g->builtin_types.entry_u32;
   5999     LLVMValueRef indices[] = {
   6000         LLVMConstNull(get_llvm_type(g, u32)),
   6001         LLVMConstInt(get_llvm_type(g, u32), err_union_err_index, false),
   6002     };
   6003     return LLVMConstInBoundsGEP(base_ptr, indices, 2);
   6004 }
   6005 
   6006 static LLVMValueRef gen_const_ptr_err_union_payload_recursive(CodeGen *g, ConstExprValue *err_union_const_val) {
   6007     ConstParent *parent = &err_union_const_val->parent;
   6008     LLVMValueRef base_ptr = gen_parent_ptr(g, err_union_const_val, parent);
   6009 
   6010     ZigType *u32 = g->builtin_types.entry_u32;
   6011     LLVMValueRef indices[] = {
   6012         LLVMConstNull(get_llvm_type(g, u32)),
   6013         LLVMConstInt(get_llvm_type(g, u32), err_union_payload_index, false),
   6014     };
   6015     return LLVMConstInBoundsGEP(base_ptr, indices, 2);
   6016 }
   6017 
   6018 static LLVMValueRef gen_const_ptr_optional_payload_recursive(CodeGen *g, ConstExprValue *optional_const_val) {
   6019     ConstParent *parent = &optional_const_val->parent;
   6020     LLVMValueRef base_ptr = gen_parent_ptr(g, optional_const_val, parent);
   6021 
   6022     ZigType *u32 = g->builtin_types.entry_u32;
   6023     LLVMValueRef indices[] = {
   6024         LLVMConstNull(get_llvm_type(g, u32)),
   6025         LLVMConstInt(get_llvm_type(g, u32), maybe_child_index, false),
   6026     };
   6027     return LLVMConstInBoundsGEP(base_ptr, indices, 2);
   6028 }
   6029 
   6030 static LLVMValueRef gen_const_ptr_union_recursive(CodeGen *g, ConstExprValue *union_const_val) {
   6031     ConstParent *parent = &union_const_val->parent;
   6032     LLVMValueRef base_ptr = gen_parent_ptr(g, union_const_val, parent);
   6033 
   6034     ZigType *u32 = g->builtin_types.entry_u32;
   6035     LLVMValueRef indices[] = {
   6036         LLVMConstNull(get_llvm_type(g, u32)),
   6037         LLVMConstInt(get_llvm_type(g, u32), 0, false), // TODO test const union with more aligned tag type than payload
   6038     };
   6039     return LLVMConstInBoundsGEP(base_ptr, indices, 2);
   6040 }
   6041 
   6042 static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, ConstExprValue *const_val) {
   6043     switch (const_val->special) {
   6044         case ConstValSpecialRuntime:
   6045             zig_unreachable();
   6046         case ConstValSpecialUndef:
   6047             return LLVMConstInt(big_int_type_ref, 0, false);
   6048         case ConstValSpecialStatic:
   6049             break;
   6050     }
   6051 
   6052     ZigType *type_entry = const_val->type;
   6053     assert(type_has_bits(type_entry));
   6054     switch (type_entry->id) {
   6055         case ZigTypeIdInvalid:
   6056         case ZigTypeIdMetaType:
   6057         case ZigTypeIdUnreachable:
   6058         case ZigTypeIdComptimeFloat:
   6059         case ZigTypeIdComptimeInt:
   6060         case ZigTypeIdEnumLiteral:
   6061         case ZigTypeIdUndefined:
   6062         case ZigTypeIdNull:
   6063         case ZigTypeIdErrorUnion:
   6064         case ZigTypeIdErrorSet:
   6065         case ZigTypeIdBoundFn:
   6066         case ZigTypeIdArgTuple:
   6067         case ZigTypeIdVoid:
   6068         case ZigTypeIdOpaque:
   6069             zig_unreachable();
   6070         case ZigTypeIdBool:
   6071             return LLVMConstInt(big_int_type_ref, const_val->data.x_bool ? 1 : 0, false);
   6072         case ZigTypeIdEnum:
   6073             {
   6074                 assert(type_entry->data.enumeration.decl_node->data.container_decl.init_arg_expr != nullptr);
   6075                 LLVMValueRef int_val = gen_const_val(g, const_val, "");
   6076                 return LLVMConstZExt(int_val, big_int_type_ref);
   6077             }
   6078         case ZigTypeIdInt:
   6079             {
   6080                 LLVMValueRef int_val = gen_const_val(g, const_val, "");
   6081                 return LLVMConstZExt(int_val, big_int_type_ref);
   6082             }
   6083         case ZigTypeIdFloat:
   6084             {
   6085                 LLVMValueRef float_val = gen_const_val(g, const_val, "");
   6086                 LLVMValueRef int_val = LLVMConstFPToUI(float_val,
   6087                         LLVMIntType((unsigned)type_entry->data.floating.bit_count));
   6088                 return LLVMConstZExt(int_val, big_int_type_ref);
   6089             }
   6090         case ZigTypeIdPointer:
   6091         case ZigTypeIdFn:
   6092         case ZigTypeIdOptional:
   6093             {
   6094                 LLVMValueRef ptr_val = gen_const_val(g, const_val, "");
   6095                 LLVMValueRef ptr_size_int_val = LLVMConstPtrToInt(ptr_val, g->builtin_types.entry_usize->llvm_type);
   6096                 return LLVMConstZExt(ptr_size_int_val, big_int_type_ref);
   6097             }
   6098         case ZigTypeIdArray: {
   6099             LLVMValueRef val = LLVMConstInt(big_int_type_ref, 0, false);
   6100             if (const_val->data.x_array.special == ConstArraySpecialUndef) {
   6101                 return val;
   6102             }
   6103             expand_undef_array(g, const_val);
   6104             bool is_big_endian = g->is_big_endian; // TODO get endianness from struct type
   6105             uint32_t packed_bits_size = type_size_bits(g, type_entry->data.array.child_type);
   6106             size_t used_bits = 0;
   6107             for (size_t i = 0; i < type_entry->data.array.len; i += 1) {
   6108                 ConstExprValue *elem_val = &const_val->data.x_array.data.s_none.elements[i];
   6109                 LLVMValueRef child_val = pack_const_int(g, big_int_type_ref, elem_val);
   6110 
   6111                 if (is_big_endian) {
   6112                     LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, packed_bits_size, false);
   6113                     val = LLVMConstShl(val, shift_amt);
   6114                     val = LLVMConstOr(val, child_val);
   6115                 } else {
   6116                     LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, used_bits, false);
   6117                     LLVMValueRef child_val_shifted = LLVMConstShl(child_val, shift_amt);
   6118                     val = LLVMConstOr(val, child_val_shifted);
   6119                     used_bits += packed_bits_size;
   6120                 }
   6121             }
   6122             return val;
   6123         }
   6124         case ZigTypeIdVector:
   6125             zig_panic("TODO bit pack a vector");
   6126         case ZigTypeIdUnion:
   6127             zig_panic("TODO bit pack a union");
   6128         case ZigTypeIdStruct:
   6129             {
   6130                 assert(type_entry->data.structure.layout == ContainerLayoutPacked);
   6131                 bool is_big_endian = g->is_big_endian; // TODO get endianness from struct type
   6132 
   6133                 LLVMValueRef val = LLVMConstInt(big_int_type_ref, 0, false);
   6134                 size_t used_bits = 0;
   6135                 for (size_t i = 0; i < type_entry->data.structure.src_field_count; i += 1) {
   6136                     TypeStructField *field = &type_entry->data.structure.fields[i];
   6137                     if (field->gen_index == SIZE_MAX) {
   6138                         continue;
   6139                     }
   6140                     LLVMValueRef child_val = pack_const_int(g, big_int_type_ref, &const_val->data.x_struct.fields[i]);
   6141                     uint32_t packed_bits_size = type_size_bits(g, field->type_entry);
   6142                     if (is_big_endian) {
   6143                         LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, packed_bits_size, false);
   6144                         val = LLVMConstShl(val, shift_amt);
   6145                         val = LLVMConstOr(val, child_val);
   6146                     } else {
   6147                         LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, used_bits, false);
   6148                         LLVMValueRef child_val_shifted = LLVMConstShl(child_val, shift_amt);
   6149                         val = LLVMConstOr(val, child_val_shifted);
   6150                         used_bits += packed_bits_size;
   6151                     }
   6152                 }
   6153                 return val;
   6154             }
   6155         case ZigTypeIdFnFrame:
   6156             zig_panic("TODO bit pack an async function frame");
   6157         case ZigTypeIdAnyFrame:
   6158             zig_panic("TODO bit pack an anyframe");
   6159     }
   6160     zig_unreachable();
   6161 }
   6162 
   6163 // We have this because union constants can't be represented by the official union type,
   6164 // and this property bubbles up in whatever aggregate type contains a union constant
   6165 static bool is_llvm_value_unnamed_type(CodeGen *g, ZigType *type_entry, LLVMValueRef val) {
   6166     return LLVMTypeOf(val) != get_llvm_type(g, type_entry);
   6167 }
   6168 
   6169 static LLVMValueRef gen_const_val_ptr(CodeGen *g, ConstExprValue *const_val, const char *name) {
   6170     switch (const_val->data.x_ptr.special) {
   6171         case ConstPtrSpecialInvalid:
   6172         case ConstPtrSpecialDiscard:
   6173             zig_unreachable();
   6174         case ConstPtrSpecialRef:
   6175             {
   6176                 assert(const_val->global_refs != nullptr);
   6177                 ConstExprValue *pointee = const_val->data.x_ptr.data.ref.pointee;
   6178                 render_const_val(g, pointee, "");
   6179                 render_const_val_global(g, pointee, "");
   6180                 const_val->global_refs->llvm_value = LLVMConstBitCast(pointee->global_refs->llvm_global,
   6181                         get_llvm_type(g, const_val->type));
   6182                 return const_val->global_refs->llvm_value;
   6183             }
   6184         case ConstPtrSpecialBaseArray:
   6185             {
   6186                 assert(const_val->global_refs != nullptr);
   6187                 ConstExprValue *array_const_val = const_val->data.x_ptr.data.base_array.array_val;
   6188                 assert(array_const_val->type->id == ZigTypeIdArray);
   6189                 if (!type_has_bits(array_const_val->type)) {
   6190                     // make this a null pointer
   6191                     ZigType *usize = g->builtin_types.entry_usize;
   6192                     const_val->global_refs->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
   6193                             get_llvm_type(g, const_val->type));
   6194                     return const_val->global_refs->llvm_value;
   6195                 }
   6196                 size_t elem_index = const_val->data.x_ptr.data.base_array.elem_index;
   6197                 LLVMValueRef uncasted_ptr_val = gen_const_ptr_array_recursive(g, array_const_val, elem_index);
   6198                 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
   6199                 const_val->global_refs->llvm_value = ptr_val;
   6200                 return ptr_val;
   6201             }
   6202         case ConstPtrSpecialBaseStruct:
   6203             {
   6204                 assert(const_val->global_refs != nullptr);
   6205                 ConstExprValue *struct_const_val = const_val->data.x_ptr.data.base_struct.struct_val;
   6206                 assert(struct_const_val->type->id == ZigTypeIdStruct);
   6207                 if (!type_has_bits(struct_const_val->type)) {
   6208                     // make this a null pointer
   6209                     ZigType *usize = g->builtin_types.entry_usize;
   6210                     const_val->global_refs->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
   6211                             get_llvm_type(g, const_val->type));
   6212                     return const_val->global_refs->llvm_value;
   6213                 }
   6214                 size_t src_field_index = const_val->data.x_ptr.data.base_struct.field_index;
   6215                 size_t gen_field_index = struct_const_val->type->data.structure.fields[src_field_index].gen_index;
   6216                 LLVMValueRef uncasted_ptr_val = gen_const_ptr_struct_recursive(g, struct_const_val,
   6217                         gen_field_index);
   6218                 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
   6219                 const_val->global_refs->llvm_value = ptr_val;
   6220                 return ptr_val;
   6221             }
   6222         case ConstPtrSpecialBaseErrorUnionCode:
   6223             {
   6224                 assert(const_val->global_refs != nullptr);
   6225                 ConstExprValue *err_union_const_val = const_val->data.x_ptr.data.base_err_union_code.err_union_val;
   6226                 assert(err_union_const_val->type->id == ZigTypeIdErrorUnion);
   6227                 if (!type_has_bits(err_union_const_val->type)) {
   6228                     // make this a null pointer
   6229                     ZigType *usize = g->builtin_types.entry_usize;
   6230                     const_val->global_refs->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
   6231                             get_llvm_type(g, const_val->type));
   6232                     return const_val->global_refs->llvm_value;
   6233                 }
   6234                 LLVMValueRef uncasted_ptr_val = gen_const_ptr_err_union_code_recursive(g, err_union_const_val);
   6235                 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
   6236                 const_val->global_refs->llvm_value = ptr_val;
   6237                 return ptr_val;
   6238             }
   6239         case ConstPtrSpecialBaseErrorUnionPayload:
   6240             {
   6241                 assert(const_val->global_refs != nullptr);
   6242                 ConstExprValue *err_union_const_val = const_val->data.x_ptr.data.base_err_union_payload.err_union_val;
   6243                 assert(err_union_const_val->type->id == ZigTypeIdErrorUnion);
   6244                 if (!type_has_bits(err_union_const_val->type)) {
   6245                     // make this a null pointer
   6246                     ZigType *usize = g->builtin_types.entry_usize;
   6247                     const_val->global_refs->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
   6248                             get_llvm_type(g, const_val->type));
   6249                     return const_val->global_refs->llvm_value;
   6250                 }
   6251                 LLVMValueRef uncasted_ptr_val = gen_const_ptr_err_union_payload_recursive(g, err_union_const_val);
   6252                 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
   6253                 const_val->global_refs->llvm_value = ptr_val;
   6254                 return ptr_val;
   6255             }
   6256         case ConstPtrSpecialBaseOptionalPayload:
   6257             {
   6258                 assert(const_val->global_refs != nullptr);
   6259                 ConstExprValue *optional_const_val = const_val->data.x_ptr.data.base_optional_payload.optional_val;
   6260                 assert(optional_const_val->type->id == ZigTypeIdOptional);
   6261                 if (!type_has_bits(optional_const_val->type)) {
   6262                     // make this a null pointer
   6263                     ZigType *usize = g->builtin_types.entry_usize;
   6264                     const_val->global_refs->llvm_value = LLVMConstIntToPtr(LLVMConstNull(usize->llvm_type),
   6265                             get_llvm_type(g, const_val->type));
   6266                     return const_val->global_refs->llvm_value;
   6267                 }
   6268                 LLVMValueRef uncasted_ptr_val = gen_const_ptr_optional_payload_recursive(g, optional_const_val);
   6269                 LLVMValueRef ptr_val = LLVMConstBitCast(uncasted_ptr_val, get_llvm_type(g, const_val->type));
   6270                 const_val->global_refs->llvm_value = ptr_val;
   6271                 return ptr_val;
   6272             }
   6273         case ConstPtrSpecialHardCodedAddr:
   6274             {
   6275                 assert(const_val->global_refs != nullptr);
   6276                 uint64_t addr_value = const_val->data.x_ptr.data.hard_coded_addr.addr;
   6277                 ZigType *usize = g->builtin_types.entry_usize;
   6278                 const_val->global_refs->llvm_value = LLVMConstIntToPtr(
   6279                         LLVMConstInt(usize->llvm_type, addr_value, false), get_llvm_type(g, const_val->type));
   6280                 return const_val->global_refs->llvm_value;
   6281             }
   6282         case ConstPtrSpecialFunction:
   6283             return LLVMConstBitCast(fn_llvm_value(g, const_val->data.x_ptr.data.fn.fn_entry),
   6284                     get_llvm_type(g, const_val->type));
   6285         case ConstPtrSpecialNull:
   6286             return LLVMConstNull(get_llvm_type(g, const_val->type));
   6287     }
   6288     zig_unreachable();
   6289 }
   6290 
   6291 static LLVMValueRef gen_const_val_err_set(CodeGen *g, ConstExprValue *const_val, const char *name) {
   6292     uint64_t value = (const_val->data.x_err_set == nullptr) ? 0 : const_val->data.x_err_set->value;
   6293     return LLVMConstInt(get_llvm_type(g, g->builtin_types.entry_global_error_set), value, false);
   6294 }
   6295 
   6296 static LLVMValueRef gen_const_val(CodeGen *g, ConstExprValue *const_val, const char *name) {
   6297     Error err;
   6298 
   6299     ZigType *type_entry = const_val->type;
   6300     assert(type_has_bits(type_entry));
   6301 
   6302     switch (const_val->special) {
   6303         case ConstValSpecialRuntime:
   6304             zig_unreachable();
   6305         case ConstValSpecialUndef:
   6306             return LLVMGetUndef(get_llvm_type(g, type_entry));
   6307         case ConstValSpecialStatic:
   6308             break;
   6309     }
   6310 
   6311     if ((err = type_resolve(g, type_entry, ResolveStatusLLVMFull)))
   6312         zig_unreachable();
   6313 
   6314     switch (type_entry->id) {
   6315         case ZigTypeIdInt:
   6316             return bigint_to_llvm_const(get_llvm_type(g, type_entry), &const_val->data.x_bigint);
   6317         case ZigTypeIdErrorSet:
   6318             return gen_const_val_err_set(g, const_val, name);
   6319         case ZigTypeIdFloat:
   6320             switch (type_entry->data.floating.bit_count) {
   6321                 case 16:
   6322                     return LLVMConstReal(get_llvm_type(g, type_entry), zig_f16_to_double(const_val->data.x_f16));
   6323                 case 32:
   6324                     return LLVMConstReal(get_llvm_type(g, type_entry), const_val->data.x_f32);
   6325                 case 64:
   6326                     return LLVMConstReal(get_llvm_type(g, type_entry), const_val->data.x_f64);
   6327                 case 128:
   6328                     {
   6329                         // TODO make sure this is correct on big endian targets too
   6330                         uint8_t buf[16];
   6331                         memcpy(buf, &const_val->data.x_f128, 16);
   6332                         LLVMValueRef as_int = LLVMConstIntOfArbitraryPrecision(LLVMInt128Type(), 2,
   6333                                 (uint64_t*)buf);
   6334                         return LLVMConstBitCast(as_int, get_llvm_type(g, type_entry));
   6335                     }
   6336                 default:
   6337                     zig_unreachable();
   6338             }
   6339         case ZigTypeIdBool:
   6340             if (const_val->data.x_bool) {
   6341                 return LLVMConstAllOnes(LLVMInt1Type());
   6342             } else {
   6343                 return LLVMConstNull(LLVMInt1Type());
   6344             }
   6345         case ZigTypeIdOptional:
   6346             {
   6347                 ZigType *child_type = type_entry->data.maybe.child_type;
   6348                 if (!type_has_bits(child_type)) {
   6349                     return LLVMConstInt(LLVMInt1Type(), const_val->data.x_optional ? 1 : 0, false);
   6350                 } else if (get_codegen_ptr_type(type_entry) != nullptr) {
   6351                     return gen_const_val_ptr(g, const_val, name);
   6352                 } else if (child_type->id == ZigTypeIdErrorSet) {
   6353                     return gen_const_val_err_set(g, const_val, name);
   6354                 } else {
   6355                     LLVMValueRef child_val;
   6356                     LLVMValueRef maybe_val;
   6357                     bool make_unnamed_struct;
   6358                     if (const_val->data.x_optional) {
   6359                         child_val = gen_const_val(g, const_val->data.x_optional, "");
   6360                         maybe_val = LLVMConstAllOnes(LLVMInt1Type());
   6361 
   6362                         make_unnamed_struct = is_llvm_value_unnamed_type(g, const_val->type, child_val);
   6363                     } else {
   6364                         child_val = LLVMGetUndef(get_llvm_type(g, child_type));
   6365                         maybe_val = LLVMConstNull(LLVMInt1Type());
   6366 
   6367                         make_unnamed_struct = false;
   6368                     }
   6369                     LLVMValueRef fields[] = {
   6370                         child_val,
   6371                         maybe_val,
   6372                     };
   6373                     if (make_unnamed_struct) {
   6374                         return LLVMConstStruct(fields, 2, false);
   6375                     } else {
   6376                         return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, 2);
   6377                     }
   6378                 }
   6379             }
   6380         case ZigTypeIdStruct:
   6381             {
   6382                 LLVMValueRef *fields = allocate<LLVMValueRef>(type_entry->data.structure.gen_field_count);
   6383                 size_t src_field_count = type_entry->data.structure.src_field_count;
   6384                 bool make_unnamed_struct = false;
   6385                 assert(type_entry->data.structure.resolve_status == ResolveStatusLLVMFull);
   6386                 if (type_entry->data.structure.layout == ContainerLayoutPacked) {
   6387                     size_t src_field_index = 0;
   6388                     while (src_field_index < src_field_count) {
   6389                         TypeStructField *type_struct_field = &type_entry->data.structure.fields[src_field_index];
   6390                         if (type_struct_field->gen_index == SIZE_MAX) {
   6391                             src_field_index += 1;
   6392                             continue;
   6393                         }
   6394 
   6395                         size_t src_field_index_end = src_field_index + 1;
   6396                         for (; src_field_index_end < src_field_count; src_field_index_end += 1) {
   6397                             TypeStructField *it_field = &type_entry->data.structure.fields[src_field_index_end];
   6398                             if (it_field->gen_index != type_struct_field->gen_index)
   6399                                 break;
   6400                         }
   6401 
   6402                         if (src_field_index + 1 == src_field_index_end) {
   6403                             ConstExprValue *field_val = &const_val->data.x_struct.fields[src_field_index];
   6404                             LLVMValueRef val = gen_const_val(g, field_val, "");
   6405                             fields[type_struct_field->gen_index] = val;
   6406                             make_unnamed_struct = make_unnamed_struct || is_llvm_value_unnamed_type(g, field_val->type, val);
   6407                         } else {
   6408                             bool is_big_endian = g->is_big_endian; // TODO get endianness from struct type
   6409                             LLVMTypeRef big_int_type_ref = LLVMStructGetTypeAtIndex(get_llvm_type(g, type_entry),
   6410                                     (unsigned)type_struct_field->gen_index);
   6411                             LLVMValueRef val = LLVMConstInt(big_int_type_ref, 0, false);
   6412                             size_t used_bits = 0;
   6413                             for (size_t i = src_field_index; i < src_field_index_end; i += 1) {
   6414                                 TypeStructField *it_field = &type_entry->data.structure.fields[i];
   6415                                 if (it_field->gen_index == SIZE_MAX) {
   6416                                     continue;
   6417                                 }
   6418                                 LLVMValueRef child_val = pack_const_int(g, big_int_type_ref,
   6419                                         &const_val->data.x_struct.fields[i]);
   6420                                 uint32_t packed_bits_size = type_size_bits(g, it_field->type_entry);
   6421                                 if (is_big_endian) {
   6422                                     LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref,
   6423                                             packed_bits_size, false);
   6424                                     val = LLVMConstShl(val, shift_amt);
   6425                                     val = LLVMConstOr(val, child_val);
   6426                                 } else {
   6427                                     LLVMValueRef shift_amt = LLVMConstInt(big_int_type_ref, used_bits, false);
   6428                                     LLVMValueRef child_val_shifted = LLVMConstShl(child_val, shift_amt);
   6429                                     val = LLVMConstOr(val, child_val_shifted);
   6430                                     used_bits += packed_bits_size;
   6431                                 }
   6432                             }
   6433                             fields[type_struct_field->gen_index] = val;
   6434                         }
   6435 
   6436                         src_field_index = src_field_index_end;
   6437                     }
   6438                 } else {
   6439                     for (uint32_t i = 0; i < src_field_count; i += 1) {
   6440                         TypeStructField *type_struct_field = &type_entry->data.structure.fields[i];
   6441                         if (type_struct_field->gen_index == SIZE_MAX) {
   6442                             continue;
   6443                         }
   6444                         ConstExprValue *field_val = &const_val->data.x_struct.fields[i];
   6445                         assert(field_val->type != nullptr);
   6446                         if ((err = ensure_const_val_repr(nullptr, g, nullptr, field_val,
   6447                                         type_struct_field->type_entry)))
   6448                         {
   6449                             zig_unreachable();
   6450                         }
   6451 
   6452                         LLVMValueRef val = gen_const_val(g, field_val, "");
   6453                         fields[type_struct_field->gen_index] = val;
   6454                         make_unnamed_struct = make_unnamed_struct || is_llvm_value_unnamed_type(g, field_val->type, val);
   6455 
   6456                         size_t end_pad_gen_index = (i + 1 < src_field_count) ?
   6457                             type_entry->data.structure.fields[i + 1].gen_index :
   6458                             type_entry->data.structure.gen_field_count;
   6459                         size_t next_offset = (i + 1 < src_field_count) ?
   6460                             type_entry->data.structure.fields[i + 1].offset : type_entry->abi_size;
   6461                         if (end_pad_gen_index != SIZE_MAX) {
   6462                             for (size_t gen_i = type_struct_field->gen_index + 1; gen_i < end_pad_gen_index;
   6463                                     gen_i += 1)
   6464                             {
   6465                                 size_t pad_bytes = next_offset -
   6466                                     (type_struct_field->offset + type_struct_field->type_entry->abi_size);
   6467                                 LLVMTypeRef llvm_array_type = LLVMArrayType(LLVMInt8Type(), pad_bytes);
   6468                                 fields[gen_i] = LLVMGetUndef(llvm_array_type);
   6469                             }
   6470                         }
   6471                     }
   6472                 }
   6473                 if (make_unnamed_struct) {
   6474                     return LLVMConstStruct(fields, type_entry->data.structure.gen_field_count,
   6475                         type_entry->data.structure.layout == ContainerLayoutPacked);
   6476                 } else {
   6477                     return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, type_entry->data.structure.gen_field_count);
   6478                 }
   6479             }
   6480         case ZigTypeIdArray:
   6481             {
   6482                 uint64_t len = type_entry->data.array.len;
   6483                 switch (const_val->data.x_array.special) {
   6484                     case ConstArraySpecialUndef:
   6485                         return LLVMGetUndef(get_llvm_type(g, type_entry));
   6486                     case ConstArraySpecialNone: {
   6487                         LLVMValueRef *values = allocate<LLVMValueRef>(len);
   6488                         LLVMTypeRef element_type_ref = get_llvm_type(g, type_entry->data.array.child_type);
   6489                         bool make_unnamed_struct = false;
   6490                         for (uint64_t i = 0; i < len; i += 1) {
   6491                             ConstExprValue *elem_value = &const_val->data.x_array.data.s_none.elements[i];
   6492                             LLVMValueRef val = gen_const_val(g, elem_value, "");
   6493                             values[i] = val;
   6494                             make_unnamed_struct = make_unnamed_struct || is_llvm_value_unnamed_type(g, elem_value->type, val);
   6495                         }
   6496                         if (make_unnamed_struct) {
   6497                             return LLVMConstStruct(values, len, true);
   6498                         } else {
   6499                             return LLVMConstArray(element_type_ref, values, (unsigned)len);
   6500                         }
   6501                     }
   6502                     case ConstArraySpecialBuf: {
   6503                         Buf *buf = const_val->data.x_array.data.s_buf;
   6504                         return LLVMConstString(buf_ptr(buf), (unsigned)buf_len(buf), true);
   6505                     }
   6506                 }
   6507                 zig_unreachable();
   6508             }
   6509         case ZigTypeIdVector: {
   6510             uint32_t len = type_entry->data.vector.len;
   6511             switch (const_val->data.x_array.special) {
   6512                 case ConstArraySpecialUndef:
   6513                     return LLVMGetUndef(get_llvm_type(g, type_entry));
   6514                 case ConstArraySpecialNone: {
   6515                     LLVMValueRef *values = allocate<LLVMValueRef>(len);
   6516                     for (uint64_t i = 0; i < len; i += 1) {
   6517                         ConstExprValue *elem_value = &const_val->data.x_array.data.s_none.elements[i];
   6518                         values[i] = gen_const_val(g, elem_value, "");
   6519                     }
   6520                     return LLVMConstVector(values, len);
   6521                 }
   6522                 case ConstArraySpecialBuf: {
   6523                     Buf *buf = const_val->data.x_array.data.s_buf;
   6524                     assert(buf_len(buf) == len);
   6525                     LLVMValueRef *values = allocate<LLVMValueRef>(len);
   6526                     for (uint64_t i = 0; i < len; i += 1) {
   6527                         values[i] = LLVMConstInt(g->builtin_types.entry_u8->llvm_type, buf_ptr(buf)[i], false);
   6528                     }
   6529                     return LLVMConstVector(values, len);
   6530                 }
   6531             }
   6532             zig_unreachable();
   6533         }
   6534         case ZigTypeIdUnion:
   6535             {
   6536                 // Force type_entry->data.unionation.union_llvm_type to get resolved
   6537                 (void)get_llvm_type(g, type_entry);
   6538 
   6539                 if (type_entry->data.unionation.gen_field_count == 0) {
   6540                     if (type_entry->data.unionation.tag_type == nullptr) {
   6541                         return nullptr;
   6542                     } else {
   6543                         return bigint_to_llvm_const(get_llvm_type(g, type_entry->data.unionation.tag_type),
   6544                             &const_val->data.x_union.tag);
   6545                     }
   6546                 }
   6547 
   6548                 LLVMTypeRef union_type_ref = type_entry->data.unionation.union_llvm_type;
   6549                 assert(union_type_ref != nullptr);
   6550 
   6551                 LLVMValueRef union_value_ref;
   6552                 bool make_unnamed_struct;
   6553                 ConstExprValue *payload_value = const_val->data.x_union.payload;
   6554                 if (payload_value == nullptr || !type_has_bits(payload_value->type)) {
   6555                     if (type_entry->data.unionation.gen_tag_index == SIZE_MAX)
   6556                         return LLVMGetUndef(get_llvm_type(g, type_entry));
   6557 
   6558                     union_value_ref = LLVMGetUndef(union_type_ref);
   6559                     make_unnamed_struct = false;
   6560                 } else {
   6561                     uint64_t field_type_bytes = LLVMStoreSizeOfType(g->target_data_ref,
   6562                             get_llvm_type(g, payload_value->type));
   6563                     uint64_t pad_bytes = type_entry->data.unionation.union_abi_size - field_type_bytes;
   6564                     LLVMValueRef correctly_typed_value = gen_const_val(g, payload_value, "");
   6565                     make_unnamed_struct = is_llvm_value_unnamed_type(g, payload_value->type, correctly_typed_value) ||
   6566                         payload_value->type != type_entry->data.unionation.most_aligned_union_member;
   6567 
   6568                     {
   6569                         if (pad_bytes == 0) {
   6570                             union_value_ref = correctly_typed_value;
   6571                         } else {
   6572                             LLVMValueRef fields[2];
   6573                             fields[0] = correctly_typed_value;
   6574                             fields[1] = LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), (unsigned)pad_bytes));
   6575                             if (make_unnamed_struct || type_entry->data.unionation.gen_tag_index != SIZE_MAX) {
   6576                                 union_value_ref = LLVMConstStruct(fields, 2, false);
   6577                             } else {
   6578                                 union_value_ref = LLVMConstNamedStruct(union_type_ref, fields, 2);
   6579                             }
   6580                         }
   6581                     }
   6582 
   6583                     if (type_entry->data.unionation.gen_tag_index == SIZE_MAX) {
   6584                         return union_value_ref;
   6585                     }
   6586                 }
   6587 
   6588                 LLVMValueRef tag_value = bigint_to_llvm_const(
   6589                         get_llvm_type(g, type_entry->data.unionation.tag_type),
   6590                         &const_val->data.x_union.tag);
   6591 
   6592                 LLVMValueRef fields[3];
   6593                 fields[type_entry->data.unionation.gen_union_index] = union_value_ref;
   6594                 fields[type_entry->data.unionation.gen_tag_index] = tag_value;
   6595 
   6596                 if (make_unnamed_struct) {
   6597                     LLVMValueRef result = LLVMConstStruct(fields, 2, false);
   6598                     uint64_t last_field_offset = LLVMOffsetOfElement(g->target_data_ref, LLVMTypeOf(result), 1);
   6599                     uint64_t end_offset = last_field_offset +
   6600                         LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(fields[1]));
   6601                     uint64_t expected_sz = LLVMStoreSizeOfType(g->target_data_ref, get_llvm_type(g, type_entry));
   6602                     unsigned pad_sz = expected_sz - end_offset;
   6603                     if (pad_sz != 0) {
   6604                         fields[2] = LLVMGetUndef(LLVMArrayType(LLVMInt8Type(), pad_sz));
   6605                         result = LLVMConstStruct(fields, 3, false);
   6606                     }
   6607                     uint64_t actual_sz = LLVMStoreSizeOfType(g->target_data_ref, LLVMTypeOf(result));
   6608                     assert(actual_sz == expected_sz);
   6609                     return result;
   6610                 } else {
   6611                     return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, 2);
   6612                 }
   6613 
   6614             }
   6615 
   6616         case ZigTypeIdEnum:
   6617             return bigint_to_llvm_const(get_llvm_type(g, type_entry), &const_val->data.x_enum_tag);
   6618         case ZigTypeIdFn:
   6619             if (const_val->data.x_ptr.special == ConstPtrSpecialFunction) {
   6620                 assert(const_val->data.x_ptr.mut == ConstPtrMutComptimeConst);
   6621                 return fn_llvm_value(g, const_val->data.x_ptr.data.fn.fn_entry);
   6622             } else if (const_val->data.x_ptr.special == ConstPtrSpecialHardCodedAddr) {
   6623                 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   6624                 uint64_t addr = const_val->data.x_ptr.data.hard_coded_addr.addr;
   6625                 return LLVMConstIntToPtr(LLVMConstInt(usize_type_ref, addr, false), get_llvm_type(g, type_entry));
   6626             } else {
   6627                 zig_unreachable();
   6628             }
   6629         case ZigTypeIdPointer:
   6630             return gen_const_val_ptr(g, const_val, name);
   6631         case ZigTypeIdErrorUnion:
   6632             {
   6633                 ZigType *payload_type = type_entry->data.error_union.payload_type;
   6634                 ZigType *err_set_type = type_entry->data.error_union.err_set_type;
   6635                 if (!type_has_bits(payload_type)) {
   6636                     assert(type_has_bits(err_set_type));
   6637                     ErrorTableEntry *err_set = const_val->data.x_err_union.error_set->data.x_err_set;
   6638                     uint64_t value = (err_set == nullptr) ? 0 : err_set->value;
   6639                     return LLVMConstInt(get_llvm_type(g, g->err_tag_type), value, false);
   6640                 } else if (!type_has_bits(err_set_type)) {
   6641                     assert(type_has_bits(payload_type));
   6642                     return gen_const_val(g, const_val->data.x_err_union.payload, "");
   6643                 } else {
   6644                     LLVMValueRef err_tag_value;
   6645                     LLVMValueRef err_payload_value;
   6646                     bool make_unnamed_struct;
   6647                     ErrorTableEntry *err_set = const_val->data.x_err_union.error_set->data.x_err_set;
   6648                     if (err_set != nullptr) {
   6649                         err_tag_value = LLVMConstInt(get_llvm_type(g, g->err_tag_type), err_set->value, false);
   6650                         err_payload_value = LLVMConstNull(get_llvm_type(g, payload_type));
   6651                         make_unnamed_struct = false;
   6652                     } else {
   6653                         err_tag_value = LLVMConstNull(get_llvm_type(g, g->err_tag_type));
   6654                         ConstExprValue *payload_val = const_val->data.x_err_union.payload;
   6655                         err_payload_value = gen_const_val(g, payload_val, "");
   6656                         make_unnamed_struct = is_llvm_value_unnamed_type(g, payload_val->type, err_payload_value);
   6657                     }
   6658                     LLVMValueRef fields[3];
   6659                     fields[err_union_err_index] = err_tag_value;
   6660                     fields[err_union_payload_index] = err_payload_value;
   6661                     size_t field_count = 2;
   6662                     if (type_entry->data.error_union.pad_llvm_type != nullptr) {
   6663                         fields[2] = LLVMGetUndef(type_entry->data.error_union.pad_llvm_type);
   6664                         field_count = 3;
   6665                     }
   6666                     if (make_unnamed_struct) {
   6667                         return LLVMConstStruct(fields, field_count, false);
   6668                     } else {
   6669                         return LLVMConstNamedStruct(get_llvm_type(g, type_entry), fields, field_count);
   6670                     }
   6671                 }
   6672             }
   6673         case ZigTypeIdVoid:
   6674             return nullptr;
   6675         case ZigTypeIdInvalid:
   6676         case ZigTypeIdMetaType:
   6677         case ZigTypeIdUnreachable:
   6678         case ZigTypeIdComptimeFloat:
   6679         case ZigTypeIdComptimeInt:
   6680         case ZigTypeIdEnumLiteral:
   6681         case ZigTypeIdUndefined:
   6682         case ZigTypeIdNull:
   6683         case ZigTypeIdBoundFn:
   6684         case ZigTypeIdArgTuple:
   6685         case ZigTypeIdOpaque:
   6686             zig_unreachable();
   6687         case ZigTypeIdFnFrame:
   6688             zig_panic("TODO");
   6689         case ZigTypeIdAnyFrame:
   6690             zig_panic("TODO");
   6691     }
   6692     zig_unreachable();
   6693 }
   6694 
   6695 static void render_const_val(CodeGen *g, ConstExprValue *const_val, const char *name) {
   6696     if (!const_val->global_refs)
   6697         const_val->global_refs = allocate<ConstGlobalRefs>(1);
   6698     if (!const_val->global_refs->llvm_value)
   6699         const_val->global_refs->llvm_value = gen_const_val(g, const_val, name);
   6700 
   6701     if (const_val->global_refs->llvm_global)
   6702         LLVMSetInitializer(const_val->global_refs->llvm_global, const_val->global_refs->llvm_value);
   6703 }
   6704 
   6705 static void render_const_val_global(CodeGen *g, ConstExprValue *const_val, const char *name) {
   6706     if (!const_val->global_refs)
   6707         const_val->global_refs = allocate<ConstGlobalRefs>(1);
   6708 
   6709     if (!const_val->global_refs->llvm_global) {
   6710         LLVMTypeRef type_ref = const_val->global_refs->llvm_value ?
   6711             LLVMTypeOf(const_val->global_refs->llvm_value) : get_llvm_type(g, const_val->type);
   6712         LLVMValueRef global_value = LLVMAddGlobal(g->module, type_ref, name);
   6713         LLVMSetLinkage(global_value, LLVMInternalLinkage);
   6714         LLVMSetGlobalConstant(global_value, true);
   6715         LLVMSetUnnamedAddr(global_value, true);
   6716         LLVMSetAlignment(global_value, (const_val->global_refs->align == 0) ?
   6717                 get_abi_alignment(g, const_val->type) : const_val->global_refs->align);
   6718 
   6719         const_val->global_refs->llvm_global = global_value;
   6720     }
   6721 
   6722     if (const_val->global_refs->llvm_value)
   6723         LLVMSetInitializer(const_val->global_refs->llvm_global, const_val->global_refs->llvm_value);
   6724 }
   6725 
   6726 static void generate_error_name_table(CodeGen *g) {
   6727     if (g->err_name_table != nullptr || !g->generate_error_name_table || g->errors_by_index.length == 1) {
   6728         return;
   6729     }
   6730 
   6731     assert(g->errors_by_index.length > 0);
   6732 
   6733     ZigType *u8_ptr_type = get_pointer_to_type_extra(g, g->builtin_types.entry_u8, true, false,
   6734             PtrLenUnknown, get_abi_alignment(g, g->builtin_types.entry_u8), 0, 0, false);
   6735     ZigType *str_type = get_slice_type(g, u8_ptr_type);
   6736 
   6737     LLVMValueRef *values = allocate<LLVMValueRef>(g->errors_by_index.length);
   6738     values[0] = LLVMGetUndef(get_llvm_type(g, str_type));
   6739     for (size_t i = 1; i < g->errors_by_index.length; i += 1) {
   6740         ErrorTableEntry *err_entry = g->errors_by_index.at(i);
   6741         Buf *name = &err_entry->name;
   6742 
   6743         g->largest_err_name_len = max(g->largest_err_name_len, buf_len(name));
   6744 
   6745         LLVMValueRef str_init = LLVMConstString(buf_ptr(name), (unsigned)buf_len(name), true);
   6746         LLVMValueRef str_global = LLVMAddGlobal(g->module, LLVMTypeOf(str_init), "");
   6747         LLVMSetInitializer(str_global, str_init);
   6748         LLVMSetLinkage(str_global, LLVMPrivateLinkage);
   6749         LLVMSetGlobalConstant(str_global, true);
   6750         LLVMSetUnnamedAddr(str_global, true);
   6751         LLVMSetAlignment(str_global, LLVMABIAlignmentOfType(g->target_data_ref, LLVMTypeOf(str_init)));
   6752 
   6753         LLVMValueRef fields[] = {
   6754             LLVMConstBitCast(str_global, get_llvm_type(g, u8_ptr_type)),
   6755             LLVMConstInt(g->builtin_types.entry_usize->llvm_type, buf_len(name), false),
   6756         };
   6757         values[i] = LLVMConstNamedStruct(get_llvm_type(g, str_type), fields, 2);
   6758     }
   6759 
   6760     LLVMValueRef err_name_table_init = LLVMConstArray(get_llvm_type(g, str_type), values, (unsigned)g->errors_by_index.length);
   6761 
   6762     g->err_name_table = LLVMAddGlobal(g->module, LLVMTypeOf(err_name_table_init),
   6763             buf_ptr(get_mangled_name(g, buf_create_from_str("__zig_err_name_table"), false)));
   6764     LLVMSetInitializer(g->err_name_table, err_name_table_init);
   6765     LLVMSetLinkage(g->err_name_table, LLVMPrivateLinkage);
   6766     LLVMSetGlobalConstant(g->err_name_table, true);
   6767     LLVMSetUnnamedAddr(g->err_name_table, true);
   6768     LLVMSetAlignment(g->err_name_table, LLVMABIAlignmentOfType(g->target_data_ref, LLVMTypeOf(err_name_table_init)));
   6769 }
   6770 
   6771 static void build_all_basic_blocks(CodeGen *g, ZigFn *fn) {
   6772     IrExecutable *executable = &fn->analyzed_executable;
   6773     assert(executable->basic_block_list.length > 0);
   6774     LLVMValueRef fn_val = fn_llvm_value(g, fn);
   6775     LLVMBasicBlockRef first_bb = nullptr;
   6776     if (fn_is_async(fn)) {
   6777         first_bb = LLVMAppendBasicBlock(fn_val, "AsyncSwitch");
   6778         g->cur_preamble_llvm_block = first_bb;
   6779     }
   6780     for (size_t block_i = 0; block_i < executable->basic_block_list.length; block_i += 1) {
   6781         IrBasicBlock *bb = executable->basic_block_list.at(block_i);
   6782         bb->llvm_block = LLVMAppendBasicBlock(fn_val, bb->name_hint);
   6783     }
   6784     if (first_bb == nullptr) {
   6785         first_bb = executable->basic_block_list.at(0)->llvm_block;
   6786     }
   6787     LLVMPositionBuilderAtEnd(g->builder, first_bb);
   6788 }
   6789 
   6790 static void gen_global_var(CodeGen *g, ZigVar *var, LLVMValueRef init_val,
   6791     ZigType *type_entry)
   6792 {
   6793     if (g->strip_debug_symbols) {
   6794         return;
   6795     }
   6796 
   6797     assert(var->gen_is_const);
   6798     assert(type_entry);
   6799 
   6800     ZigType *import = get_scope_import(var->parent_scope);
   6801     assert(import);
   6802 
   6803     bool is_local_to_unit = true;
   6804     ZigLLVMCreateGlobalVariable(g->dbuilder, get_di_scope(g, var->parent_scope), buf_ptr(&var->name),
   6805         buf_ptr(&var->name), import->data.structure.root_struct->di_file,
   6806         (unsigned)(var->decl_node->line + 1),
   6807         get_llvm_di_type(g, type_entry), is_local_to_unit);
   6808 
   6809     // TODO ^^ make an actual global variable
   6810 }
   6811 
   6812 static void validate_inline_fns(CodeGen *g) {
   6813     for (size_t i = 0; i < g->inline_fns.length; i += 1) {
   6814         ZigFn *fn_entry = g->inline_fns.at(i);
   6815         LLVMValueRef fn_val = LLVMGetNamedFunction(g->module, fn_entry->llvm_name);
   6816         if (fn_val != nullptr) {
   6817             add_node_error(g, fn_entry->proto_node, buf_sprintf("unable to inline function"));
   6818         }
   6819     }
   6820     report_errors_and_maybe_exit(g);
   6821 }
   6822 
   6823 static void set_global_tls(CodeGen *g, ZigVar *var, LLVMValueRef global_value) {
   6824     bool is_extern = var->decl_node->data.variable_declaration.is_extern;
   6825     bool is_export = var->decl_node->data.variable_declaration.is_export;
   6826     bool is_internal_linkage = !is_extern && !is_export;
   6827     if (var->is_thread_local && (!g->is_single_threaded || !is_internal_linkage)) {
   6828         LLVMSetThreadLocalMode(global_value, LLVMGeneralDynamicTLSModel);
   6829     }
   6830 }
   6831 
   6832 static void do_code_gen(CodeGen *g) {
   6833     assert(!g->errors.length);
   6834 
   6835     generate_error_name_table(g);
   6836 
   6837     // Generate module level variables
   6838     for (size_t i = 0; i < g->global_vars.length; i += 1) {
   6839         TldVar *tld_var = g->global_vars.at(i);
   6840         ZigVar *var = tld_var->var;
   6841 
   6842         if (var->var_type->id == ZigTypeIdComptimeFloat) {
   6843             // Generate debug info for it but that's it.
   6844             ConstExprValue *const_val = var->const_value;
   6845             assert(const_val->special != ConstValSpecialRuntime);
   6846             if (const_val->type != var->var_type) {
   6847                 zig_panic("TODO debug info for var with ptr casted value");
   6848             }
   6849             ZigType *var_type = g->builtin_types.entry_f128;
   6850             ConstExprValue coerced_value = {};
   6851             coerced_value.special = ConstValSpecialStatic;
   6852             coerced_value.type = var_type;
   6853             coerced_value.data.x_f128 = bigfloat_to_f128(&const_val->data.x_bigfloat);
   6854             LLVMValueRef init_val = gen_const_val(g, &coerced_value, "");
   6855             gen_global_var(g, var, init_val, var_type);
   6856             continue;
   6857         }
   6858 
   6859         if (var->var_type->id == ZigTypeIdComptimeInt) {
   6860             // Generate debug info for it but that's it.
   6861             ConstExprValue *const_val = var->const_value;
   6862             assert(const_val->special != ConstValSpecialRuntime);
   6863             if (const_val->type != var->var_type) {
   6864                 zig_panic("TODO debug info for var with ptr casted value");
   6865             }
   6866             size_t bits_needed = bigint_bits_needed(&const_val->data.x_bigint);
   6867             if (bits_needed < 8) {
   6868                 bits_needed = 8;
   6869             }
   6870             ZigType *var_type = get_int_type(g, const_val->data.x_bigint.is_negative, bits_needed);
   6871             LLVMValueRef init_val = bigint_to_llvm_const(get_llvm_type(g, var_type), &const_val->data.x_bigint);
   6872             gen_global_var(g, var, init_val, var_type);
   6873             continue;
   6874         }
   6875 
   6876         if (!type_has_bits(var->var_type))
   6877             continue;
   6878 
   6879         assert(var->decl_node);
   6880 
   6881         GlobalLinkageId linkage;
   6882         Buf *unmangled_name = &var->name;
   6883         Buf *symbol_name;
   6884         if (var->export_list.length == 0) {
   6885             if (var->decl_node->data.variable_declaration.is_extern) {
   6886                 symbol_name = unmangled_name;
   6887                 linkage = GlobalLinkageIdStrong;
   6888             } else {
   6889                 symbol_name = get_mangled_name(g, unmangled_name, false);
   6890                 linkage = GlobalLinkageIdInternal;
   6891             }
   6892         } else {
   6893             GlobalExport *global_export = &var->export_list.items[0];
   6894             symbol_name = &global_export->name;
   6895             linkage = global_export->linkage;
   6896         }
   6897 
   6898         LLVMValueRef global_value;
   6899         bool externally_initialized = var->decl_node->data.variable_declaration.expr == nullptr;
   6900         if (externally_initialized) {
   6901             LLVMValueRef existing_llvm_var = LLVMGetNamedGlobal(g->module, buf_ptr(symbol_name));
   6902             if (existing_llvm_var) {
   6903                 global_value = LLVMConstBitCast(existing_llvm_var,
   6904                         LLVMPointerType(get_llvm_type(g, var->var_type), 0));
   6905             } else {
   6906                 global_value = LLVMAddGlobal(g->module, get_llvm_type(g, var->var_type), buf_ptr(symbol_name));
   6907                 // TODO debug info for the extern variable
   6908 
   6909                 LLVMSetLinkage(global_value, to_llvm_linkage(linkage));
   6910                 maybe_import_dll(g, global_value, GlobalLinkageIdStrong);
   6911                 LLVMSetAlignment(global_value, var->align_bytes);
   6912                 LLVMSetGlobalConstant(global_value, var->gen_is_const);
   6913                 set_global_tls(g, var, global_value);
   6914             }
   6915         } else {
   6916             bool exported = (linkage != GlobalLinkageIdInternal);
   6917             render_const_val(g, var->const_value, buf_ptr(symbol_name));
   6918             render_const_val_global(g, var->const_value, buf_ptr(symbol_name));
   6919             global_value = var->const_value->global_refs->llvm_global;
   6920 
   6921             if (exported) {
   6922                 LLVMSetLinkage(global_value, to_llvm_linkage(linkage));
   6923                 maybe_export_dll(g, global_value, GlobalLinkageIdStrong);
   6924             }
   6925             if (tld_var->section_name) {
   6926                 LLVMSetSection(global_value, buf_ptr(tld_var->section_name));
   6927             }
   6928             LLVMSetAlignment(global_value, var->align_bytes);
   6929 
   6930             // TODO debug info for function pointers
   6931             // Here we use const_value->type because that's the type of the llvm global,
   6932             // which we const ptr cast upon use to whatever it needs to be.
   6933             if (var->gen_is_const && var->const_value->type->id != ZigTypeIdFn) {
   6934                 gen_global_var(g, var, var->const_value->global_refs->llvm_value, var->const_value->type);
   6935             }
   6936 
   6937             LLVMSetGlobalConstant(global_value, var->gen_is_const);
   6938             set_global_tls(g, var, global_value);
   6939         }
   6940 
   6941         var->value_ref = global_value;
   6942 
   6943         for (size_t export_i = 1; export_i < var->export_list.length; export_i += 1) {
   6944             GlobalExport *global_export = &var->export_list.items[export_i];
   6945             LLVMAddAlias(g->module, LLVMTypeOf(var->value_ref), var->value_ref, buf_ptr(&global_export->name));
   6946         }
   6947     }
   6948 
   6949     // Generate function definitions.
   6950     for (size_t fn_i = 0; fn_i < g->fn_defs.length; fn_i += 1) {
   6951         ZigFn *fn_table_entry = g->fn_defs.at(fn_i);
   6952         FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id;
   6953         CallingConvention cc = fn_type_id->cc;
   6954         bool is_c_abi = cc == CallingConventionC;
   6955         bool want_sret = want_first_arg_sret(g, fn_type_id);
   6956 
   6957         LLVMValueRef fn = fn_llvm_value(g, fn_table_entry);
   6958         g->cur_fn = fn_table_entry;
   6959         g->cur_fn_val = fn;
   6960 
   6961         build_all_basic_blocks(g, fn_table_entry);
   6962         clear_debug_source_node(g);
   6963 
   6964         bool is_async = fn_is_async(fn_table_entry);
   6965 
   6966         if (is_async) {
   6967             g->cur_frame_ptr = LLVMGetParam(fn, 0);
   6968         } else {
   6969             if (want_sret) {
   6970                 g->cur_ret_ptr = LLVMGetParam(fn, 0);
   6971             } else if (handle_is_ptr(fn_type_id->return_type)) {
   6972                 g->cur_ret_ptr = build_alloca(g, fn_type_id->return_type, "result", 0);
   6973                 // TODO add debug info variable for this
   6974             } else {
   6975                 g->cur_ret_ptr = nullptr;
   6976             }
   6977         }
   6978 
   6979         uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry);
   6980         bool have_err_ret_trace_arg = err_ret_trace_arg_index != UINT32_MAX;
   6981         if (have_err_ret_trace_arg) {
   6982             g->cur_err_ret_trace_val_arg = LLVMGetParam(fn, err_ret_trace_arg_index);
   6983         } else {
   6984             g->cur_err_ret_trace_val_arg = nullptr;
   6985         }
   6986 
   6987         // error return tracing setup
   6988         bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn &&
   6989             !is_async && !have_err_ret_trace_arg;
   6990         LLVMValueRef err_ret_array_val = nullptr;
   6991         if (have_err_ret_trace_stack) {
   6992             ZigType *array_type = get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count);
   6993             err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type));
   6994 
   6995             (void)get_llvm_type(g, get_stack_trace_type(g));
   6996             g->cur_err_ret_trace_val_stack = build_alloca(g, get_stack_trace_type(g), "error_return_trace",
   6997                     get_abi_alignment(g, g->stack_trace_type));
   6998         } else {
   6999             g->cur_err_ret_trace_val_stack = nullptr;
   7000         }
   7001 
   7002         if (!is_async) {
   7003             // allocate temporary stack data
   7004             for (size_t alloca_i = 0; alloca_i < fn_table_entry->alloca_gen_list.length; alloca_i += 1) {
   7005                 IrInstructionAllocaGen *instruction = fn_table_entry->alloca_gen_list.at(alloca_i);
   7006                 ZigType *ptr_type = instruction->base.value.type;
   7007                 assert(ptr_type->id == ZigTypeIdPointer);
   7008                 ZigType *child_type = ptr_type->data.pointer.child_type;
   7009                 if (type_resolve(g, child_type, ResolveStatusSizeKnown))
   7010                     zig_unreachable();
   7011                 if (!type_has_bits(child_type))
   7012                     continue;
   7013                 if (instruction->base.ref_count == 0)
   7014                     continue;
   7015                 if (instruction->base.value.special != ConstValSpecialRuntime) {
   7016                     if (const_ptr_pointee(nullptr, g, &instruction->base.value, nullptr)->special !=
   7017                             ConstValSpecialRuntime)
   7018                     {
   7019                         continue;
   7020                     }
   7021                 }
   7022                 if (type_resolve(g, child_type, ResolveStatusLLVMFull))
   7023                     zig_unreachable();
   7024                 instruction->base.llvm_value = build_alloca(g, child_type, instruction->name_hint,
   7025                         get_ptr_align(g, ptr_type));
   7026             }
   7027         }
   7028 
   7029         ZigType *import = get_scope_import(&fn_table_entry->fndef_scope->base);
   7030         unsigned gen_i_init = want_sret ? 1 : 0;
   7031 
   7032         // create debug variable declarations for variables and allocate all local variables
   7033         FnWalk fn_walk_var = {};
   7034         fn_walk_var.id = FnWalkIdVars;
   7035         fn_walk_var.data.vars.import = import;
   7036         fn_walk_var.data.vars.fn = fn_table_entry;
   7037         fn_walk_var.data.vars.llvm_fn = fn;
   7038         fn_walk_var.data.vars.gen_i = gen_i_init;
   7039         for (size_t var_i = 0; var_i < fn_table_entry->variable_list.length; var_i += 1) {
   7040             ZigVar *var = fn_table_entry->variable_list.at(var_i);
   7041 
   7042             if (!type_has_bits(var->var_type)) {
   7043                 continue;
   7044             }
   7045             if (ir_get_var_is_comptime(var))
   7046                 continue;
   7047             switch (type_requires_comptime(g, var->var_type)) {
   7048                 case ReqCompTimeInvalid:
   7049                     zig_unreachable();
   7050                 case ReqCompTimeYes:
   7051                     continue;
   7052                 case ReqCompTimeNo:
   7053                     break;
   7054             }
   7055 
   7056             if (var->src_arg_index == SIZE_MAX) {
   7057                 var->di_loc_var = ZigLLVMCreateAutoVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
   7058                         buf_ptr(&var->name), import->data.structure.root_struct->di_file, (unsigned)(var->decl_node->line + 1),
   7059                         get_llvm_di_type(g, var->var_type), !g->strip_debug_symbols, 0);
   7060 
   7061             } else if (is_c_abi) {
   7062                 fn_walk_var.data.vars.var = var;
   7063                 iter_function_params_c_abi(g, fn_table_entry->type_entry, &fn_walk_var, var->src_arg_index);
   7064             } else if (!is_async) {
   7065                 ZigType *gen_type;
   7066                 FnGenParamInfo *gen_info = &fn_table_entry->type_entry->data.fn.gen_param_info[var->src_arg_index];
   7067                 assert(gen_info->gen_index != SIZE_MAX);
   7068 
   7069                 if (handle_is_ptr(var->var_type)) {
   7070                     if (gen_info->is_byval) {
   7071                         gen_type = var->var_type;
   7072                     } else {
   7073                         gen_type = gen_info->type;
   7074                     }
   7075                     var->value_ref = LLVMGetParam(fn, gen_info->gen_index);
   7076                 } else {
   7077                     gen_type = var->var_type;
   7078                     var->value_ref = build_alloca(g, var->var_type, buf_ptr(&var->name), var->align_bytes);
   7079                 }
   7080                 if (var->decl_node) {
   7081                     var->di_loc_var = ZigLLVMCreateParameterVariable(g->dbuilder, get_di_scope(g, var->parent_scope),
   7082                         buf_ptr(&var->name), import->data.structure.root_struct->di_file,
   7083                         (unsigned)(var->decl_node->line + 1),
   7084                         get_llvm_di_type(g, gen_type), !g->strip_debug_symbols, 0, (unsigned)(gen_info->gen_index+1));
   7085                 }
   7086 
   7087             }
   7088         }
   7089 
   7090         // finishing error return trace setup. we have to do this after all the allocas.
   7091         if (have_err_ret_trace_stack) {
   7092             ZigType *usize = g->builtin_types.entry_usize;
   7093             size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index;
   7094             LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, "");
   7095             gen_store_untyped(g, LLVMConstNull(usize->llvm_type), index_field_ptr, 0, false);
   7096 
   7097             size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index;
   7098             LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, "");
   7099 
   7100             ZigType *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry;
   7101             size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index;
   7102             LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, "");
   7103             LLVMValueRef zero = LLVMConstNull(usize->llvm_type);
   7104             LLVMValueRef indices[] = {zero, zero};
   7105             LLVMValueRef err_ret_array_val_elem0_ptr = LLVMBuildInBoundsGEP(g->builder, err_ret_array_val,
   7106                     indices, 2, "");
   7107             ZigType *ptr_ptr_usize_type = get_pointer_to_type(g, get_pointer_to_type(g, usize, false), false);
   7108             gen_store(g, err_ret_array_val_elem0_ptr, ptr_field_ptr, ptr_ptr_usize_type);
   7109 
   7110             size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index;
   7111             LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, "");
   7112             gen_store(g, LLVMConstInt(usize->llvm_type, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false));
   7113         }
   7114 
   7115         if (is_async) {
   7116             (void)get_llvm_type(g, fn_table_entry->frame_type);
   7117             g->cur_resume_block_count = 0;
   7118 
   7119             LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
   7120             LLVMValueRef size_val = LLVMConstInt(usize_type_ref, fn_table_entry->frame_type->abi_size, false);
   7121             ZigLLVMFunctionSetPrefixData(fn_table_entry->llvm_value, size_val);
   7122 
   7123             if (!g->strip_debug_symbols) {
   7124                 AstNode *source_node = fn_table_entry->proto_node;
   7125                 ZigLLVMSetCurrentDebugLocation(g->builder, (int)source_node->line + 1,
   7126                         (int)source_node->column + 1, get_di_scope(g, fn_table_entry->child_scope));
   7127             }
   7128             IrExecutable *executable = &fn_table_entry->analyzed_executable;
   7129             LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
   7130             LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
   7131             gen_assertion_scope(g, PanicMsgIdBadResume, fn_table_entry->child_scope);
   7132 
   7133             LLVMPositionBuilderAtEnd(g->builder, g->cur_preamble_llvm_block);
   7134             render_async_spills(g);
   7135             g->cur_async_awaiter_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_awaiter_index, "");
   7136             LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_resume_index, "");
   7137             g->cur_async_resume_index_ptr = resume_index_ptr;
   7138 
   7139             if (type_has_bits(fn_type_id->return_type)) {
   7140                 LLVMValueRef cur_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr, frame_ret_start, "");
   7141                 g->cur_ret_ptr = LLVMBuildLoad(g->builder, cur_ret_ptr_ptr, "");
   7142             }
   7143             uint32_t trace_field_index_stack = UINT32_MAX;
   7144             if (codegen_fn_has_err_ret_tracing_stack(g, fn_table_entry, true)) {
   7145                 trace_field_index_stack = frame_index_trace_stack(g, fn_type_id);
   7146                 g->cur_err_ret_trace_val_stack = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
   7147                         trace_field_index_stack, "");
   7148             }
   7149 
   7150             LLVMValueRef resume_index = LLVMBuildLoad(g->builder, resume_index_ptr, "");
   7151             LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, resume_index, bad_resume_block, 4);
   7152             g->cur_async_switch_instr = switch_instr;
   7153 
   7154             LLVMValueRef zero = LLVMConstNull(usize_type_ref);
   7155             IrBasicBlock *entry_block = executable->basic_block_list.at(0);
   7156             LLVMAddCase(switch_instr, zero, entry_block->llvm_block);
   7157             g->cur_resume_block_count += 1;
   7158             LLVMPositionBuilderAtEnd(g->builder, entry_block->llvm_block);
   7159             if (trace_field_index_stack != UINT32_MAX) {
   7160                 if (codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type)) {
   7161                     LLVMValueRef trace_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
   7162                             frame_index_trace_arg(g, fn_type_id->return_type), "");
   7163                     LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(trace_ptr_ptr)));
   7164                     LLVMBuildStore(g->builder, zero_ptr, trace_ptr_ptr);
   7165                 }
   7166 
   7167                 LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
   7168                         trace_field_index_stack, "");
   7169                 LLVMValueRef addrs_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_frame_ptr,
   7170                         trace_field_index_stack + 1, "");
   7171 
   7172                 gen_init_stack_trace(g, trace_field_ptr, addrs_field_ptr);
   7173             }
   7174             render_async_var_decls(g, entry_block->instruction_list.at(0)->scope);
   7175         } else {
   7176             // create debug variable declarations for parameters
   7177             // rely on the first variables in the variable_list being parameters.
   7178             FnWalk fn_walk_init = {};
   7179             fn_walk_init.id = FnWalkIdInits;
   7180             fn_walk_init.data.inits.fn = fn_table_entry;
   7181             fn_walk_init.data.inits.llvm_fn = fn;
   7182             fn_walk_init.data.inits.gen_i = gen_i_init;
   7183             walk_function_params(g, fn_table_entry->type_entry, &fn_walk_init);
   7184         }
   7185 
   7186         ir_render(g, fn_table_entry);
   7187 
   7188     }
   7189 
   7190     assert(!g->errors.length);
   7191 
   7192     if (buf_len(&g->global_asm) != 0) {
   7193         LLVMSetModuleInlineAsm(g->module, buf_ptr(&g->global_asm));
   7194     }
   7195 
   7196     ZigLLVMDIBuilderFinalize(g->dbuilder);
   7197 
   7198     if (g->verbose_llvm_ir) {
   7199         fflush(stderr);
   7200         LLVMDumpModule(g->module);
   7201     }
   7202 
   7203 #ifndef NDEBUG
   7204     char *error = nullptr;
   7205     LLVMVerifyModule(g->module, LLVMAbortProcessAction, &error);
   7206 #endif
   7207 }
   7208 
   7209 static void zig_llvm_emit_output(CodeGen *g) {
   7210     bool is_small = g->build_mode == BuildModeSmallRelease;
   7211 
   7212     Buf *output_path = &g->o_file_output_path;
   7213     char *err_msg = nullptr;
   7214     switch (g->emit_file_type) {
   7215         case EmitFileTypeBinary:
   7216             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
   7217                         ZigLLVM_EmitBinary, &err_msg, g->build_mode == BuildModeDebug, is_small,
   7218                         g->enable_time_report))
   7219             {
   7220                 zig_panic("unable to write object file %s: %s", buf_ptr(output_path), err_msg);
   7221             }
   7222             validate_inline_fns(g);
   7223             g->link_objects.append(output_path);
   7224             if (g->bundle_compiler_rt && (g->out_type == OutTypeObj ||
   7225                 (g->out_type == OutTypeLib && !g->is_dynamic)))
   7226             {
   7227                 zig_link_add_compiler_rt(g);
   7228             }
   7229             break;
   7230 
   7231         case EmitFileTypeAssembly:
   7232             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
   7233                         ZigLLVM_EmitAssembly, &err_msg, g->build_mode == BuildModeDebug, is_small,
   7234                         g->enable_time_report))
   7235             {
   7236                 zig_panic("unable to write assembly file %s: %s", buf_ptr(output_path), err_msg);
   7237             }
   7238             validate_inline_fns(g);
   7239             break;
   7240 
   7241         case EmitFileTypeLLVMIr:
   7242             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
   7243                         ZigLLVM_EmitLLVMIr, &err_msg, g->build_mode == BuildModeDebug, is_small,
   7244                         g->enable_time_report))
   7245             {
   7246                 zig_panic("unable to write llvm-ir file %s: %s", buf_ptr(output_path), err_msg);
   7247             }
   7248             validate_inline_fns(g);
   7249             break;
   7250 
   7251         default:
   7252             zig_unreachable();
   7253     }
   7254 }
   7255 
   7256 struct CIntTypeInfo {
   7257     CIntType id;
   7258     const char *name;
   7259     bool is_signed;
   7260 };
   7261 
   7262 static const CIntTypeInfo c_int_type_infos[] = {
   7263     {CIntTypeShort, "c_short", true},
   7264     {CIntTypeUShort, "c_ushort", false},
   7265     {CIntTypeInt, "c_int", true},
   7266     {CIntTypeUInt, "c_uint", false},
   7267     {CIntTypeLong, "c_long", true},
   7268     {CIntTypeULong, "c_ulong", false},
   7269     {CIntTypeLongLong, "c_longlong", true},
   7270     {CIntTypeULongLong, "c_ulonglong", false},
   7271 };
   7272 
   7273 static const bool is_signed_list[] = { false, true, };
   7274 
   7275 struct GlobalLinkageValue {
   7276     GlobalLinkageId id;
   7277     const char *name;
   7278 };
   7279 
   7280 static const GlobalLinkageValue global_linkage_values[] = {
   7281     {GlobalLinkageIdInternal, "Internal"},
   7282     {GlobalLinkageIdStrong, "Strong"},
   7283     {GlobalLinkageIdWeak, "Weak"},
   7284     {GlobalLinkageIdLinkOnce, "LinkOnce"},
   7285 };
   7286 
   7287 static void add_fp_entry(CodeGen *g, const char *name, uint32_t bit_count, LLVMTypeRef type_ref,
   7288         ZigType **field)
   7289 {
   7290     ZigType *entry = new_type_table_entry(ZigTypeIdFloat);
   7291     entry->llvm_type = type_ref;
   7292     entry->size_in_bits = 8*LLVMStoreSizeOfType(g->target_data_ref, entry->llvm_type);
   7293     entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
   7294     entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
   7295     buf_init_from_str(&entry->name, name);
   7296     entry->data.floating.bit_count = bit_count;
   7297 
   7298     entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
   7299             entry->size_in_bits, ZigLLVMEncoding_DW_ATE_float());
   7300     *field = entry;
   7301     g->primitive_type_table.put(&entry->name, entry);
   7302 }
   7303 
   7304 static void define_builtin_types(CodeGen *g) {
   7305     {
   7306         // if this type is anywhere in the AST, we should never hit codegen.
   7307         ZigType *entry = new_type_table_entry(ZigTypeIdInvalid);
   7308         buf_init_from_str(&entry->name, "(invalid)");
   7309         g->builtin_types.entry_invalid = entry;
   7310     }
   7311     {
   7312         ZigType *entry = new_type_table_entry(ZigTypeIdComptimeFloat);
   7313         buf_init_from_str(&entry->name, "comptime_float");
   7314         g->builtin_types.entry_num_lit_float = entry;
   7315         g->primitive_type_table.put(&entry->name, entry);
   7316     }
   7317     {
   7318         ZigType *entry = new_type_table_entry(ZigTypeIdComptimeInt);
   7319         buf_init_from_str(&entry->name, "comptime_int");
   7320         g->builtin_types.entry_num_lit_int = entry;
   7321         g->primitive_type_table.put(&entry->name, entry);
   7322     }
   7323     {
   7324         ZigType *entry = new_type_table_entry(ZigTypeIdEnumLiteral);
   7325         buf_init_from_str(&entry->name, "(enum literal)");
   7326         g->builtin_types.entry_enum_literal = entry;
   7327     }
   7328     {
   7329         ZigType *entry = new_type_table_entry(ZigTypeIdUndefined);
   7330         buf_init_from_str(&entry->name, "(undefined)");
   7331         g->builtin_types.entry_undef = entry;
   7332     }
   7333     {
   7334         ZigType *entry = new_type_table_entry(ZigTypeIdNull);
   7335         buf_init_from_str(&entry->name, "(null)");
   7336         g->builtin_types.entry_null = entry;
   7337     }
   7338     {
   7339         ZigType *entry = new_type_table_entry(ZigTypeIdArgTuple);
   7340         buf_init_from_str(&entry->name, "(args)");
   7341         g->builtin_types.entry_arg_tuple = entry;
   7342     }
   7343 
   7344     for (size_t i = 0; i < array_length(c_int_type_infos); i += 1) {
   7345         const CIntTypeInfo *info = &c_int_type_infos[i];
   7346         uint32_t size_in_bits = target_c_type_size_in_bits(g->zig_target, info->id);
   7347         bool is_signed = info->is_signed;
   7348 
   7349         ZigType *entry = new_type_table_entry(ZigTypeIdInt);
   7350         entry->llvm_type = LLVMIntType(size_in_bits);
   7351         entry->size_in_bits = size_in_bits;
   7352         entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
   7353         entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
   7354 
   7355         buf_init_from_str(&entry->name, info->name);
   7356 
   7357         entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
   7358                 size_in_bits, is_signed ? ZigLLVMEncoding_DW_ATE_signed() : ZigLLVMEncoding_DW_ATE_unsigned());
   7359         entry->data.integral.is_signed = is_signed;
   7360         entry->data.integral.bit_count = size_in_bits;
   7361         g->primitive_type_table.put(&entry->name, entry);
   7362 
   7363         get_c_int_type_ptr(g, info->id)[0] = entry;
   7364     }
   7365 
   7366     {
   7367         ZigType *entry = new_type_table_entry(ZigTypeIdBool);
   7368         entry->llvm_type = LLVMInt1Type();
   7369         entry->size_in_bits = 1;
   7370         entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
   7371         entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
   7372         buf_init_from_str(&entry->name, "bool");
   7373         entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
   7374                 entry->size_in_bits, ZigLLVMEncoding_DW_ATE_boolean());
   7375         g->builtin_types.entry_bool = entry;
   7376         g->primitive_type_table.put(&entry->name, entry);
   7377     }
   7378 
   7379     for (size_t sign_i = 0; sign_i < array_length(is_signed_list); sign_i += 1) {
   7380         bool is_signed = is_signed_list[sign_i];
   7381 
   7382         ZigType *entry = new_type_table_entry(ZigTypeIdInt);
   7383         entry->llvm_type = LLVMIntType(g->pointer_size_bytes * 8);
   7384         entry->size_in_bits = g->pointer_size_bytes * 8;
   7385         entry->abi_size = LLVMABISizeOfType(g->target_data_ref, entry->llvm_type);
   7386         entry->abi_align = LLVMABIAlignmentOfType(g->target_data_ref, entry->llvm_type);
   7387 
   7388         const char u_or_i = is_signed ? 'i' : 'u';
   7389         buf_resize(&entry->name, 0);
   7390         buf_appendf(&entry->name, "%csize", u_or_i);
   7391 
   7392         entry->data.integral.is_signed = is_signed;
   7393         entry->data.integral.bit_count = g->pointer_size_bytes * 8;
   7394 
   7395         entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
   7396                 entry->size_in_bits,
   7397                 is_signed ? ZigLLVMEncoding_DW_ATE_signed() : ZigLLVMEncoding_DW_ATE_unsigned());
   7398         g->primitive_type_table.put(&entry->name, entry);
   7399 
   7400         if (is_signed) {
   7401             g->builtin_types.entry_isize = entry;
   7402         } else {
   7403             g->builtin_types.entry_usize = entry;
   7404         }
   7405     }
   7406 
   7407     add_fp_entry(g, "f16", 16, LLVMHalfType(), &g->builtin_types.entry_f16);
   7408     add_fp_entry(g, "f32", 32, LLVMFloatType(), &g->builtin_types.entry_f32);
   7409     add_fp_entry(g, "f64", 64, LLVMDoubleType(), &g->builtin_types.entry_f64);
   7410     add_fp_entry(g, "f128", 128, LLVMFP128Type(), &g->builtin_types.entry_f128);
   7411     add_fp_entry(g, "c_longdouble", 80, LLVMX86FP80Type(), &g->builtin_types.entry_c_longdouble);
   7412 
   7413     {
   7414         ZigType *entry = new_type_table_entry(ZigTypeIdVoid);
   7415         entry->llvm_type = LLVMVoidType();
   7416         buf_init_from_str(&entry->name, "void");
   7417         entry->llvm_di_type = ZigLLVMCreateDebugBasicType(g->dbuilder, buf_ptr(&entry->name),
   7418                 0,
   7419                 ZigLLVMEncoding_DW_ATE_signed());
   7420         g->builtin_types.entry_void = entry;
   7421         g->primitive_type_table.put(&entry->name, entry);
   7422     }
   7423     {
   7424         ZigType *entry = new_type_table_entry(ZigTypeIdUnreachable);
   7425         entry->llvm_type = LLVMVoidType();
   7426         buf_init_from_str(&entry->name, "noreturn");
   7427         entry->llvm_di_type = g->builtin_types.entry_void->llvm_di_type;
   7428         g->builtin_types.entry_unreachable = entry;
   7429         g->primitive_type_table.put(&entry->name, entry);
   7430     }
   7431     {
   7432         ZigType *entry = new_type_table_entry(ZigTypeIdMetaType);
   7433         buf_init_from_str(&entry->name, "type");
   7434         g->builtin_types.entry_type = entry;
   7435         g->primitive_type_table.put(&entry->name, entry);
   7436     }
   7437 
   7438     g->builtin_types.entry_u8 = get_int_type(g, false, 8);
   7439     g->builtin_types.entry_u16 = get_int_type(g, false, 16);
   7440     g->builtin_types.entry_u29 = get_int_type(g, false, 29);
   7441     g->builtin_types.entry_u32 = get_int_type(g, false, 32);
   7442     g->builtin_types.entry_u64 = get_int_type(g, false, 64);
   7443     g->builtin_types.entry_i8 = get_int_type(g, true, 8);
   7444     g->builtin_types.entry_i32 = get_int_type(g, true, 32);
   7445     g->builtin_types.entry_i64 = get_int_type(g, true, 64);
   7446 
   7447     {
   7448         g->builtin_types.entry_c_void = get_opaque_type(g, nullptr, nullptr, "c_void",
   7449                 buf_create_from_str("c_void"));
   7450         g->primitive_type_table.put(&g->builtin_types.entry_c_void->name, g->builtin_types.entry_c_void);
   7451     }
   7452 
   7453     {
   7454         ZigType *entry = new_type_table_entry(ZigTypeIdErrorSet);
   7455         buf_init_from_str(&entry->name, "anyerror");
   7456         entry->data.error_set.err_count = UINT32_MAX;
   7457 
   7458         // TODO https://github.com/ziglang/zig/issues/786
   7459         g->err_tag_type = g->builtin_types.entry_u16;
   7460 
   7461         entry->size_in_bits = g->err_tag_type->size_in_bits;
   7462         entry->abi_align = g->err_tag_type->abi_align;
   7463         entry->abi_size = g->err_tag_type->abi_size;
   7464 
   7465         g->builtin_types.entry_global_error_set = entry;
   7466 
   7467         g->errors_by_index.append(nullptr);
   7468 
   7469         g->primitive_type_table.put(&entry->name, entry);
   7470     }
   7471 }
   7472 
   7473 static BuiltinFnEntry *create_builtin_fn(CodeGen *g, BuiltinFnId id, const char *name, size_t count) {
   7474     BuiltinFnEntry *builtin_fn = allocate<BuiltinFnEntry>(1);
   7475     buf_init_from_str(&builtin_fn->name, name);
   7476     builtin_fn->id = id;
   7477     builtin_fn->param_count = count;
   7478     g->builtin_fn_table.put(&builtin_fn->name, builtin_fn);
   7479     return builtin_fn;
   7480 }
   7481 
   7482 static void define_builtin_fns(CodeGen *g) {
   7483     create_builtin_fn(g, BuiltinFnIdBreakpoint, "breakpoint", 0);
   7484     create_builtin_fn(g, BuiltinFnIdReturnAddress, "returnAddress", 0);
   7485     create_builtin_fn(g, BuiltinFnIdMemcpy, "memcpy", 3);
   7486     create_builtin_fn(g, BuiltinFnIdMemset, "memset", 3);
   7487     create_builtin_fn(g, BuiltinFnIdSizeof, "sizeOf", 1);
   7488     create_builtin_fn(g, BuiltinFnIdAlignOf, "alignOf", 1);
   7489     create_builtin_fn(g, BuiltinFnIdMemberCount, "memberCount", 1);
   7490     create_builtin_fn(g, BuiltinFnIdMemberType, "memberType", 2);
   7491     create_builtin_fn(g, BuiltinFnIdMemberName, "memberName", 2);
   7492     create_builtin_fn(g, BuiltinFnIdField, "field", 2);
   7493     create_builtin_fn(g, BuiltinFnIdTypeInfo, "typeInfo", 1);
   7494     create_builtin_fn(g, BuiltinFnIdHasField, "hasField", 2);
   7495     create_builtin_fn(g, BuiltinFnIdTypeof, "typeOf", 1); // TODO rename to TypeOf
   7496     create_builtin_fn(g, BuiltinFnIdAddWithOverflow, "addWithOverflow", 4);
   7497     create_builtin_fn(g, BuiltinFnIdSubWithOverflow, "subWithOverflow", 4);
   7498     create_builtin_fn(g, BuiltinFnIdMulWithOverflow, "mulWithOverflow", 4);
   7499     create_builtin_fn(g, BuiltinFnIdShlWithOverflow, "shlWithOverflow", 4);
   7500     create_builtin_fn(g, BuiltinFnIdCInclude, "cInclude", 1);
   7501     create_builtin_fn(g, BuiltinFnIdCDefine, "cDefine", 2);
   7502     create_builtin_fn(g, BuiltinFnIdCUndef, "cUndef", 1);
   7503     create_builtin_fn(g, BuiltinFnIdCtz, "ctz", 2);
   7504     create_builtin_fn(g, BuiltinFnIdClz, "clz", 2);
   7505     create_builtin_fn(g, BuiltinFnIdPopCount, "popCount", 2);
   7506     create_builtin_fn(g, BuiltinFnIdBswap, "byteSwap", 2);
   7507     create_builtin_fn(g, BuiltinFnIdBitReverse, "bitReverse", 2);
   7508     create_builtin_fn(g, BuiltinFnIdImport, "import", 1);
   7509     create_builtin_fn(g, BuiltinFnIdCImport, "cImport", 1);
   7510     create_builtin_fn(g, BuiltinFnIdErrName, "errorName", 1);
   7511     create_builtin_fn(g, BuiltinFnIdTypeName, "typeName", 1);
   7512     create_builtin_fn(g, BuiltinFnIdEmbedFile, "embedFile", 1);
   7513     create_builtin_fn(g, BuiltinFnIdCmpxchgWeak, "cmpxchgWeak", 6);
   7514     create_builtin_fn(g, BuiltinFnIdCmpxchgStrong, "cmpxchgStrong", 6);
   7515     create_builtin_fn(g, BuiltinFnIdFence, "fence", 1);
   7516     create_builtin_fn(g, BuiltinFnIdTruncate, "truncate", 2);
   7517     create_builtin_fn(g, BuiltinFnIdIntCast, "intCast", 2);
   7518     create_builtin_fn(g, BuiltinFnIdFloatCast, "floatCast", 2);
   7519     create_builtin_fn(g, BuiltinFnIdIntToFloat, "intToFloat", 2);
   7520     create_builtin_fn(g, BuiltinFnIdFloatToInt, "floatToInt", 2);
   7521     create_builtin_fn(g, BuiltinFnIdBoolToInt, "boolToInt", 1);
   7522     create_builtin_fn(g, BuiltinFnIdErrToInt, "errorToInt", 1);
   7523     create_builtin_fn(g, BuiltinFnIdIntToErr, "intToError", 1);
   7524     create_builtin_fn(g, BuiltinFnIdEnumToInt, "enumToInt", 1);
   7525     create_builtin_fn(g, BuiltinFnIdIntToEnum, "intToEnum", 2);
   7526     create_builtin_fn(g, BuiltinFnIdCompileErr, "compileError", 1);
   7527     create_builtin_fn(g, BuiltinFnIdCompileLog, "compileLog", SIZE_MAX);
   7528     create_builtin_fn(g, BuiltinFnIdIntType, "IntType", 2); // TODO rename to Int
   7529     create_builtin_fn(g, BuiltinFnIdVectorType, "Vector", 2);
   7530     create_builtin_fn(g, BuiltinFnIdSetCold, "setCold", 1);
   7531     create_builtin_fn(g, BuiltinFnIdSetRuntimeSafety, "setRuntimeSafety", 1);
   7532     create_builtin_fn(g, BuiltinFnIdSetFloatMode, "setFloatMode", 1);
   7533     create_builtin_fn(g, BuiltinFnIdPanic, "panic", 1);
   7534     create_builtin_fn(g, BuiltinFnIdPtrCast, "ptrCast", 2);
   7535     create_builtin_fn(g, BuiltinFnIdBitCast, "bitCast", 2);
   7536     create_builtin_fn(g, BuiltinFnIdIntToPtr, "intToPtr", 2);
   7537     create_builtin_fn(g, BuiltinFnIdPtrToInt, "ptrToInt", 1);
   7538     create_builtin_fn(g, BuiltinFnIdTagName, "tagName", 1);
   7539     create_builtin_fn(g, BuiltinFnIdTagType, "TagType", 1);
   7540     create_builtin_fn(g, BuiltinFnIdFieldParentPtr, "fieldParentPtr", 3);
   7541     create_builtin_fn(g, BuiltinFnIdByteOffsetOf, "byteOffsetOf", 2);
   7542     create_builtin_fn(g, BuiltinFnIdBitOffsetOf, "bitOffsetOf", 2);
   7543     create_builtin_fn(g, BuiltinFnIdDivExact, "divExact", 2);
   7544     create_builtin_fn(g, BuiltinFnIdDivTrunc, "divTrunc", 2);
   7545     create_builtin_fn(g, BuiltinFnIdDivFloor, "divFloor", 2);
   7546     create_builtin_fn(g, BuiltinFnIdRem, "rem", 2);
   7547     create_builtin_fn(g, BuiltinFnIdMod, "mod", 2);
   7548     create_builtin_fn(g, BuiltinFnIdSqrt, "sqrt", 2);
   7549     create_builtin_fn(g, BuiltinFnIdSin, "sin", 2);
   7550     create_builtin_fn(g, BuiltinFnIdCos, "cos", 2);
   7551     create_builtin_fn(g, BuiltinFnIdExp, "exp", 2);
   7552     create_builtin_fn(g, BuiltinFnIdExp2, "exp2", 2);
   7553     create_builtin_fn(g, BuiltinFnIdLn, "ln", 2);
   7554     create_builtin_fn(g, BuiltinFnIdLog2, "log2", 2);
   7555     create_builtin_fn(g, BuiltinFnIdLog10, "log10", 2);
   7556     create_builtin_fn(g, BuiltinFnIdFabs, "fabs", 2);
   7557     create_builtin_fn(g, BuiltinFnIdFloor, "floor", 2);
   7558     create_builtin_fn(g, BuiltinFnIdCeil, "ceil", 2);
   7559     create_builtin_fn(g, BuiltinFnIdTrunc, "trunc", 2);
   7560     create_builtin_fn(g, BuiltinFnIdNearbyInt, "nearbyInt", 2);
   7561     create_builtin_fn(g, BuiltinFnIdRound, "round", 2);
   7562     create_builtin_fn(g, BuiltinFnIdMulAdd, "mulAdd", 4);
   7563     create_builtin_fn(g, BuiltinFnIdInlineCall, "inlineCall", SIZE_MAX);
   7564     create_builtin_fn(g, BuiltinFnIdNoInlineCall, "noInlineCall", SIZE_MAX);
   7565     create_builtin_fn(g, BuiltinFnIdNewStackCall, "newStackCall", SIZE_MAX);
   7566     create_builtin_fn(g, BuiltinFnIdAsyncCall, "asyncCall", SIZE_MAX);
   7567     create_builtin_fn(g, BuiltinFnIdTypeId, "typeId", 1);
   7568     create_builtin_fn(g, BuiltinFnIdShlExact, "shlExact", 2);
   7569     create_builtin_fn(g, BuiltinFnIdShrExact, "shrExact", 2);
   7570     create_builtin_fn(g, BuiltinFnIdSetEvalBranchQuota, "setEvalBranchQuota", 1);
   7571     create_builtin_fn(g, BuiltinFnIdAlignCast, "alignCast", 2);
   7572     create_builtin_fn(g, BuiltinFnIdOpaqueType, "OpaqueType", 0);
   7573     create_builtin_fn(g, BuiltinFnIdSetAlignStack, "setAlignStack", 1);
   7574     create_builtin_fn(g, BuiltinFnIdArgType, "ArgType", 2);
   7575     create_builtin_fn(g, BuiltinFnIdExport, "export", 3);
   7576     create_builtin_fn(g, BuiltinFnIdErrorReturnTrace, "errorReturnTrace", 0);
   7577     create_builtin_fn(g, BuiltinFnIdAtomicRmw, "atomicRmw", 5);
   7578     create_builtin_fn(g, BuiltinFnIdAtomicLoad, "atomicLoad", 3);
   7579     create_builtin_fn(g, BuiltinFnIdErrSetCast, "errSetCast", 2);
   7580     create_builtin_fn(g, BuiltinFnIdToBytes, "sliceToBytes", 1);
   7581     create_builtin_fn(g, BuiltinFnIdFromBytes, "bytesToSlice", 2);
   7582     create_builtin_fn(g, BuiltinFnIdThis, "This", 0);
   7583     create_builtin_fn(g, BuiltinFnIdHasDecl, "hasDecl", 2);
   7584     create_builtin_fn(g, BuiltinFnIdUnionInit, "unionInit", 3);
   7585     create_builtin_fn(g, BuiltinFnIdFrameHandle, "frame", 0);
   7586     create_builtin_fn(g, BuiltinFnIdFrameType, "Frame", 1);
   7587     create_builtin_fn(g, BuiltinFnIdFrameAddress, "frameAddress", 0);
   7588     create_builtin_fn(g, BuiltinFnIdFrameSize, "frameSize", 1);
   7589 }
   7590 
   7591 static const char *bool_to_str(bool b) {
   7592     return b ? "true" : "false";
   7593 }
   7594 
   7595 static const char *build_mode_to_str(BuildMode build_mode) {
   7596     switch (build_mode) {
   7597         case BuildModeDebug: return "Mode.Debug";
   7598         case BuildModeSafeRelease: return "Mode.ReleaseSafe";
   7599         case BuildModeFastRelease: return "Mode.ReleaseFast";
   7600         case BuildModeSmallRelease: return "Mode.ReleaseSmall";
   7601     }
   7602     zig_unreachable();
   7603 }
   7604 
   7605 static const char *subsystem_to_str(TargetSubsystem subsystem) {
   7606     switch (subsystem) {
   7607         case TargetSubsystemConsole: return "Console";
   7608         case TargetSubsystemWindows: return "Windows";
   7609         case TargetSubsystemPosix: return "Posix";
   7610         case TargetSubsystemNative: return "Native";
   7611         case TargetSubsystemEfiApplication: return "EfiApplication";
   7612         case TargetSubsystemEfiBootServiceDriver: return "EfiBootServiceDriver";
   7613         case TargetSubsystemEfiRom: return "EfiRom";
   7614         case TargetSubsystemEfiRuntimeDriver: return "EfiRuntimeDriver";
   7615         case TargetSubsystemAuto: zig_unreachable();
   7616     }
   7617     zig_unreachable();
   7618 }
   7619 
   7620 static bool detect_dynamic_link(CodeGen *g) {
   7621     if (g->is_dynamic)
   7622         return true;
   7623     if (g->zig_target->os == OsFreestanding)
   7624         return false;
   7625     if (target_requires_pic(g->zig_target, g->libc_link_lib != nullptr))
   7626         return true;
   7627     // If there are no dynamic libraries then we can disable PIC
   7628     for (size_t i = 0; i < g->link_libs_list.length; i += 1) {
   7629         LinkLib *link_lib = g->link_libs_list.at(i);
   7630         if (target_is_libc_lib_name(g->zig_target, buf_ptr(link_lib->name)))
   7631             continue;
   7632         return true;
   7633     }
   7634     return false;
   7635 }
   7636 
   7637 static bool detect_pic(CodeGen *g) {
   7638     if (target_requires_pic(g->zig_target, g->libc_link_lib != nullptr))
   7639         return true;
   7640     switch (g->want_pic) {
   7641         case WantPICDisabled:
   7642             return false;
   7643         case WantPICEnabled:
   7644             return true;
   7645         case WantPICAuto:
   7646             return g->have_dynamic_link;
   7647     }
   7648     zig_unreachable();
   7649 }
   7650 
   7651 static bool detect_stack_probing(CodeGen *g) {
   7652     if (!target_supports_stack_probing(g->zig_target))
   7653         return false;
   7654     switch (g->want_stack_check) {
   7655         case WantStackCheckDisabled:
   7656             return false;
   7657         case WantStackCheckEnabled:
   7658             return true;
   7659         case WantStackCheckAuto:
   7660             return g->build_mode == BuildModeSafeRelease || g->build_mode == BuildModeDebug;
   7661     }
   7662     zig_unreachable();
   7663 }
   7664 
   7665 // Returns TargetSubsystemAuto to mean "no subsystem"
   7666 TargetSubsystem detect_subsystem(CodeGen *g) {
   7667     if (g->subsystem != TargetSubsystemAuto)
   7668         return g->subsystem;
   7669     if (g->zig_target->os == OsWindows) {
   7670         if (g->have_dllmain_crt_startup || (g->out_type == OutTypeLib && g->is_dynamic))
   7671             return TargetSubsystemAuto;
   7672         if (g->have_c_main || g->have_pub_main || g->is_test_build)
   7673             return TargetSubsystemConsole;
   7674         if (g->have_winmain || g->have_winmain_crt_startup)
   7675             return TargetSubsystemWindows;
   7676     } else if (g->zig_target->os == OsUefi) {
   7677         return TargetSubsystemEfiApplication;
   7678     }
   7679     return TargetSubsystemAuto;
   7680 }
   7681 
   7682 static bool detect_single_threaded(CodeGen *g) {
   7683     if (g->want_single_threaded)
   7684         return true;
   7685     if (target_is_single_threaded(g->zig_target)) {
   7686         return true;
   7687     }
   7688     return false;
   7689 }
   7690 
   7691 static bool detect_err_ret_tracing(CodeGen *g) {
   7692     return !g->strip_debug_symbols &&
   7693         g->build_mode != BuildModeFastRelease &&
   7694         g->build_mode != BuildModeSmallRelease;
   7695 }
   7696 
   7697 Buf *codegen_generate_builtin_source(CodeGen *g) {
   7698     g->have_dynamic_link = detect_dynamic_link(g);
   7699     g->have_pic = detect_pic(g);
   7700     g->have_stack_probing = detect_stack_probing(g);
   7701     g->is_single_threaded = detect_single_threaded(g);
   7702     g->have_err_ret_tracing = detect_err_ret_tracing(g);
   7703 
   7704     Buf *contents = buf_alloc();
   7705 
   7706     // NOTE: when editing this file, you may need to make modifications to the
   7707     // cache input parameters in define_builtin_compile_vars
   7708 
   7709     // Modifications to this struct must be coordinated with code that does anything with
   7710     // g->stack_trace_type. There are hard-coded references to the field indexes.
   7711     buf_append_str(contents,
   7712         "pub const StackTrace = struct {\n"
   7713         "    index: usize,\n"
   7714         "    instruction_addresses: []usize,\n"
   7715         "};\n\n");
   7716 
   7717     buf_append_str(contents, "pub const PanicFn = fn([]const u8, ?*StackTrace) noreturn;\n\n");
   7718 
   7719     const char *cur_os = nullptr;
   7720     {
   7721         buf_appendf(contents, "pub const Os = enum {\n");
   7722         uint32_t field_count = (uint32_t)target_os_count();
   7723         for (uint32_t i = 0; i < field_count; i += 1) {
   7724             Os os_type = target_os_enum(i);
   7725             const char *name = target_os_name(os_type);
   7726             buf_appendf(contents, "    %s,\n", name);
   7727 
   7728             if (os_type == g->zig_target->os) {
   7729                 g->target_os_index = i;
   7730                 cur_os = name;
   7731             }
   7732         }
   7733         buf_appendf(contents, "};\n\n");
   7734     }
   7735     assert(cur_os != nullptr);
   7736 
   7737     const char *cur_arch = nullptr;
   7738     {
   7739         buf_appendf(contents, "pub const Arch = union(enum) {\n");
   7740         uint32_t field_count = (uint32_t)target_arch_count();
   7741         for (uint32_t arch_i = 0; arch_i < field_count; arch_i += 1) {
   7742             ZigLLVM_ArchType arch = target_arch_enum(arch_i);
   7743             const char *arch_name = target_arch_name(arch);
   7744             SubArchList sub_arch_list = target_subarch_list(arch);
   7745             if (sub_arch_list == SubArchListNone) {
   7746                 buf_appendf(contents, "    %s,\n", arch_name);
   7747                 if (arch == g->zig_target->arch) {
   7748                     g->target_arch_index = arch_i;
   7749                     cur_arch = buf_ptr(buf_sprintf("Arch.%s", arch_name));
   7750                 }
   7751             } else {
   7752                 const char *sub_arch_list_name = target_subarch_list_name(sub_arch_list);
   7753                 buf_appendf(contents, "    %s: %s,\n", arch_name, sub_arch_list_name);
   7754                 if (arch == g->zig_target->arch) {
   7755                     size_t sub_count = target_subarch_count(sub_arch_list);
   7756                     for (size_t sub_i = 0; sub_i < sub_count; sub_i += 1) {
   7757                         ZigLLVM_SubArchType sub = target_subarch_enum(sub_arch_list, sub_i);
   7758                         if (sub == g->zig_target->sub_arch) {
   7759                             g->target_sub_arch_index = sub_i;
   7760                             cur_arch = buf_ptr(buf_sprintf("Arch{ .%s = Arch.%s.%s }",
   7761                                         arch_name, sub_arch_list_name, target_subarch_name(sub)));
   7762                         }
   7763                     }
   7764                 }
   7765             }
   7766         }
   7767 
   7768         uint32_t list_count = target_subarch_list_count();
   7769         // start at index 1 to skip None
   7770         for (uint32_t list_i = 1; list_i < list_count; list_i += 1) {
   7771             SubArchList sub_arch_list = target_subarch_list_enum(list_i);
   7772             const char *subarch_list_name = target_subarch_list_name(sub_arch_list);
   7773             buf_appendf(contents, "    pub const %s = enum {\n", subarch_list_name);
   7774             size_t sub_count = target_subarch_count(sub_arch_list);
   7775             for (size_t sub_i = 0; sub_i < sub_count; sub_i += 1) {
   7776                 ZigLLVM_SubArchType sub = target_subarch_enum(sub_arch_list, sub_i);
   7777                 buf_appendf(contents, "        %s,\n", target_subarch_name(sub));
   7778             }
   7779             buf_appendf(contents, "    };\n");
   7780         }
   7781         buf_appendf(contents, "};\n\n");
   7782     }
   7783     assert(cur_arch != nullptr);
   7784 
   7785     const char *cur_abi = nullptr;
   7786     {
   7787         buf_appendf(contents, "pub const Abi = enum {\n");
   7788         uint32_t field_count = (uint32_t)target_abi_count();
   7789         for (uint32_t i = 0; i < field_count; i += 1) {
   7790             ZigLLVM_EnvironmentType abi = target_abi_enum(i);
   7791             const char *name = target_abi_name(abi);
   7792             buf_appendf(contents, "    %s,\n", name);
   7793 
   7794             if (abi == g->zig_target->abi) {
   7795                 g->target_abi_index = i;
   7796                 cur_abi = name;
   7797             }
   7798         }
   7799         buf_appendf(contents, "};\n\n");
   7800     }
   7801     assert(cur_abi != nullptr);
   7802 
   7803     const char *cur_obj_fmt = nullptr;
   7804     {
   7805         buf_appendf(contents, "pub const ObjectFormat = enum {\n");
   7806         uint32_t field_count = (uint32_t)target_oformat_count();
   7807         for (uint32_t i = 0; i < field_count; i += 1) {
   7808             ZigLLVM_ObjectFormatType oformat = target_oformat_enum(i);
   7809             const char *name = target_oformat_name(oformat);
   7810             buf_appendf(contents, "    %s,\n", name);
   7811 
   7812             ZigLLVM_ObjectFormatType target_oformat = target_object_format(g->zig_target);
   7813             if (oformat == target_oformat) {
   7814                 g->target_oformat_index = i;
   7815                 cur_obj_fmt = name;
   7816             }
   7817         }
   7818 
   7819         buf_appendf(contents, "};\n\n");
   7820     }
   7821     assert(cur_obj_fmt != nullptr);
   7822 
   7823     {
   7824         buf_appendf(contents, "pub const GlobalLinkage = enum {\n");
   7825         uint32_t field_count = array_length(global_linkage_values);
   7826         for (uint32_t i = 0; i < field_count; i += 1) {
   7827             const GlobalLinkageValue *value = &global_linkage_values[i];
   7828             buf_appendf(contents, "    %s,\n", value->name);
   7829         }
   7830         buf_appendf(contents, "};\n\n");
   7831     }
   7832     {
   7833         buf_appendf(contents,
   7834             "pub const AtomicOrder = enum {\n"
   7835             "    Unordered,\n"
   7836             "    Monotonic,\n"
   7837             "    Acquire,\n"
   7838             "    Release,\n"
   7839             "    AcqRel,\n"
   7840             "    SeqCst,\n"
   7841             "};\n\n");
   7842     }
   7843     {
   7844         buf_appendf(contents,
   7845             "pub const AtomicRmwOp = enum {\n"
   7846             "    Xchg,\n"
   7847             "    Add,\n"
   7848             "    Sub,\n"
   7849             "    And,\n"
   7850             "    Nand,\n"
   7851             "    Or,\n"
   7852             "    Xor,\n"
   7853             "    Max,\n"
   7854             "    Min,\n"
   7855             "};\n\n");
   7856     }
   7857     {
   7858         buf_appendf(contents,
   7859             "pub const Mode = enum {\n"
   7860             "    Debug,\n"
   7861             "    ReleaseSafe,\n"
   7862             "    ReleaseFast,\n"
   7863             "    ReleaseSmall,\n"
   7864             "};\n\n");
   7865     }
   7866     {
   7867         buf_appendf(contents, "pub const TypeId = enum {\n");
   7868         size_t field_count = type_id_len();
   7869         for (size_t i = 0; i < field_count; i += 1) {
   7870             const ZigTypeId id = type_id_at_index(i);
   7871             buf_appendf(contents, "    %s,\n", type_id_name(id));
   7872         }
   7873         buf_appendf(contents, "};\n\n");
   7874     }
   7875     {
   7876         buf_appendf(contents,
   7877             "pub const TypeInfo = union(TypeId) {\n"
   7878             "    Type: void,\n"
   7879             "    Void: void,\n"
   7880             "    Bool: void,\n"
   7881             "    NoReturn: void,\n"
   7882             "    Int: Int,\n"
   7883             "    Float: Float,\n"
   7884             "    Pointer: Pointer,\n"
   7885             "    Array: Array,\n"
   7886             "    Struct: Struct,\n"
   7887             "    ComptimeFloat: void,\n"
   7888             "    ComptimeInt: void,\n"
   7889             "    Undefined: void,\n"
   7890             "    Null: void,\n"
   7891             "    Optional: Optional,\n"
   7892             "    ErrorUnion: ErrorUnion,\n"
   7893             "    ErrorSet: ErrorSet,\n"
   7894             "    Enum: Enum,\n"
   7895             "    Union: Union,\n"
   7896             "    Fn: Fn,\n"
   7897             "    BoundFn: Fn,\n"
   7898             "    ArgTuple: void,\n"
   7899             "    Opaque: void,\n"
   7900             "    Frame: void,\n"
   7901             "    AnyFrame: AnyFrame,\n"
   7902             "    Vector: Vector,\n"
   7903             "    EnumLiteral: void,\n"
   7904             "\n\n"
   7905             "    pub const Int = struct {\n"
   7906             "        is_signed: bool,\n"
   7907             "        bits: comptime_int,\n"
   7908             "    };\n"
   7909             "\n"
   7910             "    pub const Float = struct {\n"
   7911             "        bits: comptime_int,\n"
   7912             "    };\n"
   7913             "\n"
   7914             "    pub const Pointer = struct {\n"
   7915             "        size: Size,\n"
   7916             "        is_const: bool,\n"
   7917             "        is_volatile: bool,\n"
   7918             "        alignment: comptime_int,\n"
   7919             "        child: type,\n"
   7920             "        is_allowzero: bool,\n"
   7921             "\n"
   7922             "        pub const Size = enum {\n"
   7923             "            One,\n"
   7924             "            Many,\n"
   7925             "            Slice,\n"
   7926             "            C,\n"
   7927             "        };\n"
   7928             "    };\n"
   7929             "\n"
   7930             "    pub const Array = struct {\n"
   7931             "        len: comptime_int,\n"
   7932             "        child: type,\n"
   7933             "    };\n"
   7934             "\n"
   7935             "    pub const ContainerLayout = enum {\n"
   7936             "        Auto,\n"
   7937             "        Extern,\n"
   7938             "        Packed,\n"
   7939             "    };\n"
   7940             "\n"
   7941             "    pub const StructField = struct {\n"
   7942             "        name: []const u8,\n"
   7943             "        offset: ?comptime_int,\n"
   7944             "        field_type: type,\n"
   7945             "    };\n"
   7946             "\n"
   7947             "    pub const Struct = struct {\n"
   7948             "        layout: ContainerLayout,\n"
   7949             "        fields: []StructField,\n"
   7950             "        decls: []Declaration,\n"
   7951             "    };\n"
   7952             "\n"
   7953             "    pub const Optional = struct {\n"
   7954             "        child: type,\n"
   7955             "    };\n"
   7956             "\n"
   7957             "    pub const ErrorUnion = struct {\n"
   7958             "        error_set: type,\n"
   7959             "        payload: type,\n"
   7960             "    };\n"
   7961             "\n"
   7962             "    pub const Error = struct {\n"
   7963             "        name: []const u8,\n"
   7964             "        value: comptime_int,\n"
   7965             "    };\n"
   7966             "\n"
   7967             "    pub const ErrorSet = ?[]Error;\n"
   7968             "\n"
   7969             "    pub const EnumField = struct {\n"
   7970             "        name: []const u8,\n"
   7971             "        value: comptime_int,\n"
   7972             "    };\n"
   7973             "\n"
   7974             "    pub const Enum = struct {\n"
   7975             "        layout: ContainerLayout,\n"
   7976             "        tag_type: type,\n"
   7977             "        fields: []EnumField,\n"
   7978             "        decls: []Declaration,\n"
   7979             "    };\n"
   7980             "\n"
   7981             "    pub const UnionField = struct {\n"
   7982             "        name: []const u8,\n"
   7983             "        enum_field: ?EnumField,\n"
   7984             "        field_type: type,\n"
   7985             "    };\n"
   7986             "\n"
   7987             "    pub const Union = struct {\n"
   7988             "        layout: ContainerLayout,\n"
   7989             "        tag_type: ?type,\n"
   7990             "        fields: []UnionField,\n"
   7991             "        decls: []Declaration,\n"
   7992             "    };\n"
   7993             "\n"
   7994             "    pub const CallingConvention = enum {\n"
   7995             "        Unspecified,\n"
   7996             "        C,\n"
   7997             "        Cold,\n"
   7998             "        Naked,\n"
   7999             "        Stdcall,\n"
   8000             "        Async,\n"
   8001             "    };\n"
   8002             "\n"
   8003             "    pub const FnArg = struct {\n"
   8004             "        is_generic: bool,\n"
   8005             "        is_noalias: bool,\n"
   8006             "        arg_type: ?type,\n"
   8007             "    };\n"
   8008             "\n"
   8009             "    pub const Fn = struct {\n"
   8010             "        calling_convention: CallingConvention,\n"
   8011             "        is_generic: bool,\n"
   8012             "        is_var_args: bool,\n"
   8013             "        return_type: ?type,\n"
   8014             "        args: []FnArg,\n"
   8015             "    };\n"
   8016             "\n"
   8017             "    pub const AnyFrame = struct {\n"
   8018             "        child: ?type,\n"
   8019             "    };\n"
   8020             "\n"
   8021             "    pub const Vector = struct {\n"
   8022             "        len: comptime_int,\n"
   8023             "        child: type,\n"
   8024             "    };\n"
   8025             "\n"
   8026             "    pub const Declaration = struct {\n"
   8027             "        name: []const u8,\n"
   8028             "        is_pub: bool,\n"
   8029             "        data: Data,\n"
   8030             "\n"
   8031             "        pub const Data = union(enum) {\n"
   8032             "            Type: type,\n"
   8033             "            Var: type,\n"
   8034             "            Fn: FnDecl,\n"
   8035             "\n"
   8036             "            pub const FnDecl = struct {\n"
   8037             "                fn_type: type,\n"
   8038             "                inline_type: Inline,\n"
   8039             "                calling_convention: CallingConvention,\n"
   8040             "                is_var_args: bool,\n"
   8041             "                is_extern: bool,\n"
   8042             "                is_export: bool,\n"
   8043             "                lib_name: ?[]const u8,\n"
   8044             "                return_type: type,\n"
   8045             "                arg_names: [][] const u8,\n"
   8046             "\n"
   8047             "                pub const Inline = enum {\n"
   8048             "                    Auto,\n"
   8049             "                    Always,\n"
   8050             "                    Never,\n"
   8051             "                };\n"
   8052             "            };\n"
   8053             "        };\n"
   8054             "    };\n"
   8055             "};\n\n");
   8056         assert(ContainerLayoutAuto == 0);
   8057         assert(ContainerLayoutExtern == 1);
   8058         assert(ContainerLayoutPacked == 2);
   8059 
   8060         assert(CallingConventionUnspecified == 0);
   8061         assert(CallingConventionC == 1);
   8062         assert(CallingConventionCold == 2);
   8063         assert(CallingConventionNaked == 3);
   8064         assert(CallingConventionStdcall == 4);
   8065         assert(CallingConventionAsync == 5);
   8066 
   8067         assert(FnInlineAuto == 0);
   8068         assert(FnInlineAlways == 1);
   8069         assert(FnInlineNever == 2);
   8070     }
   8071     {
   8072         buf_appendf(contents,
   8073             "pub const FloatMode = enum {\n"
   8074             "    Strict,\n"
   8075             "    Optimized,\n"
   8076             "};\n\n");
   8077         assert(FloatModeStrict == 0);
   8078         assert(FloatModeOptimized == 1);
   8079     }
   8080     {
   8081         buf_appendf(contents,
   8082             "pub const Endian = enum {\n"
   8083             "    Big,\n"
   8084             "    Little,\n"
   8085             "};\n\n");
   8086         //assert(EndianBig == 0);
   8087         //assert(EndianLittle == 1);
   8088     }
   8089     {
   8090         buf_appendf(contents,
   8091             "pub const Version = struct {\n"
   8092             "    major: u32,\n"
   8093             "    minor: u32,\n"
   8094             "    patch: u32,\n"
   8095             "};\n\n");
   8096     }
   8097     {
   8098         buf_appendf(contents,
   8099         "pub const SubSystem = enum {\n"
   8100         "    Console,\n"
   8101         "    Windows,\n"
   8102         "    Posix,\n"
   8103         "    Native,\n"
   8104         "    EfiApplication,\n"
   8105         "    EfiBootServiceDriver,\n"
   8106         "    EfiRom,\n"
   8107         "    EfiRuntimeDriver,\n"
   8108         "};\n\n");
   8109 
   8110         assert(TargetSubsystemConsole == 0);
   8111         assert(TargetSubsystemWindows == 1);
   8112         assert(TargetSubsystemPosix == 2);
   8113         assert(TargetSubsystemNative == 3);
   8114         assert(TargetSubsystemEfiApplication == 4);
   8115         assert(TargetSubsystemEfiBootServiceDriver == 5);
   8116         assert(TargetSubsystemEfiRom == 6);
   8117         assert(TargetSubsystemEfiRuntimeDriver == 7);
   8118     }
   8119     {
   8120         const char *endian_str = g->is_big_endian ? "Endian.Big" : "Endian.Little";
   8121         buf_appendf(contents, "pub const endian = %s;\n", endian_str);
   8122     }
   8123     buf_appendf(contents, "pub const is_test = %s;\n", bool_to_str(g->is_test_build));
   8124     buf_appendf(contents, "pub const single_threaded = %s;\n", bool_to_str(g->is_single_threaded));
   8125     buf_appendf(contents, "pub const os = Os.%s;\n", cur_os);
   8126     buf_appendf(contents, "pub const arch = %s;\n", cur_arch);
   8127     buf_appendf(contents, "pub const abi = Abi.%s;\n", cur_abi);
   8128     if (g->libc_link_lib != nullptr && g->zig_target->glibc_version != nullptr) {
   8129         buf_appendf(contents,
   8130             "pub const glibc_version: ?Version = Version{.major = %d, .minor = %d, .patch = %d};\n",
   8131                 g->zig_target->glibc_version->major,
   8132                 g->zig_target->glibc_version->minor,
   8133                 g->zig_target->glibc_version->patch);
   8134     } else {
   8135         buf_appendf(contents, "pub const glibc_version: ?Version = null;\n");
   8136     }
   8137     buf_appendf(contents, "pub const object_format = ObjectFormat.%s;\n", cur_obj_fmt);
   8138     buf_appendf(contents, "pub const mode = %s;\n", build_mode_to_str(g->build_mode));
   8139     buf_appendf(contents, "pub const link_libc = %s;\n", bool_to_str(g->libc_link_lib != nullptr));
   8140     buf_appendf(contents, "pub const have_error_return_tracing = %s;\n", bool_to_str(g->have_err_ret_tracing));
   8141     buf_appendf(contents, "pub const valgrind_support = %s;\n", bool_to_str(want_valgrind_support(g)));
   8142     buf_appendf(contents, "pub const position_independent_code = %s;\n", bool_to_str(g->have_pic));
   8143     buf_appendf(contents, "pub const strip_debug_info = %s;\n", bool_to_str(g->strip_debug_symbols));
   8144 
   8145     {
   8146         TargetSubsystem detected_subsystem = detect_subsystem(g);
   8147         if (detected_subsystem != TargetSubsystemAuto) {
   8148             buf_appendf(contents, "pub const subsystem = SubSystem.%s;\n", subsystem_to_str(detected_subsystem));
   8149         }
   8150     }
   8151 
   8152     if (g->is_test_build) {
   8153         buf_appendf(contents,
   8154             "const TestFn = struct {\n"
   8155                 "name: []const u8,\n"
   8156                 "func: fn()anyerror!void,\n"
   8157             "};\n"
   8158             "pub const test_functions = {}; // overwritten later\n"
   8159         );
   8160     }
   8161 
   8162     return contents;
   8163 }
   8164 
   8165 static ZigPackage *create_test_runner_pkg(CodeGen *g) {
   8166     return codegen_create_package(g, buf_ptr(g->zig_std_special_dir), "test_runner.zig", "std.special");
   8167 }
   8168 
   8169 static ZigPackage *create_panic_pkg(CodeGen *g) {
   8170     return codegen_create_package(g, buf_ptr(g->zig_std_special_dir), "panic.zig", "std.special");
   8171 }
   8172 
   8173 static Error define_builtin_compile_vars(CodeGen *g) {
   8174     if (g->std_package == nullptr)
   8175         return ErrorNone;
   8176 
   8177     Error err;
   8178 
   8179     Buf *manifest_dir = buf_alloc();
   8180     os_path_join(get_stage1_cache_path(), buf_create_from_str("builtin"), manifest_dir);
   8181 
   8182     CacheHash cache_hash;
   8183     cache_init(&cache_hash, manifest_dir);
   8184 
   8185     Buf *compiler_id;
   8186     if ((err = get_compiler_id(&compiler_id)))
   8187         return err;
   8188 
   8189     // Only a few things affect builtin.zig
   8190     cache_buf(&cache_hash, compiler_id);
   8191     cache_int(&cache_hash, g->build_mode);
   8192     cache_bool(&cache_hash, g->strip_debug_symbols);
   8193     cache_bool(&cache_hash, g->is_test_build);
   8194     cache_bool(&cache_hash, g->is_single_threaded);
   8195     cache_int(&cache_hash, g->zig_target->is_native);
   8196     cache_int(&cache_hash, g->zig_target->arch);
   8197     cache_int(&cache_hash, g->zig_target->sub_arch);
   8198     cache_int(&cache_hash, g->zig_target->vendor);
   8199     cache_int(&cache_hash, g->zig_target->os);
   8200     cache_int(&cache_hash, g->zig_target->abi);
   8201     if (g->zig_target->glibc_version != nullptr) {
   8202         cache_int(&cache_hash, g->zig_target->glibc_version->major);
   8203         cache_int(&cache_hash, g->zig_target->glibc_version->minor);
   8204         cache_int(&cache_hash, g->zig_target->glibc_version->patch);
   8205     }
   8206     cache_bool(&cache_hash, g->have_err_ret_tracing);
   8207     cache_bool(&cache_hash, g->libc_link_lib != nullptr);
   8208     cache_bool(&cache_hash, g->valgrind_support);
   8209     cache_int(&cache_hash, detect_subsystem(g));
   8210 
   8211     Buf digest = BUF_INIT;
   8212     buf_resize(&digest, 0);
   8213     if ((err = cache_hit(&cache_hash, &digest))) {
   8214         // Treat an invalid format error as a cache miss.
   8215         if (err != ErrorInvalidFormat)
   8216             return err;
   8217     }
   8218 
   8219     // We should always get a cache hit because there are no
   8220     // files in the input hash.
   8221     assert(buf_len(&digest) != 0);
   8222 
   8223     Buf *this_dir = buf_alloc();
   8224     os_path_join(manifest_dir, &digest, this_dir);
   8225 
   8226     if ((err = os_make_path(this_dir)))
   8227         return err;
   8228 
   8229     const char *builtin_zig_basename = "builtin.zig";
   8230     Buf *builtin_zig_path = buf_alloc();
   8231     os_path_join(this_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path);
   8232 
   8233     bool hit;
   8234     if ((err = os_file_exists(builtin_zig_path, &hit)))
   8235         return err;
   8236     Buf *contents;
   8237     if (hit) {
   8238         contents = buf_alloc();
   8239         if ((err = os_fetch_file_path(builtin_zig_path, contents))) {
   8240             fprintf(stderr, "Unable to open '%s': %s\n", buf_ptr(builtin_zig_path), err_str(err));
   8241             exit(1);
   8242         }
   8243     } else {
   8244         contents = codegen_generate_builtin_source(g);
   8245         if ((err = os_write_file(builtin_zig_path, contents))) {
   8246             fprintf(stderr, "Unable to write file '%s': %s\n", buf_ptr(builtin_zig_path), err_str(err));
   8247             exit(1);
   8248         }
   8249     }
   8250 
   8251     assert(g->root_package);
   8252     assert(g->std_package);
   8253     g->compile_var_package = new_package(buf_ptr(this_dir), builtin_zig_basename, "builtin");
   8254     g->root_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package);
   8255     g->std_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package);
   8256     g->std_package->package_table.put(buf_create_from_str("std"), g->std_package);
   8257     ZigPackage *root_pkg;
   8258     if (g->is_test_build) {
   8259         if (g->test_runner_package == nullptr) {
   8260             g->test_runner_package = create_test_runner_pkg(g);
   8261         }
   8262         root_pkg = g->test_runner_package;
   8263     } else {
   8264         root_pkg = g->root_package;
   8265     }
   8266     g->std_package->package_table.put(buf_create_from_str("root"), root_pkg);
   8267     g->compile_var_import = add_source_file(g, g->compile_var_package, builtin_zig_path, contents,
   8268             SourceKindPkgMain);
   8269 
   8270     return ErrorNone;
   8271 }
   8272 
   8273 static void init(CodeGen *g) {
   8274     if (g->module)
   8275         return;
   8276 
   8277     g->have_dynamic_link = detect_dynamic_link(g);
   8278     g->have_pic = detect_pic(g);
   8279     g->have_stack_probing = detect_stack_probing(g);
   8280     g->is_single_threaded = detect_single_threaded(g);
   8281     g->have_err_ret_tracing = detect_err_ret_tracing(g);
   8282 
   8283     if (target_is_single_threaded(g->zig_target)) {
   8284         g->is_single_threaded = true;
   8285     }
   8286 
   8287     assert(g->root_out_name);
   8288     g->module = LLVMModuleCreateWithName(buf_ptr(g->root_out_name));
   8289 
   8290     LLVMSetTarget(g->module, buf_ptr(&g->llvm_triple_str));
   8291 
   8292     if (target_object_format(g->zig_target) == ZigLLVM_COFF) {
   8293         ZigLLVMAddModuleCodeViewFlag(g->module);
   8294     } else {
   8295         ZigLLVMAddModuleDebugInfoFlag(g->module);
   8296     }
   8297 
   8298     LLVMTargetRef target_ref;
   8299     char *err_msg = nullptr;
   8300     if (LLVMGetTargetFromTriple(buf_ptr(&g->llvm_triple_str), &target_ref, &err_msg)) {
   8301         fprintf(stderr,
   8302             "Zig is expecting LLVM to understand this target: '%s'\n"
   8303             "However LLVM responded with: \"%s\"\n"
   8304             "Zig is unable to continue. This is a bug in Zig:\n"
   8305             "https://github.com/ziglang/zig/issues/438\n"
   8306         , buf_ptr(&g->llvm_triple_str), err_msg);
   8307         exit(1);
   8308     }
   8309 
   8310     bool is_optimized = g->build_mode != BuildModeDebug;
   8311     LLVMCodeGenOptLevel opt_level = is_optimized ? LLVMCodeGenLevelAggressive : LLVMCodeGenLevelNone;
   8312 
   8313     LLVMRelocMode reloc_mode;
   8314     if (g->have_pic) {
   8315         reloc_mode = LLVMRelocPIC;
   8316     } else if (g->have_dynamic_link) {
   8317         reloc_mode = LLVMRelocDynamicNoPic;
   8318     } else {
   8319         reloc_mode = LLVMRelocStatic;
   8320     }
   8321 
   8322     const char *target_specific_cpu_args;
   8323     const char *target_specific_features;
   8324     if (g->zig_target->is_native) {
   8325         // LLVM creates invalid binaries on Windows sometimes.
   8326         // See https://github.com/ziglang/zig/issues/508
   8327         // As a workaround we do not use target native features on Windows.
   8328         if (g->zig_target->os == OsWindows || g->zig_target->os == OsUefi) {
   8329             target_specific_cpu_args = "";
   8330             target_specific_features = "";
   8331         } else {
   8332             target_specific_cpu_args = ZigLLVMGetHostCPUName();
   8333             target_specific_features = ZigLLVMGetNativeFeatures();
   8334         }
   8335     } else if (target_is_riscv(g->zig_target)) {
   8336         // TODO https://github.com/ziglang/zig/issues/2883
   8337         target_specific_cpu_args = "";
   8338         target_specific_features = "+a";
   8339     } else {
   8340         target_specific_cpu_args = "";
   8341         target_specific_features = "";
   8342     }
   8343 
   8344     g->target_machine = ZigLLVMCreateTargetMachine(target_ref, buf_ptr(&g->llvm_triple_str),
   8345             target_specific_cpu_args, target_specific_features, opt_level, reloc_mode,
   8346             LLVMCodeModelDefault, g->function_sections);
   8347 
   8348     g->target_data_ref = LLVMCreateTargetDataLayout(g->target_machine);
   8349 
   8350     char *layout_str = LLVMCopyStringRepOfTargetData(g->target_data_ref);
   8351     LLVMSetDataLayout(g->module, layout_str);
   8352 
   8353 
   8354     assert(g->pointer_size_bytes == LLVMPointerSize(g->target_data_ref));
   8355     g->is_big_endian = (LLVMByteOrder(g->target_data_ref) == LLVMBigEndian);
   8356 
   8357     g->builder = LLVMCreateBuilder();
   8358     g->dbuilder = ZigLLVMCreateDIBuilder(g->module, true);
   8359 
   8360     // Don't use ZIG_VERSION_STRING here, llvm misparses it when it includes
   8361     // the git revision.
   8362     Buf *producer = buf_sprintf("zig %d.%d.%d", ZIG_VERSION_MAJOR, ZIG_VERSION_MINOR, ZIG_VERSION_PATCH);
   8363     const char *flags = "";
   8364     unsigned runtime_version = 0;
   8365     ZigLLVMDIFile *compile_unit_file = ZigLLVMCreateFile(g->dbuilder, buf_ptr(g->root_out_name),
   8366             buf_ptr(&g->root_package->root_src_dir));
   8367     g->compile_unit = ZigLLVMCreateCompileUnit(g->dbuilder, ZigLLVMLang_DW_LANG_C99(),
   8368             compile_unit_file, buf_ptr(producer), is_optimized, flags, runtime_version,
   8369             "", 0, !g->strip_debug_symbols);
   8370 
   8371     // This is for debug stuff that doesn't have a real file.
   8372     g->dummy_di_file = nullptr;
   8373 
   8374     define_builtin_types(g);
   8375 
   8376     IrInstruction *sentinel_instructions = allocate<IrInstruction>(2);
   8377     g->invalid_instruction = &sentinel_instructions[0];
   8378     g->invalid_instruction->value.type = g->builtin_types.entry_invalid;
   8379     g->invalid_instruction->value.global_refs = allocate<ConstGlobalRefs>(1);
   8380 
   8381     g->unreach_instruction = &sentinel_instructions[1];
   8382     g->unreach_instruction->value.type = g->builtin_types.entry_unreachable;
   8383     g->unreach_instruction->value.global_refs = allocate<ConstGlobalRefs>(1);
   8384 
   8385     g->const_void_val.special = ConstValSpecialStatic;
   8386     g->const_void_val.type = g->builtin_types.entry_void;
   8387     g->const_void_val.global_refs = allocate<ConstGlobalRefs>(1);
   8388 
   8389     {
   8390         ConstGlobalRefs *global_refs = allocate<ConstGlobalRefs>(PanicMsgIdCount);
   8391         for (size_t i = 0; i < PanicMsgIdCount; i += 1) {
   8392             g->panic_msg_vals[i].global_refs = &global_refs[i];
   8393         }
   8394     }
   8395 
   8396     define_builtin_fns(g);
   8397     Error err;
   8398     if ((err = define_builtin_compile_vars(g))) {
   8399         fprintf(stderr, "Unable to create builtin.zig: %s\n", err_str(err));
   8400         exit(1);
   8401     }
   8402 }
   8403 
   8404 static void detect_dynamic_linker(CodeGen *g) {
   8405     if (g->dynamic_linker_path != nullptr)
   8406         return;
   8407     if (!g->have_dynamic_link)
   8408         return;
   8409     if (g->out_type == OutTypeObj || (g->out_type == OutTypeLib && !g->is_dynamic))
   8410         return;
   8411 
   8412     const char *standard_ld_path = target_dynamic_linker(g->zig_target);
   8413     if (standard_ld_path == nullptr)
   8414         return;
   8415 
   8416     if (g->zig_target->is_native) {
   8417         // target_dynamic_linker is usually correct. However on some systems, such as NixOS
   8418         // it will be incorrect. See if we can do better by looking at what zig's own
   8419         // dynamic linker path is.
   8420         g->dynamic_linker_path = get_self_dynamic_linker_path();
   8421         if (g->dynamic_linker_path != nullptr)
   8422             return;
   8423 
   8424         // If Zig is statically linked, such as via distributed binary static builds, the above
   8425         // trick won't work. What are we left with? Try to run the system C compiler and get
   8426         // it to tell us the dynamic linker path
   8427 #if defined(ZIG_OS_LINUX)
   8428         {
   8429             Error err;
   8430             Buf *result = buf_alloc();
   8431             for (size_t i = 0; possible_ld_names[i] != NULL; i += 1) {
   8432                 const char *lib_name = possible_ld_names[i];
   8433                 if ((err = zig_libc_cc_print_file_name(lib_name, result, false, true))) {
   8434                     if (err != ErrorCCompilerCannotFindFile && err != ErrorNoCCompilerInstalled) {
   8435                         fprintf(stderr, "Unable to detect native dynamic linker: %s\n", err_str(err));
   8436                         exit(1);
   8437                     }
   8438                     continue;
   8439                 }
   8440                 g->dynamic_linker_path = result;
   8441                 return;
   8442             }
   8443         }
   8444 #endif
   8445     }
   8446 
   8447     g->dynamic_linker_path = buf_create_from_str(standard_ld_path);
   8448 }
   8449 
   8450 static void detect_libc(CodeGen *g) {
   8451     Error err;
   8452 
   8453     if (g->libc != nullptr || g->libc_link_lib == nullptr)
   8454         return;
   8455 
   8456     if (target_can_build_libc(g->zig_target)) {
   8457         const char *generic_name = target_libc_generic_name(g->zig_target);
   8458         const char *arch_name = target_arch_name(g->zig_target->arch);
   8459         const char *abi_name = target_abi_name(g->zig_target->abi);
   8460         if (target_is_musl(g->zig_target)) {
   8461             // musl has some overrides. its headers are ABI-agnostic and so they all have the "musl" ABI name.
   8462             abi_name = "musl";
   8463             // some architectures are handled by the same set of headers
   8464             arch_name = target_arch_musl_name(g->zig_target->arch);
   8465         }
   8466         Buf *arch_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "%s-%s-%s",
   8467                 buf_ptr(g->zig_lib_dir), arch_name, target_os_name(g->zig_target->os), abi_name);
   8468         Buf *generic_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "generic-%s",
   8469                 buf_ptr(g->zig_lib_dir), generic_name);
   8470         Buf *arch_os_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "%s-%s-any",
   8471                 buf_ptr(g->zig_lib_dir), target_arch_name(g->zig_target->arch), target_os_name(g->zig_target->os));
   8472         Buf *generic_os_include_dir = buf_sprintf("%s" OS_SEP "libc" OS_SEP "include" OS_SEP "any-%s-any",
   8473                 buf_ptr(g->zig_lib_dir), target_os_name(g->zig_target->os));
   8474 
   8475         g->libc_include_dir_len = 4;
   8476         g->libc_include_dir_list = allocate<Buf*>(g->libc_include_dir_len);
   8477         g->libc_include_dir_list[0] = arch_include_dir;
   8478         g->libc_include_dir_list[1] = generic_include_dir;
   8479         g->libc_include_dir_list[2] = arch_os_include_dir;
   8480         g->libc_include_dir_list[3] = generic_os_include_dir;
   8481         return;
   8482     }
   8483 
   8484     if (g->zig_target->is_native) {
   8485         g->libc = allocate<ZigLibCInstallation>(1);
   8486 
   8487         // Look for zig-cache/native_libc.txt
   8488         Buf *native_libc_txt = buf_alloc();
   8489         os_path_join(g->cache_dir, buf_create_from_str("native_libc.txt"), native_libc_txt);
   8490         if ((err = zig_libc_parse(g->libc, native_libc_txt, g->zig_target, false))) {
   8491             if ((err = zig_libc_find_native(g->libc, true))) {
   8492                 fprintf(stderr,
   8493                     "Unable to link against libc: Unable to find libc installation: %s\n"
   8494                     "See `zig libc --help` for more details.\n", err_str(err));
   8495                 exit(1);
   8496             }
   8497             if ((err = os_make_path(g->cache_dir))) {
   8498                 fprintf(stderr, "Unable to create %s directory: %s\n",
   8499                     buf_ptr(g->cache_dir), err_str(err));
   8500                 exit(1);
   8501             }
   8502             Buf *native_libc_tmp = buf_sprintf("%s.tmp", buf_ptr(native_libc_txt));
   8503             FILE *file = fopen(buf_ptr(native_libc_tmp), "wb");
   8504             if (file == nullptr) {
   8505                 fprintf(stderr, "Unable to open %s: %s\n", buf_ptr(native_libc_tmp), strerror(errno));
   8506                 exit(1);
   8507             }
   8508             zig_libc_render(g->libc, file);
   8509             if (fclose(file) != 0) {
   8510                 fprintf(stderr, "Unable to save %s: %s\n", buf_ptr(native_libc_tmp), strerror(errno));
   8511                 exit(1);
   8512             }
   8513             if ((err = os_rename(native_libc_tmp, native_libc_txt))) {
   8514                 fprintf(stderr, "Unable to create %s: %s\n", buf_ptr(native_libc_txt), err_str(err));
   8515                 exit(1);
   8516             }
   8517         }
   8518         bool want_sys_dir = !buf_eql_buf(&g->libc->include_dir, &g->libc->sys_include_dir);
   8519         size_t dir_count = 1 + want_sys_dir;
   8520         g->libc_include_dir_len = dir_count;
   8521         g->libc_include_dir_list = allocate<Buf*>(dir_count);
   8522         g->libc_include_dir_list[0] = &g->libc->include_dir;
   8523         if (want_sys_dir) {
   8524             g->libc_include_dir_list[1] = &g->libc->sys_include_dir;
   8525         }
   8526     } else if ((g->out_type == OutTypeExe || (g->out_type == OutTypeLib && g->is_dynamic)) &&
   8527         !target_os_is_darwin(g->zig_target->os))
   8528     {
   8529         Buf triple_buf = BUF_INIT;
   8530         target_triple_zig(&triple_buf, g->zig_target);
   8531         fprintf(stderr,
   8532             "Zig is unable to provide a libc for the chosen target '%s'.\n"
   8533             "The target is non-native, so Zig also cannot use the native libc installation.\n"
   8534             "Choose a target which has a libc available (see `zig targets`), or\n"
   8535             "provide a libc installation text file (see `zig libc --help`).\n", buf_ptr(&triple_buf));
   8536         exit(1);
   8537     }
   8538 }
   8539 
   8540 // does not add the "cc" arg
   8541 void add_cc_args(CodeGen *g, ZigList<const char *> &args, const char *out_dep_path, bool translate_c) {
   8542     if (translate_c) {
   8543         args.append("-x");
   8544         args.append("c");
   8545     }
   8546 
   8547     if (out_dep_path != nullptr) {
   8548         args.append("-MD");
   8549         args.append("-MV");
   8550         args.append("-MF");
   8551         args.append(out_dep_path);
   8552     }
   8553 
   8554     args.append("-nostdinc");
   8555     args.append("-fno-spell-checking");
   8556 
   8557     if (g->function_sections) {
   8558         args.append("-ffunction-sections");
   8559     }
   8560 
   8561     if (translate_c) {
   8562         // this gives us access to preprocessing entities, presumably at
   8563         // the cost of performance
   8564         args.append("-Xclang");
   8565         args.append("-detailed-preprocessing-record");
   8566     } else {
   8567         switch (g->err_color) {
   8568             case ErrColorAuto:
   8569                 break;
   8570             case ErrColorOff:
   8571                 args.append("-fno-color-diagnostics");
   8572                 args.append("-fno-caret-diagnostics");
   8573                 break;
   8574             case ErrColorOn:
   8575                 args.append("-fcolor-diagnostics");
   8576                 args.append("-fcaret-diagnostics");
   8577                 break;
   8578         }
   8579     }
   8580 
   8581     // According to Rich Felker libc headers are supposed to go before C language headers.
   8582     // However as noted by @dimenus, appending libc headers before c_headers breaks intrinsics
   8583     // and other compiler specific items.
   8584     args.append("-isystem");
   8585     args.append(buf_ptr(g->zig_c_headers_dir));
   8586 
   8587     for (size_t i = 0; i < g->libc_include_dir_len; i += 1) {
   8588         Buf *include_dir = g->libc_include_dir_list[i];
   8589         args.append("-isystem");
   8590         args.append(buf_ptr(include_dir));
   8591     }
   8592 
   8593     if (g->zig_target->is_native) {
   8594         args.append("-march=native");
   8595     } else {
   8596         args.append("-target");
   8597         args.append(buf_ptr(&g->llvm_triple_str));
   8598 
   8599         if (target_is_musl(g->zig_target) && target_is_riscv(g->zig_target)) {
   8600             // Musl depends on atomic instructions, which are disabled by default in Clang/LLVM's
   8601             // cross compilation CPU info for RISCV.
   8602             // TODO: https://github.com/ziglang/zig/issues/2883
   8603             args.append("-Xclang");
   8604             args.append("-target-feature");
   8605             args.append("-Xclang");
   8606             args.append("+a");
   8607         }
   8608     }
   8609     if (g->zig_target->os == OsFreestanding) {
   8610         args.append("-ffreestanding");
   8611     }
   8612 
   8613     // windows.h has files such as pshpack1.h which do #pragma packing, triggering a clang warning.
   8614     // So for this target, we disable this warning.
   8615     if (g->zig_target->os == OsWindows && target_abi_is_gnu(g->zig_target->abi)) {
   8616         args.append("-Wno-pragma-pack");
   8617     }
   8618 
   8619     if (!g->strip_debug_symbols) {
   8620         args.append("-g");
   8621     }
   8622 
   8623     if (codegen_have_frame_pointer(g)) {
   8624         args.append("-fno-omit-frame-pointer");
   8625     } else {
   8626         args.append("-fomit-frame-pointer");
   8627     }
   8628 
   8629     switch (g->build_mode) {
   8630         case BuildModeDebug:
   8631             // windows c runtime requires -D_DEBUG if using debug libraries
   8632             args.append("-D_DEBUG");
   8633 
   8634             if (g->libc_link_lib != nullptr) {
   8635                 args.append("-fstack-protector-strong");
   8636                 args.append("--param");
   8637                 args.append("ssp-buffer-size=4");
   8638             } else {
   8639                 args.append("-fno-stack-protector");
   8640             }
   8641             break;
   8642         case BuildModeSafeRelease:
   8643             // See the comment in the BuildModeFastRelease case for why we pass -O2 rather
   8644             // than -O3 here.
   8645             args.append("-O2");
   8646             if (g->libc_link_lib != nullptr) {
   8647                 args.append("-D_FORTIFY_SOURCE=2");
   8648                 args.append("-fstack-protector-strong");
   8649                 args.append("--param");
   8650                 args.append("ssp-buffer-size=4");
   8651             } else {
   8652                 args.append("-fno-stack-protector");
   8653             }
   8654             break;
   8655         case BuildModeFastRelease:
   8656             args.append("-DNDEBUG");
   8657             // Here we pass -O2 rather than -O3 because, although we do the equivalent of
   8658             // -O3 in Zig code, the justification for the difference here is that Zig
   8659             // has better detection and prevention of undefined behavior, so -O3 is safer for
   8660             // Zig code than it is for C code. Also, C programmers are used to their code
   8661             // running in -O2 and thus the -O3 path has been tested less.
   8662             args.append("-O2");
   8663             args.append("-fno-stack-protector");
   8664             break;
   8665         case BuildModeSmallRelease:
   8666             args.append("-DNDEBUG");
   8667             args.append("-Os");
   8668             args.append("-fno-stack-protector");
   8669             break;
   8670     }
   8671 
   8672     if (target_supports_fpic(g->zig_target) && g->have_pic) {
   8673         args.append("-fPIC");
   8674     }
   8675 
   8676     for (size_t arg_i = 0; arg_i < g->clang_argv_len; arg_i += 1) {
   8677         args.append(g->clang_argv[arg_i]);
   8678     }
   8679 
   8680 }
   8681 
   8682 void codegen_translate_c(CodeGen *g, Buf *full_path, FILE *out_file, bool use_userland_implementation) {
   8683     Error err;
   8684     Buf *src_basename = buf_alloc();
   8685     Buf *src_dirname = buf_alloc();
   8686     os_path_split(full_path, src_dirname, src_basename);
   8687 
   8688     Buf noextname = BUF_INIT;
   8689     os_path_extname(src_basename, &noextname, nullptr);
   8690 
   8691     detect_libc(g);
   8692 
   8693     init(g);
   8694 
   8695     Stage2TranslateMode trans_mode = buf_ends_with_str(full_path, ".h") ?
   8696         Stage2TranslateModeImport : Stage2TranslateModeTranslate;
   8697 
   8698 
   8699     ZigList<const char *> clang_argv = {0};
   8700     add_cc_args(g, clang_argv, nullptr, true);
   8701 
   8702     clang_argv.append(buf_ptr(full_path));
   8703 
   8704     if (g->verbose_cc) {
   8705         fprintf(stderr, "clang");
   8706         for (size_t i = 0; i < clang_argv.length; i += 1) {
   8707             fprintf(stderr, " %s", clang_argv.at(i));
   8708         }
   8709         fprintf(stderr, "\n");
   8710     }
   8711 
   8712     clang_argv.append(nullptr); // to make the [start...end] argument work
   8713 
   8714     const char *resources_path = buf_ptr(g->zig_c_headers_dir);
   8715     Stage2ErrorMsg *errors_ptr;
   8716     size_t errors_len;
   8717     Stage2Ast *ast;
   8718     AstNode *root_node;
   8719 
   8720     if (use_userland_implementation) {
   8721         err = stage2_translate_c(&ast, &errors_ptr, &errors_len,
   8722                         &clang_argv.at(0), &clang_argv.last(), trans_mode, resources_path);
   8723     } else {
   8724         err = parse_h_file(g, &root_node, &errors_ptr, &errors_len, &clang_argv.at(0), &clang_argv.last(),
   8725                 trans_mode, resources_path);
   8726     }
   8727 
   8728     if (err == ErrorCCompileErrors && errors_len > 0) {
   8729         for (size_t i = 0; i < errors_len; i += 1) {
   8730             Stage2ErrorMsg *clang_err = &errors_ptr[i];
   8731             ErrorMsg *err_msg = err_msg_create_with_offset(
   8732                     clang_err->filename_ptr ?
   8733                         buf_create_from_mem(clang_err->filename_ptr, clang_err->filename_len) : buf_alloc(),
   8734                     clang_err->line, clang_err->column, clang_err->offset, clang_err->source,
   8735                     buf_create_from_mem(clang_err->msg_ptr, clang_err->msg_len));
   8736             print_err_msg(err_msg, g->err_color);
   8737         }
   8738         exit(1);
   8739     }
   8740 
   8741     if (err) {
   8742         fprintf(stderr, "unable to parse C file: %s\n", err_str(err));
   8743         exit(1);
   8744     }
   8745 
   8746 
   8747     if (use_userland_implementation) {
   8748         stage2_render_ast(ast, out_file);
   8749     } else {
   8750         ast_render(out_file, root_node, 4);
   8751     }
   8752 }
   8753 
   8754 static ZigType *add_special_code(CodeGen *g, ZigPackage *package, const char *basename) {
   8755     Buf *code_basename = buf_create_from_str(basename);
   8756     Buf path_to_code_src = BUF_INIT;
   8757     os_path_join(g->zig_std_special_dir, code_basename, &path_to_code_src);
   8758 
   8759     Buf *resolve_paths[] = {&path_to_code_src};
   8760     Buf *resolved_path = buf_alloc();
   8761     *resolved_path = os_path_resolve(resolve_paths, 1);
   8762     Buf *import_code = buf_alloc();
   8763     Error err;
   8764     if ((err = file_fetch(g, resolved_path, import_code))) {
   8765         zig_panic("unable to open '%s': %s\n", buf_ptr(&path_to_code_src), err_str(err));
   8766     }
   8767 
   8768     return add_source_file(g, package, resolved_path, import_code, SourceKindPkgMain);
   8769 }
   8770 
   8771 static ZigPackage *create_start_pkg(CodeGen *g, ZigPackage *pkg_with_main) {
   8772     ZigPackage *package = codegen_create_package(g, buf_ptr(g->zig_std_special_dir), "start.zig", "std.special");
   8773     package->package_table.put(buf_create_from_str("root"), pkg_with_main);
   8774     return package;
   8775 }
   8776 
   8777 static void create_test_compile_var_and_add_test_runner(CodeGen *g) {
   8778     Error err;
   8779 
   8780     assert(g->is_test_build);
   8781 
   8782     if (g->test_fns.length == 0) {
   8783         fprintf(stderr, "No tests to run.\n");
   8784         exit(0);
   8785     }
   8786 
   8787     ZigType *fn_type = get_test_fn_type(g);
   8788 
   8789     ConstExprValue *test_fn_type_val = get_builtin_value(g, "TestFn");
   8790     assert(test_fn_type_val->type->id == ZigTypeIdMetaType);
   8791     ZigType *struct_type = test_fn_type_val->data.x_type;
   8792     if ((err = type_resolve(g, struct_type, ResolveStatusSizeKnown)))
   8793         zig_unreachable();
   8794 
   8795     ConstExprValue *test_fn_array = create_const_vals(1);
   8796     test_fn_array->type = get_array_type(g, struct_type, g->test_fns.length);
   8797     test_fn_array->special = ConstValSpecialStatic;
   8798     test_fn_array->data.x_array.data.s_none.elements = create_const_vals(g->test_fns.length);
   8799 
   8800     for (size_t i = 0; i < g->test_fns.length; i += 1) {
   8801         ZigFn *test_fn_entry = g->test_fns.at(i);
   8802 
   8803         ConstExprValue *this_val = &test_fn_array->data.x_array.data.s_none.elements[i];
   8804         this_val->special = ConstValSpecialStatic;
   8805         this_val->type = struct_type;
   8806         this_val->parent.id = ConstParentIdArray;
   8807         this_val->parent.data.p_array.array_val = test_fn_array;
   8808         this_val->parent.data.p_array.elem_index = i;
   8809         this_val->data.x_struct.fields = create_const_vals(2);
   8810 
   8811         ConstExprValue *name_field = &this_val->data.x_struct.fields[0];
   8812         ConstExprValue *name_array_val = create_const_str_lit(g, &test_fn_entry->symbol_name);
   8813         init_const_slice(g, name_field, name_array_val, 0, buf_len(&test_fn_entry->symbol_name), true);
   8814 
   8815         ConstExprValue *fn_field = &this_val->data.x_struct.fields[1];
   8816         fn_field->type = fn_type;
   8817         fn_field->special = ConstValSpecialStatic;
   8818         fn_field->data.x_ptr.special = ConstPtrSpecialFunction;
   8819         fn_field->data.x_ptr.mut = ConstPtrMutComptimeConst;
   8820         fn_field->data.x_ptr.data.fn.fn_entry = test_fn_entry;
   8821     }
   8822 
   8823     ConstExprValue *test_fn_slice = create_const_slice(g, test_fn_array, 0, g->test_fns.length, true);
   8824 
   8825     update_compile_var(g, buf_create_from_str("test_functions"), test_fn_slice);
   8826     assert(g->test_runner_package != nullptr);
   8827     g->test_runner_import = add_special_code(g, g->test_runner_package, "test_runner.zig");
   8828 }
   8829 
   8830 static Buf *get_resolved_root_src_path(CodeGen *g) {
   8831     // TODO memoize
   8832     if (buf_len(&g->root_package->root_src_path) == 0)
   8833         return nullptr;
   8834 
   8835     Buf rel_full_path = BUF_INIT;
   8836     os_path_join(&g->root_package->root_src_dir, &g->root_package->root_src_path, &rel_full_path);
   8837 
   8838     Buf *resolved_path = buf_alloc();
   8839     Buf *resolve_paths[] = {&rel_full_path};
   8840     *resolved_path = os_path_resolve(resolve_paths, 1);
   8841 
   8842     return resolved_path;
   8843 }
   8844 
   8845 static bool want_startup_code(CodeGen *g) {
   8846     // Test builds get handled separately.
   8847     if (g->is_test_build)
   8848         return false;
   8849 
   8850     // start code does not handle UEFI target
   8851     if (g->zig_target->os == OsUefi)
   8852         return false;
   8853 
   8854     // WASM freestanding can still have an entry point but other freestanding targets do not.
   8855     if (g->zig_target->os == OsFreestanding && !target_is_wasm(g->zig_target))
   8856         return false;
   8857 
   8858     // Declaring certain export functions means skipping the start code
   8859     if (g->have_c_main || g->have_winmain || g->have_winmain_crt_startup)
   8860         return false;
   8861 
   8862     // If there is a pub main in the root source file, that means we need start code.
   8863     if (g->have_pub_main)
   8864         return true;
   8865 
   8866     if (g->out_type == OutTypeExe) {
   8867         // For build-exe, we might add start code even though there is no pub main, so that the
   8868         // programmer gets the "no pub main" compile error. However if linking libc and there is
   8869         // a C source file, that might have main().
   8870         return g->c_source_files.length == 0 || g->libc_link_lib == nullptr;
   8871     }
   8872 
   8873     // For objects and libraries, and we don't have pub main, no start code.
   8874     return false;
   8875 }
   8876 
   8877 static void gen_root_source(CodeGen *g) {
   8878     Buf *resolved_path = get_resolved_root_src_path(g);
   8879     if (resolved_path == nullptr)
   8880         return;
   8881 
   8882     Buf *source_code = buf_alloc();
   8883     Error err;
   8884     // No need for using the caching system for this file fetch because it is handled
   8885     // separately.
   8886     if ((err = os_fetch_file_path(resolved_path, source_code))) {
   8887         fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(resolved_path), err_str(err));
   8888         exit(1);
   8889     }
   8890 
   8891     ZigType *root_import_alias = add_source_file(g, g->root_package, resolved_path, source_code, SourceKindRoot);
   8892     assert(root_import_alias == g->root_import);
   8893 
   8894     assert(g->root_out_name);
   8895     assert(g->out_type != OutTypeUnknown);
   8896 
   8897     if (!g->is_dummy_so) {
   8898         // Zig has lazy top level definitions. Here we semantically analyze the panic function.
   8899         ZigType *import_with_panic;
   8900         if (g->have_pub_panic) {
   8901             import_with_panic = g->root_import;
   8902         } else {
   8903             g->panic_package = create_panic_pkg(g);
   8904             import_with_panic = add_special_code(g, g->panic_package, "panic.zig");
   8905         }
   8906         Tld *panic_tld = find_decl(g, &get_container_scope(import_with_panic)->base, buf_create_from_str("panic"));
   8907         assert(panic_tld != nullptr);
   8908         resolve_top_level_decl(g, panic_tld, nullptr);
   8909     }
   8910 
   8911 
   8912     if (!g->error_during_imports) {
   8913         semantic_analyze(g);
   8914     }
   8915     report_errors_and_maybe_exit(g);
   8916 
   8917     if (want_startup_code(g)) {
   8918         g->start_import = add_special_code(g, create_start_pkg(g, g->root_package), "start.zig");
   8919     }
   8920     if (g->zig_target->os == OsWindows && !g->have_dllmain_crt_startup &&
   8921             g->out_type == OutTypeLib && g->is_dynamic)
   8922     {
   8923         g->start_import = add_special_code(g, create_start_pkg(g, g->root_package), "start_lib.zig");
   8924     }
   8925 
   8926     if (!g->error_during_imports) {
   8927         semantic_analyze(g);
   8928     }
   8929     if (g->is_test_build) {
   8930         create_test_compile_var_and_add_test_runner(g);
   8931         g->start_import = add_special_code(g, create_start_pkg(g, g->test_runner_package), "start.zig");
   8932 
   8933         if (!g->error_during_imports) {
   8934             semantic_analyze(g);
   8935         }
   8936     }
   8937 
   8938     if (!g->is_dummy_so) {
   8939         typecheck_panic_fn(g, g->panic_tld_fn, g->panic_fn);
   8940     }
   8941 
   8942     report_errors_and_maybe_exit(g);
   8943 
   8944 }
   8945 
   8946 static void print_zig_cc_cmd(ZigList<const char *> *args) {
   8947     for (size_t arg_i = 0; arg_i < args->length; arg_i += 1) {
   8948         const char *space_str = (arg_i == 0) ? "" : " ";
   8949         fprintf(stderr, "%s%s", space_str, args->at(arg_i));
   8950     }
   8951     fprintf(stderr, "\n");
   8952 }
   8953 
   8954 // Caller should delete the file when done or rename it into a better location.
   8955 static Error get_tmp_filename(CodeGen *g, Buf *out, Buf *suffix) {
   8956     Error err;
   8957     buf_resize(out, 0);
   8958     os_path_join(g->cache_dir, buf_create_from_str("tmp" OS_SEP), out);
   8959     if ((err = os_make_path(out))) {
   8960         return err;
   8961     }
   8962     const char base64[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_";
   8963     assert(array_length(base64) == 64 + 1);
   8964     for (size_t i = 0; i < 12; i += 1) {
   8965         buf_append_char(out, base64[rand() % 64]);
   8966     }
   8967     buf_append_char(out, '-');
   8968     buf_append_buf(out, suffix);
   8969     return ErrorNone;
   8970 }
   8971 
   8972 Error create_c_object_cache(CodeGen *g, CacheHash **out_cache_hash, bool verbose) {
   8973     Error err;
   8974     CacheHash *cache_hash = allocate<CacheHash>(1);
   8975     Buf *manifest_dir = buf_sprintf("%s" OS_SEP CACHE_HASH_SUBDIR, buf_ptr(g->cache_dir));
   8976     cache_init(cache_hash, manifest_dir);
   8977 
   8978     Buf *compiler_id;
   8979     if ((err = get_compiler_id(&compiler_id))) {
   8980         if (verbose) {
   8981             fprintf(stderr, "unable to get compiler id: %s\n", err_str(err));
   8982         }
   8983         return err;
   8984     }
   8985     cache_buf(cache_hash, compiler_id);
   8986     cache_int(cache_hash, g->err_color);
   8987     cache_buf(cache_hash, g->zig_c_headers_dir);
   8988     cache_list_of_buf(cache_hash, g->libc_include_dir_list, g->libc_include_dir_len);
   8989     cache_int(cache_hash, g->zig_target->is_native);
   8990     cache_int(cache_hash, g->zig_target->arch);
   8991     cache_int(cache_hash, g->zig_target->sub_arch);
   8992     cache_int(cache_hash, g->zig_target->vendor);
   8993     cache_int(cache_hash, g->zig_target->os);
   8994     cache_int(cache_hash, g->zig_target->abi);
   8995     cache_bool(cache_hash, g->strip_debug_symbols);
   8996     cache_int(cache_hash, g->build_mode);
   8997     cache_bool(cache_hash, g->have_pic);
   8998     cache_bool(cache_hash, want_valgrind_support(g));
   8999     cache_bool(cache_hash, g->function_sections);
   9000     for (size_t arg_i = 0; arg_i < g->clang_argv_len; arg_i += 1) {
   9001         cache_str(cache_hash, g->clang_argv[arg_i]);
   9002     }
   9003 
   9004     *out_cache_hash = cache_hash;
   9005     return ErrorNone;
   9006 }
   9007 
   9008 // returns true if it was a cache miss
   9009 static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) {
   9010     Error err;
   9011 
   9012     Buf *artifact_dir;
   9013     Buf *o_final_path;
   9014 
   9015     Buf *o_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR, buf_ptr(g->cache_dir));
   9016 
   9017     Buf *c_source_file = buf_create_from_str(c_file->source_path);
   9018     Buf *c_source_basename = buf_alloc();
   9019     os_path_split(c_source_file, nullptr, c_source_basename);
   9020     Buf *final_o_basename = buf_alloc();
   9021     os_path_extname(c_source_basename, final_o_basename, nullptr);
   9022     buf_append_str(final_o_basename, target_o_file_ext(g->zig_target));
   9023 
   9024     CacheHash *cache_hash;
   9025     if ((err = create_c_object_cache(g, &cache_hash, true))) {
   9026         // Already printed error; verbose = true
   9027         exit(1);
   9028     }
   9029     cache_file(cache_hash, c_source_file);
   9030 
   9031     // Note: not directory args, just args that always have a file next
   9032     static const char *file_args[] = {
   9033         "-include",
   9034     };
   9035     for (size_t arg_i = 0; arg_i < c_file->args.length; arg_i += 1) {
   9036         const char *arg = c_file->args.at(arg_i);
   9037         cache_str(cache_hash, arg);
   9038         for (size_t file_arg_i = 0; file_arg_i < array_length(file_args); file_arg_i += 1) {
   9039             if (strcmp(arg, file_args[file_arg_i]) == 0 && arg_i + 1 < c_file->args.length) {
   9040                 arg_i += 1;
   9041                 cache_file(cache_hash, buf_create_from_str(c_file->args.at(arg_i)));
   9042             }
   9043         }
   9044     }
   9045 
   9046     Buf digest = BUF_INIT;
   9047     buf_resize(&digest, 0);
   9048     if ((err = cache_hit(cache_hash, &digest))) {
   9049         if (err != ErrorInvalidFormat) {
   9050             if (err == ErrorCacheUnavailable) {
   9051                 // already printed error
   9052             } else {
   9053                 fprintf(stderr, "unable to check cache when compiling C object: %s\n", err_str(err));
   9054             }
   9055             exit(1);
   9056         }
   9057     }
   9058     bool is_cache_miss = (buf_len(&digest) == 0);
   9059     if (is_cache_miss) {
   9060         // we can't know the digest until we do the C compiler invocation, so we
   9061         // need a tmp filename.
   9062         Buf *out_obj_path = buf_alloc();
   9063         if ((err = get_tmp_filename(g, out_obj_path, final_o_basename))) {
   9064             fprintf(stderr, "unable to create tmp dir: %s\n", err_str(err));
   9065             exit(1);
   9066         }
   9067 
   9068         Termination term;
   9069         ZigList<const char *> args = {};
   9070         args.append(buf_ptr(self_exe_path));
   9071         args.append("cc");
   9072 
   9073         Buf *out_dep_path = buf_sprintf("%s.d", buf_ptr(out_obj_path));
   9074         add_cc_args(g, args, buf_ptr(out_dep_path), false);
   9075 
   9076         args.append("-o");
   9077         args.append(buf_ptr(out_obj_path));
   9078 
   9079         args.append("-c");
   9080         args.append(buf_ptr(c_source_file));
   9081 
   9082         for (size_t arg_i = 0; arg_i < c_file->args.length; arg_i += 1) {
   9083             args.append(c_file->args.at(arg_i));
   9084         }
   9085 
   9086         if (g->verbose_cc) {
   9087             print_zig_cc_cmd(&args);
   9088         }
   9089         os_spawn_process(args, &term);
   9090         if (term.how != TerminationIdClean || term.code != 0) {
   9091             fprintf(stderr, "\nThe following command failed:\n");
   9092             print_zig_cc_cmd(&args);
   9093             exit(1);
   9094         }
   9095 
   9096         // add the files depended on to the cache system
   9097         if ((err = cache_add_dep_file(cache_hash, out_dep_path, true))) {
   9098             // Don't treat the absence of the .d file as a fatal error, the
   9099             // compiler may not produce one eg. when compiling .s files
   9100             if (err != ErrorFileNotFound) {
   9101                 fprintf(stderr, "Failed to add C source dependencies to cache: %s\n", err_str(err));
   9102                 exit(1);
   9103             }
   9104         }
   9105         if (err != ErrorFileNotFound) {
   9106             os_delete_file(out_dep_path);
   9107         }
   9108 
   9109         if ((err = cache_final(cache_hash, &digest))) {
   9110             fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err));
   9111             exit(1);
   9112         }
   9113         artifact_dir = buf_alloc();
   9114         os_path_join(o_dir, &digest, artifact_dir);
   9115         if ((err = os_make_path(artifact_dir))) {
   9116             fprintf(stderr, "Unable to create output directory '%s': %s",
   9117                     buf_ptr(artifact_dir), err_str(err));
   9118             exit(1);
   9119         }
   9120         o_final_path = buf_alloc();
   9121         os_path_join(artifact_dir, final_o_basename, o_final_path);
   9122         if ((err = os_rename(out_obj_path, o_final_path))) {
   9123             fprintf(stderr, "Unable to rename object: %s\n", err_str(err));
   9124             exit(1);
   9125         }
   9126     } else {
   9127         // cache hit
   9128         artifact_dir = buf_alloc();
   9129         os_path_join(o_dir, &digest, artifact_dir);
   9130         o_final_path = buf_alloc();
   9131         os_path_join(artifact_dir, final_o_basename, o_final_path);
   9132     }
   9133 
   9134     g->link_objects.append(o_final_path);
   9135     g->caches_to_release.append(cache_hash);
   9136 }
   9137 
   9138 // returns true if we had any cache misses
   9139 static void gen_c_objects(CodeGen *g) {
   9140     Error err;
   9141 
   9142     if (g->c_source_files.length == 0)
   9143         return;
   9144 
   9145     Buf *self_exe_path = buf_alloc();
   9146     if ((err = os_self_exe_path(self_exe_path))) {
   9147         fprintf(stderr, "Unable to get self exe path: %s\n", err_str(err));
   9148         exit(1);
   9149     }
   9150 
   9151     codegen_add_time_event(g, "Compile C Code");
   9152 
   9153     for (size_t c_file_i = 0; c_file_i < g->c_source_files.length; c_file_i += 1) {
   9154         CFile *c_file = g->c_source_files.at(c_file_i);
   9155         gen_c_object(g, self_exe_path, c_file);
   9156     }
   9157 }
   9158 
   9159 void codegen_add_object(CodeGen *g, Buf *object_path) {
   9160     g->link_objects.append(object_path);
   9161 }
   9162 
   9163 // Must be coordinated with with CIntType enum
   9164 static const char *c_int_type_names[] = {
   9165     "short",
   9166     "unsigned short",
   9167     "int",
   9168     "unsigned int",
   9169     "long",
   9170     "unsigned long",
   9171     "long long",
   9172     "unsigned long long",
   9173 };
   9174 
   9175 struct GenH {
   9176     ZigList<ZigType *> types_to_declare;
   9177 };
   9178 
   9179 static void prepend_c_type_to_decl_list(CodeGen *g, GenH *gen_h, ZigType *type_entry) {
   9180     if (type_entry->gen_h_loop_flag)
   9181         return;
   9182     type_entry->gen_h_loop_flag = true;
   9183 
   9184     switch (type_entry->id) {
   9185         case ZigTypeIdInvalid:
   9186         case ZigTypeIdMetaType:
   9187         case ZigTypeIdComptimeFloat:
   9188         case ZigTypeIdComptimeInt:
   9189         case ZigTypeIdEnumLiteral:
   9190         case ZigTypeIdUndefined:
   9191         case ZigTypeIdNull:
   9192         case ZigTypeIdBoundFn:
   9193         case ZigTypeIdArgTuple:
   9194         case ZigTypeIdErrorUnion:
   9195         case ZigTypeIdErrorSet:
   9196         case ZigTypeIdFnFrame:
   9197         case ZigTypeIdAnyFrame:
   9198             zig_unreachable();
   9199         case ZigTypeIdVoid:
   9200         case ZigTypeIdUnreachable:
   9201         case ZigTypeIdBool:
   9202         case ZigTypeIdInt:
   9203         case ZigTypeIdFloat:
   9204             return;
   9205         case ZigTypeIdOpaque:
   9206             gen_h->types_to_declare.append(type_entry);
   9207             return;
   9208         case ZigTypeIdStruct:
   9209             for (uint32_t i = 0; i < type_entry->data.structure.src_field_count; i += 1) {
   9210                 TypeStructField *field = &type_entry->data.structure.fields[i];
   9211                 prepend_c_type_to_decl_list(g, gen_h, field->type_entry);
   9212             }
   9213             gen_h->types_to_declare.append(type_entry);
   9214             return;
   9215         case ZigTypeIdUnion:
   9216             for (uint32_t i = 0; i < type_entry->data.unionation.src_field_count; i += 1) {
   9217                 TypeUnionField *field = &type_entry->data.unionation.fields[i];
   9218                 prepend_c_type_to_decl_list(g, gen_h, field->type_entry);
   9219             }
   9220             gen_h->types_to_declare.append(type_entry);
   9221             return;
   9222         case ZigTypeIdEnum:
   9223             prepend_c_type_to_decl_list(g, gen_h, type_entry->data.enumeration.tag_int_type);
   9224             gen_h->types_to_declare.append(type_entry);
   9225             return;
   9226         case ZigTypeIdPointer:
   9227             prepend_c_type_to_decl_list(g, gen_h, type_entry->data.pointer.child_type);
   9228             return;
   9229         case ZigTypeIdArray:
   9230             prepend_c_type_to_decl_list(g, gen_h, type_entry->data.array.child_type);
   9231             return;
   9232         case ZigTypeIdVector:
   9233             prepend_c_type_to_decl_list(g, gen_h, type_entry->data.vector.elem_type);
   9234             return;
   9235         case ZigTypeIdOptional:
   9236             prepend_c_type_to_decl_list(g, gen_h, type_entry->data.maybe.child_type);
   9237             return;
   9238         case ZigTypeIdFn:
   9239             for (size_t i = 0; i < type_entry->data.fn.fn_type_id.param_count; i += 1) {
   9240                 prepend_c_type_to_decl_list(g, gen_h, type_entry->data.fn.fn_type_id.param_info[i].type);
   9241             }
   9242             prepend_c_type_to_decl_list(g, gen_h, type_entry->data.fn.fn_type_id.return_type);
   9243             return;
   9244     }
   9245 }
   9246 
   9247 static void get_c_type(CodeGen *g, GenH *gen_h, ZigType *type_entry, Buf *out_buf) {
   9248     assert(type_entry);
   9249 
   9250     for (size_t i = 0; i < array_length(c_int_type_names); i += 1) {
   9251         if (type_entry == g->builtin_types.entry_c_int[i]) {
   9252             buf_init_from_str(out_buf, c_int_type_names[i]);
   9253             return;
   9254         }
   9255     }
   9256     if (type_entry == g->builtin_types.entry_c_longdouble) {
   9257         buf_init_from_str(out_buf, "long double");
   9258         return;
   9259     }
   9260     if (type_entry == g->builtin_types.entry_c_void) {
   9261         buf_init_from_str(out_buf, "void");
   9262         return;
   9263     }
   9264     if (type_entry == g->builtin_types.entry_isize) {
   9265         g->c_want_stdint = true;
   9266         buf_init_from_str(out_buf, "intptr_t");
   9267         return;
   9268     }
   9269     if (type_entry == g->builtin_types.entry_usize) {
   9270         g->c_want_stdint = true;
   9271         buf_init_from_str(out_buf, "uintptr_t");
   9272         return;
   9273     }
   9274 
   9275     prepend_c_type_to_decl_list(g, gen_h, type_entry);
   9276 
   9277     switch (type_entry->id) {
   9278         case ZigTypeIdVoid:
   9279             buf_init_from_str(out_buf, "void");
   9280             break;
   9281         case ZigTypeIdBool:
   9282             buf_init_from_str(out_buf, "bool");
   9283             g->c_want_stdbool = true;
   9284             break;
   9285         case ZigTypeIdUnreachable:
   9286             buf_init_from_str(out_buf, "__attribute__((__noreturn__)) void");
   9287             break;
   9288         case ZigTypeIdFloat:
   9289             switch (type_entry->data.floating.bit_count) {
   9290                 case 32:
   9291                     buf_init_from_str(out_buf, "float");
   9292                     break;
   9293                 case 64:
   9294                     buf_init_from_str(out_buf, "double");
   9295                     break;
   9296                 case 80:
   9297                     buf_init_from_str(out_buf, "__float80");
   9298                     break;
   9299                 case 128:
   9300                     buf_init_from_str(out_buf, "__float128");
   9301                     break;
   9302                 default:
   9303                     zig_unreachable();
   9304             }
   9305             break;
   9306         case ZigTypeIdInt:
   9307             g->c_want_stdint = true;
   9308             buf_resize(out_buf, 0);
   9309             buf_appendf(out_buf, "%sint%" PRIu32 "_t",
   9310                     type_entry->data.integral.is_signed ? "" : "u",
   9311                     type_entry->data.integral.bit_count);
   9312             break;
   9313         case ZigTypeIdPointer:
   9314             {
   9315                 Buf child_buf = BUF_INIT;
   9316                 ZigType *child_type = type_entry->data.pointer.child_type;
   9317                 get_c_type(g, gen_h, child_type, &child_buf);
   9318 
   9319                 const char *const_str = type_entry->data.pointer.is_const ? "const " : "";
   9320                 buf_resize(out_buf, 0);
   9321                 buf_appendf(out_buf, "%s%s *", const_str, buf_ptr(&child_buf));
   9322                 break;
   9323             }
   9324         case ZigTypeIdOptional:
   9325             {
   9326                 ZigType *child_type = type_entry->data.maybe.child_type;
   9327                 if (!type_has_bits(child_type)) {
   9328                     buf_init_from_str(out_buf, "bool");
   9329                     return;
   9330                 } else if (type_is_nonnull_ptr(child_type)) {
   9331                     return get_c_type(g, gen_h, child_type, out_buf);
   9332                 } else {
   9333                     zig_unreachable();
   9334                 }
   9335             }
   9336         case ZigTypeIdStruct:
   9337         case ZigTypeIdOpaque:
   9338             {
   9339                 buf_init_from_str(out_buf, "struct ");
   9340                 buf_append_buf(out_buf, type_h_name(type_entry));
   9341                 return;
   9342             }
   9343         case ZigTypeIdUnion:
   9344             {
   9345                 buf_init_from_str(out_buf, "union ");
   9346                 buf_append_buf(out_buf, type_h_name(type_entry));
   9347                 return;
   9348             }
   9349         case ZigTypeIdEnum:
   9350             {
   9351                 buf_init_from_str(out_buf, "enum ");
   9352                 buf_append_buf(out_buf, type_h_name(type_entry));
   9353                 return;
   9354             }
   9355         case ZigTypeIdArray:
   9356             {
   9357                 ZigTypeArray *array_data = &type_entry->data.array;
   9358 
   9359                 Buf *child_buf = buf_alloc();
   9360                 get_c_type(g, gen_h, array_data->child_type, child_buf);
   9361 
   9362                 buf_resize(out_buf, 0);
   9363                 buf_appendf(out_buf, "%s", buf_ptr(child_buf));
   9364                 return;
   9365             }
   9366         case ZigTypeIdVector:
   9367             zig_panic("TODO implement get_c_type for vector types");
   9368         case ZigTypeIdErrorUnion:
   9369         case ZigTypeIdErrorSet:
   9370         case ZigTypeIdFn:
   9371             zig_panic("TODO implement get_c_type for more types");
   9372         case ZigTypeIdInvalid:
   9373         case ZigTypeIdMetaType:
   9374         case ZigTypeIdBoundFn:
   9375         case ZigTypeIdComptimeFloat:
   9376         case ZigTypeIdComptimeInt:
   9377         case ZigTypeIdEnumLiteral:
   9378         case ZigTypeIdUndefined:
   9379         case ZigTypeIdNull:
   9380         case ZigTypeIdArgTuple:
   9381         case ZigTypeIdFnFrame:
   9382         case ZigTypeIdAnyFrame:
   9383             zig_unreachable();
   9384     }
   9385 }
   9386 
   9387 static const char *preprocessor_alphabet1 = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
   9388 static const char *preprocessor_alphabet2 = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
   9389 
   9390 static bool need_to_preprocessor_mangle(Buf *src) {
   9391     for (size_t i = 0; i < buf_len(src); i += 1) {
   9392         const char *alphabet = (i == 0) ? preprocessor_alphabet1 : preprocessor_alphabet2;
   9393         uint8_t byte = buf_ptr(src)[i];
   9394         if (strchr(alphabet, byte) == nullptr) {
   9395             return true;
   9396         }
   9397     }
   9398     return false;
   9399 }
   9400 
   9401 static Buf *preprocessor_mangle(Buf *src) {
   9402     if (!need_to_preprocessor_mangle(src)) {
   9403         return buf_create_from_buf(src);
   9404     }
   9405     Buf *result = buf_alloc();
   9406     for (size_t i = 0; i < buf_len(src); i += 1) {
   9407         const char *alphabet = (i == 0) ? preprocessor_alphabet1 : preprocessor_alphabet2;
   9408         uint8_t byte = buf_ptr(src)[i];
   9409         if (strchr(alphabet, byte) == nullptr) {
   9410             // perform escape
   9411             buf_appendf(result, "_%02x_", byte);
   9412         } else {
   9413             buf_append_char(result, byte);
   9414         }
   9415     }
   9416     return result;
   9417 }
   9418 
   9419 static void gen_h_file(CodeGen *g) {
   9420     GenH gen_h_data = {0};
   9421     GenH *gen_h = &gen_h_data;
   9422 
   9423     assert(!g->is_test_build);
   9424     assert(!g->disable_gen_h);
   9425 
   9426     Buf *out_h_path = buf_sprintf("%s" OS_SEP "%s.h", buf_ptr(g->output_dir), buf_ptr(g->root_out_name));
   9427 
   9428     FILE *out_h = fopen(buf_ptr(out_h_path), "wb");
   9429     if (!out_h)
   9430         zig_panic("unable to open %s: %s\n", buf_ptr(out_h_path), strerror(errno));
   9431 
   9432     Buf *export_macro = nullptr;
   9433     if (g->is_dynamic) {
   9434         export_macro = preprocessor_mangle(buf_sprintf("%s_EXPORT", buf_ptr(g->root_out_name)));
   9435         buf_upcase(export_macro);
   9436     }
   9437 
   9438     Buf *extern_c_macro = preprocessor_mangle(buf_sprintf("%s_EXTERN_C", buf_ptr(g->root_out_name)));
   9439     buf_upcase(extern_c_macro);
   9440 
   9441     Buf h_buf = BUF_INIT;
   9442     buf_resize(&h_buf, 0);
   9443     for (size_t fn_def_i = 0; fn_def_i < g->fn_defs.length; fn_def_i += 1) {
   9444         ZigFn *fn_table_entry = g->fn_defs.at(fn_def_i);
   9445 
   9446         if (fn_table_entry->export_list.length == 0)
   9447             continue;
   9448 
   9449         FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id;
   9450 
   9451         Buf return_type_c = BUF_INIT;
   9452         get_c_type(g, gen_h, fn_type_id->return_type, &return_type_c);
   9453 
   9454         Buf *symbol_name;
   9455         if (fn_table_entry->export_list.length == 0) {
   9456             symbol_name = &fn_table_entry->symbol_name;
   9457         } else {
   9458             GlobalExport *fn_export = &fn_table_entry->export_list.items[0];
   9459             symbol_name = &fn_export->name;
   9460         }
   9461 
   9462         buf_appendf(&h_buf, "%s %s %s(",
   9463             buf_ptr(g->is_dynamic ? export_macro : extern_c_macro),
   9464             buf_ptr(&return_type_c),
   9465             buf_ptr(symbol_name));
   9466 
   9467         Buf param_type_c = BUF_INIT;
   9468         if (fn_type_id->param_count > 0) {
   9469             for (size_t param_i = 0; param_i < fn_type_id->param_count; param_i += 1) {
   9470                 FnTypeParamInfo *param_info = &fn_type_id->param_info[param_i];
   9471                 AstNode *param_decl_node = get_param_decl_node(fn_table_entry, param_i);
   9472                 Buf *param_name = param_decl_node->data.param_decl.name;
   9473 
   9474                 const char *comma_str = (param_i == 0) ? "" : ", ";
   9475                 const char *restrict_str = param_info->is_noalias ? "restrict" : "";
   9476                 get_c_type(g, gen_h, param_info->type, &param_type_c);
   9477 
   9478                 if (param_info->type->id == ZigTypeIdArray) {
   9479                     // Arrays decay to pointers
   9480                     buf_appendf(&h_buf, "%s%s%s %s[]", comma_str, buf_ptr(&param_type_c),
   9481                             restrict_str, buf_ptr(param_name));
   9482                 } else {
   9483                     buf_appendf(&h_buf, "%s%s%s %s", comma_str, buf_ptr(&param_type_c),
   9484                             restrict_str, buf_ptr(param_name));
   9485                 }
   9486             }
   9487             buf_appendf(&h_buf, ")");
   9488         } else {
   9489             buf_appendf(&h_buf, "void)");
   9490         }
   9491 
   9492         buf_appendf(&h_buf, ";\n");
   9493 
   9494     }
   9495 
   9496     Buf *ifdef_dance_name = preprocessor_mangle(buf_sprintf("%s_H", buf_ptr(g->root_out_name)));
   9497     buf_upcase(ifdef_dance_name);
   9498 
   9499     fprintf(out_h, "#ifndef %s\n", buf_ptr(ifdef_dance_name));
   9500     fprintf(out_h, "#define %s\n\n", buf_ptr(ifdef_dance_name));
   9501 
   9502     if (g->c_want_stdbool)
   9503         fprintf(out_h, "#include <stdbool.h>\n");
   9504     if (g->c_want_stdint)
   9505         fprintf(out_h, "#include <stdint.h>\n");
   9506 
   9507     fprintf(out_h, "\n");
   9508 
   9509     fprintf(out_h, "#ifdef __cplusplus\n");
   9510     fprintf(out_h, "#define %s extern \"C\"\n", buf_ptr(extern_c_macro));
   9511     fprintf(out_h, "#else\n");
   9512     fprintf(out_h, "#define %s\n", buf_ptr(extern_c_macro));
   9513     fprintf(out_h, "#endif\n");
   9514     fprintf(out_h, "\n");
   9515 
   9516     if (g->is_dynamic) {
   9517         fprintf(out_h, "#if defined(_WIN32)\n");
   9518         fprintf(out_h, "#define %s %s __declspec(dllimport)\n", buf_ptr(export_macro), buf_ptr(extern_c_macro));
   9519         fprintf(out_h, "#else\n");
   9520         fprintf(out_h, "#define %s %s __attribute__((visibility (\"default\")))\n",
   9521             buf_ptr(export_macro), buf_ptr(extern_c_macro));
   9522         fprintf(out_h, "#endif\n");
   9523         fprintf(out_h, "\n");
   9524     }
   9525 
   9526     for (size_t type_i = 0; type_i < gen_h->types_to_declare.length; type_i += 1) {
   9527         ZigType *type_entry = gen_h->types_to_declare.at(type_i);
   9528         switch (type_entry->id) {
   9529             case ZigTypeIdInvalid:
   9530             case ZigTypeIdMetaType:
   9531             case ZigTypeIdVoid:
   9532             case ZigTypeIdBool:
   9533             case ZigTypeIdUnreachable:
   9534             case ZigTypeIdInt:
   9535             case ZigTypeIdFloat:
   9536             case ZigTypeIdPointer:
   9537             case ZigTypeIdComptimeFloat:
   9538             case ZigTypeIdComptimeInt:
   9539             case ZigTypeIdEnumLiteral:
   9540             case ZigTypeIdArray:
   9541             case ZigTypeIdUndefined:
   9542             case ZigTypeIdNull:
   9543             case ZigTypeIdErrorUnion:
   9544             case ZigTypeIdErrorSet:
   9545             case ZigTypeIdBoundFn:
   9546             case ZigTypeIdArgTuple:
   9547             case ZigTypeIdOptional:
   9548             case ZigTypeIdFn:
   9549             case ZigTypeIdVector:
   9550             case ZigTypeIdFnFrame:
   9551             case ZigTypeIdAnyFrame:
   9552                 zig_unreachable();
   9553 
   9554             case ZigTypeIdEnum:
   9555                 if (type_entry->data.enumeration.layout == ContainerLayoutExtern) {
   9556                     fprintf(out_h, "enum %s {\n", buf_ptr(type_h_name(type_entry)));
   9557                     for (uint32_t field_i = 0; field_i < type_entry->data.enumeration.src_field_count; field_i += 1) {
   9558                         TypeEnumField *enum_field = &type_entry->data.enumeration.fields[field_i];
   9559                         Buf *value_buf = buf_alloc();
   9560                         bigint_append_buf(value_buf, &enum_field->value, 10);
   9561                         fprintf(out_h, "    %s = %s", buf_ptr(enum_field->name), buf_ptr(value_buf));
   9562                         if (field_i != type_entry->data.enumeration.src_field_count - 1) {
   9563                             fprintf(out_h, ",");
   9564                         }
   9565                         fprintf(out_h, "\n");
   9566                     }
   9567                     fprintf(out_h, "};\n\n");
   9568                 } else {
   9569                     fprintf(out_h, "enum %s;\n", buf_ptr(type_h_name(type_entry)));
   9570                 }
   9571                 break;
   9572             case ZigTypeIdStruct:
   9573                 if (type_entry->data.structure.layout == ContainerLayoutExtern) {
   9574                     fprintf(out_h, "struct %s {\n", buf_ptr(type_h_name(type_entry)));
   9575                     for (uint32_t field_i = 0; field_i < type_entry->data.structure.src_field_count; field_i += 1) {
   9576                         TypeStructField *struct_field = &type_entry->data.structure.fields[field_i];
   9577 
   9578                         Buf *type_name_buf = buf_alloc();
   9579                         get_c_type(g, gen_h, struct_field->type_entry, type_name_buf);
   9580 
   9581                         if (struct_field->type_entry->id == ZigTypeIdArray) {
   9582                             fprintf(out_h, "    %s %s[%" ZIG_PRI_u64 "];\n", buf_ptr(type_name_buf),
   9583                                     buf_ptr(struct_field->name),
   9584                                     struct_field->type_entry->data.array.len);
   9585                         } else {
   9586                             fprintf(out_h, "    %s %s;\n", buf_ptr(type_name_buf), buf_ptr(struct_field->name));
   9587                         }
   9588 
   9589                     }
   9590                     fprintf(out_h, "};\n\n");
   9591                 } else {
   9592                     fprintf(out_h, "struct %s;\n", buf_ptr(type_h_name(type_entry)));
   9593                 }
   9594                 break;
   9595             case ZigTypeIdUnion:
   9596                 if (type_entry->data.unionation.layout == ContainerLayoutExtern) {
   9597                     fprintf(out_h, "union %s {\n", buf_ptr(type_h_name(type_entry)));
   9598                     for (uint32_t field_i = 0; field_i < type_entry->data.unionation.src_field_count; field_i += 1) {
   9599                         TypeUnionField *union_field = &type_entry->data.unionation.fields[field_i];
   9600 
   9601                         Buf *type_name_buf = buf_alloc();
   9602                         get_c_type(g, gen_h, union_field->type_entry, type_name_buf);
   9603                         fprintf(out_h, "    %s %s;\n", buf_ptr(type_name_buf), buf_ptr(union_field->name));
   9604                     }
   9605                     fprintf(out_h, "};\n\n");
   9606                 } else {
   9607                     fprintf(out_h, "union %s;\n", buf_ptr(type_h_name(type_entry)));
   9608                 }
   9609                 break;
   9610             case ZigTypeIdOpaque:
   9611                 fprintf(out_h, "struct %s;\n\n", buf_ptr(type_h_name(type_entry)));
   9612                 break;
   9613         }
   9614     }
   9615 
   9616     fprintf(out_h, "%s", buf_ptr(&h_buf));
   9617 
   9618     fprintf(out_h, "\n#endif\n");
   9619 
   9620     if (fclose(out_h))
   9621         zig_panic("unable to close h file: %s", strerror(errno));
   9622 }
   9623 
   9624 void codegen_print_timing_report(CodeGen *g, FILE *f) {
   9625     double start_time = g->timing_events.at(0).time;
   9626     double end_time = g->timing_events.last().time;
   9627     double total = end_time - start_time;
   9628     fprintf(f, "%20s%12s%12s%12s%12s\n", "Name", "Start", "End", "Duration", "Percent");
   9629     for (size_t i = 0; i < g->timing_events.length - 1; i += 1) {
   9630         TimeEvent *te = &g->timing_events.at(i);
   9631         TimeEvent *next_te = &g->timing_events.at(i + 1);
   9632         fprintf(f, "%20s%12.4f%12.4f%12.4f%12.4f\n", te->name,
   9633                 te->time - start_time,
   9634                 next_te->time - start_time,
   9635                 next_te->time - te->time,
   9636                 (next_te->time - te->time) / total);
   9637     }
   9638     fprintf(f, "%20s%12.4f%12.4f%12.4f%12.4f\n", "Total", 0.0, total, total, 1.0);
   9639 }
   9640 
   9641 void codegen_add_time_event(CodeGen *g, const char *name) {
   9642     OsTimeStamp timestamp = os_timestamp_monotonic();
   9643     double seconds = (double)timestamp.sec;
   9644     seconds += ((double)timestamp.nsec) / 1000000000.0;
   9645     g->timing_events.append({seconds, name});
   9646 }
   9647 
   9648 static void add_cache_pkg(CodeGen *g, CacheHash *ch, ZigPackage *pkg) {
   9649     if (buf_len(&pkg->root_src_path) == 0)
   9650         return;
   9651     pkg->added_to_cache = true;
   9652 
   9653     Buf *rel_full_path = buf_alloc();
   9654     os_path_join(&pkg->root_src_dir, &pkg->root_src_path, rel_full_path);
   9655     cache_file(ch, rel_full_path);
   9656 
   9657     auto it = pkg->package_table.entry_iterator();
   9658     for (;;) {
   9659         auto *entry = it.next();
   9660         if (!entry)
   9661             break;
   9662 
   9663         if (!pkg->added_to_cache) {
   9664             cache_buf(ch, entry->key);
   9665             add_cache_pkg(g, ch, entry->value);
   9666         }
   9667     }
   9668 }
   9669 
   9670 // Called before init()
   9671 // is_cache_hit takes into account gen_c_objects
   9672 static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) {
   9673     Error err;
   9674 
   9675     Buf *compiler_id;
   9676     if ((err = get_compiler_id(&compiler_id)))
   9677         return err;
   9678 
   9679     CacheHash *ch = &g->cache_hash;
   9680     cache_init(ch, manifest_dir);
   9681 
   9682     add_cache_pkg(g, ch, g->root_package);
   9683     if (g->linker_script != nullptr) {
   9684         cache_file(ch, buf_create_from_str(g->linker_script));
   9685     }
   9686     cache_buf(ch, compiler_id);
   9687     cache_buf(ch, g->root_out_name);
   9688     cache_buf(ch, g->zig_lib_dir);
   9689     cache_buf(ch, g->zig_std_dir);
   9690     cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length);
   9691     cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length);
   9692     cache_list_of_buf(ch, g->rpath_list.items, g->rpath_list.length);
   9693     cache_list_of_buf(ch, g->forbidden_libs.items, g->forbidden_libs.length);
   9694     cache_int(ch, g->build_mode);
   9695     cache_int(ch, g->out_type);
   9696     cache_bool(ch, g->zig_target->is_native);
   9697     cache_int(ch, g->zig_target->arch);
   9698     cache_int(ch, g->zig_target->sub_arch);
   9699     cache_int(ch, g->zig_target->vendor);
   9700     cache_int(ch, g->zig_target->os);
   9701     cache_int(ch, g->zig_target->abi);
   9702     if (g->zig_target->glibc_version != nullptr) {
   9703         cache_int(ch, g->zig_target->glibc_version->major);
   9704         cache_int(ch, g->zig_target->glibc_version->minor);
   9705         cache_int(ch, g->zig_target->glibc_version->patch);
   9706     }
   9707     cache_int(ch, detect_subsystem(g));
   9708     cache_bool(ch, g->strip_debug_symbols);
   9709     cache_bool(ch, g->is_test_build);
   9710     if (g->is_test_build) {
   9711         cache_buf_opt(ch, g->test_filter);
   9712         cache_buf_opt(ch, g->test_name_prefix);
   9713     }
   9714     cache_bool(ch, g->is_single_threaded);
   9715     cache_bool(ch, g->linker_rdynamic);
   9716     cache_bool(ch, g->each_lib_rpath);
   9717     cache_bool(ch, g->disable_gen_h);
   9718     cache_bool(ch, g->bundle_compiler_rt);
   9719     cache_bool(ch, want_valgrind_support(g));
   9720     cache_bool(ch, g->have_pic);
   9721     cache_bool(ch, g->have_dynamic_link);
   9722     cache_bool(ch, g->have_stack_probing);
   9723     cache_bool(ch, g->is_dummy_so);
   9724     cache_bool(ch, g->function_sections);
   9725     cache_buf_opt(ch, g->mmacosx_version_min);
   9726     cache_buf_opt(ch, g->mios_version_min);
   9727     cache_usize(ch, g->version_major);
   9728     cache_usize(ch, g->version_minor);
   9729     cache_usize(ch, g->version_patch);
   9730     cache_list_of_str(ch, g->llvm_argv, g->llvm_argv_len);
   9731     cache_list_of_str(ch, g->clang_argv, g->clang_argv_len);
   9732     cache_list_of_str(ch, g->lib_dirs.items, g->lib_dirs.length);
   9733     if (g->libc) {
   9734         cache_buf(ch, &g->libc->include_dir);
   9735         cache_buf(ch, &g->libc->sys_include_dir);
   9736         cache_buf(ch, &g->libc->crt_dir);
   9737         cache_buf(ch, &g->libc->msvc_lib_dir);
   9738         cache_buf(ch, &g->libc->kernel32_lib_dir);
   9739     }
   9740     cache_buf_opt(ch, g->dynamic_linker_path);
   9741     cache_buf_opt(ch, g->version_script_path);
   9742 
   9743     // gen_c_objects appends objects to g->link_objects which we want to include in the hash
   9744     gen_c_objects(g);
   9745     cache_list_of_file(ch, g->link_objects.items, g->link_objects.length);
   9746 
   9747     buf_resize(digest, 0);
   9748     if ((err = cache_hit(ch, digest))) {
   9749         if (err != ErrorInvalidFormat)
   9750             return err;
   9751     }
   9752 
   9753     if (ch->manifest_file_path != nullptr) {
   9754         g->caches_to_release.append(ch);
   9755     }
   9756 
   9757     return ErrorNone;
   9758 }
   9759 
   9760 static bool need_llvm_module(CodeGen *g) {
   9761     return buf_len(&g->root_package->root_src_path) != 0;
   9762 }
   9763 
   9764 static void resolve_out_paths(CodeGen *g) {
   9765     assert(g->output_dir != nullptr);
   9766     assert(g->root_out_name != nullptr);
   9767 
   9768     Buf *out_basename = buf_create_from_buf(g->root_out_name);
   9769     Buf *o_basename = buf_create_from_buf(g->root_out_name);
   9770     switch (g->emit_file_type) {
   9771         case EmitFileTypeBinary: {
   9772             switch (g->out_type) {
   9773                 case OutTypeUnknown:
   9774                     zig_unreachable();
   9775                 case OutTypeObj:
   9776                     if (g->enable_cache && g->link_objects.length == 1 && !need_llvm_module(g)) {
   9777                         buf_init_from_buf(&g->output_file_path, g->link_objects.at(0));
   9778                         return;
   9779                     }
   9780                     if (need_llvm_module(g) && g->link_objects.length != 0 && !g->enable_cache &&
   9781                         buf_eql_buf(o_basename, out_basename))
   9782                     {
   9783                         // make it not collide with main output object
   9784                         buf_append_str(o_basename, ".root");
   9785                     }
   9786                     buf_append_str(o_basename, target_o_file_ext(g->zig_target));
   9787                     buf_append_str(out_basename, target_o_file_ext(g->zig_target));
   9788                     break;
   9789                 case OutTypeExe:
   9790                     buf_append_str(o_basename, target_o_file_ext(g->zig_target));
   9791                     buf_append_str(out_basename, target_exe_file_ext(g->zig_target));
   9792                     break;
   9793                 case OutTypeLib:
   9794                     buf_append_str(o_basename, target_o_file_ext(g->zig_target));
   9795                     buf_resize(out_basename, 0);
   9796                     buf_append_str(out_basename, target_lib_file_prefix(g->zig_target));
   9797                     buf_append_buf(out_basename, g->root_out_name);
   9798                     buf_append_str(out_basename, target_lib_file_ext(g->zig_target, !g->is_dynamic,
   9799                                 g->version_major, g->version_minor, g->version_patch));
   9800                     break;
   9801             }
   9802             break;
   9803         }
   9804         case EmitFileTypeAssembly: {
   9805             const char *asm_ext = target_asm_file_ext(g->zig_target);
   9806             buf_append_str(o_basename, asm_ext);
   9807             buf_append_str(out_basename, asm_ext);
   9808             break;
   9809         }
   9810         case EmitFileTypeLLVMIr: {
   9811             const char *llvm_ir_ext = target_llvm_ir_file_ext(g->zig_target);
   9812             buf_append_str(o_basename, llvm_ir_ext);
   9813             buf_append_str(out_basename, llvm_ir_ext);
   9814             break;
   9815         }
   9816     }
   9817 
   9818     os_path_join(g->output_dir, o_basename, &g->o_file_output_path);
   9819     os_path_join(g->output_dir, out_basename, &g->output_file_path);
   9820 }
   9821 
   9822 void codegen_build_and_link(CodeGen *g) {
   9823     Error err;
   9824     assert(g->out_type != OutTypeUnknown);
   9825 
   9826     if (!g->enable_cache && g->output_dir == nullptr) {
   9827         g->output_dir = buf_create_from_str(".");
   9828     }
   9829 
   9830     g->have_dynamic_link = detect_dynamic_link(g);
   9831     g->have_pic = detect_pic(g);
   9832     g->is_single_threaded = detect_single_threaded(g);
   9833     g->have_err_ret_tracing = detect_err_ret_tracing(g);
   9834     detect_libc(g);
   9835     detect_dynamic_linker(g);
   9836 
   9837     Buf digest = BUF_INIT;
   9838     if (g->enable_cache) {
   9839         Buf *manifest_dir = buf_alloc();
   9840         os_path_join(g->cache_dir, buf_create_from_str(CACHE_HASH_SUBDIR), manifest_dir);
   9841 
   9842         if ((err = check_cache(g, manifest_dir, &digest))) {
   9843             if (err == ErrorCacheUnavailable) {
   9844                 // message already printed
   9845             } else if (err == ErrorNotDir) {
   9846                 fprintf(stderr, "Unable to check cache: %s is not a directory\n",
   9847                     buf_ptr(manifest_dir));
   9848             } else {
   9849                 fprintf(stderr, "Unable to check cache: %s: %s\n", buf_ptr(manifest_dir), err_str(err));
   9850             }
   9851             exit(1);
   9852         }
   9853     } else {
   9854         // There is a call to this in check_cache
   9855         gen_c_objects(g);
   9856     }
   9857 
   9858     if (g->enable_cache && buf_len(&digest) != 0) {
   9859         g->output_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s",
   9860                 buf_ptr(g->cache_dir), buf_ptr(&digest));
   9861         resolve_out_paths(g);
   9862     } else {
   9863         if (need_llvm_module(g)) {
   9864             init(g);
   9865 
   9866             codegen_add_time_event(g, "Semantic Analysis");
   9867 
   9868             gen_root_source(g);
   9869 
   9870         }
   9871         if (g->enable_cache) {
   9872             if (buf_len(&digest) == 0) {
   9873                 if ((err = cache_final(&g->cache_hash, &digest))) {
   9874                     fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err));
   9875                     exit(1);
   9876                 }
   9877             }
   9878             g->output_dir = buf_sprintf("%s" OS_SEP CACHE_OUT_SUBDIR OS_SEP "%s",
   9879                     buf_ptr(g->cache_dir), buf_ptr(&digest));
   9880 
   9881             if ((err = os_make_path(g->output_dir))) {
   9882                 fprintf(stderr, "Unable to create output directory: %s\n", err_str(err));
   9883                 exit(1);
   9884             }
   9885         }
   9886         resolve_out_paths(g);
   9887 
   9888         if (need_llvm_module(g)) {
   9889             codegen_add_time_event(g, "Code Generation");
   9890 
   9891             do_code_gen(g);
   9892             codegen_add_time_event(g, "LLVM Emit Output");
   9893             zig_llvm_emit_output(g);
   9894 
   9895             if (!g->disable_gen_h && (g->out_type == OutTypeObj || g->out_type == OutTypeLib)) {
   9896                 codegen_add_time_event(g, "Generate .h");
   9897                 gen_h_file(g);
   9898             }
   9899         }
   9900 
   9901         // If we're outputting assembly or llvm IR we skip linking.
   9902         // If we're making a library or executable we must link.
   9903         // If there is more than one object, we have to link them (with -r).
   9904         // Finally, if we didn't make an object from zig source, and we don't have caching enabled,
   9905         // then we have an object from C source that we must copy to the output dir which we do with a -r link.
   9906         if (g->emit_file_type == EmitFileTypeBinary && (g->out_type != OutTypeObj || g->link_objects.length > 1 ||
   9907                     (!need_llvm_module(g) && !g->enable_cache)))
   9908         {
   9909             codegen_link(g);
   9910         }
   9911     }
   9912 
   9913     codegen_release_caches(g);
   9914     codegen_add_time_event(g, "Done");
   9915 }
   9916 
   9917 void codegen_release_caches(CodeGen *g) {
   9918     while (g->caches_to_release.length != 0) {
   9919         cache_release(g->caches_to_release.pop());
   9920     }
   9921 }
   9922 
   9923 ZigPackage *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path,
   9924         const char *pkg_path)
   9925 {
   9926     init(g);
   9927     ZigPackage *pkg = new_package(root_src_dir, root_src_path, pkg_path);
   9928     if (g->std_package != nullptr) {
   9929         assert(g->compile_var_package != nullptr);
   9930         pkg->package_table.put(buf_create_from_str("std"), g->std_package);
   9931 
   9932         ZigPackage *main_pkg = g->is_test_build ? g->test_runner_package : g->root_package;
   9933         pkg->package_table.put(buf_create_from_str("root"), main_pkg);
   9934 
   9935         pkg->package_table.put(buf_create_from_str("builtin"), g->compile_var_package);
   9936     }
   9937     return pkg;
   9938 }
   9939 
   9940 CodeGen *create_child_codegen(CodeGen *parent_gen, Buf *root_src_path, OutType out_type,
   9941         ZigLibCInstallation *libc)
   9942 {
   9943     CodeGen *child_gen = codegen_create(nullptr, root_src_path, parent_gen->zig_target, out_type,
   9944         parent_gen->build_mode, parent_gen->zig_lib_dir, parent_gen->zig_std_dir, libc, get_stage1_cache_path(),
   9945         false);
   9946     child_gen->disable_gen_h = true;
   9947     child_gen->want_stack_check = WantStackCheckDisabled;
   9948     child_gen->verbose_tokenize = parent_gen->verbose_tokenize;
   9949     child_gen->verbose_ast = parent_gen->verbose_ast;
   9950     child_gen->verbose_link = parent_gen->verbose_link;
   9951     child_gen->verbose_ir = parent_gen->verbose_ir;
   9952     child_gen->verbose_llvm_ir = parent_gen->verbose_llvm_ir;
   9953     child_gen->verbose_cimport = parent_gen->verbose_cimport;
   9954     child_gen->verbose_cc = parent_gen->verbose_cc;
   9955     child_gen->llvm_argv = parent_gen->llvm_argv;
   9956     child_gen->dynamic_linker_path = parent_gen->dynamic_linker_path;
   9957 
   9958     codegen_set_strip(child_gen, parent_gen->strip_debug_symbols);
   9959     child_gen->want_pic = parent_gen->have_pic ? WantPICEnabled : WantPICDisabled;
   9960     child_gen->valgrind_support = ValgrindSupportDisabled;
   9961 
   9962     codegen_set_errmsg_color(child_gen, parent_gen->err_color);
   9963 
   9964     codegen_set_mmacosx_version_min(child_gen, parent_gen->mmacosx_version_min);
   9965     codegen_set_mios_version_min(child_gen, parent_gen->mios_version_min);
   9966 
   9967     child_gen->enable_cache = true;
   9968 
   9969     return child_gen;
   9970 }
   9971 
   9972 CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget *target,
   9973     OutType out_type, BuildMode build_mode, Buf *override_lib_dir, Buf *override_std_dir,
   9974     ZigLibCInstallation *libc, Buf *cache_dir, bool is_test_build)
   9975 {
   9976     CodeGen *g = allocate<CodeGen>(1);
   9977 
   9978     codegen_add_time_event(g, "Initialize");
   9979 
   9980     g->subsystem = TargetSubsystemAuto;
   9981     g->libc = libc;
   9982     g->zig_target = target;
   9983     g->cache_dir = cache_dir;
   9984 
   9985     if (override_lib_dir == nullptr) {
   9986         g->zig_lib_dir = get_zig_lib_dir();
   9987     } else {
   9988         g->zig_lib_dir = override_lib_dir;
   9989     }
   9990 
   9991     if (override_std_dir == nullptr) {
   9992         g->zig_std_dir = buf_alloc();
   9993         os_path_join(g->zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir);
   9994     } else {
   9995         g->zig_std_dir = override_std_dir;
   9996     }
   9997 
   9998     g->zig_c_headers_dir = buf_alloc();
   9999     os_path_join(g->zig_lib_dir, buf_create_from_str("include"), g->zig_c_headers_dir);
  10000 
  10001     g->build_mode = build_mode;
  10002     g->out_type = out_type;
  10003     g->import_table.init(32);
  10004     g->builtin_fn_table.init(32);
  10005     g->primitive_type_table.init(32);
  10006     g->type_table.init(32);
  10007     g->fn_type_table.init(32);
  10008     g->error_table.init(16);
  10009     g->generic_table.init(16);
  10010     g->llvm_fn_table.init(16);
  10011     g->memoized_fn_eval_table.init(16);
  10012     g->exported_symbol_names.init(8);
  10013     g->external_prototypes.init(8);
  10014     g->string_literals_table.init(16);
  10015     g->type_info_cache.init(32);
  10016     g->is_test_build = is_test_build;
  10017     g->is_single_threaded = false;
  10018     buf_resize(&g->global_asm, 0);
  10019 
  10020     for (size_t i = 0; i < array_length(symbols_that_llvm_depends_on); i += 1) {
  10021         g->external_prototypes.put(buf_create_from_str(symbols_that_llvm_depends_on[i]), nullptr);
  10022     }
  10023 
  10024     if (root_src_path) {
  10025         Buf *root_pkg_path;
  10026         Buf *rel_root_src_path;
  10027         if (main_pkg_path == nullptr) {
  10028             Buf *src_basename = buf_alloc();
  10029             Buf *src_dir = buf_alloc();
  10030             os_path_split(root_src_path, src_dir, src_basename);
  10031 
  10032             if (buf_len(src_basename) == 0) {
  10033                 fprintf(stderr, "Invalid root source path: %s\n", buf_ptr(root_src_path));
  10034                 exit(1);
  10035             }
  10036             root_pkg_path = src_dir;
  10037             rel_root_src_path = src_basename;
  10038         } else {
  10039             Buf resolved_root_src_path = os_path_resolve(&root_src_path, 1);
  10040             Buf resolved_main_pkg_path = os_path_resolve(&main_pkg_path, 1);
  10041 
  10042             if (!buf_starts_with_buf(&resolved_root_src_path, &resolved_main_pkg_path)) {
  10043                 fprintf(stderr, "Root source path '%s' outside main package path '%s'",
  10044                         buf_ptr(root_src_path), buf_ptr(main_pkg_path));
  10045                 exit(1);
  10046             }
  10047             root_pkg_path = main_pkg_path;
  10048             rel_root_src_path = buf_create_from_mem(
  10049                     buf_ptr(&resolved_root_src_path) + buf_len(&resolved_main_pkg_path) + 1,
  10050                     buf_len(&resolved_root_src_path) - buf_len(&resolved_main_pkg_path) - 1);
  10051         }
  10052 
  10053         g->root_package = new_package(buf_ptr(root_pkg_path), buf_ptr(rel_root_src_path), "");
  10054         g->std_package = new_package(buf_ptr(g->zig_std_dir), "std.zig", "std");
  10055         g->root_package->package_table.put(buf_create_from_str("std"), g->std_package);
  10056     } else {
  10057         g->root_package = new_package(".", "", "");
  10058     }
  10059 
  10060     g->root_package->package_table.put(buf_create_from_str("root"), g->root_package);
  10061 
  10062     g->zig_std_special_dir = buf_alloc();
  10063     os_path_join(g->zig_std_dir, buf_sprintf("special"), g->zig_std_special_dir);
  10064 
  10065     assert(target != nullptr);
  10066     if (!target->is_native) {
  10067         g->each_lib_rpath = false;
  10068     } else {
  10069         g->each_lib_rpath = true;
  10070 
  10071         if (target_os_is_darwin(g->zig_target->os)) {
  10072             init_darwin_native(g);
  10073         }
  10074 
  10075     }
  10076 
  10077     if (target_os_requires_libc(g->zig_target->os)) {
  10078         g->libc_link_lib = create_link_lib(buf_create_from_str("c"));
  10079         g->link_libs_list.append(g->libc_link_lib);
  10080     }
  10081 
  10082     target_triple_llvm(&g->llvm_triple_str, g->zig_target);
  10083     g->pointer_size_bytes = target_arch_pointer_bit_width(g->zig_target->arch) / 8;
  10084 
  10085     if (!target_has_debug_info(g->zig_target)) {
  10086         g->strip_debug_symbols = true;
  10087     }
  10088 
  10089     return g;
  10090 }
  10091 
  10092 bool codegen_fn_has_err_ret_tracing_arg(CodeGen *g, ZigType *return_type) {
  10093     return g->have_err_ret_tracing &&
  10094         (return_type->id == ZigTypeIdErrorUnion ||
  10095          return_type->id == ZigTypeIdErrorSet);
  10096 }
  10097 
  10098 bool codegen_fn_has_err_ret_tracing_stack(CodeGen *g, ZigFn *fn, bool is_async) {
  10099     if (is_async) {
  10100         return g->have_err_ret_tracing && (fn->calls_or_awaits_errorable_fn ||
  10101             codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type));
  10102     } else {
  10103         return g->have_err_ret_tracing && fn->calls_or_awaits_errorable_fn &&
  10104             !codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type);
  10105     }
  10106 }
	motiejus/zig fork of https://codeberg.org/ziglang/zig
	git clone https://git.jakstys.lt/motiejus/zig.git
	Log \| Tree \| Refs \| README \| LICENSE