atomics.zig (26388B) - Raw
1 const std = @import("std"); 2 const builtin = @import("builtin"); 3 const common = @import("./common.zig"); 4 const cpu = builtin.cpu; 5 const arch = cpu.arch; 6 const linkage = common.linkage; 7 const visibility = common.visibility; 8 pub const panic = common.panic; 9 10 // This parameter is true iff the target architecture supports the bare minimum 11 // to implement the atomic load/store intrinsics. 12 // Some architectures support atomic load/stores but no CAS, but we ignore this 13 // detail to keep the export logic clean and because we need some kind of CAS to 14 // implement the spinlocks. 15 const supports_atomic_ops = switch (arch) { 16 .msp430, .avr, .bpfel, .bpfeb => false, 17 .arm, .armeb, .thumb, .thumbeb => 18 // The ARM v6m ISA has no ldrex/strex and so it's impossible to do CAS 19 // operations (unless we're targeting Linux, the kernel provides a way to 20 // perform CAS operations). 21 // XXX: The Linux code path is not implemented yet. 22 !builtin.cpu.has(.arm, .has_v6m), 23 else => true, 24 }; 25 26 // The size (in bytes) of the biggest object that the architecture can 27 // load/store atomically. 28 // Objects bigger than this threshold require the use of a lock. 29 const largest_atomic_size = switch (arch) { 30 // On SPARC systems that lacks CAS and/or swap instructions, the only 31 // available atomic operation is a test-and-set (`ldstub`), so we force 32 // every atomic memory access to go through the lock. 33 .sparc => if (builtin.cpu.has(.sparc, .hasleoncasa)) @sizeOf(usize) else 0, 34 35 // XXX: On x86/x86_64 we could check the presence of cmpxchg8b/cmpxchg16b 36 // and set this parameter accordingly. 37 else => @sizeOf(usize), 38 }; 39 40 // The size (in bytes) of the smallest atomic object that the architecture can 41 // perform fetch/exchange atomically. Note, this does not encompass load and store. 42 // Objects smaller than this threshold are implemented in terms of compare-exchange 43 // of a larger value. 44 const smallest_atomic_fetch_exch_size = switch (arch) { 45 // On AMDGCN, there are no instructions for atomic operations other than load and store 46 // (as of LLVM 15), and so these need to be implemented in terms of atomic CAS. 47 .amdgcn => @sizeOf(u32), 48 else => @sizeOf(u8), 49 }; 50 51 const cache_line_size = 64; 52 53 const SpinlockTable = struct { 54 // Allocate ~4096 bytes of memory for the spinlock table 55 const max_spinlocks = 64; 56 57 const Spinlock = struct { 58 // SPARC ldstub instruction will write a 255 into the memory location. 59 // We'll use that as a sign that the lock is currently held. 60 // See also: Section B.7 in SPARCv8 spec & A.29 in SPARCv9 spec. 61 const sparc_lock: type = enum(u8) { Unlocked = 0, Locked = 255 }; 62 const other_lock: type = enum(usize) { Unlocked = 0, Locked }; 63 64 // Prevent false sharing by providing enough padding between two 65 // consecutive spinlock elements 66 v: if (arch.isSPARC()) sparc_lock else other_lock align(cache_line_size) = .Unlocked, 67 68 fn acquire(self: *@This()) void { 69 while (true) { 70 const flag = if (comptime arch.isSPARC()) flag: { 71 break :flag asm volatile ("ldstub [%[addr]], %[flag]" 72 : [flag] "=r" (-> @TypeOf(self.v)), 73 : [addr] "r" (&self.v), 74 : .{ .memory = true }); 75 } else flag: { 76 break :flag @atomicRmw(@TypeOf(self.v), &self.v, .Xchg, .Locked, .acquire); 77 }; 78 79 switch (flag) { 80 .Unlocked => break, 81 .Locked => {}, 82 } 83 } 84 } 85 fn release(self: *@This()) void { 86 if (comptime arch.isSPARC()) { 87 _ = asm volatile ("clrb [%[addr]]" 88 : 89 : [addr] "r" (&self.v), 90 : .{ .memory = true }); 91 } else { 92 @atomicStore(@TypeOf(self.v), &self.v, .Unlocked, .release); 93 } 94 } 95 }; 96 97 list: [max_spinlocks]Spinlock = [_]Spinlock{.{}} ** max_spinlocks, 98 99 // The spinlock table behaves as a really simple hash table, mapping 100 // addresses to spinlocks. The mapping is not unique but that's only a 101 // performance problem as the lock will be contended by more than a pair of 102 // threads. 103 fn get(self: *@This(), address: usize) *Spinlock { 104 var sl = &self.list[(address >> 3) % max_spinlocks]; 105 sl.acquire(); 106 return sl; 107 } 108 }; 109 110 var spinlocks: SpinlockTable = SpinlockTable{}; 111 112 // The following builtins do not respect the specified memory model and instead 113 // uses seq_cst, the strongest one, for simplicity sake. 114 115 // Generic version of GCC atomic builtin functions. 116 // Those work on any object no matter the pointer alignment nor its size. 117 118 fn __atomic_load(size: u32, src: [*]u8, dest: [*]u8, model: i32) callconv(.c) void { 119 _ = model; 120 var sl = spinlocks.get(@intFromPtr(src)); 121 defer sl.release(); 122 @memcpy(dest[0..size], src); 123 } 124 125 fn __atomic_store(size: u32, dest: [*]u8, src: [*]u8, model: i32) callconv(.c) void { 126 _ = model; 127 var sl = spinlocks.get(@intFromPtr(dest)); 128 defer sl.release(); 129 @memcpy(dest[0..size], src); 130 } 131 132 fn __atomic_exchange(size: u32, ptr: [*]u8, val: [*]u8, old: [*]u8, model: i32) callconv(.c) void { 133 _ = model; 134 var sl = spinlocks.get(@intFromPtr(ptr)); 135 defer sl.release(); 136 @memcpy(old[0..size], ptr); 137 @memcpy(ptr[0..size], val); 138 } 139 140 fn __atomic_compare_exchange( 141 size: u32, 142 ptr: [*]u8, 143 expected: [*]u8, 144 desired: [*]u8, 145 success: i32, 146 failure: i32, 147 ) callconv(.c) i32 { 148 _ = success; 149 _ = failure; 150 var sl = spinlocks.get(@intFromPtr(ptr)); 151 defer sl.release(); 152 for (ptr[0..size], 0..) |b, i| { 153 if (expected[i] != b) break; 154 } else { 155 // The two objects, ptr and expected, are equal 156 @memcpy(ptr[0..size], desired); 157 return 1; 158 } 159 @memcpy(expected[0..size], ptr); 160 return 0; 161 } 162 163 // Specialized versions of the GCC atomic builtin functions. 164 // LLVM emits those iff the object size is known and the pointers are correctly 165 // aligned. 166 inline fn atomic_load_N(comptime T: type, src: *T, model: i32) T { 167 _ = model; 168 if (@sizeOf(T) > largest_atomic_size) { 169 var sl = spinlocks.get(@intFromPtr(src)); 170 defer sl.release(); 171 return src.*; 172 } else { 173 return @atomicLoad(T, src, .seq_cst); 174 } 175 } 176 177 fn __atomic_load_1(src: *u8, model: i32) callconv(.c) u8 { 178 return atomic_load_N(u8, src, model); 179 } 180 181 fn __atomic_load_2(src: *u16, model: i32) callconv(.c) u16 { 182 return atomic_load_N(u16, src, model); 183 } 184 185 fn __atomic_load_4(src: *u32, model: i32) callconv(.c) u32 { 186 return atomic_load_N(u32, src, model); 187 } 188 189 fn __atomic_load_8(src: *u64, model: i32) callconv(.c) u64 { 190 return atomic_load_N(u64, src, model); 191 } 192 193 fn __atomic_load_16(src: *u128, model: i32) callconv(.c) u128 { 194 return atomic_load_N(u128, src, model); 195 } 196 197 inline fn atomic_store_N(comptime T: type, dst: *T, value: T, model: i32) void { 198 _ = model; 199 if (@sizeOf(T) > largest_atomic_size) { 200 var sl = spinlocks.get(@intFromPtr(dst)); 201 defer sl.release(); 202 dst.* = value; 203 } else { 204 @atomicStore(T, dst, value, .seq_cst); 205 } 206 } 207 208 fn __atomic_store_1(dst: *u8, value: u8, model: i32) callconv(.c) void { 209 return atomic_store_N(u8, dst, value, model); 210 } 211 212 fn __atomic_store_2(dst: *u16, value: u16, model: i32) callconv(.c) void { 213 return atomic_store_N(u16, dst, value, model); 214 } 215 216 fn __atomic_store_4(dst: *u32, value: u32, model: i32) callconv(.c) void { 217 return atomic_store_N(u32, dst, value, model); 218 } 219 220 fn __atomic_store_8(dst: *u64, value: u64, model: i32) callconv(.c) void { 221 return atomic_store_N(u64, dst, value, model); 222 } 223 224 fn __atomic_store_16(dst: *u128, value: u128, model: i32) callconv(.c) void { 225 return atomic_store_N(u128, dst, value, model); 226 } 227 228 fn wideUpdate(comptime T: type, ptr: *T, val: T, update: anytype) T { 229 const WideAtomic = std.meta.Int(.unsigned, smallest_atomic_fetch_exch_size * 8); 230 231 const addr = @intFromPtr(ptr); 232 const wide_addr = addr & ~(@as(T, smallest_atomic_fetch_exch_size) - 1); 233 const wide_ptr: *align(smallest_atomic_fetch_exch_size) WideAtomic = @alignCast(@as(*WideAtomic, @ptrFromInt(wide_addr))); 234 235 const inner_offset = addr & (@as(T, smallest_atomic_fetch_exch_size) - 1); 236 const inner_shift = @as(std.math.Log2Int(T), @intCast(inner_offset * 8)); 237 238 const mask = @as(WideAtomic, std.math.maxInt(T)) << inner_shift; 239 240 var wide_old = @atomicLoad(WideAtomic, wide_ptr, .seq_cst); 241 while (true) { 242 const old = @as(T, @truncate((wide_old & mask) >> inner_shift)); 243 const new = update(val, old); 244 const wide_new = wide_old & ~mask | (@as(WideAtomic, new) << inner_shift); 245 if (@cmpxchgWeak(WideAtomic, wide_ptr, wide_old, wide_new, .seq_cst, .seq_cst)) |new_wide_old| { 246 wide_old = new_wide_old; 247 } else { 248 return old; 249 } 250 } 251 } 252 253 inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T { 254 _ = model; 255 if (@sizeOf(T) > largest_atomic_size) { 256 var sl = spinlocks.get(@intFromPtr(ptr)); 257 defer sl.release(); 258 const value = ptr.*; 259 ptr.* = val; 260 return value; 261 } else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) { 262 // Machine does not support this type, but it does support a larger type. 263 const Updater = struct { 264 fn update(new: T, old: T) T { 265 _ = old; 266 return new; 267 } 268 }; 269 return wideUpdate(T, ptr, val, Updater.update); 270 } else { 271 return @atomicRmw(T, ptr, .Xchg, val, .seq_cst); 272 } 273 } 274 275 fn __atomic_exchange_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 276 return atomic_exchange_N(u8, ptr, val, model); 277 } 278 279 fn __atomic_exchange_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 280 return atomic_exchange_N(u16, ptr, val, model); 281 } 282 283 fn __atomic_exchange_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 284 return atomic_exchange_N(u32, ptr, val, model); 285 } 286 287 fn __atomic_exchange_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 288 return atomic_exchange_N(u64, ptr, val, model); 289 } 290 291 fn __atomic_exchange_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 292 return atomic_exchange_N(u128, ptr, val, model); 293 } 294 295 inline fn atomic_compare_exchange_N( 296 comptime T: type, 297 ptr: *T, 298 expected: *T, 299 desired: T, 300 success: i32, 301 failure: i32, 302 ) i32 { 303 _ = success; 304 _ = failure; 305 if (@sizeOf(T) > largest_atomic_size) { 306 var sl = spinlocks.get(@intFromPtr(ptr)); 307 defer sl.release(); 308 const value = ptr.*; 309 if (value == expected.*) { 310 ptr.* = desired; 311 return 1; 312 } 313 expected.* = value; 314 return 0; 315 } else { 316 if (@cmpxchgStrong(T, ptr, expected.*, desired, .seq_cst, .seq_cst)) |old_value| { 317 expected.* = old_value; 318 return 0; 319 } 320 return 1; 321 } 322 } 323 324 fn __atomic_compare_exchange_1(ptr: *u8, expected: *u8, desired: u8, success: i32, failure: i32) callconv(.c) i32 { 325 return atomic_compare_exchange_N(u8, ptr, expected, desired, success, failure); 326 } 327 328 fn __atomic_compare_exchange_2(ptr: *u16, expected: *u16, desired: u16, success: i32, failure: i32) callconv(.c) i32 { 329 return atomic_compare_exchange_N(u16, ptr, expected, desired, success, failure); 330 } 331 332 fn __atomic_compare_exchange_4(ptr: *u32, expected: *u32, desired: u32, success: i32, failure: i32) callconv(.c) i32 { 333 return atomic_compare_exchange_N(u32, ptr, expected, desired, success, failure); 334 } 335 336 fn __atomic_compare_exchange_8(ptr: *u64, expected: *u64, desired: u64, success: i32, failure: i32) callconv(.c) i32 { 337 return atomic_compare_exchange_N(u64, ptr, expected, desired, success, failure); 338 } 339 340 fn __atomic_compare_exchange_16(ptr: *u128, expected: *u128, desired: u128, success: i32, failure: i32) callconv(.c) i32 { 341 return atomic_compare_exchange_N(u128, ptr, expected, desired, success, failure); 342 } 343 344 inline fn fetch_op_N(comptime T: type, comptime op: std.builtin.AtomicRmwOp, ptr: *T, val: T, model: i32) T { 345 _ = model; 346 const Updater = struct { 347 fn update(new: T, old: T) T { 348 return switch (op) { 349 .Add => old +% new, 350 .Sub => old -% new, 351 .And => old & new, 352 .Nand => ~(old & new), 353 .Or => old | new, 354 .Xor => old ^ new, 355 .Max => @max(old, new), 356 .Min => @min(old, new), 357 else => @compileError("unsupported atomic op"), 358 }; 359 } 360 }; 361 362 if (@sizeOf(T) > largest_atomic_size) { 363 var sl = spinlocks.get(@intFromPtr(ptr)); 364 defer sl.release(); 365 366 const value = ptr.*; 367 ptr.* = Updater.update(val, value); 368 return value; 369 } else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) { 370 // Machine does not support this type, but it does support a larger type. 371 return wideUpdate(T, ptr, val, Updater.update); 372 } 373 374 return @atomicRmw(T, ptr, op, val, .seq_cst); 375 } 376 377 fn __atomic_fetch_add_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 378 return fetch_op_N(u8, .Add, ptr, val, model); 379 } 380 381 fn __atomic_fetch_add_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 382 return fetch_op_N(u16, .Add, ptr, val, model); 383 } 384 385 fn __atomic_fetch_add_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 386 return fetch_op_N(u32, .Add, ptr, val, model); 387 } 388 389 fn __atomic_fetch_add_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 390 return fetch_op_N(u64, .Add, ptr, val, model); 391 } 392 393 fn __atomic_fetch_add_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 394 return fetch_op_N(u128, .Add, ptr, val, model); 395 } 396 397 fn __atomic_fetch_sub_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 398 return fetch_op_N(u8, .Sub, ptr, val, model); 399 } 400 401 fn __atomic_fetch_sub_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 402 return fetch_op_N(u16, .Sub, ptr, val, model); 403 } 404 405 fn __atomic_fetch_sub_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 406 return fetch_op_N(u32, .Sub, ptr, val, model); 407 } 408 409 fn __atomic_fetch_sub_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 410 return fetch_op_N(u64, .Sub, ptr, val, model); 411 } 412 413 fn __atomic_fetch_sub_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 414 return fetch_op_N(u128, .Sub, ptr, val, model); 415 } 416 417 fn __atomic_fetch_and_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 418 return fetch_op_N(u8, .And, ptr, val, model); 419 } 420 421 fn __atomic_fetch_and_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 422 return fetch_op_N(u16, .And, ptr, val, model); 423 } 424 425 fn __atomic_fetch_and_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 426 return fetch_op_N(u32, .And, ptr, val, model); 427 } 428 429 fn __atomic_fetch_and_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 430 return fetch_op_N(u64, .And, ptr, val, model); 431 } 432 433 fn __atomic_fetch_and_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 434 return fetch_op_N(u128, .And, ptr, val, model); 435 } 436 437 fn __atomic_fetch_or_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 438 return fetch_op_N(u8, .Or, ptr, val, model); 439 } 440 441 fn __atomic_fetch_or_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 442 return fetch_op_N(u16, .Or, ptr, val, model); 443 } 444 445 fn __atomic_fetch_or_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 446 return fetch_op_N(u32, .Or, ptr, val, model); 447 } 448 449 fn __atomic_fetch_or_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 450 return fetch_op_N(u64, .Or, ptr, val, model); 451 } 452 453 fn __atomic_fetch_or_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 454 return fetch_op_N(u128, .Or, ptr, val, model); 455 } 456 457 fn __atomic_fetch_xor_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 458 return fetch_op_N(u8, .Xor, ptr, val, model); 459 } 460 461 fn __atomic_fetch_xor_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 462 return fetch_op_N(u16, .Xor, ptr, val, model); 463 } 464 465 fn __atomic_fetch_xor_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 466 return fetch_op_N(u32, .Xor, ptr, val, model); 467 } 468 469 fn __atomic_fetch_xor_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 470 return fetch_op_N(u64, .Xor, ptr, val, model); 471 } 472 473 fn __atomic_fetch_xor_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 474 return fetch_op_N(u128, .Xor, ptr, val, model); 475 } 476 477 fn __atomic_fetch_nand_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 478 return fetch_op_N(u8, .Nand, ptr, val, model); 479 } 480 481 fn __atomic_fetch_nand_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 482 return fetch_op_N(u16, .Nand, ptr, val, model); 483 } 484 485 fn __atomic_fetch_nand_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 486 return fetch_op_N(u32, .Nand, ptr, val, model); 487 } 488 489 fn __atomic_fetch_nand_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 490 return fetch_op_N(u64, .Nand, ptr, val, model); 491 } 492 493 fn __atomic_fetch_nand_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 494 return fetch_op_N(u128, .Nand, ptr, val, model); 495 } 496 497 fn __atomic_fetch_umax_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 498 return fetch_op_N(u8, .Max, ptr, val, model); 499 } 500 501 fn __atomic_fetch_umax_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 502 return fetch_op_N(u16, .Max, ptr, val, model); 503 } 504 505 fn __atomic_fetch_umax_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 506 return fetch_op_N(u32, .Max, ptr, val, model); 507 } 508 509 fn __atomic_fetch_umax_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 510 return fetch_op_N(u64, .Max, ptr, val, model); 511 } 512 513 fn __atomic_fetch_umax_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 514 return fetch_op_N(u128, .Max, ptr, val, model); 515 } 516 517 fn __atomic_fetch_umin_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 { 518 return fetch_op_N(u8, .Min, ptr, val, model); 519 } 520 521 fn __atomic_fetch_umin_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 { 522 return fetch_op_N(u16, .Min, ptr, val, model); 523 } 524 525 fn __atomic_fetch_umin_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 { 526 return fetch_op_N(u32, .Min, ptr, val, model); 527 } 528 529 fn __atomic_fetch_umin_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 { 530 return fetch_op_N(u64, .Min, ptr, val, model); 531 } 532 533 fn __atomic_fetch_umin_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 { 534 return fetch_op_N(u128, .Min, ptr, val, model); 535 } 536 537 comptime { 538 if (supports_atomic_ops and builtin.object_format != .c) { 539 @export(&__atomic_load, .{ .name = "__atomic_load", .linkage = linkage, .visibility = visibility }); 540 @export(&__atomic_store, .{ .name = "__atomic_store", .linkage = linkage, .visibility = visibility }); 541 @export(&__atomic_exchange, .{ .name = "__atomic_exchange", .linkage = linkage, .visibility = visibility }); 542 @export(&__atomic_compare_exchange, .{ .name = "__atomic_compare_exchange", .linkage = linkage, .visibility = visibility }); 543 544 @export(&__atomic_fetch_add_1, .{ .name = "__atomic_fetch_add_1", .linkage = linkage, .visibility = visibility }); 545 @export(&__atomic_fetch_add_2, .{ .name = "__atomic_fetch_add_2", .linkage = linkage, .visibility = visibility }); 546 @export(&__atomic_fetch_add_4, .{ .name = "__atomic_fetch_add_4", .linkage = linkage, .visibility = visibility }); 547 @export(&__atomic_fetch_add_8, .{ .name = "__atomic_fetch_add_8", .linkage = linkage, .visibility = visibility }); 548 @export(&__atomic_fetch_add_16, .{ .name = "__atomic_fetch_add_16", .linkage = linkage, .visibility = visibility }); 549 550 @export(&__atomic_fetch_sub_1, .{ .name = "__atomic_fetch_sub_1", .linkage = linkage, .visibility = visibility }); 551 @export(&__atomic_fetch_sub_2, .{ .name = "__atomic_fetch_sub_2", .linkage = linkage, .visibility = visibility }); 552 @export(&__atomic_fetch_sub_4, .{ .name = "__atomic_fetch_sub_4", .linkage = linkage, .visibility = visibility }); 553 @export(&__atomic_fetch_sub_8, .{ .name = "__atomic_fetch_sub_8", .linkage = linkage, .visibility = visibility }); 554 @export(&__atomic_fetch_sub_16, .{ .name = "__atomic_fetch_sub_16", .linkage = linkage, .visibility = visibility }); 555 556 @export(&__atomic_fetch_and_1, .{ .name = "__atomic_fetch_and_1", .linkage = linkage, .visibility = visibility }); 557 @export(&__atomic_fetch_and_2, .{ .name = "__atomic_fetch_and_2", .linkage = linkage, .visibility = visibility }); 558 @export(&__atomic_fetch_and_4, .{ .name = "__atomic_fetch_and_4", .linkage = linkage, .visibility = visibility }); 559 @export(&__atomic_fetch_and_8, .{ .name = "__atomic_fetch_and_8", .linkage = linkage, .visibility = visibility }); 560 @export(&__atomic_fetch_and_16, .{ .name = "__atomic_fetch_and_16", .linkage = linkage, .visibility = visibility }); 561 562 @export(&__atomic_fetch_or_1, .{ .name = "__atomic_fetch_or_1", .linkage = linkage, .visibility = visibility }); 563 @export(&__atomic_fetch_or_2, .{ .name = "__atomic_fetch_or_2", .linkage = linkage, .visibility = visibility }); 564 @export(&__atomic_fetch_or_4, .{ .name = "__atomic_fetch_or_4", .linkage = linkage, .visibility = visibility }); 565 @export(&__atomic_fetch_or_8, .{ .name = "__atomic_fetch_or_8", .linkage = linkage, .visibility = visibility }); 566 @export(&__atomic_fetch_or_16, .{ .name = "__atomic_fetch_or_16", .linkage = linkage, .visibility = visibility }); 567 568 @export(&__atomic_fetch_xor_1, .{ .name = "__atomic_fetch_xor_1", .linkage = linkage, .visibility = visibility }); 569 @export(&__atomic_fetch_xor_2, .{ .name = "__atomic_fetch_xor_2", .linkage = linkage, .visibility = visibility }); 570 @export(&__atomic_fetch_xor_4, .{ .name = "__atomic_fetch_xor_4", .linkage = linkage, .visibility = visibility }); 571 @export(&__atomic_fetch_xor_8, .{ .name = "__atomic_fetch_xor_8", .linkage = linkage, .visibility = visibility }); 572 @export(&__atomic_fetch_xor_16, .{ .name = "__atomic_fetch_xor_16", .linkage = linkage, .visibility = visibility }); 573 574 @export(&__atomic_fetch_nand_1, .{ .name = "__atomic_fetch_nand_1", .linkage = linkage, .visibility = visibility }); 575 @export(&__atomic_fetch_nand_2, .{ .name = "__atomic_fetch_nand_2", .linkage = linkage, .visibility = visibility }); 576 @export(&__atomic_fetch_nand_4, .{ .name = "__atomic_fetch_nand_4", .linkage = linkage, .visibility = visibility }); 577 @export(&__atomic_fetch_nand_8, .{ .name = "__atomic_fetch_nand_8", .linkage = linkage, .visibility = visibility }); 578 @export(&__atomic_fetch_nand_16, .{ .name = "__atomic_fetch_nand_16", .linkage = linkage, .visibility = visibility }); 579 580 @export(&__atomic_fetch_umax_1, .{ .name = "__atomic_fetch_umax_1", .linkage = linkage, .visibility = visibility }); 581 @export(&__atomic_fetch_umax_2, .{ .name = "__atomic_fetch_umax_2", .linkage = linkage, .visibility = visibility }); 582 @export(&__atomic_fetch_umax_4, .{ .name = "__atomic_fetch_umax_4", .linkage = linkage, .visibility = visibility }); 583 @export(&__atomic_fetch_umax_8, .{ .name = "__atomic_fetch_umax_8", .linkage = linkage, .visibility = visibility }); 584 @export(&__atomic_fetch_umax_16, .{ .name = "__atomic_fetch_umax_16", .linkage = linkage, .visibility = visibility }); 585 586 @export(&__atomic_fetch_umin_1, .{ .name = "__atomic_fetch_umin_1", .linkage = linkage, .visibility = visibility }); 587 @export(&__atomic_fetch_umin_2, .{ .name = "__atomic_fetch_umin_2", .linkage = linkage, .visibility = visibility }); 588 @export(&__atomic_fetch_umin_4, .{ .name = "__atomic_fetch_umin_4", .linkage = linkage, .visibility = visibility }); 589 @export(&__atomic_fetch_umin_8, .{ .name = "__atomic_fetch_umin_8", .linkage = linkage, .visibility = visibility }); 590 @export(&__atomic_fetch_umin_16, .{ .name = "__atomic_fetch_umin_16", .linkage = linkage, .visibility = visibility }); 591 592 @export(&__atomic_load_1, .{ .name = "__atomic_load_1", .linkage = linkage, .visibility = visibility }); 593 @export(&__atomic_load_2, .{ .name = "__atomic_load_2", .linkage = linkage, .visibility = visibility }); 594 @export(&__atomic_load_4, .{ .name = "__atomic_load_4", .linkage = linkage, .visibility = visibility }); 595 @export(&__atomic_load_8, .{ .name = "__atomic_load_8", .linkage = linkage, .visibility = visibility }); 596 @export(&__atomic_load_16, .{ .name = "__atomic_load_16", .linkage = linkage, .visibility = visibility }); 597 598 @export(&__atomic_store_1, .{ .name = "__atomic_store_1", .linkage = linkage, .visibility = visibility }); 599 @export(&__atomic_store_2, .{ .name = "__atomic_store_2", .linkage = linkage, .visibility = visibility }); 600 @export(&__atomic_store_4, .{ .name = "__atomic_store_4", .linkage = linkage, .visibility = visibility }); 601 @export(&__atomic_store_8, .{ .name = "__atomic_store_8", .linkage = linkage, .visibility = visibility }); 602 @export(&__atomic_store_16, .{ .name = "__atomic_store_16", .linkage = linkage, .visibility = visibility }); 603 604 @export(&__atomic_exchange_1, .{ .name = "__atomic_exchange_1", .linkage = linkage, .visibility = visibility }); 605 @export(&__atomic_exchange_2, .{ .name = "__atomic_exchange_2", .linkage = linkage, .visibility = visibility }); 606 @export(&__atomic_exchange_4, .{ .name = "__atomic_exchange_4", .linkage = linkage, .visibility = visibility }); 607 @export(&__atomic_exchange_8, .{ .name = "__atomic_exchange_8", .linkage = linkage, .visibility = visibility }); 608 @export(&__atomic_exchange_16, .{ .name = "__atomic_exchange_16", .linkage = linkage, .visibility = visibility }); 609 610 @export(&__atomic_compare_exchange_1, .{ .name = "__atomic_compare_exchange_1", .linkage = linkage, .visibility = visibility }); 611 @export(&__atomic_compare_exchange_2, .{ .name = "__atomic_compare_exchange_2", .linkage = linkage, .visibility = visibility }); 612 @export(&__atomic_compare_exchange_4, .{ .name = "__atomic_compare_exchange_4", .linkage = linkage, .visibility = visibility }); 613 @export(&__atomic_compare_exchange_8, .{ .name = "__atomic_compare_exchange_8", .linkage = linkage, .visibility = visibility }); 614 @export(&__atomic_compare_exchange_16, .{ .name = "__atomic_compare_exchange_16", .linkage = linkage, .visibility = visibility }); 615 } 616 }