atomic.h (30286B) - Raw
1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2008 Marcel Moolenaar 5 * Copyright (c) 2001 Benno Rice 6 * Copyright (c) 2001 David E. O'Brien 7 * Copyright (c) 1998 Doug Rabson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #ifndef _MACHINE_ATOMIC_H_ 33 #define _MACHINE_ATOMIC_H_ 34 35 #include <sys/atomic_common.h> 36 37 #ifndef __powerpc64__ 38 #include <sys/_atomic64e.h> 39 #endif 40 41 /* 42 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 43 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 44 * of this file. See also Appendix B.2 of Book II of the architecture manual. 45 * 46 * Note that not all Book-E processors accept the light-weight sync variant. 47 * In particular, early models of E500 cores are known to wedge. Bank on all 48 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 49 * to use the heavier-weight sync. 50 */ 51 52 #ifdef __powerpc64__ 53 #define mb() __asm __volatile("sync" : : : "memory") 54 #define rmb() __asm __volatile("lwsync" : : : "memory") 55 #define wmb() __asm __volatile("lwsync" : : : "memory") 56 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 57 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 58 #else 59 #define mb() __asm __volatile("sync" : : : "memory") 60 #define rmb() __asm __volatile("sync" : : : "memory") 61 #define wmb() __asm __volatile("sync" : : : "memory") 62 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 63 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 64 #endif 65 66 static __inline void 67 powerpc_lwsync(void) 68 { 69 70 #ifdef __powerpc64__ 71 __asm __volatile("lwsync" : : : "memory"); 72 #else 73 __asm __volatile("sync" : : : "memory"); 74 #endif 75 } 76 77 /* 78 * atomic_add(p, v) 79 * { *p += v; } 80 */ 81 82 #define __atomic_add_int(p, v, t) \ 83 __asm __volatile( \ 84 "1: lwarx %0, 0, %2\n" \ 85 " add %0, %3, %0\n" \ 86 " stwcx. %0, 0, %2\n" \ 87 " bne- 1b\n" \ 88 : "=&r" (t), "=m" (*p) \ 89 : "r" (p), "r" (v), "m" (*p) \ 90 : "cr0", "memory") \ 91 /* __atomic_add_int */ 92 93 #ifdef __powerpc64__ 94 #define __atomic_add_long(p, v, t) \ 95 __asm __volatile( \ 96 "1: ldarx %0, 0, %2\n" \ 97 " add %0, %3, %0\n" \ 98 " stdcx. %0, 0, %2\n" \ 99 " bne- 1b\n" \ 100 : "=&r" (t), "=m" (*p) \ 101 : "r" (p), "r" (v), "m" (*p) \ 102 : "cr0", "memory") \ 103 /* __atomic_add_long */ 104 #else 105 #define __atomic_add_long(p, v, t) \ 106 __asm __volatile( \ 107 "1: lwarx %0, 0, %2\n" \ 108 " add %0, %3, %0\n" \ 109 " stwcx. %0, 0, %2\n" \ 110 " bne- 1b\n" \ 111 : "=&r" (t), "=m" (*p) \ 112 : "r" (p), "r" (v), "m" (*p) \ 113 : "cr0", "memory") \ 114 /* __atomic_add_long */ 115 #endif 116 117 #define _ATOMIC_ADD(type) \ 118 static __inline void \ 119 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 120 u_##type t; \ 121 __atomic_add_##type(p, v, t); \ 122 } \ 123 \ 124 static __inline void \ 125 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 126 u_##type t; \ 127 __atomic_add_##type(p, v, t); \ 128 __ATOMIC_ACQ(); \ 129 } \ 130 \ 131 static __inline void \ 132 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 133 u_##type t; \ 134 __ATOMIC_REL(); \ 135 __atomic_add_##type(p, v, t); \ 136 } \ 137 /* _ATOMIC_ADD */ 138 139 _ATOMIC_ADD(int) 140 _ATOMIC_ADD(long) 141 142 #define atomic_add_32 atomic_add_int 143 #define atomic_add_acq_32 atomic_add_acq_int 144 #define atomic_add_rel_32 atomic_add_rel_int 145 146 #ifdef __powerpc64__ 147 #define atomic_add_64 atomic_add_long 148 #define atomic_add_acq_64 atomic_add_acq_long 149 #define atomic_add_rel_64 atomic_add_rel_long 150 151 #define atomic_add_ptr atomic_add_long 152 #define atomic_add_acq_ptr atomic_add_acq_long 153 #define atomic_add_rel_ptr atomic_add_rel_long 154 #else 155 #define atomic_add_ptr atomic_add_int 156 #define atomic_add_acq_ptr atomic_add_acq_int 157 #define atomic_add_rel_ptr atomic_add_rel_int 158 #endif 159 #undef _ATOMIC_ADD 160 #undef __atomic_add_long 161 #undef __atomic_add_int 162 163 /* 164 * atomic_clear(p, v) 165 * { *p &= ~v; } 166 */ 167 168 #define __atomic_clear_int(p, v, t) \ 169 __asm __volatile( \ 170 "1: lwarx %0, 0, %2\n" \ 171 " andc %0, %0, %3\n" \ 172 " stwcx. %0, 0, %2\n" \ 173 " bne- 1b\n" \ 174 : "=&r" (t), "=m" (*p) \ 175 : "r" (p), "r" (v), "m" (*p) \ 176 : "cr0", "memory") \ 177 /* __atomic_clear_int */ 178 179 #ifdef __powerpc64__ 180 #define __atomic_clear_long(p, v, t) \ 181 __asm __volatile( \ 182 "1: ldarx %0, 0, %2\n" \ 183 " andc %0, %0, %3\n" \ 184 " stdcx. %0, 0, %2\n" \ 185 " bne- 1b\n" \ 186 : "=&r" (t), "=m" (*p) \ 187 : "r" (p), "r" (v), "m" (*p) \ 188 : "cr0", "memory") \ 189 /* __atomic_clear_long */ 190 #else 191 #define __atomic_clear_long(p, v, t) \ 192 __asm __volatile( \ 193 "1: lwarx %0, 0, %2\n" \ 194 " andc %0, %0, %3\n" \ 195 " stwcx. %0, 0, %2\n" \ 196 " bne- 1b\n" \ 197 : "=&r" (t), "=m" (*p) \ 198 : "r" (p), "r" (v), "m" (*p) \ 199 : "cr0", "memory") \ 200 /* __atomic_clear_long */ 201 #endif 202 203 #define _ATOMIC_CLEAR(type) \ 204 static __inline void \ 205 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 206 u_##type t; \ 207 __atomic_clear_##type(p, v, t); \ 208 } \ 209 \ 210 static __inline void \ 211 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 212 u_##type t; \ 213 __atomic_clear_##type(p, v, t); \ 214 __ATOMIC_ACQ(); \ 215 } \ 216 \ 217 static __inline void \ 218 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 219 u_##type t; \ 220 __ATOMIC_REL(); \ 221 __atomic_clear_##type(p, v, t); \ 222 } \ 223 /* _ATOMIC_CLEAR */ 224 225 _ATOMIC_CLEAR(int) 226 _ATOMIC_CLEAR(long) 227 228 #define atomic_clear_32 atomic_clear_int 229 #define atomic_clear_acq_32 atomic_clear_acq_int 230 #define atomic_clear_rel_32 atomic_clear_rel_int 231 232 #ifdef __powerpc64__ 233 #define atomic_clear_64 atomic_clear_long 234 #define atomic_clear_acq_64 atomic_clear_acq_long 235 #define atomic_clear_rel_64 atomic_clear_rel_long 236 237 #define atomic_clear_ptr atomic_clear_long 238 #define atomic_clear_acq_ptr atomic_clear_acq_long 239 #define atomic_clear_rel_ptr atomic_clear_rel_long 240 #else 241 #define atomic_clear_ptr atomic_clear_int 242 #define atomic_clear_acq_ptr atomic_clear_acq_int 243 #define atomic_clear_rel_ptr atomic_clear_rel_int 244 #endif 245 #undef _ATOMIC_CLEAR 246 #undef __atomic_clear_long 247 #undef __atomic_clear_int 248 249 /* 250 * atomic_cmpset(p, o, n) 251 */ 252 /* TODO -- see below */ 253 254 /* 255 * atomic_load_acq(p) 256 */ 257 /* TODO -- see below */ 258 259 /* 260 * atomic_readandclear(p) 261 */ 262 /* TODO -- see below */ 263 264 /* 265 * atomic_set(p, v) 266 * { *p |= v; } 267 */ 268 269 #define __atomic_set_int(p, v, t) \ 270 __asm __volatile( \ 271 "1: lwarx %0, 0, %2\n" \ 272 " or %0, %3, %0\n" \ 273 " stwcx. %0, 0, %2\n" \ 274 " bne- 1b\n" \ 275 : "=&r" (t), "=m" (*p) \ 276 : "r" (p), "r" (v), "m" (*p) \ 277 : "cr0", "memory") \ 278 /* __atomic_set_int */ 279 280 #ifdef __powerpc64__ 281 #define __atomic_set_long(p, v, t) \ 282 __asm __volatile( \ 283 "1: ldarx %0, 0, %2\n" \ 284 " or %0, %3, %0\n" \ 285 " stdcx. %0, 0, %2\n" \ 286 " bne- 1b\n" \ 287 : "=&r" (t), "=m" (*p) \ 288 : "r" (p), "r" (v), "m" (*p) \ 289 : "cr0", "memory") \ 290 /* __atomic_set_long */ 291 #else 292 #define __atomic_set_long(p, v, t) \ 293 __asm __volatile( \ 294 "1: lwarx %0, 0, %2\n" \ 295 " or %0, %3, %0\n" \ 296 " stwcx. %0, 0, %2\n" \ 297 " bne- 1b\n" \ 298 : "=&r" (t), "=m" (*p) \ 299 : "r" (p), "r" (v), "m" (*p) \ 300 : "cr0", "memory") \ 301 /* __atomic_set_long */ 302 #endif 303 304 #define _ATOMIC_SET(type) \ 305 static __inline void \ 306 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 307 u_##type t; \ 308 __atomic_set_##type(p, v, t); \ 309 } \ 310 \ 311 static __inline void \ 312 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 313 u_##type t; \ 314 __atomic_set_##type(p, v, t); \ 315 __ATOMIC_ACQ(); \ 316 } \ 317 \ 318 static __inline void \ 319 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 320 u_##type t; \ 321 __ATOMIC_REL(); \ 322 __atomic_set_##type(p, v, t); \ 323 } \ 324 /* _ATOMIC_SET */ 325 326 _ATOMIC_SET(int) 327 _ATOMIC_SET(long) 328 329 #define atomic_set_32 atomic_set_int 330 #define atomic_set_acq_32 atomic_set_acq_int 331 #define atomic_set_rel_32 atomic_set_rel_int 332 333 #ifdef __powerpc64__ 334 #define atomic_set_64 atomic_set_long 335 #define atomic_set_acq_64 atomic_set_acq_long 336 #define atomic_set_rel_64 atomic_set_rel_long 337 338 #define atomic_set_ptr atomic_set_long 339 #define atomic_set_acq_ptr atomic_set_acq_long 340 #define atomic_set_rel_ptr atomic_set_rel_long 341 #else 342 #define atomic_set_ptr atomic_set_int 343 #define atomic_set_acq_ptr atomic_set_acq_int 344 #define atomic_set_rel_ptr atomic_set_rel_int 345 #endif 346 #undef _ATOMIC_SET 347 #undef __atomic_set_long 348 #undef __atomic_set_int 349 350 /* 351 * atomic_subtract(p, v) 352 * { *p -= v; } 353 */ 354 355 #define __atomic_subtract_int(p, v, t) \ 356 __asm __volatile( \ 357 "1: lwarx %0, 0, %2\n" \ 358 " subf %0, %3, %0\n" \ 359 " stwcx. %0, 0, %2\n" \ 360 " bne- 1b\n" \ 361 : "=&r" (t), "=m" (*p) \ 362 : "r" (p), "r" (v), "m" (*p) \ 363 : "cr0", "memory") \ 364 /* __atomic_subtract_int */ 365 366 #ifdef __powerpc64__ 367 #define __atomic_subtract_long(p, v, t) \ 368 __asm __volatile( \ 369 "1: ldarx %0, 0, %2\n" \ 370 " subf %0, %3, %0\n" \ 371 " stdcx. %0, 0, %2\n" \ 372 " bne- 1b\n" \ 373 : "=&r" (t), "=m" (*p) \ 374 : "r" (p), "r" (v), "m" (*p) \ 375 : "cr0", "memory") \ 376 /* __atomic_subtract_long */ 377 #else 378 #define __atomic_subtract_long(p, v, t) \ 379 __asm __volatile( \ 380 "1: lwarx %0, 0, %2\n" \ 381 " subf %0, %3, %0\n" \ 382 " stwcx. %0, 0, %2\n" \ 383 " bne- 1b\n" \ 384 : "=&r" (t), "=m" (*p) \ 385 : "r" (p), "r" (v), "m" (*p) \ 386 : "cr0", "memory") \ 387 /* __atomic_subtract_long */ 388 #endif 389 390 #define _ATOMIC_SUBTRACT(type) \ 391 static __inline void \ 392 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 393 u_##type t; \ 394 __atomic_subtract_##type(p, v, t); \ 395 } \ 396 \ 397 static __inline void \ 398 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 399 u_##type t; \ 400 __atomic_subtract_##type(p, v, t); \ 401 __ATOMIC_ACQ(); \ 402 } \ 403 \ 404 static __inline void \ 405 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 406 u_##type t; \ 407 __ATOMIC_REL(); \ 408 __atomic_subtract_##type(p, v, t); \ 409 } \ 410 /* _ATOMIC_SUBTRACT */ 411 412 _ATOMIC_SUBTRACT(int) 413 _ATOMIC_SUBTRACT(long) 414 415 #define atomic_subtract_32 atomic_subtract_int 416 #define atomic_subtract_acq_32 atomic_subtract_acq_int 417 #define atomic_subtract_rel_32 atomic_subtract_rel_int 418 419 #ifdef __powerpc64__ 420 #define atomic_subtract_64 atomic_subtract_long 421 #define atomic_subtract_acq_64 atomic_subract_acq_long 422 #define atomic_subtract_rel_64 atomic_subtract_rel_long 423 424 #define atomic_subtract_ptr atomic_subtract_long 425 #define atomic_subtract_acq_ptr atomic_subtract_acq_long 426 #define atomic_subtract_rel_ptr atomic_subtract_rel_long 427 #else 428 #define atomic_subtract_ptr atomic_subtract_int 429 #define atomic_subtract_acq_ptr atomic_subtract_acq_int 430 #define atomic_subtract_rel_ptr atomic_subtract_rel_int 431 #endif 432 #undef _ATOMIC_SUBTRACT 433 #undef __atomic_subtract_long 434 #undef __atomic_subtract_int 435 436 /* 437 * atomic_store_rel(p, v) 438 */ 439 /* TODO -- see below */ 440 441 /* 442 * Old/original implementations that still need revisiting. 443 */ 444 445 static __inline u_int 446 atomic_readandclear_int(volatile u_int *addr) 447 { 448 u_int result,temp; 449 450 __asm __volatile ( 451 "\tsync\n" /* drain writes */ 452 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 453 "li %1, 0\n\t" /* load new value */ 454 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 455 "bne- 1b\n\t" /* spin if failed */ 456 : "=&r"(result), "=&r"(temp), "=m" (*addr) 457 : "r" (addr), "m" (*addr) 458 : "cr0", "memory"); 459 460 return (result); 461 } 462 463 #ifdef __powerpc64__ 464 static __inline u_long 465 atomic_readandclear_long(volatile u_long *addr) 466 { 467 u_long result,temp; 468 469 __asm __volatile ( 470 "\tsync\n" /* drain writes */ 471 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 472 "li %1, 0\n\t" /* load new value */ 473 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 474 "bne- 1b\n\t" /* spin if failed */ 475 : "=&r"(result), "=&r"(temp), "=m" (*addr) 476 : "r" (addr), "m" (*addr) 477 : "cr0", "memory"); 478 479 return (result); 480 } 481 #endif 482 483 #define atomic_readandclear_32 atomic_readandclear_int 484 485 #ifdef __powerpc64__ 486 #define atomic_readandclear_64 atomic_readandclear_long 487 488 #define atomic_readandclear_ptr atomic_readandclear_long 489 #else 490 static __inline u_long 491 atomic_readandclear_long(volatile u_long *addr) 492 { 493 494 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 495 } 496 497 #define atomic_readandclear_ptr atomic_readandclear_int 498 #endif 499 500 /* 501 * We assume that a = b will do atomic loads and stores. 502 */ 503 #define ATOMIC_STORE_LOAD(TYPE) \ 504 static __inline u_##TYPE \ 505 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 506 { \ 507 u_##TYPE v; \ 508 \ 509 v = *p; \ 510 powerpc_lwsync(); \ 511 return (v); \ 512 } \ 513 \ 514 static __inline void \ 515 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 516 { \ 517 \ 518 powerpc_lwsync(); \ 519 *p = v; \ 520 } 521 522 ATOMIC_STORE_LOAD(int) 523 524 #define atomic_load_acq_32 atomic_load_acq_int 525 #define atomic_store_rel_32 atomic_store_rel_int 526 527 #ifdef __powerpc64__ 528 ATOMIC_STORE_LOAD(long) 529 530 #define atomic_load_acq_64 atomic_load_acq_long 531 #define atomic_store_rel_64 atomic_store_rel_long 532 533 #define atomic_load_acq_ptr atomic_load_acq_long 534 #define atomic_store_rel_ptr atomic_store_rel_long 535 #else 536 static __inline u_long 537 atomic_load_acq_long(volatile u_long *addr) 538 { 539 540 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 541 } 542 543 static __inline void 544 atomic_store_rel_long(volatile u_long *addr, u_long val) 545 { 546 547 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 548 } 549 550 #define atomic_load_acq_ptr atomic_load_acq_int 551 #define atomic_store_rel_ptr atomic_store_rel_int 552 #endif 553 #undef ATOMIC_STORE_LOAD 554 555 /* 556 * Atomically compare the value stored at *p with cmpval and if the 557 * two values are equal, update the value of *p with newval. Returns 558 * zero if the compare failed, nonzero otherwise. 559 */ 560 #ifdef ISA_206_ATOMICS 561 static __inline int 562 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) 563 { 564 int ret; 565 566 __asm __volatile ( 567 "1:\tlbarx %0, 0, %2\n\t" /* load old value */ 568 "cmplw %3, %0\n\t" /* compare */ 569 "bne- 2f\n\t" /* exit if not equal */ 570 "stbcx. %4, 0, %2\n\t" /* attempt to store */ 571 "bne- 1b\n\t" /* spin if failed */ 572 "li %0, 1\n\t" /* success - retval = 1 */ 573 "b 3f\n\t" /* we've succeeded */ 574 "2:\n\t" 575 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 576 "li %0, 0\n\t" /* failure - retval = 0 */ 577 "3:\n\t" 578 : "=&r" (ret), "=m" (*p) 579 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 580 : "cr0", "memory"); 581 582 return (ret); 583 } 584 585 static __inline int 586 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) 587 { 588 int ret; 589 590 __asm __volatile ( 591 "1:\tlharx %0, 0, %2\n\t" /* load old value */ 592 "cmplw %3, %0\n\t" /* compare */ 593 "bne- 2f\n\t" /* exit if not equal */ 594 "sthcx. %4, 0, %2\n\t" /* attempt to store */ 595 "bne- 1b\n\t" /* spin if failed */ 596 "li %0, 1\n\t" /* success - retval = 1 */ 597 "b 3f\n\t" /* we've succeeded */ 598 "2:\n\t" 599 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 600 "li %0, 0\n\t" /* failure - retval = 0 */ 601 "3:\n\t" 602 : "=&r" (ret), "=m" (*p) 603 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 604 : "cr0", "memory"); 605 606 return (ret); 607 } 608 #else 609 static __inline int 610 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval, 611 uint32_t mask) 612 { 613 int ret; 614 uint32_t tmp; 615 616 __asm __volatile ( 617 "1:\tlwarx %2, 0, %3\n\t" /* load old value */ 618 "and %0, %2, %7\n\t" 619 "cmplw %4, %0\n\t" /* compare */ 620 "bne- 2f\n\t" /* exit if not equal */ 621 "andc %2, %2, %7\n\t" 622 "or %2, %2, %5\n\t" 623 "stwcx. %2, 0, %3\n\t" /* attempt to store */ 624 "bne- 1b\n\t" /* spin if failed */ 625 "li %0, 1\n\t" /* success - retval = 1 */ 626 "b 3f\n\t" /* we've succeeded */ 627 "2:\n\t" 628 "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */ 629 "li %0, 0\n\t" /* failure - retval = 0 */ 630 "3:\n\t" 631 : "=&r" (ret), "=m" (*p), "+&r" (tmp) 632 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p), 633 "r" (mask) 634 : "cr0", "memory"); 635 636 return (ret); 637 } 638 639 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m) 640 #endif 641 642 static __inline int 643 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 644 { 645 int ret; 646 647 __asm __volatile ( 648 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 649 "cmplw %3, %0\n\t" /* compare */ 650 "bne- 2f\n\t" /* exit if not equal */ 651 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 652 "bne- 1b\n\t" /* spin if failed */ 653 "li %0, 1\n\t" /* success - retval = 1 */ 654 "b 3f\n\t" /* we've succeeded */ 655 "2:\n\t" 656 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 657 "li %0, 0\n\t" /* failure - retval = 0 */ 658 "3:\n\t" 659 : "=&r" (ret), "=m" (*p) 660 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 661 : "cr0", "memory"); 662 663 return (ret); 664 } 665 static __inline int 666 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 667 { 668 int ret; 669 670 __asm __volatile ( 671 #ifdef __powerpc64__ 672 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 673 "cmpld %3, %0\n\t" /* compare */ 674 "bne- 2f\n\t" /* exit if not equal */ 675 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 676 #else 677 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 678 "cmplw %3, %0\n\t" /* compare */ 679 "bne- 2f\n\t" /* exit if not equal */ 680 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 681 #endif 682 "bne- 1b\n\t" /* spin if failed */ 683 "li %0, 1\n\t" /* success - retval = 1 */ 684 "b 3f\n\t" /* we've succeeded */ 685 "2:\n\t" 686 #ifdef __powerpc64__ 687 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 688 #else 689 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 690 #endif 691 "li %0, 0\n\t" /* failure - retval = 0 */ 692 "3:\n\t" 693 : "=&r" (ret), "=m" (*p) 694 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 695 : "cr0", "memory"); 696 697 return (ret); 698 } 699 700 #define ATOMIC_CMPSET_ACQ_REL(type) \ 701 static __inline int \ 702 atomic_cmpset_acq_##type(volatile u_##type *p, \ 703 u_##type cmpval, u_##type newval)\ 704 {\ 705 u_##type retval; \ 706 retval = atomic_cmpset_##type(p, cmpval, newval);\ 707 __ATOMIC_ACQ();\ 708 return (retval);\ 709 }\ 710 static __inline int \ 711 atomic_cmpset_rel_##type(volatile u_##type *p, \ 712 u_##type cmpval, u_##type newval)\ 713 {\ 714 __ATOMIC_REL();\ 715 return (atomic_cmpset_##type(p, cmpval, newval));\ 716 }\ 717 struct hack 718 719 ATOMIC_CMPSET_ACQ_REL(int); 720 ATOMIC_CMPSET_ACQ_REL(long); 721 722 #ifdef ISA_206_ATOMICS 723 #define atomic_cmpset_8 atomic_cmpset_char 724 #endif 725 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char 726 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char 727 728 #ifdef ISA_206_ATOMICS 729 #define atomic_cmpset_16 atomic_cmpset_short 730 #endif 731 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short 732 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short 733 734 #define atomic_cmpset_32 atomic_cmpset_int 735 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int 736 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int 737 738 #ifdef __powerpc64__ 739 #define atomic_cmpset_64 atomic_cmpset_long 740 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long 741 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long 742 743 #define atomic_cmpset_ptr atomic_cmpset_long 744 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 745 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 746 #else 747 #define atomic_cmpset_ptr atomic_cmpset_int 748 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 749 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 750 #endif 751 752 /* 753 * Atomically compare the value stored at *p with *cmpval and if the 754 * two values are equal, update the value of *p with newval. Returns 755 * zero if the compare failed and sets *cmpval to the read value from *p, 756 * nonzero otherwise. 757 */ 758 #ifdef ISA_206_ATOMICS 759 static __inline int 760 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) 761 { 762 int ret; 763 764 __asm __volatile ( 765 "lbarx %0, 0, %3\n\t" /* load old value */ 766 "cmplw %4, %0\n\t" /* compare */ 767 "bne- 1f\n\t" /* exit if not equal */ 768 "stbcx. %5, 0, %3\n\t" /* attempt to store */ 769 "bne- 1f\n\t" /* exit if failed */ 770 "li %0, 1\n\t" /* success - retval = 1 */ 771 "b 2f\n\t" /* we've succeeded */ 772 "1:\n\t" 773 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 774 "stbx %0, 0, %7\n\t" 775 "li %0, 0\n\t" /* failure - retval = 0 */ 776 "2:\n\t" 777 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 778 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 779 : "cr0", "memory"); 780 781 return (ret); 782 } 783 784 static __inline int 785 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) 786 { 787 int ret; 788 789 __asm __volatile ( 790 "lharx %0, 0, %3\n\t" /* load old value */ 791 "cmplw %4, %0\n\t" /* compare */ 792 "bne- 1f\n\t" /* exit if not equal */ 793 "sthcx. %5, 0, %3\n\t" /* attempt to store */ 794 "bne- 1f\n\t" /* exit if failed */ 795 "li %0, 1\n\t" /* success - retval = 1 */ 796 "b 2f\n\t" /* we've succeeded */ 797 "1:\n\t" 798 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 799 "sthx %0, 0, %7\n\t" 800 "li %0, 0\n\t" /* failure - retval = 0 */ 801 "2:\n\t" 802 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 803 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 804 : "cr0", "memory"); 805 806 return (ret); 807 } 808 #endif /* ISA_206_ATOMICS */ 809 810 static __inline int 811 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) 812 { 813 int ret; 814 815 __asm __volatile ( 816 "lwarx %0, 0, %3\n\t" /* load old value */ 817 "cmplw %4, %0\n\t" /* compare */ 818 "bne- 1f\n\t" /* exit if not equal */ 819 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 820 "bne- 1f\n\t" /* exit if failed */ 821 "li %0, 1\n\t" /* success - retval = 1 */ 822 "b 2f\n\t" /* we've succeeded */ 823 "1:\n\t" 824 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 825 "stwx %0, 0, %7\n\t" 826 "li %0, 0\n\t" /* failure - retval = 0 */ 827 "2:\n\t" 828 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 829 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 830 : "cr0", "memory"); 831 832 return (ret); 833 } 834 static __inline int 835 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) 836 { 837 int ret; 838 839 __asm __volatile ( 840 #ifdef __powerpc64__ 841 "ldarx %0, 0, %3\n\t" /* load old value */ 842 "cmpld %4, %0\n\t" /* compare */ 843 "bne- 1f\n\t" /* exit if not equal */ 844 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 845 #else 846 "lwarx %0, 0, %3\n\t" /* load old value */ 847 "cmplw %4, %0\n\t" /* compare */ 848 "bne- 1f\n\t" /* exit if not equal */ 849 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 850 #endif 851 "bne- 1f\n\t" /* exit if failed */ 852 "li %0, 1\n\t" /* success - retval = 1 */ 853 "b 2f\n\t" /* we've succeeded */ 854 "1:\n\t" 855 #ifdef __powerpc64__ 856 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 857 "stdx %0, 0, %7\n\t" 858 #else 859 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 860 "stwx %0, 0, %7\n\t" 861 #endif 862 "li %0, 0\n\t" /* failure - retval = 0 */ 863 "2:\n\t" 864 : "=&r" (ret), "=m" (*p), "=m" (*cmpval) 865 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) 866 : "cr0", "memory"); 867 868 return (ret); 869 } 870 871 #define ATOMIC_FCMPSET_ACQ_REL(type) \ 872 static __inline int \ 873 atomic_fcmpset_acq_##type(volatile u_##type *p, \ 874 u_##type *cmpval, u_##type newval)\ 875 {\ 876 u_##type retval; \ 877 retval = atomic_fcmpset_##type(p, cmpval, newval);\ 878 __ATOMIC_ACQ();\ 879 return (retval);\ 880 }\ 881 static __inline int \ 882 atomic_fcmpset_rel_##type(volatile u_##type *p, \ 883 u_##type *cmpval, u_##type newval)\ 884 {\ 885 __ATOMIC_REL();\ 886 return (atomic_fcmpset_##type(p, cmpval, newval));\ 887 }\ 888 struct hack 889 890 ATOMIC_FCMPSET_ACQ_REL(int); 891 ATOMIC_FCMPSET_ACQ_REL(long); 892 893 #ifdef ISA_206_ATOMICS 894 #define atomic_fcmpset_8 atomic_fcmpset_char 895 #endif 896 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char 897 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char 898 899 #ifdef ISA_206_ATOMICS 900 #define atomic_fcmpset_16 atomic_fcmpset_short 901 #endif 902 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short 903 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short 904 905 #define atomic_fcmpset_32 atomic_fcmpset_int 906 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int 907 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int 908 909 #ifdef __powerpc64__ 910 #define atomic_fcmpset_64 atomic_fcmpset_long 911 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long 912 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long 913 914 #define atomic_fcmpset_ptr atomic_fcmpset_long 915 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long 916 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long 917 #else 918 #define atomic_fcmpset_ptr atomic_fcmpset_int 919 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int 920 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int 921 #endif 922 923 static __inline u_int 924 atomic_fetchadd_int(volatile u_int *p, u_int v) 925 { 926 u_int value; 927 928 do { 929 value = *p; 930 } while (!atomic_cmpset_int(p, value, value + v)); 931 return (value); 932 } 933 934 static __inline u_long 935 atomic_fetchadd_long(volatile u_long *p, u_long v) 936 { 937 u_long value; 938 939 do { 940 value = *p; 941 } while (!atomic_cmpset_long(p, value, value + v)); 942 return (value); 943 } 944 945 static __inline u_int 946 atomic_swap_32(volatile u_int *p, u_int v) 947 { 948 u_int prev; 949 950 __asm __volatile( 951 "1: lwarx %0,0,%2\n" 952 " stwcx. %3,0,%2\n" 953 " bne- 1b\n" 954 : "=&r" (prev), "+m" (*(volatile u_int *)p) 955 : "r" (p), "r" (v) 956 : "cr0", "memory"); 957 958 return (prev); 959 } 960 961 #ifdef __powerpc64__ 962 static __inline u_long 963 atomic_swap_64(volatile u_long *p, u_long v) 964 { 965 u_long prev; 966 967 __asm __volatile( 968 "1: ldarx %0,0,%2\n" 969 " stdcx. %3,0,%2\n" 970 " bne- 1b\n" 971 : "=&r" (prev), "+m" (*(volatile u_long *)p) 972 : "r" (p), "r" (v) 973 : "cr0", "memory"); 974 975 return (prev); 976 } 977 #endif 978 979 #define atomic_fetchadd_32 atomic_fetchadd_int 980 #define atomic_swap_int atomic_swap_32 981 982 #ifdef __powerpc64__ 983 #define atomic_fetchadd_64 atomic_fetchadd_long 984 #define atomic_swap_long atomic_swap_64 985 #define atomic_swap_ptr atomic_swap_64 986 #else 987 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) 988 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) 989 #endif 990 991 static __inline int 992 atomic_testandset_int(volatile u_int *p, u_int v) 993 { 994 u_int m = (1u << (v & 0x1f)); 995 u_int res; 996 u_int tmp; 997 998 __asm __volatile( 999 "1: lwarx %0,0,%3\n" 1000 " and %1,%0,%4\n" 1001 " or %0,%0,%4\n" 1002 " stwcx. %0,0,%3\n" 1003 " bne- 1b\n" 1004 : "=&r"(tmp), "=&r"(res), "+m"(*p) 1005 : "r"(p), "r"(m) 1006 : "cr0", "memory"); 1007 1008 return (res != 0); 1009 } 1010 1011 static __inline int 1012 atomic_testandclear_int(volatile u_int *p, u_int v) 1013 { 1014 u_int m = (1u << (v & 0x1f)); 1015 u_int res; 1016 u_int tmp; 1017 1018 __asm __volatile( 1019 "1: lwarx %0,0,%3\n" 1020 " and %1,%0,%4\n" 1021 " andc %0,%0,%4\n" 1022 " stwcx. %0,0,%3\n" 1023 " bne- 1b\n" 1024 : "=&r"(tmp), "=&r"(res), "+m"(*p) 1025 : "r"(p), "r"(m) 1026 : "cr0", "memory"); 1027 1028 return (res != 0); 1029 } 1030 1031 #ifdef __powerpc64__ 1032 static __inline int 1033 atomic_testandset_long(volatile u_long *p, u_int v) 1034 { 1035 u_long m = (1ul << (v & 0x3f)); 1036 u_long res; 1037 u_long tmp; 1038 1039 __asm __volatile( 1040 "1: ldarx %0,0,%3\n" 1041 " and %1,%0,%4\n" 1042 " or %0,%0,%4\n" 1043 " stdcx. %0,0,%3\n" 1044 " bne- 1b\n" 1045 : "=&r"(tmp), "=&r"(res), "+m"(*(volatile u_long *)p) 1046 : "r"(p), "r"(m) 1047 : "cr0", "memory"); 1048 1049 return (res != 0); 1050 } 1051 1052 static __inline int 1053 atomic_testandclear_long(volatile u_long *p, u_int v) 1054 { 1055 u_long m = (1ul << (v & 0x3f)); 1056 u_long res; 1057 u_long tmp; 1058 1059 __asm __volatile( 1060 "1: ldarx %0,0,%3\n" 1061 " and %1,%0,%4\n" 1062 " andc %0,%0,%4\n" 1063 " stdcx. %0,0,%3\n" 1064 " bne- 1b\n" 1065 : "=&r"(tmp), "=&r"(res), "+m"(*p) 1066 : "r"(p), "r"(m) 1067 : "cr0", "memory"); 1068 1069 return (res != 0); 1070 } 1071 #else 1072 static __inline int 1073 atomic_testandset_long(volatile u_long *p, u_int v) 1074 { 1075 return (atomic_testandset_int((volatile u_int *)p, v)); 1076 } 1077 1078 static __inline int 1079 atomic_testandclear_long(volatile u_long *p, u_int v) 1080 { 1081 return (atomic_testandclear_int((volatile u_int *)p, v)); 1082 } 1083 #endif 1084 1085 #define atomic_testandclear_32 atomic_testandclear_int 1086 #define atomic_testandset_32 atomic_testandset_int 1087 1088 static __inline int 1089 atomic_testandset_acq_long(volatile u_long *p, u_int v) 1090 { 1091 u_int a = atomic_testandset_long(p, v); 1092 __ATOMIC_ACQ(); 1093 return (a); 1094 } 1095 1096 #define atomic_testandclear_int atomic_testandclear_int 1097 #define atomic_testandset_int atomic_testandset_int 1098 #define atomic_testandclear_long atomic_testandclear_long 1099 #define atomic_testandset_long atomic_testandset_long 1100 #define atomic_testandset_acq_long atomic_testandset_acq_long 1101 1102 static __inline void 1103 atomic_thread_fence_acq(void) 1104 { 1105 1106 powerpc_lwsync(); 1107 } 1108 1109 static __inline void 1110 atomic_thread_fence_rel(void) 1111 { 1112 1113 powerpc_lwsync(); 1114 } 1115 1116 static __inline void 1117 atomic_thread_fence_acq_rel(void) 1118 { 1119 1120 powerpc_lwsync(); 1121 } 1122 1123 static __inline void 1124 atomic_thread_fence_seq_cst(void) 1125 { 1126 1127 __asm __volatile("sync" : : : "memory"); 1128 } 1129 1130 #ifndef ISA_206_ATOMICS 1131 #include <sys/_atomic_subword.h> 1132 #define atomic_cmpset_char atomic_cmpset_8 1133 #define atomic_cmpset_short atomic_cmpset_16 1134 #define atomic_fcmpset_char atomic_fcmpset_8 1135 #define atomic_fcmpset_short atomic_fcmpset_16 1136 #endif 1137 1138 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ 1139 ATOMIC_CMPSET_ACQ_REL(char); 1140 ATOMIC_CMPSET_ACQ_REL(short); 1141 1142 ATOMIC_FCMPSET_ACQ_REL(char); 1143 ATOMIC_FCMPSET_ACQ_REL(short); 1144 1145 #undef __ATOMIC_REL 1146 #undef __ATOMIC_ACQ 1147 1148 #endif /* ! _MACHINE_ATOMIC_H_ */