intrin.h (18596B) - Raw
1 /* ===-------- intrin.h ---------------------------------------------------=== 2 * 3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 * See https://llvm.org/LICENSE.txt for license information. 5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 * 7 *===-----------------------------------------------------------------------=== 8 */ 9 10 /* Only include this if we're compiling for the windows platform. */ 11 #ifndef _MSC_VER 12 #include_next <intrin.h> 13 #else 14 15 #ifndef __INTRIN_H 16 #define __INTRIN_H 17 18 #include <intrin0.h> 19 20 /* First include the standard intrinsics. */ 21 #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__)) 22 #include <x86intrin.h> 23 #endif 24 25 #if defined(__arm__) 26 #include <armintr.h> 27 #endif 28 29 #if defined(__aarch64__) || defined(__arm64ec__) 30 #include <arm64intr.h> 31 #endif 32 33 /* For the definition of jmp_buf. */ 34 #if __STDC_HOSTED__ 35 #include <setjmp.h> 36 #endif 37 38 /* Define the default attributes for the functions in this file. */ 39 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 40 41 #if __x86_64__ 42 #define __LPTRINT_TYPE__ __int64 43 #else 44 #define __LPTRINT_TYPE__ long 45 #endif 46 47 #ifdef __cplusplus 48 extern "C" { 49 #endif 50 51 #if defined(__MMX__) 52 /* And the random ones that aren't in those files. */ 53 __m64 _m_from_float(float); 54 float _m_to_float(__m64); 55 #endif 56 57 /* Other assorted instruction intrinsics. */ 58 void __addfsbyte(unsigned long, unsigned char); 59 void __addfsdword(unsigned long, unsigned long); 60 void __addfsword(unsigned long, unsigned short); 61 void __code_seg(const char *); 62 void __cpuid(int[4], int); 63 void __cpuidex(int[4], int, int); 64 __int64 __emul(int, int); 65 unsigned __int64 __emulu(unsigned int, unsigned int); 66 unsigned int __getcallerseflags(void); 67 void __halt(void); 68 unsigned char __inbyte(unsigned short); 69 void __inbytestring(unsigned short, unsigned char *, unsigned long); 70 void __incfsbyte(unsigned long); 71 void __incfsdword(unsigned long); 72 void __incfsword(unsigned long); 73 unsigned long __indword(unsigned short); 74 void __indwordstring(unsigned short, unsigned long *, unsigned long); 75 void __int2c(void); 76 void __invlpg(void *); 77 unsigned short __inword(unsigned short); 78 void __inwordstring(unsigned short, unsigned short *, unsigned long); 79 void __lidt(void *); 80 unsigned __int64 __ll_lshift(unsigned __int64, int); 81 __int64 __ll_rshift(__int64, int); 82 void __movsb(unsigned char *, unsigned char const *, size_t); 83 void __movsd(unsigned long *, unsigned long const *, size_t); 84 void __movsw(unsigned short *, unsigned short const *, size_t); 85 void __nop(void); 86 void __nvreg_restore_fence(void); 87 void __nvreg_save_fence(void); 88 void __outbyte(unsigned short, unsigned char); 89 void __outbytestring(unsigned short, unsigned char *, unsigned long); 90 void __outdword(unsigned short, unsigned long); 91 void __outdwordstring(unsigned short, unsigned long *, unsigned long); 92 void __outword(unsigned short, unsigned short); 93 void __outwordstring(unsigned short, unsigned short *, unsigned long); 94 unsigned long __readcr0(void); 95 unsigned long __readcr2(void); 96 unsigned __LPTRINT_TYPE__ __readcr3(void); 97 unsigned __LPTRINT_TYPE__ __readcr4(void); 98 unsigned __int64 __readcr8(void); 99 unsigned int __readdr(unsigned int); 100 #ifdef __i386__ 101 unsigned char __readfsbyte(unsigned long); 102 unsigned short __readfsword(unsigned long); 103 unsigned long __readfsdword(unsigned long); 104 unsigned __int64 __readfsqword(unsigned long); 105 #endif 106 unsigned __int64 __readmsr(unsigned long); 107 unsigned __int64 __readpmc(unsigned long); 108 unsigned long __segmentlimit(unsigned long); 109 void __sidt(void *); 110 void __stosb(unsigned char *, unsigned char, size_t); 111 void __stosd(unsigned long *, unsigned long, size_t); 112 void __stosw(unsigned short *, unsigned short, size_t); 113 void __svm_clgi(void); 114 void __svm_invlpga(void *, int); 115 void __svm_skinit(int); 116 void __svm_stgi(void); 117 void __svm_vmload(size_t); 118 void __svm_vmrun(size_t); 119 void __svm_vmsave(size_t); 120 void __ud2(void); 121 unsigned __int64 __ull_rshift(unsigned __int64, int); 122 void __vmx_off(void); 123 void __vmx_vmptrst(unsigned __int64 *); 124 void __wbinvd(void); 125 void __writecr0(unsigned int); 126 void __writecr3(unsigned __INTPTR_TYPE__); 127 void __writecr4(unsigned __INTPTR_TYPE__); 128 void __writecr8(unsigned __int64); 129 void __writedr(unsigned int, unsigned int); 130 void __writefsbyte(unsigned long, unsigned char); 131 void __writefsdword(unsigned long, unsigned long); 132 void __writefsqword(unsigned long, unsigned __int64); 133 void __writefsword(unsigned long, unsigned short); 134 void __writemsr(unsigned long, unsigned __int64); 135 void *_AddressOfReturnAddress(void); 136 unsigned char _bittest(long const *, long); 137 unsigned char _bittestandcomplement(long *, long); 138 unsigned char _bittestandreset(long *, long); 139 unsigned char _bittestandset(long *, long); 140 void __cdecl _disable(void); 141 void __cdecl _enable(void); 142 long _InterlockedAddLargeStatistic(__int64 volatile *_Addend, long _Value); 143 unsigned char _interlockedbittestandreset(long volatile *, long); 144 unsigned char _interlockedbittestandset(long volatile *, long); 145 void *_InterlockedCompareExchangePointer_HLEAcquire(void *volatile *, void *, 146 void *); 147 void *_InterlockedCompareExchangePointer_HLERelease(void *volatile *, void *, 148 void *); 149 long _InterlockedExchangeAdd_HLEAcquire(long volatile *, long); 150 long _InterlockedExchangeAdd_HLERelease(long volatile *, long); 151 __int64 _InterlockedExchangeAdd64_HLEAcquire(__int64 volatile *, __int64); 152 __int64 _InterlockedExchangeAdd64_HLERelease(__int64 volatile *, __int64); 153 void _ReadBarrier(void); 154 unsigned int _rorx_u32(unsigned int, const unsigned int); 155 int _sarx_i32(int, unsigned int); 156 #if __STDC_HOSTED__ 157 int __cdecl _setjmp(jmp_buf); 158 #endif 159 unsigned int _shlx_u32(unsigned int, unsigned int); 160 unsigned int _shrx_u32(unsigned int, unsigned int); 161 void _Store_HLERelease(long volatile *, long); 162 void _Store64_HLERelease(__int64 volatile *, __int64); 163 void _StorePointer_HLERelease(void *volatile *, void *); 164 void _WriteBarrier(void); 165 unsigned __int32 xbegin(void); 166 void _xend(void); 167 168 /* These additional intrinsics are turned on in x64/amd64/x86_64 mode. */ 169 #if defined(__x86_64__) && !defined(__arm64ec__) 170 void __addgsbyte(unsigned long, unsigned char); 171 void __addgsdword(unsigned long, unsigned long); 172 void __addgsqword(unsigned long, unsigned __int64); 173 void __addgsword(unsigned long, unsigned short); 174 void __faststorefence(void); 175 void __incgsbyte(unsigned long); 176 void __incgsdword(unsigned long); 177 void __incgsqword(unsigned long); 178 void __incgsword(unsigned long); 179 void __movsq(unsigned long long *, unsigned long long const *, size_t); 180 unsigned char __readgsbyte(unsigned long); 181 unsigned long __readgsdword(unsigned long); 182 unsigned __int64 __readgsqword(unsigned long); 183 unsigned short __readgsword(unsigned long); 184 void __stosq(unsigned __int64 *, unsigned __int64, size_t); 185 unsigned char __vmx_on(unsigned __int64 *); 186 unsigned char __vmx_vmclear(unsigned __int64 *); 187 unsigned char __vmx_vmlaunch(void); 188 unsigned char __vmx_vmptrld(unsigned __int64 *); 189 unsigned char __vmx_vmread(size_t, size_t *); 190 unsigned char __vmx_vmresume(void); 191 unsigned char __vmx_vmwrite(size_t, size_t); 192 void __writegsbyte(unsigned long, unsigned char); 193 void __writegsdword(unsigned long, unsigned long); 194 void __writegsqword(unsigned long, unsigned __int64); 195 void __writegsword(unsigned long, unsigned short); 196 unsigned char _bittest64(__int64 const *, __int64); 197 unsigned char _bittestandcomplement64(__int64 *, __int64); 198 unsigned char _bittestandreset64(__int64 *, __int64); 199 unsigned char _bittestandset64(__int64 *, __int64); 200 long _InterlockedAnd_np(long volatile *_Value, long _Mask); 201 short _InterlockedAnd16_np(short volatile *_Value, short _Mask); 202 __int64 _InterlockedAnd64_np(__int64 volatile *_Value, __int64 _Mask); 203 char _InterlockedAnd8_np(char volatile *_Value, char _Mask); 204 unsigned char _interlockedbittestandreset64(__int64 volatile *, __int64); 205 unsigned char _interlockedbittestandset64(__int64 volatile *, __int64); 206 long _InterlockedCompareExchange_np(long volatile *_Destination, long _Exchange, 207 long _Comparand); 208 unsigned char _InterlockedCompareExchange128_np(__int64 volatile *_Destination, 209 __int64 _ExchangeHigh, 210 __int64 _ExchangeLow, 211 __int64 *_ComparandResult); 212 short _InterlockedCompareExchange16_np(short volatile *_Destination, 213 short _Exchange, short _Comparand); 214 __int64 _InterlockedCompareExchange64_np(__int64 volatile *_Destination, 215 __int64 _Exchange, __int64 _Comparand); 216 void *_InterlockedCompareExchangePointer_np(void *volatile *_Destination, 217 void *_Exchange, void *_Comparand); 218 long _InterlockedOr_np(long volatile *_Value, long _Mask); 219 short _InterlockedOr16_np(short volatile *_Value, short _Mask); 220 __int64 _InterlockedOr64_np(__int64 volatile *_Value, __int64 _Mask); 221 char _InterlockedOr8_np(char volatile *_Value, char _Mask); 222 long _InterlockedXor_np(long volatile *_Value, long _Mask); 223 short _InterlockedXor16_np(short volatile *_Value, short _Mask); 224 __int64 _InterlockedXor64_np(__int64 volatile *_Value, __int64 _Mask); 225 char _InterlockedXor8_np(char volatile *_Value, char _Mask); 226 unsigned __int64 _rorx_u64(unsigned __int64, const unsigned int); 227 __int64 _sarx_i64(__int64, unsigned int); 228 unsigned __int64 _shlx_u64(unsigned __int64, unsigned int); 229 unsigned __int64 _shrx_u64(unsigned __int64, unsigned int); 230 __int64 __mulh(__int64, __int64); 231 unsigned __int64 __umulh(unsigned __int64, unsigned __int64); 232 __int64 _mul128(__int64, __int64, __int64 *); 233 234 #endif /* __x86_64__ */ 235 236 /*----------------------------------------------------------------------------*\ 237 |* movs, stos 238 \*----------------------------------------------------------------------------*/ 239 240 #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__)) 241 static __inline__ void __DEFAULT_FN_ATTRS __movsb(unsigned char *__dst, 242 unsigned char const *__src, 243 size_t __n) { 244 #if defined(__x86_64__) 245 __asm__ __volatile__("rep movsb" 246 : "+D"(__dst), "+S"(__src), "+c"(__n) 247 : 248 : "memory"); 249 #else 250 __asm__ __volatile__("xchg {%%esi, %1|%1, esi}\n" 251 "rep movsb\n" 252 "xchg {%%esi, %1|%1, esi}" 253 : "+D"(__dst), "+r"(__src), "+c"(__n) 254 : 255 : "memory"); 256 #endif 257 } 258 static __inline__ void __DEFAULT_FN_ATTRS __movsd(unsigned long *__dst, 259 unsigned long const *__src, 260 size_t __n) { 261 #if defined(__x86_64__) 262 __asm__ __volatile__("rep movs{l|d}" 263 : "+D"(__dst), "+S"(__src), "+c"(__n) 264 : 265 : "memory"); 266 #else 267 __asm__ __volatile__("xchg {%%esi, %1|%1, esi}\n" 268 "rep movs{l|d}\n" 269 "xchg {%%esi, %1|%1, esi}" 270 : "+D"(__dst), "+r"(__src), "+c"(__n) 271 : 272 : "memory"); 273 #endif 274 } 275 static __inline__ void __DEFAULT_FN_ATTRS __movsw(unsigned short *__dst, 276 unsigned short const *__src, 277 size_t __n) { 278 #if defined(__x86_64__) 279 __asm__ __volatile__("rep movsw" 280 : "+D"(__dst), "+S"(__src), "+c"(__n) 281 : 282 : "memory"); 283 #else 284 __asm__ __volatile__("xchg {%%esi, %1|%1, esi}\n" 285 "rep movsw\n" 286 "xchg {%%esi, %1|%1, esi}" 287 : "+D"(__dst), "+r"(__src), "+c"(__n) 288 : 289 : "memory"); 290 #endif 291 } 292 static __inline__ void __DEFAULT_FN_ATTRS __stosd(unsigned long *__dst, 293 unsigned long __x, 294 size_t __n) { 295 __asm__ __volatile__("rep stos{l|d}" 296 : "+D"(__dst), "+c"(__n) 297 : "a"(__x) 298 : "memory"); 299 } 300 static __inline__ void __DEFAULT_FN_ATTRS __stosw(unsigned short *__dst, 301 unsigned short __x, 302 size_t __n) { 303 __asm__ __volatile__("rep stosw" 304 : "+D"(__dst), "+c"(__n) 305 : "a"(__x) 306 : "memory"); 307 } 308 #endif 309 #if defined(__x86_64__) && !defined(__arm64ec__) 310 static __inline__ void __DEFAULT_FN_ATTRS __movsq( 311 unsigned long long *__dst, unsigned long long const *__src, size_t __n) { 312 __asm__ __volatile__("rep movsq" 313 : "+D"(__dst), "+S"(__src), "+c"(__n) 314 : 315 : "memory"); 316 } 317 static __inline__ void __DEFAULT_FN_ATTRS __stosq(unsigned __int64 *__dst, 318 unsigned __int64 __x, 319 size_t __n) { 320 __asm__ __volatile__("rep stosq" : "+D"(__dst), "+c"(__n) : "a"(__x) 321 : "memory"); 322 } 323 #endif 324 325 /*----------------------------------------------------------------------------*\ 326 |* Misc 327 \*----------------------------------------------------------------------------*/ 328 #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__)) 329 static __inline__ void __DEFAULT_FN_ATTRS __halt(void) { 330 __asm__ volatile("hlt"); 331 } 332 333 static __inline__ unsigned char __inbyte(unsigned short port) { 334 unsigned char ret; 335 __asm__ __volatile__("inb %w1, %b0" : "=a"(ret) : "Nd"(port)); 336 return ret; 337 } 338 339 static __inline__ unsigned short __inword(unsigned short port) { 340 unsigned short ret; 341 __asm__ __volatile__("inw %w1, %w0" : "=a"(ret) : "Nd"(port)); 342 return ret; 343 } 344 345 static __inline__ unsigned long __indword(unsigned short port) { 346 unsigned long ret; 347 __asm__ __volatile__("inl %w1, %k0" : "=a"(ret) : "Nd"(port)); 348 return ret; 349 } 350 351 static __inline__ void __outbyte(unsigned short port, unsigned char data) { 352 __asm__ __volatile__("outb %b0, %w1" : : "a"(data), "Nd"(port)); 353 } 354 355 static __inline__ void __outword(unsigned short port, unsigned short data) { 356 __asm__ __volatile__("outw %w0, %w1" : : "a"(data), "Nd"(port)); 357 } 358 359 static __inline__ void __outdword(unsigned short port, unsigned long data) { 360 __asm__ __volatile__("outl %k0, %w1" : : "a"(data), "Nd"(port)); 361 } 362 #endif 363 364 #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) 365 static __inline__ void __DEFAULT_FN_ATTRS __nop(void) { 366 __asm__ volatile("nop"); 367 } 368 #endif 369 370 /*----------------------------------------------------------------------------*\ 371 |* MS AArch64 specific 372 \*----------------------------------------------------------------------------*/ 373 #if defined(__aarch64__) || defined(__arm64ec__) 374 unsigned __int64 __getReg(int); 375 long _InterlockedAdd(long volatile *Addend, long Value); 376 __int64 _InterlockedAdd64(__int64 volatile *Addend, __int64 Value); 377 __int64 _ReadStatusReg(int); 378 void _WriteStatusReg(int, __int64); 379 380 unsigned short __cdecl _byteswap_ushort(unsigned short val); 381 unsigned long __cdecl _byteswap_ulong (unsigned long val); 382 unsigned __int64 __cdecl _byteswap_uint64(unsigned __int64 val); 383 384 __int64 __mulh(__int64 __a, __int64 __b); 385 unsigned __int64 __umulh(unsigned __int64 __a, unsigned __int64 __b); 386 387 void __break(int); 388 389 void __writex18byte(unsigned long offset, unsigned char data); 390 void __writex18word(unsigned long offset, unsigned short data); 391 void __writex18dword(unsigned long offset, unsigned long data); 392 void __writex18qword(unsigned long offset, unsigned __int64 data); 393 394 unsigned char __readx18byte(unsigned long offset); 395 unsigned short __readx18word(unsigned long offset); 396 unsigned long __readx18dword(unsigned long offset); 397 unsigned __int64 __readx18qword(unsigned long offset); 398 399 void __addx18byte(unsigned long offset, unsigned char data); 400 void __addx18word(unsigned long offset, unsigned short data); 401 void __addx18dword(unsigned long offset, unsigned long data); 402 void __addx18qword(unsigned long offset, unsigned __int64 data); 403 404 void __incx18byte(unsigned long offset); 405 void __incx18word(unsigned long offset); 406 void __incx18dword(unsigned long offset); 407 void __incx18qword(unsigned long offset); 408 409 double _CopyDoubleFromInt64(__int64); 410 float _CopyFloatFromInt32(__int32); 411 __int32 _CopyInt32FromFloat(float); 412 __int64 _CopyInt64FromDouble(double); 413 414 unsigned int _CountLeadingOnes(unsigned long); 415 unsigned int _CountLeadingOnes64(unsigned __int64); 416 unsigned int _CountLeadingSigns(long); 417 unsigned int _CountLeadingSigns64(__int64); 418 unsigned int _CountOneBits(unsigned long); 419 unsigned int _CountOneBits64(unsigned __int64); 420 421 unsigned int __hlt(unsigned int, ...); 422 423 void __cdecl __prefetch(const void *); 424 425 #endif 426 427 /*----------------------------------------------------------------------------*\ 428 |* Privileged intrinsics 429 \*----------------------------------------------------------------------------*/ 430 #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__)) 431 static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 432 __readmsr(unsigned long __register) { 433 // Loads the contents of a 64-bit model specific register (MSR) specified in 434 // the ECX register into registers EDX:EAX. The EDX register is loaded with 435 // the high-order 32 bits of the MSR and the EAX register is loaded with the 436 // low-order 32 bits. If less than 64 bits are implemented in the MSR being 437 // read, the values returned to EDX:EAX in unimplemented bit locations are 438 // undefined. 439 unsigned long __edx; 440 unsigned long __eax; 441 __asm__ ("rdmsr" : "=d"(__edx), "=a"(__eax) : "c"(__register)); 442 return (((unsigned __int64)__edx) << 32) | (unsigned __int64)__eax; 443 } 444 445 static __inline__ unsigned __LPTRINT_TYPE__ __DEFAULT_FN_ATTRS __readcr3(void) { 446 unsigned __LPTRINT_TYPE__ __cr3_val; 447 __asm__ __volatile__( 448 "mov {%%cr3, %0|%0, cr3}" 449 : "=r"(__cr3_val) 450 : 451 : "memory"); 452 return __cr3_val; 453 } 454 455 static __inline__ void __DEFAULT_FN_ATTRS 456 __writecr3(unsigned __INTPTR_TYPE__ __cr3_val) { 457 __asm__ ("mov {%0, %%cr3|cr3, %0}" : : "r"(__cr3_val) : "memory"); 458 } 459 #endif 460 461 #ifdef __cplusplus 462 } 463 #endif 464 465 #undef __LPTRINT_TYPE__ 466 467 #undef __DEFAULT_FN_ATTRS 468 469 #endif /* __INTRIN_H */ 470 #endif /* _MSC_VER */