VirtualBox

source: vbox/trunk/include/iprt/asm.h@ 33138

最後變更 在這個檔案從33138是 33138,由 vboxsync 提交於 14 年 前

iprt/asm.h: build fix.
  • 屬性 svn:eol-style 設為 native
  • 屬性 svn:keywords 設為 Author Date Id Revision
檔案大小: 128.8 KB
行 
1/** @file
2 * IPRT - Assembly Functions.
3 */
4
5/*
6 * Copyright (C) 2006-2010 Oracle Corporation
7 *
8 * This file is part of VirtualBox Open Source Edition (OSE), as
9 * available from http://www.alldomusa.eu.org. This file is free software;
10 * you can redistribute it and/or modify it under the terms of the GNU
11 * General Public License (GPL) as published by the Free Software
12 * Foundation, in version 2 as it comes in the "COPYING" file of the
13 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
14 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
15 *
16 * The contents of this file may alternatively be used under the terms
17 * of the Common Development and Distribution License Version 1.0
18 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
19 * VirtualBox OSE distribution, in which case the provisions of the
20 * CDDL are applicable instead of those of the GPL.
21 *
22 * You may elect to license modified versions of this file under the
23 * terms and conditions of either the GPL or the CDDL or both.
24 */
25
26#ifndef ___iprt_asm_h
27#define ___iprt_asm_h
28
29#include <iprt/cdefs.h>
30#include <iprt/types.h>
31#include <iprt/assert.h>
32/** @def RT_INLINE_ASM_USES_INTRIN
33 * Defined as 1 if we're using a _MSC_VER 1400.
34 * Otherwise defined as 0.
35 */
36
37/* Solaris 10 header ugliness */
38#ifdef u
39# undef u
40#endif
41
42#if defined(_MSC_VER) && RT_INLINE_ASM_USES_INTRIN
43# include <intrin.h>
44 /* Emit the intrinsics at all optimization levels. */
45# pragma intrinsic(_ReadWriteBarrier)
46# pragma intrinsic(__cpuid)
47# pragma intrinsic(__stosd)
48# pragma intrinsic(__stosw)
49# pragma intrinsic(__stosb)
50# pragma intrinsic(_BitScanForward)
51# pragma intrinsic(_BitScanReverse)
52# pragma intrinsic(_bittest)
53# pragma intrinsic(_bittestandset)
54# pragma intrinsic(_bittestandreset)
55# pragma intrinsic(_bittestandcomplement)
56# pragma intrinsic(_byteswap_ushort)
57# pragma intrinsic(_byteswap_ulong)
58# pragma intrinsic(_interlockedbittestandset)
59# pragma intrinsic(_interlockedbittestandreset)
60# pragma intrinsic(_InterlockedAnd)
61# pragma intrinsic(_InterlockedOr)
62# pragma intrinsic(_InterlockedIncrement)
63# pragma intrinsic(_InterlockedDecrement)
64# pragma intrinsic(_InterlockedExchange)
65# pragma intrinsic(_InterlockedExchangeAdd)
66# pragma intrinsic(_InterlockedCompareExchange)
67# pragma intrinsic(_InterlockedCompareExchange64)
68# ifdef RT_ARCH_AMD64
69# pragma intrinsic(__stosq)
70# pragma intrinsic(_byteswap_uint64)
71# pragma intrinsic(_InterlockedExchange64)
72# pragma intrinsic(_InterlockedExchangeAdd64)
73# pragma intrinsic(_InterlockedAnd64)
74# pragma intrinsic(_InterlockedOr64)
75# pragma intrinsic(_InterlockedIncrement64)
76# pragma intrinsic(_InterlockedDecrement64)
77# endif
78#endif
79
80
81/** @defgroup grp_rt_asm ASM - Assembly Routines
82 * @ingroup grp_rt
83 *
84 * @remarks The difference between ordered and unordered atomic operations are that
85 * the former will complete outstanding reads and writes before continuing
86 * while the latter doesn't make any promisses about the order. Ordered
87 * operations doesn't, it seems, make any 100% promise wrt to whether
88 * the operation will complete before any subsequent memory access.
89 * (please, correct if wrong.)
90 *
91 * ASMAtomicSomething operations are all ordered, while ASMAtomicUoSomething
92 * are unordered (note the Uo).
93 *
94 * @remarks Some remarks about __volatile__: Without this keyword gcc is allowed to reorder
95 * or even optimize assembler instructions away. For instance, in the following code
96 * the second rdmsr instruction is optimized away because gcc treats that instruction
97 * as deterministic:
98 *
99 * @code
100 * static inline uint64_t rdmsr_low(int idx)
101 * {
102 * uint32_t low;
103 * __asm__ ("rdmsr" : "=a"(low) : "c"(idx) : "edx");
104 * }
105 * ...
106 * uint32_t msr1 = rdmsr_low(1);
107 * foo(msr1);
108 * msr1 = rdmsr_low(1);
109 * bar(msr1);
110 * @endcode
111 *
112 * The input parameter of rdmsr_low is the same for both calls and therefore gcc will
113 * use the result of the first call as input parameter for bar() as well. For rdmsr this
114 * is not acceptable as this instruction is _not_ deterministic. This applies to reading
115 * machine status information in general.
116 *
117 * @{
118 */
119
120
121/** @def RT_INLINE_ASM_GCC_4_3_X_X86
122 * Used to work around some 4.3.x register allocation issues in this version of
123 * the compiler. So far this workaround is still required for 4.4 and 4.5. */
124#ifdef __GNUC__
125# define RT_INLINE_ASM_GCC_4_3_X_X86 (__GNUC__ == 4 && __GNUC_MINOR__ >= 3 && defined(__i386__))
126#endif
127#ifndef RT_INLINE_ASM_GCC_4_3_X_X86
128# define RT_INLINE_ASM_GCC_4_3_X_X86 0
129#endif
130
131/** @def RT_INLINE_DONT_USE_CMPXCHG8B
132 * i686-apple-darwin9-gcc-4.0.1 (GCC) 4.0.1 (Apple Inc. build 5493) screws up
133 * RTSemRWRequestWrite semsemrw-lockless-generic.cpp in release builds. PIC
134 * mode, x86.
135 *
136 * Some gcc 4.3.x versions may have register allocation issues with cmpxchg8b
137 * when in PIC mode on x86.
138 */
139#ifndef RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
140# define RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC \
141 ( (defined(PIC) || defined(__PIC__)) \
142 && defined(RT_ARCH_X86) \
143 && ( RT_INLINE_ASM_GCC_4_3_X_X86 \
144 || defined(RT_OS_DARWIN)) )
145#endif
146
147
148/** @def ASMReturnAddress
149 * Gets the return address of the current (or calling if you like) function or method.
150 */
151#ifdef _MSC_VER
152# ifdef __cplusplus
153extern "C"
154# endif
155void * _ReturnAddress(void);
156# pragma intrinsic(_ReturnAddress)
157# define ASMReturnAddress() _ReturnAddress()
158#elif defined(__GNUC__) || defined(DOXYGEN_RUNNING)
159# define ASMReturnAddress() __builtin_return_address(0)
160#else
161# error "Unsupported compiler."
162#endif
163
164
165/**
166 * Compiler memory barrier.
167 *
168 * Ensure that the compiler does not use any cached (register/tmp stack) memory
169 * values or any outstanding writes when returning from this function.
170 *
171 * This function must be used if non-volatile data is modified by a
172 * device or the VMM. Typical cases are port access, MMIO access,
173 * trapping instruction, etc.
174 */
175#if RT_INLINE_ASM_GNU_STYLE
176# define ASMCompilerBarrier() do { __asm__ __volatile__("" : : : "memory"); } while (0)
177#elif RT_INLINE_ASM_USES_INTRIN
178# define ASMCompilerBarrier() do { _ReadWriteBarrier(); } while (0)
179#else /* 2003 should have _ReadWriteBarrier() but I guess we're at 2002 level then... */
180DECLINLINE(void) ASMCompilerBarrier(void)
181{
182 __asm
183 {
184 }
185}
186#endif
187
188
189/** @def ASMBreakpoint
190 * Debugger Breakpoint.
191 * @remark In the gnu world we add a nop instruction after the int3 to
192 * force gdb to remain at the int3 source line.
193 * @remark The L4 kernel will try make sense of the breakpoint, thus the jmp.
194 * @internal
195 */
196#if RT_INLINE_ASM_GNU_STYLE
197# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
198# ifndef __L4ENV__
199# define ASMBreakpoint() do { __asm__ __volatile__("int3\n\tnop"); } while (0)
200# else
201# define ASMBreakpoint() do { __asm__ __volatile__("int3; jmp 1f; 1:"); } while (0)
202# endif
203# elif defined(RT_ARCH_SPARC64)
204# define ASMBreakpoint() do { __asm__ __volatile__("illtrap 0\n\t") } while (0) /** @todo Sparc64: this is just a wild guess. */
205# elif defined(RT_ARCH_SPARC)
206# define ASMBreakpoint() do { __asm__ __volatile__("unimp 0\n\t"); } while (0) /** @todo Sparc: this is just a wild guess (same as Sparc64, just different name). */
207# else
208# error "PORTME"
209# endif
210#else
211# define ASMBreakpoint() __debugbreak()
212#endif
213
214
215/**
216 * Spinloop hint for platforms that have these, empty function on the other
217 * platforms.
218 *
219 * x86 & AMD64: The PAUSE variant of NOP for helping hyperthreaded CPUs detecing
220 * spin locks.
221 */
222#if RT_INLINE_ASM_EXTERNAL && (defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86))
223DECLASM(void) ASMNopPause(void);
224#else
225DECLINLINE(void) ASMNopPause(void)
226{
227# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
228# if RT_INLINE_ASM_GNU_STYLE
229 __asm__ __volatile__(".byte 0xf3,0x90\n\t");
230# else
231 __asm {
232 _emit 0f3h
233 _emit 090h
234 }
235# endif
236# else
237 /* dummy */
238# endif
239}
240#endif
241
242
243/**
244 * Atomically Exchange an unsigned 8-bit value, ordered.
245 *
246 * @returns Current *pu8 value
247 * @param pu8 Pointer to the 8-bit variable to update.
248 * @param u8 The 8-bit value to assign to *pu8.
249 */
250#if RT_INLINE_ASM_EXTERNAL
251DECLASM(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8);
252#else
253DECLINLINE(uint8_t) ASMAtomicXchgU8(volatile uint8_t *pu8, uint8_t u8)
254{
255# if RT_INLINE_ASM_GNU_STYLE
256 __asm__ __volatile__("xchgb %0, %1\n\t"
257 : "=m" (*pu8),
258 "=q" (u8) /* =r - busted on g++ (GCC) 3.4.4 20050721 (Red Hat 3.4.4-2) */
259 : "1" (u8),
260 "m" (*pu8));
261# else
262 __asm
263 {
264# ifdef RT_ARCH_AMD64
265 mov rdx, [pu8]
266 mov al, [u8]
267 xchg [rdx], al
268 mov [u8], al
269# else
270 mov edx, [pu8]
271 mov al, [u8]
272 xchg [edx], al
273 mov [u8], al
274# endif
275 }
276# endif
277 return u8;
278}
279#endif
280
281
282/**
283 * Atomically Exchange a signed 8-bit value, ordered.
284 *
285 * @returns Current *pu8 value
286 * @param pi8 Pointer to the 8-bit variable to update.
287 * @param i8 The 8-bit value to assign to *pi8.
288 */
289DECLINLINE(int8_t) ASMAtomicXchgS8(volatile int8_t *pi8, int8_t i8)
290{
291 return (int8_t)ASMAtomicXchgU8((volatile uint8_t *)pi8, (uint8_t)i8);
292}
293
294
295/**
296 * Atomically Exchange a bool value, ordered.
297 *
298 * @returns Current *pf value
299 * @param pf Pointer to the 8-bit variable to update.
300 * @param f The 8-bit value to assign to *pi8.
301 */
302DECLINLINE(bool) ASMAtomicXchgBool(volatile bool *pf, bool f)
303{
304#ifdef _MSC_VER
305 return !!ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
306#else
307 return (bool)ASMAtomicXchgU8((volatile uint8_t *)pf, (uint8_t)f);
308#endif
309}
310
311
312/**
313 * Atomically Exchange an unsigned 16-bit value, ordered.
314 *
315 * @returns Current *pu16 value
316 * @param pu16 Pointer to the 16-bit variable to update.
317 * @param u16 The 16-bit value to assign to *pu16.
318 */
319#if RT_INLINE_ASM_EXTERNAL
320DECLASM(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16);
321#else
322DECLINLINE(uint16_t) ASMAtomicXchgU16(volatile uint16_t *pu16, uint16_t u16)
323{
324# if RT_INLINE_ASM_GNU_STYLE
325 __asm__ __volatile__("xchgw %0, %1\n\t"
326 : "=m" (*pu16),
327 "=r" (u16)
328 : "1" (u16),
329 "m" (*pu16));
330# else
331 __asm
332 {
333# ifdef RT_ARCH_AMD64
334 mov rdx, [pu16]
335 mov ax, [u16]
336 xchg [rdx], ax
337 mov [u16], ax
338# else
339 mov edx, [pu16]
340 mov ax, [u16]
341 xchg [edx], ax
342 mov [u16], ax
343# endif
344 }
345# endif
346 return u16;
347}
348#endif
349
350
351/**
352 * Atomically Exchange a signed 16-bit value, ordered.
353 *
354 * @returns Current *pu16 value
355 * @param pi16 Pointer to the 16-bit variable to update.
356 * @param i16 The 16-bit value to assign to *pi16.
357 */
358DECLINLINE(int16_t) ASMAtomicXchgS16(volatile int16_t *pi16, int16_t i16)
359{
360 return (int16_t)ASMAtomicXchgU16((volatile uint16_t *)pi16, (uint16_t)i16);
361}
362
363
364/**
365 * Atomically Exchange an unsigned 32-bit value, ordered.
366 *
367 * @returns Current *pu32 value
368 * @param pu32 Pointer to the 32-bit variable to update.
369 * @param u32 The 32-bit value to assign to *pu32.
370 */
371#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
372DECLASM(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32);
373#else
374DECLINLINE(uint32_t) ASMAtomicXchgU32(volatile uint32_t *pu32, uint32_t u32)
375{
376# if RT_INLINE_ASM_GNU_STYLE
377 __asm__ __volatile__("xchgl %0, %1\n\t"
378 : "=m" (*pu32),
379 "=r" (u32)
380 : "1" (u32),
381 "m" (*pu32));
382
383# elif RT_INLINE_ASM_USES_INTRIN
384 u32 = _InterlockedExchange((long *)pu32, u32);
385
386# else
387 __asm
388 {
389# ifdef RT_ARCH_AMD64
390 mov rdx, [pu32]
391 mov eax, u32
392 xchg [rdx], eax
393 mov [u32], eax
394# else
395 mov edx, [pu32]
396 mov eax, u32
397 xchg [edx], eax
398 mov [u32], eax
399# endif
400 }
401# endif
402 return u32;
403}
404#endif
405
406
407/**
408 * Atomically Exchange a signed 32-bit value, ordered.
409 *
410 * @returns Current *pu32 value
411 * @param pi32 Pointer to the 32-bit variable to update.
412 * @param i32 The 32-bit value to assign to *pi32.
413 */
414DECLINLINE(int32_t) ASMAtomicXchgS32(volatile int32_t *pi32, int32_t i32)
415{
416 return (int32_t)ASMAtomicXchgU32((volatile uint32_t *)pi32, (uint32_t)i32);
417}
418
419
420/**
421 * Atomically Exchange an unsigned 64-bit value, ordered.
422 *
423 * @returns Current *pu64 value
424 * @param pu64 Pointer to the 64-bit variable to update.
425 * @param u64 The 64-bit value to assign to *pu64.
426 */
427#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
428 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
429DECLASM(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64);
430#else
431DECLINLINE(uint64_t) ASMAtomicXchgU64(volatile uint64_t *pu64, uint64_t u64)
432{
433# if defined(RT_ARCH_AMD64)
434# if RT_INLINE_ASM_USES_INTRIN
435 u64 = _InterlockedExchange64((__int64 *)pu64, u64);
436
437# elif RT_INLINE_ASM_GNU_STYLE
438 __asm__ __volatile__("xchgq %0, %1\n\t"
439 : "=m" (*pu64),
440 "=r" (u64)
441 : "1" (u64),
442 "m" (*pu64));
443# else
444 __asm
445 {
446 mov rdx, [pu64]
447 mov rax, [u64]
448 xchg [rdx], rax
449 mov [u64], rax
450 }
451# endif
452# else /* !RT_ARCH_AMD64 */
453# if RT_INLINE_ASM_GNU_STYLE
454# if defined(PIC) || defined(__PIC__)
455 uint32_t u32EBX = (uint32_t)u64;
456 __asm__ __volatile__(/*"xchgl %%esi, %5\n\t"*/
457 "xchgl %%ebx, %3\n\t"
458 "1:\n\t"
459 "lock; cmpxchg8b (%5)\n\t"
460 "jnz 1b\n\t"
461 "movl %3, %%ebx\n\t"
462 /*"xchgl %%esi, %5\n\t"*/
463 : "=A" (u64),
464 "=m" (*pu64)
465 : "0" (*pu64),
466 "m" ( u32EBX ),
467 "c" ( (uint32_t)(u64 >> 32) ),
468 "S" (pu64));
469# else /* !PIC */
470 __asm__ __volatile__("1:\n\t"
471 "lock; cmpxchg8b %1\n\t"
472 "jnz 1b\n\t"
473 : "=A" (u64),
474 "=m" (*pu64)
475 : "0" (*pu64),
476 "b" ( (uint32_t)u64 ),
477 "c" ( (uint32_t)(u64 >> 32) ));
478# endif
479# else
480 __asm
481 {
482 mov ebx, dword ptr [u64]
483 mov ecx, dword ptr [u64 + 4]
484 mov edi, pu64
485 mov eax, dword ptr [edi]
486 mov edx, dword ptr [edi + 4]
487 retry:
488 lock cmpxchg8b [edi]
489 jnz retry
490 mov dword ptr [u64], eax
491 mov dword ptr [u64 + 4], edx
492 }
493# endif
494# endif /* !RT_ARCH_AMD64 */
495 return u64;
496}
497#endif
498
499
500/**
501 * Atomically Exchange an signed 64-bit value, ordered.
502 *
503 * @returns Current *pi64 value
504 * @param pi64 Pointer to the 64-bit variable to update.
505 * @param i64 The 64-bit value to assign to *pi64.
506 */
507DECLINLINE(int64_t) ASMAtomicXchgS64(volatile int64_t *pi64, int64_t i64)
508{
509 return (int64_t)ASMAtomicXchgU64((volatile uint64_t *)pi64, (uint64_t)i64);
510}
511
512
513/**
514 * Atomically Exchange a pointer value, ordered.
515 *
516 * @returns Current *ppv value
517 * @param ppv Pointer to the pointer variable to update.
518 * @param pv The pointer value to assign to *ppv.
519 */
520DECLINLINE(void *) ASMAtomicXchgPtr(void * volatile *ppv, const void *pv)
521{
522#if ARCH_BITS == 32
523 return (void *)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppv, (uint32_t)pv);
524#elif ARCH_BITS == 64
525 return (void *)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppv, (uint64_t)pv);
526#else
527# error "ARCH_BITS is bogus"
528#endif
529}
530
531
532/**
533 * Convenience macro for avoiding the annoying casting with ASMAtomicXchgPtr.
534 *
535 * @returns Current *pv value
536 * @param ppv Pointer to the pointer variable to update.
537 * @param pv The pointer value to assign to *ppv.
538 * @param Type The type of *ppv, sans volatile.
539 */
540#ifdef __GNUC__
541# define ASMAtomicXchgPtrT(ppv, pv, Type) \
542 __extension__ \
543 ({\
544 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
545 Type const pvTypeChecked = (pv); \
546 Type pvTypeCheckedRet = (__typeof__(*(ppv))) ASMAtomicXchgPtr((void * volatile *)ppvTypeChecked, (void *)pvTypeChecked); \
547 pvTypeCheckedRet; \
548 })
549#else
550# define ASMAtomicXchgPtrT(ppv, pv, Type) \
551 (Type)ASMAtomicXchgPtr((void * volatile *)(ppv), (void *)(pv))
552#endif
553
554
555/**
556 * Atomically Exchange a raw-mode context pointer value, ordered.
557 *
558 * @returns Current *ppv value
559 * @param ppvRC Pointer to the pointer variable to update.
560 * @param pvRC The pointer value to assign to *ppv.
561 */
562DECLINLINE(RTRCPTR) ASMAtomicXchgRCPtr(RTRCPTR volatile *ppvRC, RTRCPTR pvRC)
563{
564 return (RTRCPTR)ASMAtomicXchgU32((uint32_t volatile *)(void *)ppvRC, (uint32_t)pvRC);
565}
566
567
568/**
569 * Atomically Exchange a ring-0 pointer value, ordered.
570 *
571 * @returns Current *ppv value
572 * @param ppvR0 Pointer to the pointer variable to update.
573 * @param pvR0 The pointer value to assign to *ppv.
574 */
575DECLINLINE(RTR0PTR) ASMAtomicXchgR0Ptr(RTR0PTR volatile *ppvR0, RTR0PTR pvR0)
576{
577#if R0_ARCH_BITS == 32
578 return (RTR0PTR)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppvR0, (uint32_t)pvR0);
579#elif R0_ARCH_BITS == 64
580 return (RTR0PTR)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppvR0, (uint64_t)pvR0);
581#else
582# error "R0_ARCH_BITS is bogus"
583#endif
584}
585
586
587/**
588 * Atomically Exchange a ring-3 pointer value, ordered.
589 *
590 * @returns Current *ppv value
591 * @param ppvR3 Pointer to the pointer variable to update.
592 * @param pvR3 The pointer value to assign to *ppv.
593 */
594DECLINLINE(RTR3PTR) ASMAtomicXchgR3Ptr(RTR3PTR volatile *ppvR3, RTR3PTR pvR3)
595{
596#if R3_ARCH_BITS == 32
597 return (RTR3PTR)ASMAtomicXchgU32((volatile uint32_t *)(void *)ppvR3, (uint32_t)pvR3);
598#elif R3_ARCH_BITS == 64
599 return (RTR3PTR)ASMAtomicXchgU64((volatile uint64_t *)(void *)ppvR3, (uint64_t)pvR3);
600#else
601# error "R3_ARCH_BITS is bogus"
602#endif
603}
604
605
606/** @def ASMAtomicXchgHandle
607 * Atomically Exchange a typical IPRT handle value, ordered.
608 *
609 * @param ph Pointer to the value to update.
610 * @param hNew The new value to assigned to *pu.
611 * @param phRes Where to store the current *ph value.
612 *
613 * @remarks This doesn't currently work for all handles (like RTFILE).
614 */
615#if HC_ARCH_BITS == 32
616# define ASMAtomicXchgHandle(ph, hNew, phRes) \
617 do { \
618 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
619 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
620 *(uint32_t *)(phRes) = ASMAtomicXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
621 } while (0)
622#elif HC_ARCH_BITS == 64
623# define ASMAtomicXchgHandle(ph, hNew, phRes) \
624 do { \
625 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
626 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
627 *(uint64_t *)(phRes) = ASMAtomicXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
628 } while (0)
629#else
630# error HC_ARCH_BITS
631#endif
632
633
634/**
635 * Atomically Exchange a value which size might differ
636 * between platforms or compilers, ordered.
637 *
638 * @param pu Pointer to the variable to update.
639 * @param uNew The value to assign to *pu.
640 * @todo This is busted as its missing the result argument.
641 */
642#define ASMAtomicXchgSize(pu, uNew) \
643 do { \
644 switch (sizeof(*(pu))) { \
645 case 1: ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
646 case 2: ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
647 case 4: ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
648 case 8: ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
649 default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
650 } \
651 } while (0)
652
653/**
654 * Atomically Exchange a value which size might differ
655 * between platforms or compilers, ordered.
656 *
657 * @param pu Pointer to the variable to update.
658 * @param uNew The value to assign to *pu.
659 * @param puRes Where to store the current *pu value.
660 */
661#define ASMAtomicXchgSizeCorrect(pu, uNew, puRes) \
662 do { \
663 switch (sizeof(*(pu))) { \
664 case 1: *(uint8_t *)(puRes) = ASMAtomicXchgU8((volatile uint8_t *)(void *)(pu), (uint8_t)(uNew)); break; \
665 case 2: *(uint16_t *)(puRes) = ASMAtomicXchgU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
666 case 4: *(uint32_t *)(puRes) = ASMAtomicXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
667 case 8: *(uint64_t *)(puRes) = ASMAtomicXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
668 default: AssertMsgFailed(("ASMAtomicXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
669 } \
670 } while (0)
671
672
673
674/**
675 * Atomically Compare and Exchange an unsigned 8-bit value, ordered.
676 *
677 * @returns true if xchg was done.
678 * @returns false if xchg wasn't done.
679 *
680 * @param pu8 Pointer to the value to update.
681 * @param u8New The new value to assigned to *pu8.
682 * @param u8Old The old value to *pu8 compare with.
683 */
684#if RT_INLINE_ASM_EXTERNAL || !RT_INLINE_ASM_GNU_STYLE
685DECLASM(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, const uint8_t u8Old);
686#else
687DECLINLINE(bool) ASMAtomicCmpXchgU8(volatile uint8_t *pu8, const uint8_t u8New, uint8_t u8Old)
688{
689 uint8_t u8Ret;
690 __asm__ __volatile__("lock; cmpxchgb %3, %0\n\t"
691 "setz %1\n\t"
692 : "=m" (*pu8),
693 "=qm" (u8Ret),
694 "=a" (u8Old)
695 : "q" (u8New),
696 "2" (u8Old),
697 "m" (*pu8));
698 return (bool)u8Ret;
699}
700#endif
701
702
703/**
704 * Atomically Compare and Exchange a signed 8-bit value, ordered.
705 *
706 * @returns true if xchg was done.
707 * @returns false if xchg wasn't done.
708 *
709 * @param pi8 Pointer to the value to update.
710 * @param i8New The new value to assigned to *pi8.
711 * @param i8Old The old value to *pi8 compare with.
712 */
713DECLINLINE(bool) ASMAtomicCmpXchgS8(volatile int8_t *pi8, const int8_t i8New, const int8_t i8Old)
714{
715 return ASMAtomicCmpXchgU8((volatile uint8_t *)pi8, (const uint8_t)i8New, (const uint8_t)i8Old);
716}
717
718
719/**
720 * Atomically Compare and Exchange a bool value, ordered.
721 *
722 * @returns true if xchg was done.
723 * @returns false if xchg wasn't done.
724 *
725 * @param pf Pointer to the value to update.
726 * @param fNew The new value to assigned to *pf.
727 * @param fOld The old value to *pf compare with.
728 */
729DECLINLINE(bool) ASMAtomicCmpXchgBool(volatile bool *pf, const bool fNew, const bool fOld)
730{
731 return ASMAtomicCmpXchgU8((volatile uint8_t *)pf, (const uint8_t)fNew, (const uint8_t)fOld);
732}
733
734
735/**
736 * Atomically Compare and Exchange an unsigned 32-bit value, ordered.
737 *
738 * @returns true if xchg was done.
739 * @returns false if xchg wasn't done.
740 *
741 * @param pu32 Pointer to the value to update.
742 * @param u32New The new value to assigned to *pu32.
743 * @param u32Old The old value to *pu32 compare with.
744 */
745#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
746DECLASM(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old);
747#else
748DECLINLINE(bool) ASMAtomicCmpXchgU32(volatile uint32_t *pu32, const uint32_t u32New, uint32_t u32Old)
749{
750# if RT_INLINE_ASM_GNU_STYLE
751 uint8_t u8Ret;
752 __asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
753 "setz %1\n\t"
754 : "=m" (*pu32),
755 "=qm" (u8Ret),
756 "=a" (u32Old)
757 : "r" (u32New),
758 "2" (u32Old),
759 "m" (*pu32));
760 return (bool)u8Ret;
761
762# elif RT_INLINE_ASM_USES_INTRIN
763 return _InterlockedCompareExchange((long *)pu32, u32New, u32Old) == u32Old;
764
765# else
766 uint32_t u32Ret;
767 __asm
768 {
769# ifdef RT_ARCH_AMD64
770 mov rdx, [pu32]
771# else
772 mov edx, [pu32]
773# endif
774 mov eax, [u32Old]
775 mov ecx, [u32New]
776# ifdef RT_ARCH_AMD64
777 lock cmpxchg [rdx], ecx
778# else
779 lock cmpxchg [edx], ecx
780# endif
781 setz al
782 movzx eax, al
783 mov [u32Ret], eax
784 }
785 return !!u32Ret;
786# endif
787}
788#endif
789
790
791/**
792 * Atomically Compare and Exchange a signed 32-bit value, ordered.
793 *
794 * @returns true if xchg was done.
795 * @returns false if xchg wasn't done.
796 *
797 * @param pi32 Pointer to the value to update.
798 * @param i32New The new value to assigned to *pi32.
799 * @param i32Old The old value to *pi32 compare with.
800 */
801DECLINLINE(bool) ASMAtomicCmpXchgS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old)
802{
803 return ASMAtomicCmpXchgU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old);
804}
805
806
807/**
808 * Atomically Compare and exchange an unsigned 64-bit value, ordered.
809 *
810 * @returns true if xchg was done.
811 * @returns false if xchg wasn't done.
812 *
813 * @param pu64 Pointer to the 64-bit variable to update.
814 * @param u64New The 64-bit value to assign to *pu64.
815 * @param u64Old The value to compare with.
816 */
817#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
818 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
819DECLASM(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old);
820#else
821DECLINLINE(bool) ASMAtomicCmpXchgU64(volatile uint64_t *pu64, uint64_t u64New, uint64_t u64Old)
822{
823# if RT_INLINE_ASM_USES_INTRIN
824 return _InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old) == u64Old;
825
826# elif defined(RT_ARCH_AMD64)
827# if RT_INLINE_ASM_GNU_STYLE
828 uint8_t u8Ret;
829 __asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
830 "setz %1\n\t"
831 : "=m" (*pu64),
832 "=qm" (u8Ret),
833 "=a" (u64Old)
834 : "r" (u64New),
835 "2" (u64Old),
836 "m" (*pu64));
837 return (bool)u8Ret;
838# else
839 bool fRet;
840 __asm
841 {
842 mov rdx, [pu32]
843 mov rax, [u64Old]
844 mov rcx, [u64New]
845 lock cmpxchg [rdx], rcx
846 setz al
847 mov [fRet], al
848 }
849 return fRet;
850# endif
851# else /* !RT_ARCH_AMD64 */
852 uint32_t u32Ret;
853# if RT_INLINE_ASM_GNU_STYLE
854# if defined(PIC) || defined(__PIC__)
855 uint32_t u32EBX = (uint32_t)u64New;
856 uint32_t u32Spill;
857 __asm__ __volatile__("xchgl %%ebx, %4\n\t"
858 "lock; cmpxchg8b (%6)\n\t"
859 "setz %%al\n\t"
860 "movl %4, %%ebx\n\t"
861 "movzbl %%al, %%eax\n\t"
862 : "=a" (u32Ret),
863 "=d" (u32Spill),
864# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
865 "+m" (*pu64)
866# else
867 "=m" (*pu64)
868# endif
869 : "A" (u64Old),
870 "m" ( u32EBX ),
871 "c" ( (uint32_t)(u64New >> 32) ),
872 "S" (pu64));
873# else /* !PIC */
874 uint32_t u32Spill;
875 __asm__ __volatile__("lock; cmpxchg8b %2\n\t"
876 "setz %%al\n\t"
877 "movzbl %%al, %%eax\n\t"
878 : "=a" (u32Ret),
879 "=d" (u32Spill),
880 "+m" (*pu64)
881 : "A" (u64Old),
882 "b" ( (uint32_t)u64New ),
883 "c" ( (uint32_t)(u64New >> 32) ));
884# endif
885 return (bool)u32Ret;
886# else
887 __asm
888 {
889 mov ebx, dword ptr [u64New]
890 mov ecx, dword ptr [u64New + 4]
891 mov edi, [pu64]
892 mov eax, dword ptr [u64Old]
893 mov edx, dword ptr [u64Old + 4]
894 lock cmpxchg8b [edi]
895 setz al
896 movzx eax, al
897 mov dword ptr [u32Ret], eax
898 }
899 return !!u32Ret;
900# endif
901# endif /* !RT_ARCH_AMD64 */
902}
903#endif
904
905
906/**
907 * Atomically Compare and exchange a signed 64-bit value, ordered.
908 *
909 * @returns true if xchg was done.
910 * @returns false if xchg wasn't done.
911 *
912 * @param pi64 Pointer to the 64-bit variable to update.
913 * @param i64 The 64-bit value to assign to *pu64.
914 * @param i64Old The value to compare with.
915 */
916DECLINLINE(bool) ASMAtomicCmpXchgS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old)
917{
918 return ASMAtomicCmpXchgU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old);
919}
920
921
922/**
923 * Atomically Compare and Exchange a pointer value, ordered.
924 *
925 * @returns true if xchg was done.
926 * @returns false if xchg wasn't done.
927 *
928 * @param ppv Pointer to the value to update.
929 * @param pvNew The new value to assigned to *ppv.
930 * @param pvOld The old value to *ppv compare with.
931 */
932DECLINLINE(bool) ASMAtomicCmpXchgPtrVoid(void * volatile *ppv, const void *pvNew, const void *pvOld)
933{
934#if ARCH_BITS == 32
935 return ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld);
936#elif ARCH_BITS == 64
937 return ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld);
938#else
939# error "ARCH_BITS is bogus"
940#endif
941}
942
943
944/**
945 * Atomically Compare and Exchange a pointer value, ordered.
946 *
947 * @returns true if xchg was done.
948 * @returns false if xchg wasn't done.
949 *
950 * @param ppv Pointer to the value to update.
951 * @param pvNew The new value to assigned to *ppv.
952 * @param pvOld The old value to *ppv compare with.
953 *
954 * @remarks This is relatively type safe on GCC platforms.
955 */
956#ifdef __GNUC__
957# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
958 __extension__ \
959 ({\
960 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
961 __typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
962 __typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
963 bool fMacroRet = ASMAtomicCmpXchgPtrVoid((void * volatile *)ppvTypeChecked, \
964 (void *)pvNewTypeChecked, (void *)pvOldTypeChecked); \
965 fMacroRet; \
966 })
967#else
968# define ASMAtomicCmpXchgPtr(ppv, pvNew, pvOld) \
969 ASMAtomicCmpXchgPtrVoid((void * volatile *)(ppv), (void *)(pvNew), (void *)(pvOld))
970#endif
971
972
973/** @def ASMAtomicCmpXchgHandle
974 * Atomically Compare and Exchange a typical IPRT handle value, ordered.
975 *
976 * @param ph Pointer to the value to update.
977 * @param hNew The new value to assigned to *pu.
978 * @param hOld The old value to *pu compare with.
979 * @param fRc Where to store the result.
980 *
981 * @remarks This doesn't currently work for all handles (like RTFILE).
982 */
983#if HC_ARCH_BITS == 32
984# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
985 do { \
986 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
987 (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)(ph), (const uint32_t)(hNew), (const uint32_t)(hOld)); \
988 } while (0)
989#elif HC_ARCH_BITS == 64
990# define ASMAtomicCmpXchgHandle(ph, hNew, hOld, fRc) \
991 do { \
992 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
993 (fRc) = ASMAtomicCmpXchgU64((uint64_t volatile *)(ph), (const uint64_t)(hNew), (const uint64_t)(hOld)); \
994 } while (0)
995#else
996# error HC_ARCH_BITS
997#endif
998
999
1000/** @def ASMAtomicCmpXchgSize
1001 * Atomically Compare and Exchange a value which size might differ
1002 * between platforms or compilers, ordered.
1003 *
1004 * @param pu Pointer to the value to update.
1005 * @param uNew The new value to assigned to *pu.
1006 * @param uOld The old value to *pu compare with.
1007 * @param fRc Where to store the result.
1008 */
1009#define ASMAtomicCmpXchgSize(pu, uNew, uOld, fRc) \
1010 do { \
1011 switch (sizeof(*(pu))) { \
1012 case 4: (fRc) = ASMAtomicCmpXchgU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld)); \
1013 break; \
1014 case 8: (fRc) = ASMAtomicCmpXchgU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld)); \
1015 break; \
1016 default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1017 (fRc) = false; \
1018 break; \
1019 } \
1020 } while (0)
1021
1022
1023/**
1024 * Atomically Compare and Exchange an unsigned 32-bit value, additionally
1025 * passes back old value, ordered.
1026 *
1027 * @returns true if xchg was done.
1028 * @returns false if xchg wasn't done.
1029 *
1030 * @param pu32 Pointer to the value to update.
1031 * @param u32New The new value to assigned to *pu32.
1032 * @param u32Old The old value to *pu32 compare with.
1033 * @param pu32Old Pointer store the old value at.
1034 */
1035#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1036DECLASM(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old);
1037#else
1038DECLINLINE(bool) ASMAtomicCmpXchgExU32(volatile uint32_t *pu32, const uint32_t u32New, const uint32_t u32Old, uint32_t *pu32Old)
1039{
1040# if RT_INLINE_ASM_GNU_STYLE
1041 uint8_t u8Ret;
1042 __asm__ __volatile__("lock; cmpxchgl %3, %0\n\t"
1043 "setz %1\n\t"
1044 : "=m" (*pu32),
1045 "=qm" (u8Ret),
1046 "=a" (*pu32Old)
1047 : "r" (u32New),
1048 "a" (u32Old),
1049 "m" (*pu32));
1050 return (bool)u8Ret;
1051
1052# elif RT_INLINE_ASM_USES_INTRIN
1053 return (*pu32Old =_InterlockedCompareExchange((long *)pu32, u32New, u32Old)) == u32Old;
1054
1055# else
1056 uint32_t u32Ret;
1057 __asm
1058 {
1059# ifdef RT_ARCH_AMD64
1060 mov rdx, [pu32]
1061# else
1062 mov edx, [pu32]
1063# endif
1064 mov eax, [u32Old]
1065 mov ecx, [u32New]
1066# ifdef RT_ARCH_AMD64
1067 lock cmpxchg [rdx], ecx
1068 mov rdx, [pu32Old]
1069 mov [rdx], eax
1070# else
1071 lock cmpxchg [edx], ecx
1072 mov edx, [pu32Old]
1073 mov [edx], eax
1074# endif
1075 setz al
1076 movzx eax, al
1077 mov [u32Ret], eax
1078 }
1079 return !!u32Ret;
1080# endif
1081}
1082#endif
1083
1084
1085/**
1086 * Atomically Compare and Exchange a signed 32-bit value, additionally
1087 * passes back old value, ordered.
1088 *
1089 * @returns true if xchg was done.
1090 * @returns false if xchg wasn't done.
1091 *
1092 * @param pi32 Pointer to the value to update.
1093 * @param i32New The new value to assigned to *pi32.
1094 * @param i32Old The old value to *pi32 compare with.
1095 * @param pi32Old Pointer store the old value at.
1096 */
1097DECLINLINE(bool) ASMAtomicCmpXchgExS32(volatile int32_t *pi32, const int32_t i32New, const int32_t i32Old, int32_t *pi32Old)
1098{
1099 return ASMAtomicCmpXchgExU32((volatile uint32_t *)pi32, (uint32_t)i32New, (uint32_t)i32Old, (uint32_t *)pi32Old);
1100}
1101
1102
1103/**
1104 * Atomically Compare and exchange an unsigned 64-bit value, additionally
1105 * passing back old value, ordered.
1106 *
1107 * @returns true if xchg was done.
1108 * @returns false if xchg wasn't done.
1109 *
1110 * @param pu64 Pointer to the 64-bit variable to update.
1111 * @param u64New The 64-bit value to assign to *pu64.
1112 * @param u64Old The value to compare with.
1113 * @param pu64Old Pointer store the old value at.
1114 */
1115#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1116 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1117DECLASM(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old);
1118#else
1119DECLINLINE(bool) ASMAtomicCmpXchgExU64(volatile uint64_t *pu64, const uint64_t u64New, const uint64_t u64Old, uint64_t *pu64Old)
1120{
1121# if RT_INLINE_ASM_USES_INTRIN
1122 return (*pu64Old =_InterlockedCompareExchange64((__int64 *)pu64, u64New, u64Old)) == u64Old;
1123
1124# elif defined(RT_ARCH_AMD64)
1125# if RT_INLINE_ASM_GNU_STYLE
1126 uint8_t u8Ret;
1127 __asm__ __volatile__("lock; cmpxchgq %3, %0\n\t"
1128 "setz %1\n\t"
1129 : "=m" (*pu64),
1130 "=qm" (u8Ret),
1131 "=a" (*pu64Old)
1132 : "r" (u64New),
1133 "a" (u64Old),
1134 "m" (*pu64));
1135 return (bool)u8Ret;
1136# else
1137 bool fRet;
1138 __asm
1139 {
1140 mov rdx, [pu32]
1141 mov rax, [u64Old]
1142 mov rcx, [u64New]
1143 lock cmpxchg [rdx], rcx
1144 mov rdx, [pu64Old]
1145 mov [rdx], rax
1146 setz al
1147 mov [fRet], al
1148 }
1149 return fRet;
1150# endif
1151# else /* !RT_ARCH_AMD64 */
1152# if RT_INLINE_ASM_GNU_STYLE
1153 uint64_t u64Ret;
1154# if defined(PIC) || defined(__PIC__)
1155 /* NB: this code uses a memory clobber description, because the clean
1156 * solution with an output value for *pu64 makes gcc run out of registers.
1157 * This will cause suboptimal code, and anyone with a better solution is
1158 * welcome to improve this. */
1159 __asm__ __volatile__("xchgl %%ebx, %1\n\t"
1160 "lock; cmpxchg8b %3\n\t"
1161 "xchgl %%ebx, %1\n\t"
1162 : "=A" (u64Ret)
1163 : "DS" ((uint32_t)u64New),
1164 "c" ((uint32_t)(u64New >> 32)),
1165 "m" (*pu64),
1166 "0" (u64Old)
1167 : "memory" );
1168# else /* !PIC */
1169 __asm__ __volatile__("lock; cmpxchg8b %4\n\t"
1170 : "=A" (u64Ret),
1171 "=m" (*pu64)
1172 : "b" ((uint32_t)u64New),
1173 "c" ((uint32_t)(u64New >> 32)),
1174 "m" (*pu64),
1175 "0" (u64Old));
1176# endif
1177 *pu64Old = u64Ret;
1178 return u64Ret == u64Old;
1179# else
1180 uint32_t u32Ret;
1181 __asm
1182 {
1183 mov ebx, dword ptr [u64New]
1184 mov ecx, dword ptr [u64New + 4]
1185 mov edi, [pu64]
1186 mov eax, dword ptr [u64Old]
1187 mov edx, dword ptr [u64Old + 4]
1188 lock cmpxchg8b [edi]
1189 mov ebx, [pu64Old]
1190 mov [ebx], eax
1191 setz al
1192 movzx eax, al
1193 add ebx, 4
1194 mov [ebx], edx
1195 mov dword ptr [u32Ret], eax
1196 }
1197 return !!u32Ret;
1198# endif
1199# endif /* !RT_ARCH_AMD64 */
1200}
1201#endif
1202
1203
1204/**
1205 * Atomically Compare and exchange a signed 64-bit value, additionally
1206 * passing back old value, ordered.
1207 *
1208 * @returns true if xchg was done.
1209 * @returns false if xchg wasn't done.
1210 *
1211 * @param pi64 Pointer to the 64-bit variable to update.
1212 * @param i64 The 64-bit value to assign to *pu64.
1213 * @param i64Old The value to compare with.
1214 * @param pi64Old Pointer store the old value at.
1215 */
1216DECLINLINE(bool) ASMAtomicCmpXchgExS64(volatile int64_t *pi64, const int64_t i64, const int64_t i64Old, int64_t *pi64Old)
1217{
1218 return ASMAtomicCmpXchgExU64((volatile uint64_t *)pi64, (uint64_t)i64, (uint64_t)i64Old, (uint64_t *)pi64Old);
1219}
1220
1221/** @def ASMAtomicCmpXchgExHandle
1222 * Atomically Compare and Exchange a typical IPRT handle value, ordered.
1223 *
1224 * @param ph Pointer to the value to update.
1225 * @param hNew The new value to assigned to *pu.
1226 * @param hOld The old value to *pu compare with.
1227 * @param fRc Where to store the result.
1228 * @param phOldVal Pointer to where to store the old value.
1229 *
1230 * @remarks This doesn't currently work for all handles (like RTFILE).
1231 */
1232#if HC_ARCH_BITS == 32
1233# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1234 do { \
1235 AssertCompile(sizeof(*ph) == sizeof(uint32_t)); \
1236 AssertCompile(sizeof(*phOldVal) == sizeof(uint32_t)); \
1237 (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(puOldVal)); \
1238 } while (0)
1239#elif HC_ARCH_BITS == 64
1240# define ASMAtomicCmpXchgExHandle(ph, hNew, hOld, fRc, phOldVal) \
1241 do { \
1242 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1243 AssertCompile(sizeof(*(phOldVal)) == sizeof(uint64_t)); \
1244 (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(puOldVal)); \
1245 } while (0)
1246#else
1247# error HC_ARCH_BITS
1248#endif
1249
1250
1251/** @def ASMAtomicCmpXchgExSize
1252 * Atomically Compare and Exchange a value which size might differ
1253 * between platforms or compilers. Additionally passes back old value.
1254 *
1255 * @param pu Pointer to the value to update.
1256 * @param uNew The new value to assigned to *pu.
1257 * @param uOld The old value to *pu compare with.
1258 * @param fRc Where to store the result.
1259 * @param puOldVal Pointer to where to store the old value.
1260 */
1261#define ASMAtomicCmpXchgExSize(pu, uNew, uOld, fRc, puOldVal) \
1262 do { \
1263 switch (sizeof(*(pu))) { \
1264 case 4: (fRc) = ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew), (uint32_t)(uOld), (uint32_t *)(uOldVal)); \
1265 break; \
1266 case 8: (fRc) = ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew), (uint64_t)(uOld), (uint64_t *)(uOldVal)); \
1267 break; \
1268 default: AssertMsgFailed(("ASMAtomicCmpXchgSize: size %d is not supported\n", sizeof(*(pu)))); \
1269 (fRc) = false; \
1270 (uOldVal) = 0; \
1271 break; \
1272 } \
1273 } while (0)
1274
1275
1276/**
1277 * Atomically Compare and Exchange a pointer value, additionally
1278 * passing back old value, ordered.
1279 *
1280 * @returns true if xchg was done.
1281 * @returns false if xchg wasn't done.
1282 *
1283 * @param ppv Pointer to the value to update.
1284 * @param pvNew The new value to assigned to *ppv.
1285 * @param pvOld The old value to *ppv compare with.
1286 * @param ppvOld Pointer store the old value at.
1287 */
1288DECLINLINE(bool) ASMAtomicCmpXchgExPtrVoid(void * volatile *ppv, const void *pvNew, const void *pvOld, void **ppvOld)
1289{
1290#if ARCH_BITS == 32
1291 return ASMAtomicCmpXchgExU32((volatile uint32_t *)(void *)ppv, (uint32_t)pvNew, (uint32_t)pvOld, (uint32_t *)ppvOld);
1292#elif ARCH_BITS == 64
1293 return ASMAtomicCmpXchgExU64((volatile uint64_t *)(void *)ppv, (uint64_t)pvNew, (uint64_t)pvOld, (uint64_t *)ppvOld);
1294#else
1295# error "ARCH_BITS is bogus"
1296#endif
1297}
1298
1299
1300/**
1301 * Atomically Compare and Exchange a pointer value, additionally
1302 * passing back old value, ordered.
1303 *
1304 * @returns true if xchg was done.
1305 * @returns false if xchg wasn't done.
1306 *
1307 * @param ppv Pointer to the value to update.
1308 * @param pvNew The new value to assigned to *ppv.
1309 * @param pvOld The old value to *ppv compare with.
1310 * @param ppvOld Pointer store the old value at.
1311 *
1312 * @remarks This is relatively type safe on GCC platforms.
1313 */
1314#ifdef __GNUC__
1315# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1316 __extension__ \
1317 ({\
1318 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
1319 __typeof__(*(ppv)) const pvNewTypeChecked = (pvNew); \
1320 __typeof__(*(ppv)) const pvOldTypeChecked = (pvOld); \
1321 __typeof__(*(ppv)) * const ppvOldTypeChecked = (ppvOld); \
1322 bool fMacroRet = ASMAtomicCmpXchgExPtrVoid((void * volatile *)ppvTypeChecked, \
1323 (void *)pvNewTypeChecked, (void *)pvOldTypeChecked, \
1324 (void **)ppvOld); \
1325 fMacroRet; \
1326 })
1327#else
1328# define ASMAtomicCmpXchgExPtr(ppv, pvNew, pvOld, ppvOld) \
1329 ASMAtomicCmpXchgExPtrVoid((void * volatile *)(ppv), (void *)(pvNew), (void *)pvOld, (void **)ppvOld)
1330#endif
1331
1332
1333/**
1334 * Serialize Instruction.
1335 */
1336#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
1337DECLASM(void) ASMSerializeInstruction(void);
1338#else
1339DECLINLINE(void) ASMSerializeInstruction(void)
1340{
1341# if RT_INLINE_ASM_GNU_STYLE
1342 RTCCUINTREG xAX = 0;
1343# ifdef RT_ARCH_AMD64
1344 __asm__ ("cpuid"
1345 : "=a" (xAX)
1346 : "0" (xAX)
1347 : "rbx", "rcx", "rdx");
1348# elif (defined(PIC) || defined(__PIC__)) && defined(__i386__)
1349 __asm__ ("push %%ebx\n\t"
1350 "cpuid\n\t"
1351 "pop %%ebx\n\t"
1352 : "=a" (xAX)
1353 : "0" (xAX)
1354 : "ecx", "edx");
1355# else
1356 __asm__ ("cpuid"
1357 : "=a" (xAX)
1358 : "0" (xAX)
1359 : "ebx", "ecx", "edx");
1360# endif
1361
1362# elif RT_INLINE_ASM_USES_INTRIN
1363 int aInfo[4];
1364 __cpuid(aInfo, 0);
1365
1366# else
1367 __asm
1368 {
1369 push ebx
1370 xor eax, eax
1371 cpuid
1372 pop ebx
1373 }
1374# endif
1375}
1376#endif
1377
1378
1379/**
1380 * Memory fence, waits for any pending writes and reads to complete.
1381 */
1382DECLINLINE(void) ASMMemoryFence(void)
1383{
1384 /** @todo use mfence? check if all cpus we care for support it. */
1385 uint32_t volatile u32;
1386 ASMAtomicXchgU32(&u32, 0);
1387}
1388
1389
1390/**
1391 * Write fence, waits for any pending writes to complete.
1392 */
1393DECLINLINE(void) ASMWriteFence(void)
1394{
1395 /** @todo use sfence? check if all cpus we care for support it. */
1396 ASMMemoryFence();
1397}
1398
1399
1400/**
1401 * Read fence, waits for any pending reads to complete.
1402 */
1403DECLINLINE(void) ASMReadFence(void)
1404{
1405 /** @todo use lfence? check if all cpus we care for support it. */
1406 ASMMemoryFence();
1407}
1408
1409
1410/**
1411 * Atomically reads an unsigned 8-bit value, ordered.
1412 *
1413 * @returns Current *pu8 value
1414 * @param pu8 Pointer to the 8-bit variable to read.
1415 */
1416DECLINLINE(uint8_t) ASMAtomicReadU8(volatile uint8_t *pu8)
1417{
1418 ASMMemoryFence();
1419 return *pu8; /* byte reads are atomic on x86 */
1420}
1421
1422
1423/**
1424 * Atomically reads an unsigned 8-bit value, unordered.
1425 *
1426 * @returns Current *pu8 value
1427 * @param pu8 Pointer to the 8-bit variable to read.
1428 */
1429DECLINLINE(uint8_t) ASMAtomicUoReadU8(volatile uint8_t *pu8)
1430{
1431 return *pu8; /* byte reads are atomic on x86 */
1432}
1433
1434
1435/**
1436 * Atomically reads a signed 8-bit value, ordered.
1437 *
1438 * @returns Current *pi8 value
1439 * @param pi8 Pointer to the 8-bit variable to read.
1440 */
1441DECLINLINE(int8_t) ASMAtomicReadS8(volatile int8_t *pi8)
1442{
1443 ASMMemoryFence();
1444 return *pi8; /* byte reads are atomic on x86 */
1445}
1446
1447
1448/**
1449 * Atomically reads a signed 8-bit value, unordered.
1450 *
1451 * @returns Current *pi8 value
1452 * @param pi8 Pointer to the 8-bit variable to read.
1453 */
1454DECLINLINE(int8_t) ASMAtomicUoReadS8(volatile int8_t *pi8)
1455{
1456 return *pi8; /* byte reads are atomic on x86 */
1457}
1458
1459
1460/**
1461 * Atomically reads an unsigned 16-bit value, ordered.
1462 *
1463 * @returns Current *pu16 value
1464 * @param pu16 Pointer to the 16-bit variable to read.
1465 */
1466DECLINLINE(uint16_t) ASMAtomicReadU16(volatile uint16_t *pu16)
1467{
1468 ASMMemoryFence();
1469 Assert(!((uintptr_t)pu16 & 1));
1470 return *pu16;
1471}
1472
1473
1474/**
1475 * Atomically reads an unsigned 16-bit value, unordered.
1476 *
1477 * @returns Current *pu16 value
1478 * @param pu16 Pointer to the 16-bit variable to read.
1479 */
1480DECLINLINE(uint16_t) ASMAtomicUoReadU16(volatile uint16_t *pu16)
1481{
1482 Assert(!((uintptr_t)pu16 & 1));
1483 return *pu16;
1484}
1485
1486
1487/**
1488 * Atomically reads a signed 16-bit value, ordered.
1489 *
1490 * @returns Current *pi16 value
1491 * @param pi16 Pointer to the 16-bit variable to read.
1492 */
1493DECLINLINE(int16_t) ASMAtomicReadS16(volatile int16_t *pi16)
1494{
1495 ASMMemoryFence();
1496 Assert(!((uintptr_t)pi16 & 1));
1497 return *pi16;
1498}
1499
1500
1501/**
1502 * Atomically reads a signed 16-bit value, unordered.
1503 *
1504 * @returns Current *pi16 value
1505 * @param pi16 Pointer to the 16-bit variable to read.
1506 */
1507DECLINLINE(int16_t) ASMAtomicUoReadS16(volatile int16_t *pi16)
1508{
1509 Assert(!((uintptr_t)pi16 & 1));
1510 return *pi16;
1511}
1512
1513
1514/**
1515 * Atomically reads an unsigned 32-bit value, ordered.
1516 *
1517 * @returns Current *pu32 value
1518 * @param pu32 Pointer to the 32-bit variable to read.
1519 */
1520DECLINLINE(uint32_t) ASMAtomicReadU32(volatile uint32_t *pu32)
1521{
1522 ASMMemoryFence();
1523 Assert(!((uintptr_t)pu32 & 3));
1524 return *pu32;
1525}
1526
1527
1528/**
1529 * Atomically reads an unsigned 32-bit value, unordered.
1530 *
1531 * @returns Current *pu32 value
1532 * @param pu32 Pointer to the 32-bit variable to read.
1533 */
1534DECLINLINE(uint32_t) ASMAtomicUoReadU32(volatile uint32_t *pu32)
1535{
1536 Assert(!((uintptr_t)pu32 & 3));
1537 return *pu32;
1538}
1539
1540
1541/**
1542 * Atomically reads a signed 32-bit value, ordered.
1543 *
1544 * @returns Current *pi32 value
1545 * @param pi32 Pointer to the 32-bit variable to read.
1546 */
1547DECLINLINE(int32_t) ASMAtomicReadS32(volatile int32_t *pi32)
1548{
1549 ASMMemoryFence();
1550 Assert(!((uintptr_t)pi32 & 3));
1551 return *pi32;
1552}
1553
1554
1555/**
1556 * Atomically reads a signed 32-bit value, unordered.
1557 *
1558 * @returns Current *pi32 value
1559 * @param pi32 Pointer to the 32-bit variable to read.
1560 */
1561DECLINLINE(int32_t) ASMAtomicUoReadS32(volatile int32_t *pi32)
1562{
1563 Assert(!((uintptr_t)pi32 & 3));
1564 return *pi32;
1565}
1566
1567
1568/**
1569 * Atomically reads an unsigned 64-bit value, ordered.
1570 *
1571 * @returns Current *pu64 value
1572 * @param pu64 Pointer to the 64-bit variable to read.
1573 * The memory pointed to must be writable.
1574 * @remark This will fault if the memory is read-only!
1575 */
1576#if (RT_INLINE_ASM_EXTERNAL && !defined(RT_ARCH_AMD64)) \
1577 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1578DECLASM(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64);
1579#else
1580DECLINLINE(uint64_t) ASMAtomicReadU64(volatile uint64_t *pu64)
1581{
1582 uint64_t u64;
1583# ifdef RT_ARCH_AMD64
1584 Assert(!((uintptr_t)pu64 & 7));
1585/*# if RT_INLINE_ASM_GNU_STYLE
1586 __asm__ __volatile__( "mfence\n\t"
1587 "movq %1, %0\n\t"
1588 : "=r" (u64)
1589 : "m" (*pu64));
1590# else
1591 __asm
1592 {
1593 mfence
1594 mov rdx, [pu64]
1595 mov rax, [rdx]
1596 mov [u64], rax
1597 }
1598# endif*/
1599 ASMMemoryFence();
1600 u64 = *pu64;
1601# else /* !RT_ARCH_AMD64 */
1602# if RT_INLINE_ASM_GNU_STYLE
1603# if defined(PIC) || defined(__PIC__)
1604 uint32_t u32EBX = 0;
1605 Assert(!((uintptr_t)pu64 & 7));
1606 __asm__ __volatile__("xchgl %%ebx, %3\n\t"
1607 "lock; cmpxchg8b (%5)\n\t"
1608 "movl %3, %%ebx\n\t"
1609 : "=A" (u64),
1610# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
1611 "+m" (*pu64)
1612# else
1613 "=m" (*pu64)
1614# endif
1615 : "0" (0),
1616 "m" (u32EBX),
1617 "c" (0),
1618 "S" (pu64));
1619# else /* !PIC */
1620 __asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1621 : "=A" (u64),
1622 "+m" (*pu64)
1623 : "0" (0),
1624 "b" (0),
1625 "c" (0));
1626# endif
1627# else
1628 Assert(!((uintptr_t)pu64 & 7));
1629 __asm
1630 {
1631 xor eax, eax
1632 xor edx, edx
1633 mov edi, pu64
1634 xor ecx, ecx
1635 xor ebx, ebx
1636 lock cmpxchg8b [edi]
1637 mov dword ptr [u64], eax
1638 mov dword ptr [u64 + 4], edx
1639 }
1640# endif
1641# endif /* !RT_ARCH_AMD64 */
1642 return u64;
1643}
1644#endif
1645
1646
1647/**
1648 * Atomically reads an unsigned 64-bit value, unordered.
1649 *
1650 * @returns Current *pu64 value
1651 * @param pu64 Pointer to the 64-bit variable to read.
1652 * The memory pointed to must be writable.
1653 * @remark This will fault if the memory is read-only!
1654 */
1655#if (RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN) \
1656 || RT_INLINE_DONT_MIX_CMPXCHG8B_AND_PIC
1657DECLASM(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64);
1658#else
1659DECLINLINE(uint64_t) ASMAtomicUoReadU64(volatile uint64_t *pu64)
1660{
1661 uint64_t u64;
1662# ifdef RT_ARCH_AMD64
1663 Assert(!((uintptr_t)pu64 & 7));
1664/*# if RT_INLINE_ASM_GNU_STYLE
1665 Assert(!((uintptr_t)pu64 & 7));
1666 __asm__ __volatile__("movq %1, %0\n\t"
1667 : "=r" (u64)
1668 : "m" (*pu64));
1669# else
1670 __asm
1671 {
1672 mov rdx, [pu64]
1673 mov rax, [rdx]
1674 mov [u64], rax
1675 }
1676# endif */
1677 u64 = *pu64;
1678# else /* !RT_ARCH_AMD64 */
1679# if RT_INLINE_ASM_GNU_STYLE
1680# if defined(PIC) || defined(__PIC__)
1681 uint32_t u32EBX = 0;
1682 uint32_t u32Spill;
1683 Assert(!((uintptr_t)pu64 & 7));
1684 __asm__ __volatile__("xor %%eax,%%eax\n\t"
1685 "xor %%ecx,%%ecx\n\t"
1686 "xor %%edx,%%edx\n\t"
1687 "xchgl %%ebx, %3\n\t"
1688 "lock; cmpxchg8b (%4)\n\t"
1689 "movl %3, %%ebx\n\t"
1690 : "=A" (u64),
1691# if (__GNUC__ * 100 + __GNUC_MINOR__) >= 403
1692 "+m" (*pu64),
1693# else
1694 "=m" (*pu64),
1695# endif
1696 "=c" (u32Spill)
1697 : "m" (u32EBX),
1698 "S" (pu64));
1699# else /* !PIC */
1700 __asm__ __volatile__("lock; cmpxchg8b %1\n\t"
1701 : "=A" (u64),
1702 "+m" (*pu64)
1703 : "0" (0),
1704 "b" (0),
1705 "c" (0));
1706# endif
1707# else
1708 Assert(!((uintptr_t)pu64 & 7));
1709 __asm
1710 {
1711 xor eax, eax
1712 xor edx, edx
1713 mov edi, pu64
1714 xor ecx, ecx
1715 xor ebx, ebx
1716 lock cmpxchg8b [edi]
1717 mov dword ptr [u64], eax
1718 mov dword ptr [u64 + 4], edx
1719 }
1720# endif
1721# endif /* !RT_ARCH_AMD64 */
1722 return u64;
1723}
1724#endif
1725
1726
1727/**
1728 * Atomically reads a signed 64-bit value, ordered.
1729 *
1730 * @returns Current *pi64 value
1731 * @param pi64 Pointer to the 64-bit variable to read.
1732 * The memory pointed to must be writable.
1733 * @remark This will fault if the memory is read-only!
1734 */
1735DECLINLINE(int64_t) ASMAtomicReadS64(volatile int64_t *pi64)
1736{
1737 return (int64_t)ASMAtomicReadU64((volatile uint64_t *)pi64);
1738}
1739
1740
1741/**
1742 * Atomically reads a signed 64-bit value, unordered.
1743 *
1744 * @returns Current *pi64 value
1745 * @param pi64 Pointer to the 64-bit variable to read.
1746 * The memory pointed to must be writable.
1747 * @remark This will fault if the memory is read-only!
1748 */
1749DECLINLINE(int64_t) ASMAtomicUoReadS64(volatile int64_t *pi64)
1750{
1751 return (int64_t)ASMAtomicUoReadU64((volatile uint64_t *)pi64);
1752}
1753
1754
1755/**
1756 * Atomically reads a pointer value, ordered.
1757 *
1758 * @returns Current *pv value
1759 * @param ppv Pointer to the pointer variable to read.
1760 *
1761 * @remarks Please use ASMAtomicReadPtrT, it provides better type safety and
1762 * requires less typing (no casts).
1763 */
1764DECLINLINE(void *) ASMAtomicReadPtr(void * volatile *ppv)
1765{
1766#if ARCH_BITS == 32
1767 return (void *)ASMAtomicReadU32((volatile uint32_t *)(void *)ppv);
1768#elif ARCH_BITS == 64
1769 return (void *)ASMAtomicReadU64((volatile uint64_t *)(void *)ppv);
1770#else
1771# error "ARCH_BITS is bogus"
1772#endif
1773}
1774
1775/**
1776 * Convenience macro for avoiding the annoying casting with ASMAtomicReadPtr.
1777 *
1778 * @returns Current *pv value
1779 * @param ppv Pointer to the pointer variable to read.
1780 * @param Type The type of *ppv, sans volatile.
1781 */
1782#ifdef __GNUC__
1783# define ASMAtomicReadPtrT(ppv, Type) \
1784 __extension__ \
1785 ({\
1786 __typeof__(*(ppv)) volatile *ppvTypeChecked = (ppv); \
1787 Type pvTypeChecked = (__typeof__(*(ppv))) ASMAtomicReadPtr((void * volatile *)ppvTypeChecked); \
1788 pvTypeChecked; \
1789 })
1790#else
1791# define ASMAtomicReadPtrT(ppv, Type) \
1792 (Type)ASMAtomicReadPtr((void * volatile *)(ppv))
1793#endif
1794
1795
1796/**
1797 * Atomically reads a pointer value, unordered.
1798 *
1799 * @returns Current *pv value
1800 * @param ppv Pointer to the pointer variable to read.
1801 *
1802 * @remarks Please use ASMAtomicUoReadPtrT, it provides better type safety and
1803 * requires less typing (no casts).
1804 */
1805DECLINLINE(void *) ASMAtomicUoReadPtr(void * volatile *ppv)
1806{
1807#if ARCH_BITS == 32
1808 return (void *)ASMAtomicUoReadU32((volatile uint32_t *)(void *)ppv);
1809#elif ARCH_BITS == 64
1810 return (void *)ASMAtomicUoReadU64((volatile uint64_t *)(void *)ppv);
1811#else
1812# error "ARCH_BITS is bogus"
1813#endif
1814}
1815
1816
1817/**
1818 * Convenience macro for avoiding the annoying casting with ASMAtomicUoReadPtr.
1819 *
1820 * @returns Current *pv value
1821 * @param ppv Pointer to the pointer variable to read.
1822 * @param Type The type of *ppv, sans volatile.
1823 */
1824#ifdef __GNUC__
1825# define ASMAtomicUoReadPtrT(ppv, Type) \
1826 __extension__ \
1827 ({\
1828 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
1829 Type pvTypeChecked = (__typeof__(*(ppv))) ASMAtomicUoReadPtr((void * volatile *)ppvTypeChecked); \
1830 pvTypeChecked; \
1831 })
1832#else
1833# define ASMAtomicUoReadPtrT(ppv, Type) \
1834 (Type)ASMAtomicUoReadPtr((void * volatile *)(ppv))
1835#endif
1836
1837
1838/**
1839 * Atomically reads a boolean value, ordered.
1840 *
1841 * @returns Current *pf value
1842 * @param pf Pointer to the boolean variable to read.
1843 */
1844DECLINLINE(bool) ASMAtomicReadBool(volatile bool *pf)
1845{
1846 ASMMemoryFence();
1847 return *pf; /* byte reads are atomic on x86 */
1848}
1849
1850
1851/**
1852 * Atomically reads a boolean value, unordered.
1853 *
1854 * @returns Current *pf value
1855 * @param pf Pointer to the boolean variable to read.
1856 */
1857DECLINLINE(bool) ASMAtomicUoReadBool(volatile bool *pf)
1858{
1859 return *pf; /* byte reads are atomic on x86 */
1860}
1861
1862
1863/**
1864 * Atomically read a typical IPRT handle value, ordered.
1865 *
1866 * @param ph Pointer to the handle variable to read.
1867 * @param phRes Where to store the result.
1868 *
1869 * @remarks This doesn't currently work for all handles (like RTFILE).
1870 */
1871#if HC_ARCH_BITS == 32
1872# define ASMAtomicReadHandle(ph, phRes) \
1873 do { \
1874 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
1875 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
1876 *(uint32_t *)(phRes) = ASMAtomicReadU32((uint32_t volatile *)(ph)); \
1877 } while (0)
1878#elif HC_ARCH_BITS == 64
1879# define ASMAtomicReadHandle(ph, phRes) \
1880 do { \
1881 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1882 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
1883 *(uint64_t *)(phRes) = ASMAtomicReadU64((uint64_t volatile *)(ph)); \
1884 } while (0)
1885#else
1886# error HC_ARCH_BITS
1887#endif
1888
1889
1890/**
1891 * Atomically read a typical IPRT handle value, unordered.
1892 *
1893 * @param ph Pointer to the handle variable to read.
1894 * @param phRes Where to store the result.
1895 *
1896 * @remarks This doesn't currently work for all handles (like RTFILE).
1897 */
1898#if HC_ARCH_BITS == 32
1899# define ASMAtomicUoReadHandle(ph, phRes) \
1900 do { \
1901 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
1902 AssertCompile(sizeof(*(phRes)) == sizeof(uint32_t)); \
1903 *(uint32_t *)(phRes) = ASMAtomicUoReadU32((uint32_t volatile *)(ph)); \
1904 } while (0)
1905#elif HC_ARCH_BITS == 64
1906# define ASMAtomicUoReadHandle(ph, phRes) \
1907 do { \
1908 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
1909 AssertCompile(sizeof(*(phRes)) == sizeof(uint64_t)); \
1910 *(uint64_t *)(phRes) = ASMAtomicUoReadU64((uint64_t volatile *)(ph)); \
1911 } while (0)
1912#else
1913# error HC_ARCH_BITS
1914#endif
1915
1916
1917/**
1918 * Atomically read a value which size might differ
1919 * between platforms or compilers, ordered.
1920 *
1921 * @param pu Pointer to the variable to update.
1922 * @param puRes Where to store the result.
1923 */
1924#define ASMAtomicReadSize(pu, puRes) \
1925 do { \
1926 switch (sizeof(*(pu))) { \
1927 case 1: *(uint8_t *)(puRes) = ASMAtomicReadU8( (volatile uint8_t *)(void *)(pu)); break; \
1928 case 2: *(uint16_t *)(puRes) = ASMAtomicReadU16((volatile uint16_t *)(void *)(pu)); break; \
1929 case 4: *(uint32_t *)(puRes) = ASMAtomicReadU32((volatile uint32_t *)(void *)(pu)); break; \
1930 case 8: *(uint64_t *)(puRes) = ASMAtomicReadU64((volatile uint64_t *)(void *)(pu)); break; \
1931 default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
1932 } \
1933 } while (0)
1934
1935
1936/**
1937 * Atomically read a value which size might differ
1938 * between platforms or compilers, unordered.
1939 *
1940 * @param pu Pointer to the variable to read.
1941 * @param puRes Where to store the result.
1942 */
1943#define ASMAtomicUoReadSize(pu, puRes) \
1944 do { \
1945 switch (sizeof(*(pu))) { \
1946 case 1: *(uint8_t *)(puRes) = ASMAtomicUoReadU8( (volatile uint8_t *)(void *)(pu)); break; \
1947 case 2: *(uint16_t *)(puRes) = ASMAtomicUoReadU16((volatile uint16_t *)(void *)(pu)); break; \
1948 case 4: *(uint32_t *)(puRes) = ASMAtomicUoReadU32((volatile uint32_t *)(void *)(pu)); break; \
1949 case 8: *(uint64_t *)(puRes) = ASMAtomicUoReadU64((volatile uint64_t *)(void *)(pu)); break; \
1950 default: AssertMsgFailed(("ASMAtomicReadSize: size %d is not supported\n", sizeof(*(pu)))); \
1951 } \
1952 } while (0)
1953
1954
1955/**
1956 * Atomically writes an unsigned 8-bit value, ordered.
1957 *
1958 * @param pu8 Pointer to the 8-bit variable.
1959 * @param u8 The 8-bit value to assign to *pu8.
1960 */
1961DECLINLINE(void) ASMAtomicWriteU8(volatile uint8_t *pu8, uint8_t u8)
1962{
1963 ASMAtomicXchgU8(pu8, u8);
1964}
1965
1966
1967/**
1968 * Atomically writes an unsigned 8-bit value, unordered.
1969 *
1970 * @param pu8 Pointer to the 8-bit variable.
1971 * @param u8 The 8-bit value to assign to *pu8.
1972 */
1973DECLINLINE(void) ASMAtomicUoWriteU8(volatile uint8_t *pu8, uint8_t u8)
1974{
1975 *pu8 = u8; /* byte writes are atomic on x86 */
1976}
1977
1978
1979/**
1980 * Atomically writes a signed 8-bit value, ordered.
1981 *
1982 * @param pi8 Pointer to the 8-bit variable to read.
1983 * @param i8 The 8-bit value to assign to *pi8.
1984 */
1985DECLINLINE(void) ASMAtomicWriteS8(volatile int8_t *pi8, int8_t i8)
1986{
1987 ASMAtomicXchgS8(pi8, i8);
1988}
1989
1990
1991/**
1992 * Atomically writes a signed 8-bit value, unordered.
1993 *
1994 * @param pi8 Pointer to the 8-bit variable to read.
1995 * @param i8 The 8-bit value to assign to *pi8.
1996 */
1997DECLINLINE(void) ASMAtomicUoWriteS8(volatile int8_t *pi8, int8_t i8)
1998{
1999 *pi8 = i8; /* byte writes are atomic on x86 */
2000}
2001
2002
2003/**
2004 * Atomically writes an unsigned 16-bit value, ordered.
2005 *
2006 * @param pu16 Pointer to the 16-bit variable.
2007 * @param u16 The 16-bit value to assign to *pu16.
2008 */
2009DECLINLINE(void) ASMAtomicWriteU16(volatile uint16_t *pu16, uint16_t u16)
2010{
2011 ASMAtomicXchgU16(pu16, u16);
2012}
2013
2014
2015/**
2016 * Atomically writes an unsigned 16-bit value, unordered.
2017 *
2018 * @param pu16 Pointer to the 16-bit variable.
2019 * @param u16 The 16-bit value to assign to *pu16.
2020 */
2021DECLINLINE(void) ASMAtomicUoWriteU16(volatile uint16_t *pu16, uint16_t u16)
2022{
2023 Assert(!((uintptr_t)pu16 & 1));
2024 *pu16 = u16;
2025}
2026
2027
2028/**
2029 * Atomically writes a signed 16-bit value, ordered.
2030 *
2031 * @param pi16 Pointer to the 16-bit variable to read.
2032 * @param i16 The 16-bit value to assign to *pi16.
2033 */
2034DECLINLINE(void) ASMAtomicWriteS16(volatile int16_t *pi16, int16_t i16)
2035{
2036 ASMAtomicXchgS16(pi16, i16);
2037}
2038
2039
2040/**
2041 * Atomically writes a signed 16-bit value, unordered.
2042 *
2043 * @param pi16 Pointer to the 16-bit variable to read.
2044 * @param i16 The 16-bit value to assign to *pi16.
2045 */
2046DECLINLINE(void) ASMAtomicUoWriteS16(volatile int16_t *pi16, int16_t i16)
2047{
2048 Assert(!((uintptr_t)pi16 & 1));
2049 *pi16 = i16;
2050}
2051
2052
2053/**
2054 * Atomically writes an unsigned 32-bit value, ordered.
2055 *
2056 * @param pu32 Pointer to the 32-bit variable.
2057 * @param u32 The 32-bit value to assign to *pu32.
2058 */
2059DECLINLINE(void) ASMAtomicWriteU32(volatile uint32_t *pu32, uint32_t u32)
2060{
2061 ASMAtomicXchgU32(pu32, u32);
2062}
2063
2064
2065/**
2066 * Atomically writes an unsigned 32-bit value, unordered.
2067 *
2068 * @param pu32 Pointer to the 32-bit variable.
2069 * @param u32 The 32-bit value to assign to *pu32.
2070 */
2071DECLINLINE(void) ASMAtomicUoWriteU32(volatile uint32_t *pu32, uint32_t u32)
2072{
2073 Assert(!((uintptr_t)pu32 & 3));
2074 *pu32 = u32;
2075}
2076
2077
2078/**
2079 * Atomically writes a signed 32-bit value, ordered.
2080 *
2081 * @param pi32 Pointer to the 32-bit variable to read.
2082 * @param i32 The 32-bit value to assign to *pi32.
2083 */
2084DECLINLINE(void) ASMAtomicWriteS32(volatile int32_t *pi32, int32_t i32)
2085{
2086 ASMAtomicXchgS32(pi32, i32);
2087}
2088
2089
2090/**
2091 * Atomically writes a signed 32-bit value, unordered.
2092 *
2093 * @param pi32 Pointer to the 32-bit variable to read.
2094 * @param i32 The 32-bit value to assign to *pi32.
2095 */
2096DECLINLINE(void) ASMAtomicUoWriteS32(volatile int32_t *pi32, int32_t i32)
2097{
2098 Assert(!((uintptr_t)pi32 & 3));
2099 *pi32 = i32;
2100}
2101
2102
2103/**
2104 * Atomically writes an unsigned 64-bit value, ordered.
2105 *
2106 * @param pu64 Pointer to the 64-bit variable.
2107 * @param u64 The 64-bit value to assign to *pu64.
2108 */
2109DECLINLINE(void) ASMAtomicWriteU64(volatile uint64_t *pu64, uint64_t u64)
2110{
2111 ASMAtomicXchgU64(pu64, u64);
2112}
2113
2114
2115/**
2116 * Atomically writes an unsigned 64-bit value, unordered.
2117 *
2118 * @param pu64 Pointer to the 64-bit variable.
2119 * @param u64 The 64-bit value to assign to *pu64.
2120 */
2121DECLINLINE(void) ASMAtomicUoWriteU64(volatile uint64_t *pu64, uint64_t u64)
2122{
2123 Assert(!((uintptr_t)pu64 & 7));
2124#if ARCH_BITS == 64
2125 *pu64 = u64;
2126#else
2127 ASMAtomicXchgU64(pu64, u64);
2128#endif
2129}
2130
2131
2132/**
2133 * Atomically writes a signed 64-bit value, ordered.
2134 *
2135 * @param pi64 Pointer to the 64-bit variable.
2136 * @param i64 The 64-bit value to assign to *pi64.
2137 */
2138DECLINLINE(void) ASMAtomicWriteS64(volatile int64_t *pi64, int64_t i64)
2139{
2140 ASMAtomicXchgS64(pi64, i64);
2141}
2142
2143
2144/**
2145 * Atomically writes a signed 64-bit value, unordered.
2146 *
2147 * @param pi64 Pointer to the 64-bit variable.
2148 * @param i64 The 64-bit value to assign to *pi64.
2149 */
2150DECLINLINE(void) ASMAtomicUoWriteS64(volatile int64_t *pi64, int64_t i64)
2151{
2152 Assert(!((uintptr_t)pi64 & 7));
2153#if ARCH_BITS == 64
2154 *pi64 = i64;
2155#else
2156 ASMAtomicXchgS64(pi64, i64);
2157#endif
2158}
2159
2160
2161/**
2162 * Atomically writes a boolean value, unordered.
2163 *
2164 * @param pf Pointer to the boolean variable.
2165 * @param f The boolean value to assign to *pf.
2166 */
2167DECLINLINE(void) ASMAtomicWriteBool(volatile bool *pf, bool f)
2168{
2169 ASMAtomicWriteU8((uint8_t volatile *)pf, f);
2170}
2171
2172
2173/**
2174 * Atomically writes a boolean value, unordered.
2175 *
2176 * @param pf Pointer to the boolean variable.
2177 * @param f The boolean value to assign to *pf.
2178 */
2179DECLINLINE(void) ASMAtomicUoWriteBool(volatile bool *pf, bool f)
2180{
2181 *pf = f; /* byte writes are atomic on x86 */
2182}
2183
2184
2185/**
2186 * Atomically writes a pointer value, ordered.
2187 *
2188 * @param ppv Pointer to the pointer variable.
2189 * @param pv The pointer value to assign to *ppv.
2190 */
2191DECLINLINE(void) ASMAtomicWritePtrVoid(void * volatile *ppv, const void *pv)
2192{
2193#if ARCH_BITS == 32
2194 ASMAtomicWriteU32((volatile uint32_t *)(void *)ppv, (uint32_t)pv);
2195#elif ARCH_BITS == 64
2196 ASMAtomicWriteU64((volatile uint64_t *)(void *)ppv, (uint64_t)pv);
2197#else
2198# error "ARCH_BITS is bogus"
2199#endif
2200}
2201
2202
2203/**
2204 * Atomically writes a pointer value, ordered.
2205 *
2206 * @param ppv Pointer to the pointer variable.
2207 * @param pv The pointer value to assign to *ppv. If NULL use
2208 * ASMAtomicWriteNullPtr or you'll land in trouble.
2209 *
2210 * @remarks This is relatively type safe on GCC platforms when @a pv isn't
2211 * NULL.
2212 */
2213#ifdef __GNUC__
2214# define ASMAtomicWritePtr(ppv, pv) \
2215 do \
2216 { \
2217 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2218 __typeof__(*(ppv)) const pvTypeChecked = (pv); \
2219 \
2220 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2221 AssertCompile(sizeof(pv) == sizeof(void *)); \
2222 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2223 \
2224 ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), (void *)(pvTypeChecked)); \
2225 } while (0)
2226#else
2227# define ASMAtomicWritePtr(ppv, pv) \
2228 do \
2229 { \
2230 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2231 AssertCompile(sizeof(pv) == sizeof(void *)); \
2232 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2233 \
2234 ASMAtomicWritePtrVoid((void * volatile *)(ppv), (void *)(pv)); \
2235 } while (0)
2236#endif
2237
2238
2239/**
2240 * Atomically sets a pointer to NULL, ordered.
2241 *
2242 * @param ppv Pointer to the pointer variable that should be set to NULL.
2243 *
2244 * @remarks This is relatively type safe on GCC platforms.
2245 */
2246#ifdef __GNUC__
2247# define ASMAtomicWriteNullPtr(ppv) \
2248 do \
2249 { \
2250 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2251 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2252 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2253 ASMAtomicWritePtrVoid((void * volatile *)(ppvTypeChecked), NULL); \
2254 } while (0)
2255#else
2256# define ASMAtomicWriteNullPtr(ppv) \
2257 do \
2258 { \
2259 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2260 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2261 ASMAtomicWritePtrVoid((void * volatile *)(ppv), NULL); \
2262 } while (0)
2263#endif
2264
2265
2266/**
2267 * Atomically writes a pointer value, unordered.
2268 *
2269 * @returns Current *pv value
2270 * @param ppv Pointer to the pointer variable.
2271 * @param pv The pointer value to assign to *ppv. If NULL use
2272 * ASMAtomicUoWriteNullPtr or you'll land in trouble.
2273 *
2274 * @remarks This is relatively type safe on GCC platforms when @a pv isn't
2275 * NULL.
2276 */
2277#ifdef __GNUC__
2278# define ASMAtomicUoWritePtr(ppv, pv) \
2279 do \
2280 { \
2281 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2282 __typeof__(*(ppv)) const pvTypeChecked = (pv); \
2283 \
2284 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2285 AssertCompile(sizeof(pv) == sizeof(void *)); \
2286 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2287 \
2288 *(ppvTypeChecked) = pvTypeChecked; \
2289 } while (0)
2290#else
2291# define ASMAtomicUoWritePtr(ppv, pv) \
2292 do \
2293 { \
2294 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2295 AssertCompile(sizeof(pv) == sizeof(void *)); \
2296 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2297 *(ppv) = pv; \
2298 } while (0)
2299#endif
2300
2301
2302/**
2303 * Atomically sets a pointer to NULL, unordered.
2304 *
2305 * @param ppv Pointer to the pointer variable that should be set to NULL.
2306 *
2307 * @remarks This is relatively type safe on GCC platforms.
2308 */
2309#ifdef __GNUC__
2310# define ASMAtomicUoWriteNullPtr(ppv) \
2311 do \
2312 { \
2313 __typeof__(*(ppv)) volatile * const ppvTypeChecked = (ppv); \
2314 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2315 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2316 *(ppvTypeChecked) = NULL; \
2317 } while (0)
2318#else
2319# define ASMAtomicUoWriteNullPtr(ppv) \
2320 do \
2321 { \
2322 AssertCompile(sizeof(*ppv) == sizeof(void *)); \
2323 Assert(!( (uintptr_t)ppv & ((ARCH_BITS / 8) - 1) )); \
2324 *(ppv) = NULL; \
2325 } while (0)
2326#endif
2327
2328
2329/**
2330 * Atomically write a typical IPRT handle value, ordered.
2331 *
2332 * @param ph Pointer to the variable to update.
2333 * @param hNew The value to assign to *ph.
2334 *
2335 * @remarks This doesn't currently work for all handles (like RTFILE).
2336 */
2337#if HC_ARCH_BITS == 32
2338# define ASMAtomicWriteHandle(ph, hNew) \
2339 do { \
2340 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2341 ASMAtomicWriteU32((uint32_t volatile *)(ph), (const uint32_t)(hNew)); \
2342 } while (0)
2343#elif HC_ARCH_BITS == 64
2344# define ASMAtomicWriteHandle(ph, hNew) \
2345 do { \
2346 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2347 ASMAtomicWriteU64((uint64_t volatile *)(ph), (const uint64_t)(hNew)); \
2348 } while (0)
2349#else
2350# error HC_ARCH_BITS
2351#endif
2352
2353
2354/**
2355 * Atomically write a typical IPRT handle value, unordered.
2356 *
2357 * @param ph Pointer to the variable to update.
2358 * @param hNew The value to assign to *ph.
2359 *
2360 * @remarks This doesn't currently work for all handles (like RTFILE).
2361 */
2362#if HC_ARCH_BITS == 32
2363# define ASMAtomicUoWriteHandle(ph, hNew) \
2364 do { \
2365 AssertCompile(sizeof(*(ph)) == sizeof(uint32_t)); \
2366 ASMAtomicUoWriteU32((uint32_t volatile *)(ph), (const uint32_t)hNew); \
2367 } while (0)
2368#elif HC_ARCH_BITS == 64
2369# define ASMAtomicUoWriteHandle(ph, hNew) \
2370 do { \
2371 AssertCompile(sizeof(*(ph)) == sizeof(uint64_t)); \
2372 ASMAtomicUoWriteU64((uint64_t volatile *)(ph), (const uint64_t)hNew); \
2373 } while (0)
2374#else
2375# error HC_ARCH_BITS
2376#endif
2377
2378
2379/**
2380 * Atomically write a value which size might differ
2381 * between platforms or compilers, ordered.
2382 *
2383 * @param pu Pointer to the variable to update.
2384 * @param uNew The value to assign to *pu.
2385 */
2386#define ASMAtomicWriteSize(pu, uNew) \
2387 do { \
2388 switch (sizeof(*(pu))) { \
2389 case 1: ASMAtomicWriteU8( (volatile uint8_t *)(void *)(pu), (uint8_t )(uNew)); break; \
2390 case 2: ASMAtomicWriteU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
2391 case 4: ASMAtomicWriteU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2392 case 8: ASMAtomicWriteU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2393 default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2394 } \
2395 } while (0)
2396
2397/**
2398 * Atomically write a value which size might differ
2399 * between platforms or compilers, unordered.
2400 *
2401 * @param pu Pointer to the variable to update.
2402 * @param uNew The value to assign to *pu.
2403 */
2404#define ASMAtomicUoWriteSize(pu, uNew) \
2405 do { \
2406 switch (sizeof(*(pu))) { \
2407 case 1: ASMAtomicUoWriteU8( (volatile uint8_t *)(void *)(pu), (uint8_t )(uNew)); break; \
2408 case 2: ASMAtomicUoWriteU16((volatile uint16_t *)(void *)(pu), (uint16_t)(uNew)); break; \
2409 case 4: ASMAtomicUoWriteU32((volatile uint32_t *)(void *)(pu), (uint32_t)(uNew)); break; \
2410 case 8: ASMAtomicUoWriteU64((volatile uint64_t *)(void *)(pu), (uint64_t)(uNew)); break; \
2411 default: AssertMsgFailed(("ASMAtomicWriteSize: size %d is not supported\n", sizeof(*(pu)))); \
2412 } \
2413 } while (0)
2414
2415
2416
2417/**
2418 * Atomically exchanges and adds to a 32-bit value, ordered.
2419 *
2420 * @returns The old value.
2421 * @param pu32 Pointer to the value.
2422 * @param u32 Number to add.
2423 */
2424#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2425DECLASM(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32);
2426#else
2427DECLINLINE(uint32_t) ASMAtomicAddU32(uint32_t volatile *pu32, uint32_t u32)
2428{
2429# if RT_INLINE_ASM_USES_INTRIN
2430 u32 = _InterlockedExchangeAdd((long *)pu32, u32);
2431 return u32;
2432
2433# elif RT_INLINE_ASM_GNU_STYLE
2434 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2435 : "=r" (u32),
2436 "=m" (*pu32)
2437 : "0" (u32),
2438 "m" (*pu32)
2439 : "memory");
2440 return u32;
2441# else
2442 __asm
2443 {
2444 mov eax, [u32]
2445# ifdef RT_ARCH_AMD64
2446 mov rdx, [pu32]
2447 lock xadd [rdx], eax
2448# else
2449 mov edx, [pu32]
2450 lock xadd [edx], eax
2451# endif
2452 mov [u32], eax
2453 }
2454 return u32;
2455# endif
2456}
2457#endif
2458
2459
2460/**
2461 * Atomically exchanges and adds to a signed 32-bit value, ordered.
2462 *
2463 * @returns The old value.
2464 * @param pi32 Pointer to the value.
2465 * @param i32 Number to add.
2466 */
2467DECLINLINE(int32_t) ASMAtomicAddS32(int32_t volatile *pi32, int32_t i32)
2468{
2469 return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)i32);
2470}
2471
2472
2473/**
2474 * Atomically exchanges and adds to a 64-bit value, ordered.
2475 *
2476 * @returns The old value.
2477 * @param pu64 Pointer to the value.
2478 * @param u64 Number to add.
2479 */
2480#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2481DECLASM(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64);
2482#else
2483DECLINLINE(uint64_t) ASMAtomicAddU64(uint64_t volatile *pu64, uint64_t u64)
2484{
2485# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2486 u64 = _InterlockedExchangeAdd64((__int64 *)pu64, u64);
2487 return u64;
2488
2489# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2490 __asm__ __volatile__("lock; xaddq %0, %1\n\t"
2491 : "=r" (u64),
2492 "=m" (*pu64)
2493 : "0" (u64),
2494 "m" (*pu64)
2495 : "memory");
2496 return u64;
2497# else
2498 uint64_t u64New;
2499 for (;;)
2500 {
2501 uint64_t u64Old = ASMAtomicUoReadU64(pu64);
2502 u64New = u64Old + u64;
2503 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2504 break;
2505 ASMNopPause();
2506 }
2507 return u64New;
2508# endif
2509}
2510#endif
2511
2512
2513/**
2514 * Atomically exchanges and adds to a signed 64-bit value, ordered.
2515 *
2516 * @returns The old value.
2517 * @param pi64 Pointer to the value.
2518 * @param i64 Number to add.
2519 */
2520DECLINLINE(int64_t) ASMAtomicAddS64(int64_t volatile *pi64, int64_t i64)
2521{
2522 return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)i64);
2523}
2524
2525
2526/**
2527 * Atomically exchanges and subtracts to an unsigned 32-bit value, ordered.
2528 *
2529 * @returns The old value.
2530 * @param pu32 Pointer to the value.
2531 * @param u32 Number to subtract.
2532 */
2533DECLINLINE(uint32_t) ASMAtomicSubU32(uint32_t volatile *pu32, uint32_t u32)
2534{
2535 return ASMAtomicAddU32(pu32, (uint32_t)-(int32_t)u32);
2536}
2537
2538
2539/**
2540 * Atomically exchanges and subtracts to a signed 32-bit value, ordered.
2541 *
2542 * @returns The old value.
2543 * @param pi32 Pointer to the value.
2544 * @param i32 Number to subtract.
2545 */
2546DECLINLINE(int32_t) ASMAtomicSubS32(int32_t volatile *pi32, int32_t i32)
2547{
2548 return (int32_t)ASMAtomicAddU32((uint32_t volatile *)pi32, (uint32_t)-i32);
2549}
2550
2551
2552/**
2553 * Atomically exchanges and subtracts to an unsigned 64-bit value, ordered.
2554 *
2555 * @returns The old value.
2556 * @param pu64 Pointer to the value.
2557 * @param u64 Number to subtract.
2558 */
2559DECLINLINE(uint64_t) ASMAtomicSubU64(uint64_t volatile *pu64, uint64_t u64)
2560{
2561 return ASMAtomicAddU64(pu64, (uint64_t)-(int64_t)u64);
2562}
2563
2564
2565/**
2566 * Atomically exchanges and subtracts to a signed 64-bit value, ordered.
2567 *
2568 * @returns The old value.
2569 * @param pi64 Pointer to the value.
2570 * @param i64 Number to subtract.
2571 */
2572DECLINLINE(int64_t) ASMAtomicSubS64(int64_t volatile *pi64, int64_t i64)
2573{
2574 return (int64_t)ASMAtomicAddU64((uint64_t volatile *)pi64, (uint64_t)-i64);
2575}
2576
2577
2578/**
2579 * Atomically increment a 32-bit value, ordered.
2580 *
2581 * @returns The new value.
2582 * @param pu32 Pointer to the value to increment.
2583 */
2584#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2585DECLASM(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32);
2586#else
2587DECLINLINE(uint32_t) ASMAtomicIncU32(uint32_t volatile *pu32)
2588{
2589 uint32_t u32;
2590# if RT_INLINE_ASM_USES_INTRIN
2591 u32 = _InterlockedIncrement((long *)pu32);
2592 return u32;
2593
2594# elif RT_INLINE_ASM_GNU_STYLE
2595 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2596 : "=r" (u32),
2597 "=m" (*pu32)
2598 : "0" (1),
2599 "m" (*pu32)
2600 : "memory");
2601 return u32+1;
2602# else
2603 __asm
2604 {
2605 mov eax, 1
2606# ifdef RT_ARCH_AMD64
2607 mov rdx, [pu32]
2608 lock xadd [rdx], eax
2609# else
2610 mov edx, [pu32]
2611 lock xadd [edx], eax
2612# endif
2613 mov u32, eax
2614 }
2615 return u32+1;
2616# endif
2617}
2618#endif
2619
2620
2621/**
2622 * Atomically increment a signed 32-bit value, ordered.
2623 *
2624 * @returns The new value.
2625 * @param pi32 Pointer to the value to increment.
2626 */
2627DECLINLINE(int32_t) ASMAtomicIncS32(int32_t volatile *pi32)
2628{
2629 return (int32_t)ASMAtomicIncU32((uint32_t volatile *)pi32);
2630}
2631
2632
2633/**
2634 * Atomically increment a 64-bit value, ordered.
2635 *
2636 * @returns The new value.
2637 * @param pu64 Pointer to the value to increment.
2638 */
2639#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2640DECLASM(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64);
2641#else
2642DECLINLINE(uint64_t) ASMAtomicIncU64(uint64_t volatile *pu64)
2643{
2644# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2645 uint64_t u64;
2646 u64 = _InterlockedIncrement64((__int64 *)pu64);
2647 return u64;
2648
2649# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2650 uint64_t u64;
2651 __asm__ __volatile__("lock; xaddq %0, %1\n\t"
2652 : "=r" (u64),
2653 "=m" (*pu64)
2654 : "0" (1),
2655 "m" (*pu64)
2656 : "memory");
2657 return u64 + 1;
2658# else
2659 return ASMAtomicAddU64(pu64, 1) + 1;
2660# endif
2661}
2662#endif
2663
2664
2665/**
2666 * Atomically increment a signed 64-bit value, ordered.
2667 *
2668 * @returns The new value.
2669 * @param pi64 Pointer to the value to increment.
2670 */
2671DECLINLINE(int64_t) ASMAtomicIncS64(int64_t volatile *pi64)
2672{
2673 return (int64_t)ASMAtomicIncU64((uint64_t volatile *)pi64);
2674}
2675
2676
2677/**
2678 * Atomically decrement an unsigned 32-bit value, ordered.
2679 *
2680 * @returns The new value.
2681 * @param pu32 Pointer to the value to decrement.
2682 */
2683#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2684DECLASM(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32);
2685#else
2686DECLINLINE(uint32_t) ASMAtomicDecU32(uint32_t volatile *pu32)
2687{
2688 uint32_t u32;
2689# if RT_INLINE_ASM_USES_INTRIN
2690 u32 = _InterlockedDecrement((long *)pu32);
2691 return u32;
2692
2693# elif RT_INLINE_ASM_GNU_STYLE
2694 __asm__ __volatile__("lock; xaddl %0, %1\n\t"
2695 : "=r" (u32),
2696 "=m" (*pu32)
2697 : "0" (-1),
2698 "m" (*pu32)
2699 : "memory");
2700 return u32-1;
2701# else
2702 __asm
2703 {
2704 mov eax, -1
2705# ifdef RT_ARCH_AMD64
2706 mov rdx, [pu32]
2707 lock xadd [rdx], eax
2708# else
2709 mov edx, [pu32]
2710 lock xadd [edx], eax
2711# endif
2712 mov u32, eax
2713 }
2714 return u32-1;
2715# endif
2716}
2717#endif
2718
2719
2720/**
2721 * Atomically decrement a signed 32-bit value, ordered.
2722 *
2723 * @returns The new value.
2724 * @param pi32 Pointer to the value to decrement.
2725 */
2726DECLINLINE(int32_t) ASMAtomicDecS32(int32_t volatile *pi32)
2727{
2728 return (int32_t)ASMAtomicDecU32((uint32_t volatile *)pi32);
2729}
2730
2731
2732/**
2733 * Atomically decrement an unsigned 64-bit value, ordered.
2734 *
2735 * @returns The new value.
2736 * @param pu64 Pointer to the value to decrement.
2737 */
2738#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2739DECLASM(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64);
2740#else
2741DECLINLINE(uint64_t) ASMAtomicDecU64(uint64_t volatile *pu64)
2742{
2743# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2744 uint64_t u64 = _InterlockedDecrement64((__int64 volatile *)pu64);
2745 return u64;
2746
2747# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2748 uint64_t u64;
2749 __asm__ __volatile__("lock; xaddq %q0, %1\n\t"
2750 : "=r" (u64),
2751 "=m" (*pu64)
2752 : "0" (~(uint64_t)0),
2753 "m" (*pu64)
2754 : "memory");
2755 return u64-1;
2756# else
2757 return ASMAtomicAddU64(pu64, UINT64_MAX) - 1;
2758# endif
2759}
2760#endif
2761
2762
2763/**
2764 * Atomically decrement a signed 64-bit value, ordered.
2765 *
2766 * @returns The new value.
2767 * @param pi64 Pointer to the value to decrement.
2768 */
2769DECLINLINE(int64_t) ASMAtomicDecS64(int64_t volatile *pi64)
2770{
2771 return (int64_t)ASMAtomicDecU64((uint64_t volatile *)pi64);
2772}
2773
2774
2775/**
2776 * Atomically Or an unsigned 32-bit value, ordered.
2777 *
2778 * @param pu32 Pointer to the pointer variable to OR u32 with.
2779 * @param u32 The value to OR *pu32 with.
2780 */
2781#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2782DECLASM(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32);
2783#else
2784DECLINLINE(void) ASMAtomicOrU32(uint32_t volatile *pu32, uint32_t u32)
2785{
2786# if RT_INLINE_ASM_USES_INTRIN
2787 _InterlockedOr((long volatile *)pu32, (long)u32);
2788
2789# elif RT_INLINE_ASM_GNU_STYLE
2790 __asm__ __volatile__("lock; orl %1, %0\n\t"
2791 : "=m" (*pu32)
2792 : "ir" (u32),
2793 "m" (*pu32));
2794# else
2795 __asm
2796 {
2797 mov eax, [u32]
2798# ifdef RT_ARCH_AMD64
2799 mov rdx, [pu32]
2800 lock or [rdx], eax
2801# else
2802 mov edx, [pu32]
2803 lock or [edx], eax
2804# endif
2805 }
2806# endif
2807}
2808#endif
2809
2810
2811/**
2812 * Atomically Or a signed 32-bit value, ordered.
2813 *
2814 * @param pi32 Pointer to the pointer variable to OR u32 with.
2815 * @param i32 The value to OR *pu32 with.
2816 */
2817DECLINLINE(void) ASMAtomicOrS32(int32_t volatile *pi32, int32_t i32)
2818{
2819 ASMAtomicOrU32((uint32_t volatile *)pi32, i32);
2820}
2821
2822
2823/**
2824 * Atomically Or an unsigned 64-bit value, ordered.
2825 *
2826 * @param pu64 Pointer to the pointer variable to OR u64 with.
2827 * @param u64 The value to OR *pu64 with.
2828 */
2829#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2830DECLASM(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64);
2831#else
2832DECLINLINE(void) ASMAtomicOrU64(uint64_t volatile *pu64, uint64_t u64)
2833{
2834# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2835 _InterlockedOr64((__int64 volatile *)pu64, (__int64)u64);
2836
2837# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2838 __asm__ __volatile__("lock; orq %1, %q0\n\t"
2839 : "=m" (*pu64)
2840 : "r" (u64),
2841 "m" (*pu64));
2842# else
2843 for (;;)
2844 {
2845 uint64_t u64Old = ASMAtomicUoReadU64(pu64);
2846 uint64_t u64New = u64Old | u64;
2847 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2848 break;
2849 ASMNopPause();
2850 }
2851# endif
2852}
2853#endif
2854
2855
2856/**
2857 * Atomically Or a signed 64-bit value, ordered.
2858 *
2859 * @param pi64 Pointer to the pointer variable to OR u64 with.
2860 * @param i64 The value to OR *pu64 with.
2861 */
2862DECLINLINE(void) ASMAtomicOrS64(int64_t volatile *pi64, int64_t i64)
2863{
2864 ASMAtomicOrU64((uint64_t volatile *)pi64, i64);
2865}
2866/**
2867 * Atomically And an unsigned 32-bit value, ordered.
2868 *
2869 * @param pu32 Pointer to the pointer variable to AND u32 with.
2870 * @param u32 The value to AND *pu32 with.
2871 */
2872#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2873DECLASM(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32);
2874#else
2875DECLINLINE(void) ASMAtomicAndU32(uint32_t volatile *pu32, uint32_t u32)
2876{
2877# if RT_INLINE_ASM_USES_INTRIN
2878 _InterlockedAnd((long volatile *)pu32, u32);
2879
2880# elif RT_INLINE_ASM_GNU_STYLE
2881 __asm__ __volatile__("lock; andl %1, %0\n\t"
2882 : "=m" (*pu32)
2883 : "ir" (u32),
2884 "m" (*pu32));
2885# else
2886 __asm
2887 {
2888 mov eax, [u32]
2889# ifdef RT_ARCH_AMD64
2890 mov rdx, [pu32]
2891 lock and [rdx], eax
2892# else
2893 mov edx, [pu32]
2894 lock and [edx], eax
2895# endif
2896 }
2897# endif
2898}
2899#endif
2900
2901
2902/**
2903 * Atomically And a signed 32-bit value, ordered.
2904 *
2905 * @param pi32 Pointer to the pointer variable to AND i32 with.
2906 * @param i32 The value to AND *pi32 with.
2907 */
2908DECLINLINE(void) ASMAtomicAndS32(int32_t volatile *pi32, int32_t i32)
2909{
2910 ASMAtomicAndU32((uint32_t volatile *)pi32, (uint32_t)i32);
2911}
2912
2913
2914/**
2915 * Atomically And an unsigned 64-bit value, ordered.
2916 *
2917 * @param pu64 Pointer to the pointer variable to AND u64 with.
2918 * @param u64 The value to AND *pu64 with.
2919 */
2920#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2921DECLASM(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64);
2922#else
2923DECLINLINE(void) ASMAtomicAndU64(uint64_t volatile *pu64, uint64_t u64)
2924{
2925# if RT_INLINE_ASM_USES_INTRIN && defined(RT_ARCH_AMD64)
2926 _InterlockedAnd64((__int64 volatile *)pu64, u64);
2927
2928# elif RT_INLINE_ASM_GNU_STYLE && defined(RT_ARCH_AMD64)
2929 __asm__ __volatile__("lock; andq %1, %0\n\t"
2930 : "=m" (*pu64)
2931 : "r" (u64),
2932 "m" (*pu64));
2933# else
2934 for (;;)
2935 {
2936 uint64_t u64Old = ASMAtomicUoReadU64(pu64);
2937 uint64_t u64New = u64Old & u64;
2938 if (ASMAtomicCmpXchgU64(pu64, u64New, u64Old))
2939 break;
2940 ASMNopPause();
2941 }
2942# endif
2943}
2944#endif
2945
2946
2947/**
2948 * Atomically And a signed 64-bit value, ordered.
2949 *
2950 * @param pi64 Pointer to the pointer variable to AND i64 with.
2951 * @param i64 The value to AND *pi64 with.
2952 */
2953DECLINLINE(void) ASMAtomicAndS64(int64_t volatile *pi64, int64_t i64)
2954{
2955 ASMAtomicAndU64((uint64_t volatile *)pi64, (uint64_t)i64);
2956}
2957
2958
2959
2960/** @def RT_ASM_PAGE_SIZE
2961 * We try avoid dragging in iprt/param.h here.
2962 * @internal
2963 */
2964#if defined(RT_ARCH_SPARC64)
2965# define RT_ASM_PAGE_SIZE 0x2000
2966# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
2967# if PAGE_SIZE != 0x2000
2968# error "PAGE_SIZE is not 0x2000!"
2969# endif
2970# endif
2971#else
2972# define RT_ASM_PAGE_SIZE 0x1000
2973# if defined(PAGE_SIZE) && !defined(NT_INCLUDED)
2974# if PAGE_SIZE != 0x1000
2975# error "PAGE_SIZE is not 0x1000!"
2976# endif
2977# endif
2978#endif
2979
2980/**
2981 * Zeros a 4K memory page.
2982 *
2983 * @param pv Pointer to the memory block. This must be page aligned.
2984 */
2985#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
2986DECLASM(void) ASMMemZeroPage(volatile void *pv);
2987# else
2988DECLINLINE(void) ASMMemZeroPage(volatile void *pv)
2989{
2990# if RT_INLINE_ASM_USES_INTRIN
2991# ifdef RT_ARCH_AMD64
2992 __stosq((unsigned __int64 *)pv, 0, RT_ASM_PAGE_SIZE / 8);
2993# else
2994 __stosd((unsigned long *)pv, 0, RT_ASM_PAGE_SIZE / 4);
2995# endif
2996
2997# elif RT_INLINE_ASM_GNU_STYLE
2998 RTCCUINTREG uDummy;
2999# ifdef RT_ARCH_AMD64
3000 __asm__ __volatile__("rep stosq"
3001 : "=D" (pv),
3002 "=c" (uDummy)
3003 : "0" (pv),
3004 "c" (RT_ASM_PAGE_SIZE >> 3),
3005 "a" (0)
3006 : "memory");
3007# else
3008 __asm__ __volatile__("rep stosl"
3009 : "=D" (pv),
3010 "=c" (uDummy)
3011 : "0" (pv),
3012 "c" (RT_ASM_PAGE_SIZE >> 2),
3013 "a" (0)
3014 : "memory");
3015# endif
3016# else
3017 __asm
3018 {
3019# ifdef RT_ARCH_AMD64
3020 xor rax, rax
3021 mov ecx, 0200h
3022 mov rdi, [pv]
3023 rep stosq
3024# else
3025 xor eax, eax
3026 mov ecx, 0400h
3027 mov edi, [pv]
3028 rep stosd
3029# endif
3030 }
3031# endif
3032}
3033# endif
3034
3035
3036/**
3037 * Zeros a memory block with a 32-bit aligned size.
3038 *
3039 * @param pv Pointer to the memory block.
3040 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3041 */
3042#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3043DECLASM(void) ASMMemZero32(volatile void *pv, size_t cb);
3044#else
3045DECLINLINE(void) ASMMemZero32(volatile void *pv, size_t cb)
3046{
3047# if RT_INLINE_ASM_USES_INTRIN
3048# ifdef RT_ARCH_AMD64
3049 if (!(cb & 7))
3050 __stosq((unsigned __int64 *)pv, 0, cb / 8);
3051 else
3052# endif
3053 __stosd((unsigned long *)pv, 0, cb / 4);
3054
3055# elif RT_INLINE_ASM_GNU_STYLE
3056 __asm__ __volatile__("rep stosl"
3057 : "=D" (pv),
3058 "=c" (cb)
3059 : "0" (pv),
3060 "1" (cb >> 2),
3061 "a" (0)
3062 : "memory");
3063# else
3064 __asm
3065 {
3066 xor eax, eax
3067# ifdef RT_ARCH_AMD64
3068 mov rcx, [cb]
3069 shr rcx, 2
3070 mov rdi, [pv]
3071# else
3072 mov ecx, [cb]
3073 shr ecx, 2
3074 mov edi, [pv]
3075# endif
3076 rep stosd
3077 }
3078# endif
3079}
3080#endif
3081
3082
3083/**
3084 * Fills a memory block with a 32-bit aligned size.
3085 *
3086 * @param pv Pointer to the memory block.
3087 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3088 * @param u32 The value to fill with.
3089 */
3090#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3091DECLASM(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32);
3092#else
3093DECLINLINE(void) ASMMemFill32(volatile void *pv, size_t cb, uint32_t u32)
3094{
3095# if RT_INLINE_ASM_USES_INTRIN
3096# ifdef RT_ARCH_AMD64
3097 if (!(cb & 7))
3098 __stosq((unsigned __int64 *)pv, RT_MAKE_U64(u32, u32), cb / 8);
3099 else
3100# endif
3101 __stosd((unsigned long *)pv, u32, cb / 4);
3102
3103# elif RT_INLINE_ASM_GNU_STYLE
3104 __asm__ __volatile__("rep stosl"
3105 : "=D" (pv),
3106 "=c" (cb)
3107 : "0" (pv),
3108 "1" (cb >> 2),
3109 "a" (u32)
3110 : "memory");
3111# else
3112 __asm
3113 {
3114# ifdef RT_ARCH_AMD64
3115 mov rcx, [cb]
3116 shr rcx, 2
3117 mov rdi, [pv]
3118# else
3119 mov ecx, [cb]
3120 shr ecx, 2
3121 mov edi, [pv]
3122# endif
3123 mov eax, [u32]
3124 rep stosd
3125 }
3126# endif
3127}
3128#endif
3129
3130
3131/**
3132 * Checks if a memory page is all zeros.
3133 *
3134 * @returns true / false.
3135 *
3136 * @param pvPage Pointer to the page. Must be aligned on 16 byte
3137 * boundrary
3138 */
3139DECLINLINE(bool) ASMMemIsZeroPage(void const *pvPage)
3140{
3141# if 0 /*RT_INLINE_ASM_GNU_STYLE - this is actually slower... */
3142 union { RTCCUINTREG r; bool f; } uAX;
3143 RTCCUINTREG xCX, xDI;
3144 Assert(!((uintptr_t)pvPage & 15));
3145 __asm__ __volatile__("repe; "
3146# ifdef RT_ARCH_AMD64
3147 "scasq\n\t"
3148# else
3149 "scasl\n\t"
3150# endif
3151 "setnc %%al\n\t"
3152 : "=&c" (xCX),
3153 "=&D" (xDI),
3154 "=&a" (uAX.r)
3155 : "mr" (pvPage),
3156# ifdef RT_ARCH_AMD64
3157 "0" (RT_ASM_PAGE_SIZE/8),
3158# else
3159 "0" (RT_ASM_PAGE_SIZE/4),
3160# endif
3161 "1" (pvPage),
3162 "2" (0));
3163 return uAX.f;
3164# else
3165 uintptr_t const *puPtr = (uintptr_t const *)pvPage;
3166 int cLeft = RT_ASM_PAGE_SIZE / sizeof(uintptr_t) / 8;
3167 Assert(!((uintptr_t)pvPage & 15));
3168 for (;;)
3169 {
3170 if (puPtr[0]) return false;
3171 if (puPtr[4]) return false;
3172
3173 if (puPtr[2]) return false;
3174 if (puPtr[6]) return false;
3175
3176 if (puPtr[1]) return false;
3177 if (puPtr[5]) return false;
3178
3179 if (puPtr[3]) return false;
3180 if (puPtr[7]) return false;
3181
3182 if (!--cLeft)
3183 return true;
3184 puPtr += 8;
3185 }
3186 return true;
3187# endif
3188}
3189
3190
3191/**
3192 * Checks if a memory block is filled with the specified byte.
3193 *
3194 * This is a sort of inverted memchr.
3195 *
3196 * @returns Pointer to the byte which doesn't equal u8.
3197 * @returns NULL if all equal to u8.
3198 *
3199 * @param pv Pointer to the memory block.
3200 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3201 * @param u8 The value it's supposed to be filled with.
3202 *
3203 * @todo Fix name, it is a predicate function but it's not returning boolean!
3204 */
3205DECLINLINE(void *) ASMMemIsAll8(void const *pv, size_t cb, uint8_t u8)
3206{
3207/** @todo rewrite this in inline assembly? */
3208 uint8_t const *pb = (uint8_t const *)pv;
3209 for (; cb; cb--, pb++)
3210 if (RT_UNLIKELY(*pb != u8))
3211 return (void *)pb;
3212 return NULL;
3213}
3214
3215
3216/**
3217 * Checks if a memory block is filled with the specified 32-bit value.
3218 *
3219 * This is a sort of inverted memchr.
3220 *
3221 * @returns Pointer to the first value which doesn't equal u32.
3222 * @returns NULL if all equal to u32.
3223 *
3224 * @param pv Pointer to the memory block.
3225 * @param cb Number of bytes in the block. This MUST be aligned on 32-bit!
3226 * @param u32 The value it's supposed to be filled with.
3227 *
3228 * @todo Fix name, it is a predicate function but it's not returning boolean!
3229 */
3230DECLINLINE(uint32_t *) ASMMemIsAllU32(void const *pv, size_t cb, uint32_t u32)
3231{
3232/** @todo rewrite this in inline assembly? */
3233 uint32_t const *pu32 = (uint32_t const *)pv;
3234 for (; cb; cb -= 4, pu32++)
3235 if (RT_UNLIKELY(*pu32 != u32))
3236 return (uint32_t *)pu32;
3237 return NULL;
3238}
3239
3240
3241/**
3242 * Probes a byte pointer for read access.
3243 *
3244 * While the function will not fault if the byte is not read accessible,
3245 * the idea is to do this in a safe place like before acquiring locks
3246 * and such like.
3247 *
3248 * Also, this functions guarantees that an eager compiler is not going
3249 * to optimize the probing away.
3250 *
3251 * @param pvByte Pointer to the byte.
3252 */
3253#if RT_INLINE_ASM_EXTERNAL
3254DECLASM(uint8_t) ASMProbeReadByte(const void *pvByte);
3255#else
3256DECLINLINE(uint8_t) ASMProbeReadByte(const void *pvByte)
3257{
3258 /** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
3259 uint8_t u8;
3260# if RT_INLINE_ASM_GNU_STYLE
3261 __asm__ __volatile__("movb (%1), %0\n\t"
3262 : "=r" (u8)
3263 : "r" (pvByte));
3264# else
3265 __asm
3266 {
3267# ifdef RT_ARCH_AMD64
3268 mov rax, [pvByte]
3269 mov al, [rax]
3270# else
3271 mov eax, [pvByte]
3272 mov al, [eax]
3273# endif
3274 mov [u8], al
3275 }
3276# endif
3277 return u8;
3278}
3279#endif
3280
3281/**
3282 * Probes a buffer for read access page by page.
3283 *
3284 * While the function will fault if the buffer is not fully read
3285 * accessible, the idea is to do this in a safe place like before
3286 * acquiring locks and such like.
3287 *
3288 * Also, this functions guarantees that an eager compiler is not going
3289 * to optimize the probing away.
3290 *
3291 * @param pvBuf Pointer to the buffer.
3292 * @param cbBuf The size of the buffer in bytes. Must be >= 1.
3293 */
3294DECLINLINE(void) ASMProbeReadBuffer(const void *pvBuf, size_t cbBuf)
3295{
3296 /** @todo verify that the compiler actually doesn't optimize this away. (intel & gcc) */
3297 /* the first byte */
3298 const uint8_t *pu8 = (const uint8_t *)pvBuf;
3299 ASMProbeReadByte(pu8);
3300
3301 /* the pages in between pages. */
3302 while (cbBuf > RT_ASM_PAGE_SIZE)
3303 {
3304 ASMProbeReadByte(pu8);
3305 cbBuf -= RT_ASM_PAGE_SIZE;
3306 pu8 += RT_ASM_PAGE_SIZE;
3307 }
3308
3309 /* the last byte */
3310 ASMProbeReadByte(pu8 + cbBuf - 1);
3311}
3312
3313
3314
3315/** @defgroup grp_inline_bits Bit Operations
3316 * @{
3317 */
3318
3319
3320/**
3321 * Sets a bit in a bitmap.
3322 *
3323 * @param pvBitmap Pointer to the bitmap. This should be 32-bit aligned.
3324 * @param iBit The bit to set.
3325 *
3326 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3327 * However, doing so will yield better performance as well as avoiding
3328 * traps accessing the last bits in the bitmap.
3329 */
3330#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3331DECLASM(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit);
3332#else
3333DECLINLINE(void) ASMBitSet(volatile void *pvBitmap, int32_t iBit)
3334{
3335# if RT_INLINE_ASM_USES_INTRIN
3336 _bittestandset((long *)pvBitmap, iBit);
3337
3338# elif RT_INLINE_ASM_GNU_STYLE
3339 __asm__ __volatile__("btsl %1, %0"
3340 : "=m" (*(volatile long *)pvBitmap)
3341 : "Ir" (iBit),
3342 "m" (*(volatile long *)pvBitmap)
3343 : "memory");
3344# else
3345 __asm
3346 {
3347# ifdef RT_ARCH_AMD64
3348 mov rax, [pvBitmap]
3349 mov edx, [iBit]
3350 bts [rax], edx
3351# else
3352 mov eax, [pvBitmap]
3353 mov edx, [iBit]
3354 bts [eax], edx
3355# endif
3356 }
3357# endif
3358}
3359#endif
3360
3361
3362/**
3363 * Atomically sets a bit in a bitmap, ordered.
3364 *
3365 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3366 * the memory access isn't atomic!
3367 * @param iBit The bit to set.
3368 */
3369#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3370DECLASM(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit);
3371#else
3372DECLINLINE(void) ASMAtomicBitSet(volatile void *pvBitmap, int32_t iBit)
3373{
3374 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3375# if RT_INLINE_ASM_USES_INTRIN
3376 _interlockedbittestandset((long *)pvBitmap, iBit);
3377# elif RT_INLINE_ASM_GNU_STYLE
3378 __asm__ __volatile__("lock; btsl %1, %0"
3379 : "=m" (*(volatile long *)pvBitmap)
3380 : "Ir" (iBit),
3381 "m" (*(volatile long *)pvBitmap)
3382 : "memory");
3383# else
3384 __asm
3385 {
3386# ifdef RT_ARCH_AMD64
3387 mov rax, [pvBitmap]
3388 mov edx, [iBit]
3389 lock bts [rax], edx
3390# else
3391 mov eax, [pvBitmap]
3392 mov edx, [iBit]
3393 lock bts [eax], edx
3394# endif
3395 }
3396# endif
3397}
3398#endif
3399
3400
3401/**
3402 * Clears a bit in a bitmap.
3403 *
3404 * @param pvBitmap Pointer to the bitmap.
3405 * @param iBit The bit to clear.
3406 *
3407 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3408 * However, doing so will yield better performance as well as avoiding
3409 * traps accessing the last bits in the bitmap.
3410 */
3411#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3412DECLASM(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit);
3413#else
3414DECLINLINE(void) ASMBitClear(volatile void *pvBitmap, int32_t iBit)
3415{
3416# if RT_INLINE_ASM_USES_INTRIN
3417 _bittestandreset((long *)pvBitmap, iBit);
3418
3419# elif RT_INLINE_ASM_GNU_STYLE
3420 __asm__ __volatile__("btrl %1, %0"
3421 : "=m" (*(volatile long *)pvBitmap)
3422 : "Ir" (iBit),
3423 "m" (*(volatile long *)pvBitmap)
3424 : "memory");
3425# else
3426 __asm
3427 {
3428# ifdef RT_ARCH_AMD64
3429 mov rax, [pvBitmap]
3430 mov edx, [iBit]
3431 btr [rax], edx
3432# else
3433 mov eax, [pvBitmap]
3434 mov edx, [iBit]
3435 btr [eax], edx
3436# endif
3437 }
3438# endif
3439}
3440#endif
3441
3442
3443/**
3444 * Atomically clears a bit in a bitmap, ordered.
3445 *
3446 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3447 * the memory access isn't atomic!
3448 * @param iBit The bit to toggle set.
3449 * @remarks No memory barrier, take care on smp.
3450 */
3451#if RT_INLINE_ASM_EXTERNAL
3452DECLASM(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit);
3453#else
3454DECLINLINE(void) ASMAtomicBitClear(volatile void *pvBitmap, int32_t iBit)
3455{
3456 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3457# if RT_INLINE_ASM_GNU_STYLE
3458 __asm__ __volatile__("lock; btrl %1, %0"
3459 : "=m" (*(volatile long *)pvBitmap)
3460 : "Ir" (iBit),
3461 "m" (*(volatile long *)pvBitmap)
3462 : "memory");
3463# else
3464 __asm
3465 {
3466# ifdef RT_ARCH_AMD64
3467 mov rax, [pvBitmap]
3468 mov edx, [iBit]
3469 lock btr [rax], edx
3470# else
3471 mov eax, [pvBitmap]
3472 mov edx, [iBit]
3473 lock btr [eax], edx
3474# endif
3475 }
3476# endif
3477}
3478#endif
3479
3480
3481/**
3482 * Toggles a bit in a bitmap.
3483 *
3484 * @param pvBitmap Pointer to the bitmap.
3485 * @param iBit The bit to toggle.
3486 *
3487 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3488 * However, doing so will yield better performance as well as avoiding
3489 * traps accessing the last bits in the bitmap.
3490 */
3491#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3492DECLASM(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit);
3493#else
3494DECLINLINE(void) ASMBitToggle(volatile void *pvBitmap, int32_t iBit)
3495{
3496# if RT_INLINE_ASM_USES_INTRIN
3497 _bittestandcomplement((long *)pvBitmap, iBit);
3498# elif RT_INLINE_ASM_GNU_STYLE
3499 __asm__ __volatile__("btcl %1, %0"
3500 : "=m" (*(volatile long *)pvBitmap)
3501 : "Ir" (iBit),
3502 "m" (*(volatile long *)pvBitmap)
3503 : "memory");
3504# else
3505 __asm
3506 {
3507# ifdef RT_ARCH_AMD64
3508 mov rax, [pvBitmap]
3509 mov edx, [iBit]
3510 btc [rax], edx
3511# else
3512 mov eax, [pvBitmap]
3513 mov edx, [iBit]
3514 btc [eax], edx
3515# endif
3516 }
3517# endif
3518}
3519#endif
3520
3521
3522/**
3523 * Atomically toggles a bit in a bitmap, ordered.
3524 *
3525 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3526 * the memory access isn't atomic!
3527 * @param iBit The bit to test and set.
3528 */
3529#if RT_INLINE_ASM_EXTERNAL
3530DECLASM(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit);
3531#else
3532DECLINLINE(void) ASMAtomicBitToggle(volatile void *pvBitmap, int32_t iBit)
3533{
3534 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3535# if RT_INLINE_ASM_GNU_STYLE
3536 __asm__ __volatile__("lock; btcl %1, %0"
3537 : "=m" (*(volatile long *)pvBitmap)
3538 : "Ir" (iBit),
3539 "m" (*(volatile long *)pvBitmap)
3540 : "memory");
3541# else
3542 __asm
3543 {
3544# ifdef RT_ARCH_AMD64
3545 mov rax, [pvBitmap]
3546 mov edx, [iBit]
3547 lock btc [rax], edx
3548# else
3549 mov eax, [pvBitmap]
3550 mov edx, [iBit]
3551 lock btc [eax], edx
3552# endif
3553 }
3554# endif
3555}
3556#endif
3557
3558
3559/**
3560 * Tests and sets a bit in a bitmap.
3561 *
3562 * @returns true if the bit was set.
3563 * @returns false if the bit was clear.
3564 *
3565 * @param pvBitmap Pointer to the bitmap.
3566 * @param iBit The bit to test and set.
3567 *
3568 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3569 * However, doing so will yield better performance as well as avoiding
3570 * traps accessing the last bits in the bitmap.
3571 */
3572#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3573DECLASM(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
3574#else
3575DECLINLINE(bool) ASMBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
3576{
3577 union { bool f; uint32_t u32; uint8_t u8; } rc;
3578# if RT_INLINE_ASM_USES_INTRIN
3579 rc.u8 = _bittestandset((long *)pvBitmap, iBit);
3580
3581# elif RT_INLINE_ASM_GNU_STYLE
3582 __asm__ __volatile__("btsl %2, %1\n\t"
3583 "setc %b0\n\t"
3584 "andl $1, %0\n\t"
3585 : "=q" (rc.u32),
3586 "=m" (*(volatile long *)pvBitmap)
3587 : "Ir" (iBit),
3588 "m" (*(volatile long *)pvBitmap)
3589 : "memory");
3590# else
3591 __asm
3592 {
3593 mov edx, [iBit]
3594# ifdef RT_ARCH_AMD64
3595 mov rax, [pvBitmap]
3596 bts [rax], edx
3597# else
3598 mov eax, [pvBitmap]
3599 bts [eax], edx
3600# endif
3601 setc al
3602 and eax, 1
3603 mov [rc.u32], eax
3604 }
3605# endif
3606 return rc.f;
3607}
3608#endif
3609
3610
3611/**
3612 * Atomically tests and sets a bit in a bitmap, ordered.
3613 *
3614 * @returns true if the bit was set.
3615 * @returns false if the bit was clear.
3616 *
3617 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3618 * the memory access isn't atomic!
3619 * @param iBit The bit to set.
3620 */
3621#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3622DECLASM(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit);
3623#else
3624DECLINLINE(bool) ASMAtomicBitTestAndSet(volatile void *pvBitmap, int32_t iBit)
3625{
3626 union { bool f; uint32_t u32; uint8_t u8; } rc;
3627 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3628# if RT_INLINE_ASM_USES_INTRIN
3629 rc.u8 = _interlockedbittestandset((long *)pvBitmap, iBit);
3630# elif RT_INLINE_ASM_GNU_STYLE
3631 __asm__ __volatile__("lock; btsl %2, %1\n\t"
3632 "setc %b0\n\t"
3633 "andl $1, %0\n\t"
3634 : "=q" (rc.u32),
3635 "=m" (*(volatile long *)pvBitmap)
3636 : "Ir" (iBit),
3637 "m" (*(volatile long *)pvBitmap)
3638 : "memory");
3639# else
3640 __asm
3641 {
3642 mov edx, [iBit]
3643# ifdef RT_ARCH_AMD64
3644 mov rax, [pvBitmap]
3645 lock bts [rax], edx
3646# else
3647 mov eax, [pvBitmap]
3648 lock bts [eax], edx
3649# endif
3650 setc al
3651 and eax, 1
3652 mov [rc.u32], eax
3653 }
3654# endif
3655 return rc.f;
3656}
3657#endif
3658
3659
3660/**
3661 * Tests and clears a bit in a bitmap.
3662 *
3663 * @returns true if the bit was set.
3664 * @returns false if the bit was clear.
3665 *
3666 * @param pvBitmap Pointer to the bitmap.
3667 * @param iBit The bit to test and clear.
3668 *
3669 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3670 * However, doing so will yield better performance as well as avoiding
3671 * traps accessing the last bits in the bitmap.
3672 */
3673#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3674DECLASM(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
3675#else
3676DECLINLINE(bool) ASMBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
3677{
3678 union { bool f; uint32_t u32; uint8_t u8; } rc;
3679# if RT_INLINE_ASM_USES_INTRIN
3680 rc.u8 = _bittestandreset((long *)pvBitmap, iBit);
3681
3682# elif RT_INLINE_ASM_GNU_STYLE
3683 __asm__ __volatile__("btrl %2, %1\n\t"
3684 "setc %b0\n\t"
3685 "andl $1, %0\n\t"
3686 : "=q" (rc.u32),
3687 "=m" (*(volatile long *)pvBitmap)
3688 : "Ir" (iBit),
3689 "m" (*(volatile long *)pvBitmap)
3690 : "memory");
3691# else
3692 __asm
3693 {
3694 mov edx, [iBit]
3695# ifdef RT_ARCH_AMD64
3696 mov rax, [pvBitmap]
3697 btr [rax], edx
3698# else
3699 mov eax, [pvBitmap]
3700 btr [eax], edx
3701# endif
3702 setc al
3703 and eax, 1
3704 mov [rc.u32], eax
3705 }
3706# endif
3707 return rc.f;
3708}
3709#endif
3710
3711
3712/**
3713 * Atomically tests and clears a bit in a bitmap, ordered.
3714 *
3715 * @returns true if the bit was set.
3716 * @returns false if the bit was clear.
3717 *
3718 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3719 * the memory access isn't atomic!
3720 * @param iBit The bit to test and clear.
3721 *
3722 * @remarks No memory barrier, take care on smp.
3723 */
3724#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3725DECLASM(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit);
3726#else
3727DECLINLINE(bool) ASMAtomicBitTestAndClear(volatile void *pvBitmap, int32_t iBit)
3728{
3729 union { bool f; uint32_t u32; uint8_t u8; } rc;
3730 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3731# if RT_INLINE_ASM_USES_INTRIN
3732 rc.u8 = _interlockedbittestandreset((long *)pvBitmap, iBit);
3733
3734# elif RT_INLINE_ASM_GNU_STYLE
3735 __asm__ __volatile__("lock; btrl %2, %1\n\t"
3736 "setc %b0\n\t"
3737 "andl $1, %0\n\t"
3738 : "=q" (rc.u32),
3739 "=m" (*(volatile long *)pvBitmap)
3740 : "Ir" (iBit),
3741 "m" (*(volatile long *)pvBitmap)
3742 : "memory");
3743# else
3744 __asm
3745 {
3746 mov edx, [iBit]
3747# ifdef RT_ARCH_AMD64
3748 mov rax, [pvBitmap]
3749 lock btr [rax], edx
3750# else
3751 mov eax, [pvBitmap]
3752 lock btr [eax], edx
3753# endif
3754 setc al
3755 and eax, 1
3756 mov [rc.u32], eax
3757 }
3758# endif
3759 return rc.f;
3760}
3761#endif
3762
3763
3764/**
3765 * Tests and toggles a bit in a bitmap.
3766 *
3767 * @returns true if the bit was set.
3768 * @returns false if the bit was clear.
3769 *
3770 * @param pvBitmap Pointer to the bitmap.
3771 * @param iBit The bit to test and toggle.
3772 *
3773 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3774 * However, doing so will yield better performance as well as avoiding
3775 * traps accessing the last bits in the bitmap.
3776 */
3777#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3778DECLASM(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
3779#else
3780DECLINLINE(bool) ASMBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
3781{
3782 union { bool f; uint32_t u32; uint8_t u8; } rc;
3783# if RT_INLINE_ASM_USES_INTRIN
3784 rc.u8 = _bittestandcomplement((long *)pvBitmap, iBit);
3785
3786# elif RT_INLINE_ASM_GNU_STYLE
3787 __asm__ __volatile__("btcl %2, %1\n\t"
3788 "setc %b0\n\t"
3789 "andl $1, %0\n\t"
3790 : "=q" (rc.u32),
3791 "=m" (*(volatile long *)pvBitmap)
3792 : "Ir" (iBit),
3793 "m" (*(volatile long *)pvBitmap)
3794 : "memory");
3795# else
3796 __asm
3797 {
3798 mov edx, [iBit]
3799# ifdef RT_ARCH_AMD64
3800 mov rax, [pvBitmap]
3801 btc [rax], edx
3802# else
3803 mov eax, [pvBitmap]
3804 btc [eax], edx
3805# endif
3806 setc al
3807 and eax, 1
3808 mov [rc.u32], eax
3809 }
3810# endif
3811 return rc.f;
3812}
3813#endif
3814
3815
3816/**
3817 * Atomically tests and toggles a bit in a bitmap, ordered.
3818 *
3819 * @returns true if the bit was set.
3820 * @returns false if the bit was clear.
3821 *
3822 * @param pvBitmap Pointer to the bitmap. Must be 32-bit aligned, otherwise
3823 * the memory access isn't atomic!
3824 * @param iBit The bit to test and toggle.
3825 */
3826#if RT_INLINE_ASM_EXTERNAL
3827DECLASM(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit);
3828#else
3829DECLINLINE(bool) ASMAtomicBitTestAndToggle(volatile void *pvBitmap, int32_t iBit)
3830{
3831 union { bool f; uint32_t u32; uint8_t u8; } rc;
3832 AssertMsg(!((uintptr_t)pvBitmap & 3), ("address %p not 32-bit aligned", pvBitmap));
3833# if RT_INLINE_ASM_GNU_STYLE
3834 __asm__ __volatile__("lock; btcl %2, %1\n\t"
3835 "setc %b0\n\t"
3836 "andl $1, %0\n\t"
3837 : "=q" (rc.u32),
3838 "=m" (*(volatile long *)pvBitmap)
3839 : "Ir" (iBit),
3840 "m" (*(volatile long *)pvBitmap)
3841 : "memory");
3842# else
3843 __asm
3844 {
3845 mov edx, [iBit]
3846# ifdef RT_ARCH_AMD64
3847 mov rax, [pvBitmap]
3848 lock btc [rax], edx
3849# else
3850 mov eax, [pvBitmap]
3851 lock btc [eax], edx
3852# endif
3853 setc al
3854 and eax, 1
3855 mov [rc.u32], eax
3856 }
3857# endif
3858 return rc.f;
3859}
3860#endif
3861
3862
3863/**
3864 * Tests if a bit in a bitmap is set.
3865 *
3866 * @returns true if the bit is set.
3867 * @returns false if the bit is clear.
3868 *
3869 * @param pvBitmap Pointer to the bitmap.
3870 * @param iBit The bit to test.
3871 *
3872 * @remarks The 32-bit aligning of pvBitmap is not a strict requirement.
3873 * However, doing so will yield better performance as well as avoiding
3874 * traps accessing the last bits in the bitmap.
3875 */
3876#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
3877DECLASM(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit);
3878#else
3879DECLINLINE(bool) ASMBitTest(const volatile void *pvBitmap, int32_t iBit)
3880{
3881 union { bool f; uint32_t u32; uint8_t u8; } rc;
3882# if RT_INLINE_ASM_USES_INTRIN
3883 rc.u32 = _bittest((long *)pvBitmap, iBit);
3884# elif RT_INLINE_ASM_GNU_STYLE
3885
3886 __asm__ __volatile__("btl %2, %1\n\t"
3887 "setc %b0\n\t"
3888 "andl $1, %0\n\t"
3889 : "=q" (rc.u32)
3890 : "m" (*(const volatile long *)pvBitmap),
3891 "Ir" (iBit)
3892 : "memory");
3893# else
3894 __asm
3895 {
3896 mov edx, [iBit]
3897# ifdef RT_ARCH_AMD64
3898 mov rax, [pvBitmap]
3899 bt [rax], edx
3900# else
3901 mov eax, [pvBitmap]
3902 bt [eax], edx
3903# endif
3904 setc al
3905 and eax, 1
3906 mov [rc.u32], eax
3907 }
3908# endif
3909 return rc.f;
3910}
3911#endif
3912
3913
3914/**
3915 * Clears a bit range within a bitmap.
3916 *
3917 * @param pvBitmap Pointer to the bitmap.
3918 * @param iBitStart The First bit to clear.
3919 * @param iBitEnd The first bit not to clear.
3920 */
3921DECLINLINE(void) ASMBitClearRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
3922{
3923 if (iBitStart < iBitEnd)
3924 {
3925 volatile uint32_t *pu32 = (volatile uint32_t *)pvBitmap + (iBitStart >> 5);
3926 int iStart = iBitStart & ~31;
3927 int iEnd = iBitEnd & ~31;
3928 if (iStart == iEnd)
3929 *pu32 &= ((1 << (iBitStart & 31)) - 1) | ~((1 << (iBitEnd & 31)) - 1);
3930 else
3931 {
3932 /* bits in first dword. */
3933 if (iBitStart & 31)
3934 {
3935 *pu32 &= (1 << (iBitStart & 31)) - 1;
3936 pu32++;
3937 iBitStart = iStart + 32;
3938 }
3939
3940 /* whole dword. */
3941 if (iBitStart != iEnd)
3942 ASMMemZero32(pu32, (iEnd - iBitStart) >> 3);
3943
3944 /* bits in last dword. */
3945 if (iBitEnd & 31)
3946 {
3947 pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
3948 *pu32 &= ~((1 << (iBitEnd & 31)) - 1);
3949 }
3950 }
3951 }
3952}
3953
3954
3955/**
3956 * Sets a bit range within a bitmap.
3957 *
3958 * @param pvBitmap Pointer to the bitmap.
3959 * @param iBitStart The First bit to set.
3960 * @param iBitEnd The first bit not to set.
3961 */
3962DECLINLINE(void) ASMBitSetRange(volatile void *pvBitmap, int32_t iBitStart, int32_t iBitEnd)
3963{
3964 if (iBitStart < iBitEnd)
3965 {
3966 volatile uint32_t *pu32 = (volatile uint32_t *)pvBitmap + (iBitStart >> 5);
3967 int iStart = iBitStart & ~31;
3968 int iEnd = iBitEnd & ~31;
3969 if (iStart == iEnd)
3970 *pu32 |= ((1 << (iBitEnd - iBitStart)) - 1) << iBitStart;
3971 else
3972 {
3973 /* bits in first dword. */
3974 if (iBitStart & 31)
3975 {
3976 *pu32 |= ~((1 << (iBitStart & 31)) - 1);
3977 pu32++;
3978 iBitStart = iStart + 32;
3979 }
3980
3981 /* whole dword. */
3982 if (iBitStart != iEnd)
3983 ASMMemFill32(pu32, (iEnd - iBitStart) >> 3, ~0);
3984
3985 /* bits in last dword. */
3986 if (iBitEnd & 31)
3987 {
3988 pu32 = (volatile uint32_t *)pvBitmap + (iBitEnd >> 5);
3989 *pu32 |= (1 << (iBitEnd & 31)) - 1;
3990 }
3991 }
3992 }
3993}
3994
3995
3996/**
3997 * Finds the first clear bit in a bitmap.
3998 *
3999 * @returns Index of the first zero bit.
4000 * @returns -1 if no clear bit was found.
4001 * @param pvBitmap Pointer to the bitmap.
4002 * @param cBits The number of bits in the bitmap. Multiple of 32.
4003 */
4004#if RT_INLINE_ASM_EXTERNAL
4005DECLASM(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits);
4006#else
4007DECLINLINE(int) ASMBitFirstClear(const volatile void *pvBitmap, uint32_t cBits)
4008{
4009 if (cBits)
4010 {
4011 int32_t iBit;
4012# if RT_INLINE_ASM_GNU_STYLE
4013 RTCCUINTREG uEAX, uECX, uEDI;
4014 cBits = RT_ALIGN_32(cBits, 32);
4015 __asm__ __volatile__("repe; scasl\n\t"
4016 "je 1f\n\t"
4017# ifdef RT_ARCH_AMD64
4018 "lea -4(%%rdi), %%rdi\n\t"
4019 "xorl (%%rdi), %%eax\n\t"
4020 "subq %5, %%rdi\n\t"
4021# else
4022 "lea -4(%%edi), %%edi\n\t"
4023 "xorl (%%edi), %%eax\n\t"
4024 "subl %5, %%edi\n\t"
4025# endif
4026 "shll $3, %%edi\n\t"
4027 "bsfl %%eax, %%edx\n\t"
4028 "addl %%edi, %%edx\n\t"
4029 "1:\t\n"
4030 : "=d" (iBit),
4031 "=&c" (uECX),
4032 "=&D" (uEDI),
4033 "=&a" (uEAX)
4034 : "0" (0xffffffff),
4035 "mr" (pvBitmap),
4036 "1" (cBits >> 5),
4037 "2" (pvBitmap),
4038 "3" (0xffffffff));
4039# else
4040 cBits = RT_ALIGN_32(cBits, 32);
4041 __asm
4042 {
4043# ifdef RT_ARCH_AMD64
4044 mov rdi, [pvBitmap]
4045 mov rbx, rdi
4046# else
4047 mov edi, [pvBitmap]
4048 mov ebx, edi
4049# endif
4050 mov edx, 0ffffffffh
4051 mov eax, edx
4052 mov ecx, [cBits]
4053 shr ecx, 5
4054 repe scasd
4055 je done
4056
4057# ifdef RT_ARCH_AMD64
4058 lea rdi, [rdi - 4]
4059 xor eax, [rdi]
4060 sub rdi, rbx
4061# else
4062 lea edi, [edi - 4]
4063 xor eax, [edi]
4064 sub edi, ebx
4065# endif
4066 shl edi, 3
4067 bsf edx, eax
4068 add edx, edi
4069 done:
4070 mov [iBit], edx
4071 }
4072# endif
4073 return iBit;
4074 }
4075 return -1;
4076}
4077#endif
4078
4079
4080/**
4081 * Finds the next clear bit in a bitmap.
4082 *
4083 * @returns Index of the first zero bit.
4084 * @returns -1 if no clear bit was found.
4085 * @param pvBitmap Pointer to the bitmap.
4086 * @param cBits The number of bits in the bitmap. Multiple of 32.
4087 * @param iBitPrev The bit returned from the last search.
4088 * The search will start at iBitPrev + 1.
4089 */
4090#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4091DECLASM(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
4092#else
4093DECLINLINE(int) ASMBitNextClear(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
4094{
4095 const volatile uint32_t *pau32Bitmap = (const volatile uint32_t *)pvBitmap;
4096 int iBit = ++iBitPrev & 31;
4097 if (iBit)
4098 {
4099 /*
4100 * Inspect the 32-bit word containing the unaligned bit.
4101 */
4102 uint32_t u32 = ~pau32Bitmap[iBitPrev / 32] >> iBit;
4103
4104# if RT_INLINE_ASM_USES_INTRIN
4105 unsigned long ulBit = 0;
4106 if (_BitScanForward(&ulBit, u32))
4107 return ulBit + iBitPrev;
4108# else
4109# if RT_INLINE_ASM_GNU_STYLE
4110 __asm__ __volatile__("bsf %1, %0\n\t"
4111 "jnz 1f\n\t"
4112 "movl $-1, %0\n\t"
4113 "1:\n\t"
4114 : "=r" (iBit)
4115 : "r" (u32));
4116# else
4117 __asm
4118 {
4119 mov edx, [u32]
4120 bsf eax, edx
4121 jnz done
4122 mov eax, 0ffffffffh
4123 done:
4124 mov [iBit], eax
4125 }
4126# endif
4127 if (iBit >= 0)
4128 return iBit + iBitPrev;
4129# endif
4130
4131 /*
4132 * Skip ahead and see if there is anything left to search.
4133 */
4134 iBitPrev |= 31;
4135 iBitPrev++;
4136 if (cBits <= (uint32_t)iBitPrev)
4137 return -1;
4138 }
4139
4140 /*
4141 * 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4142 */
4143 iBit = ASMBitFirstClear(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4144 if (iBit >= 0)
4145 iBit += iBitPrev;
4146 return iBit;
4147}
4148#endif
4149
4150
4151/**
4152 * Finds the first set bit in a bitmap.
4153 *
4154 * @returns Index of the first set bit.
4155 * @returns -1 if no clear bit was found.
4156 * @param pvBitmap Pointer to the bitmap.
4157 * @param cBits The number of bits in the bitmap. Multiple of 32.
4158 */
4159#if RT_INLINE_ASM_EXTERNAL
4160DECLASM(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits);
4161#else
4162DECLINLINE(int) ASMBitFirstSet(const volatile void *pvBitmap, uint32_t cBits)
4163{
4164 if (cBits)
4165 {
4166 int32_t iBit;
4167# if RT_INLINE_ASM_GNU_STYLE
4168 RTCCUINTREG uEAX, uECX, uEDI;
4169 cBits = RT_ALIGN_32(cBits, 32);
4170 __asm__ __volatile__("repe; scasl\n\t"
4171 "je 1f\n\t"
4172# ifdef RT_ARCH_AMD64
4173 "lea -4(%%rdi), %%rdi\n\t"
4174 "movl (%%rdi), %%eax\n\t"
4175 "subq %5, %%rdi\n\t"
4176# else
4177 "lea -4(%%edi), %%edi\n\t"
4178 "movl (%%edi), %%eax\n\t"
4179 "subl %5, %%edi\n\t"
4180# endif
4181 "shll $3, %%edi\n\t"
4182 "bsfl %%eax, %%edx\n\t"
4183 "addl %%edi, %%edx\n\t"
4184 "1:\t\n"
4185 : "=d" (iBit),
4186 "=&c" (uECX),
4187 "=&D" (uEDI),
4188 "=&a" (uEAX)
4189 : "0" (0xffffffff),
4190 "mr" (pvBitmap),
4191 "1" (cBits >> 5),
4192 "2" (pvBitmap),
4193 "3" (0));
4194# else
4195 cBits = RT_ALIGN_32(cBits, 32);
4196 __asm
4197 {
4198# ifdef RT_ARCH_AMD64
4199 mov rdi, [pvBitmap]
4200 mov rbx, rdi
4201# else
4202 mov edi, [pvBitmap]
4203 mov ebx, edi
4204# endif
4205 mov edx, 0ffffffffh
4206 xor eax, eax
4207 mov ecx, [cBits]
4208 shr ecx, 5
4209 repe scasd
4210 je done
4211# ifdef RT_ARCH_AMD64
4212 lea rdi, [rdi - 4]
4213 mov eax, [rdi]
4214 sub rdi, rbx
4215# else
4216 lea edi, [edi - 4]
4217 mov eax, [edi]
4218 sub edi, ebx
4219# endif
4220 shl edi, 3
4221 bsf edx, eax
4222 add edx, edi
4223 done:
4224 mov [iBit], edx
4225 }
4226# endif
4227 return iBit;
4228 }
4229 return -1;
4230}
4231#endif
4232
4233
4234/**
4235 * Finds the next set bit in a bitmap.
4236 *
4237 * @returns Index of the next set bit.
4238 * @returns -1 if no set bit was found.
4239 * @param pvBitmap Pointer to the bitmap.
4240 * @param cBits The number of bits in the bitmap. Multiple of 32.
4241 * @param iBitPrev The bit returned from the last search.
4242 * The search will start at iBitPrev + 1.
4243 */
4244#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4245DECLASM(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev);
4246#else
4247DECLINLINE(int) ASMBitNextSet(const volatile void *pvBitmap, uint32_t cBits, uint32_t iBitPrev)
4248{
4249 const volatile uint32_t *pau32Bitmap = (const volatile uint32_t *)pvBitmap;
4250 int iBit = ++iBitPrev & 31;
4251 if (iBit)
4252 {
4253 /*
4254 * Inspect the 32-bit word containing the unaligned bit.
4255 */
4256 uint32_t u32 = pau32Bitmap[iBitPrev / 32] >> iBit;
4257
4258# if RT_INLINE_ASM_USES_INTRIN
4259 unsigned long ulBit = 0;
4260 if (_BitScanForward(&ulBit, u32))
4261 return ulBit + iBitPrev;
4262# else
4263# if RT_INLINE_ASM_GNU_STYLE
4264 __asm__ __volatile__("bsf %1, %0\n\t"
4265 "jnz 1f\n\t"
4266 "movl $-1, %0\n\t"
4267 "1:\n\t"
4268 : "=r" (iBit)
4269 : "r" (u32));
4270# else
4271 __asm
4272 {
4273 mov edx, [u32]
4274 bsf eax, edx
4275 jnz done
4276 mov eax, 0ffffffffh
4277 done:
4278 mov [iBit], eax
4279 }
4280# endif
4281 if (iBit >= 0)
4282 return iBit + iBitPrev;
4283# endif
4284
4285 /*
4286 * Skip ahead and see if there is anything left to search.
4287 */
4288 iBitPrev |= 31;
4289 iBitPrev++;
4290 if (cBits <= (uint32_t)iBitPrev)
4291 return -1;
4292 }
4293
4294 /*
4295 * 32-bit aligned search, let ASMBitFirstClear do the dirty work.
4296 */
4297 iBit = ASMBitFirstSet(&pau32Bitmap[iBitPrev / 32], cBits - iBitPrev);
4298 if (iBit >= 0)
4299 iBit += iBitPrev;
4300 return iBit;
4301}
4302#endif
4303
4304
4305/**
4306 * Finds the first bit which is set in the given 32-bit integer.
4307 * Bits are numbered from 1 (least significant) to 32.
4308 *
4309 * @returns index [1..32] of the first set bit.
4310 * @returns 0 if all bits are cleared.
4311 * @param u32 Integer to search for set bits.
4312 * @remark Similar to ffs() in BSD.
4313 */
4314#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4315DECLASM(unsigned) ASMBitFirstSetU32(uint32_t u32);
4316#else
4317DECLINLINE(unsigned) ASMBitFirstSetU32(uint32_t u32)
4318{
4319# if RT_INLINE_ASM_USES_INTRIN
4320 unsigned long iBit;
4321 if (_BitScanForward(&iBit, u32))
4322 iBit++;
4323 else
4324 iBit = 0;
4325# elif RT_INLINE_ASM_GNU_STYLE
4326 uint32_t iBit;
4327 __asm__ __volatile__("bsf %1, %0\n\t"
4328 "jnz 1f\n\t"
4329 "xorl %0, %0\n\t"
4330 "jmp 2f\n"
4331 "1:\n\t"
4332 "incl %0\n"
4333 "2:\n\t"
4334 : "=r" (iBit)
4335 : "rm" (u32));
4336# else
4337 uint32_t iBit;
4338 _asm
4339 {
4340 bsf eax, [u32]
4341 jnz found
4342 xor eax, eax
4343 jmp done
4344 found:
4345 inc eax
4346 done:
4347 mov [iBit], eax
4348 }
4349# endif
4350 return iBit;
4351}
4352#endif
4353
4354
4355/**
4356 * Finds the first bit which is set in the given 32-bit integer.
4357 * Bits are numbered from 1 (least significant) to 32.
4358 *
4359 * @returns index [1..32] of the first set bit.
4360 * @returns 0 if all bits are cleared.
4361 * @param i32 Integer to search for set bits.
4362 * @remark Similar to ffs() in BSD.
4363 */
4364DECLINLINE(unsigned) ASMBitFirstSetS32(int32_t i32)
4365{
4366 return ASMBitFirstSetU32((uint32_t)i32);
4367}
4368
4369
4370/**
4371 * Finds the last bit which is set in the given 32-bit integer.
4372 * Bits are numbered from 1 (least significant) to 32.
4373 *
4374 * @returns index [1..32] of the last set bit.
4375 * @returns 0 if all bits are cleared.
4376 * @param u32 Integer to search for set bits.
4377 * @remark Similar to fls() in BSD.
4378 */
4379#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4380DECLASM(unsigned) ASMBitLastSetU32(uint32_t u32);
4381#else
4382DECLINLINE(unsigned) ASMBitLastSetU32(uint32_t u32)
4383{
4384# if RT_INLINE_ASM_USES_INTRIN
4385 unsigned long iBit;
4386 if (_BitScanReverse(&iBit, u32))
4387 iBit++;
4388 else
4389 iBit = 0;
4390# elif RT_INLINE_ASM_GNU_STYLE
4391 uint32_t iBit;
4392 __asm__ __volatile__("bsrl %1, %0\n\t"
4393 "jnz 1f\n\t"
4394 "xorl %0, %0\n\t"
4395 "jmp 2f\n"
4396 "1:\n\t"
4397 "incl %0\n"
4398 "2:\n\t"
4399 : "=r" (iBit)
4400 : "rm" (u32));
4401# else
4402 uint32_t iBit;
4403 _asm
4404 {
4405 bsr eax, [u32]
4406 jnz found
4407 xor eax, eax
4408 jmp done
4409 found:
4410 inc eax
4411 done:
4412 mov [iBit], eax
4413 }
4414# endif
4415 return iBit;
4416}
4417#endif
4418
4419
4420/**
4421 * Finds the last bit which is set in the given 32-bit integer.
4422 * Bits are numbered from 1 (least significant) to 32.
4423 *
4424 * @returns index [1..32] of the last set bit.
4425 * @returns 0 if all bits are cleared.
4426 * @param i32 Integer to search for set bits.
4427 * @remark Similar to fls() in BSD.
4428 */
4429DECLINLINE(unsigned) ASMBitLastSetS32(int32_t i32)
4430{
4431 return ASMBitLastSetU32((uint32_t)i32);
4432}
4433
4434/**
4435 * Reverse the byte order of the given 16-bit integer.
4436 *
4437 * @returns Revert
4438 * @param u16 16-bit integer value.
4439 */
4440#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4441DECLASM(uint16_t) ASMByteSwapU16(uint16_t u16);
4442#else
4443DECLINLINE(uint16_t) ASMByteSwapU16(uint16_t u16)
4444{
4445# if RT_INLINE_ASM_USES_INTRIN
4446 u16 = _byteswap_ushort(u16);
4447# elif RT_INLINE_ASM_GNU_STYLE
4448 __asm__ ("rorw $8, %0" : "=r" (u16) : "0" (u16));
4449# else
4450 _asm
4451 {
4452 mov ax, [u16]
4453 ror ax, 8
4454 mov [u16], ax
4455 }
4456# endif
4457 return u16;
4458}
4459#endif
4460
4461
4462/**
4463 * Reverse the byte order of the given 32-bit integer.
4464 *
4465 * @returns Revert
4466 * @param u32 32-bit integer value.
4467 */
4468#if RT_INLINE_ASM_EXTERNAL && !RT_INLINE_ASM_USES_INTRIN
4469DECLASM(uint32_t) ASMByteSwapU32(uint32_t u32);
4470#else
4471DECLINLINE(uint32_t) ASMByteSwapU32(uint32_t u32)
4472{
4473# if RT_INLINE_ASM_USES_INTRIN
4474 u32 = _byteswap_ulong(u32);
4475# elif RT_INLINE_ASM_GNU_STYLE
4476 __asm__ ("bswapl %0" : "=r" (u32) : "0" (u32));
4477# else
4478 _asm
4479 {
4480 mov eax, [u32]
4481 bswap eax
4482 mov [u32], eax
4483 }
4484# endif
4485 return u32;
4486}
4487#endif
4488
4489
4490/**
4491 * Reverse the byte order of the given 64-bit integer.
4492 *
4493 * @returns Revert
4494 * @param u64 64-bit integer value.
4495 */
4496DECLINLINE(uint64_t) ASMByteSwapU64(uint64_t u64)
4497{
4498#if defined(RT_ARCH_AMD64) && RT_INLINE_ASM_USES_INTRIN
4499 u64 = _byteswap_uint64(u64);
4500#else
4501 u64 = (uint64_t)ASMByteSwapU32((uint32_t)u64) << 32
4502 | (uint64_t)ASMByteSwapU32((uint32_t)(u64 >> 32));
4503#endif
4504 return u64;
4505}
4506
4507
4508/** @} */
4509
4510
4511/** @} */
4512
4513/* KLUDGE: Play safe for now as I cannot test all solaris and linux usages. */
4514#if !defined(__cplusplus) && !defined(DEBUG)
4515# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
4516# include <iprt/asm-amd64-x86.h>
4517# endif
4518# include <iprt/asm-math.h>
4519#endif
4520
4521#endif
4522
注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette