Skip to content

Commit 059d0c1

Browse files
committed
race: Relax compare_exchange success ordering from AcqRel to Release.
See the analogous change in rust-lang/rust#131746 and the discussion in matklad#220. What is the effect of this change? Not much, because before we ever execute the `compare_exchange`, we do a load with `Ordering::Acquire`; the `compare_exchange` is in the `#[cold]` path already. Thus, this just mostly clarifies our expectations. See the non-doc comment added under the module's doc comment for the reasoning. How does this change the code gen? Consider this analogous example: ```diff #[no_mangle] fn foo1(y: &mut i32) -> bool { - let r = X.compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire).is_ok(); + let r = X.compare_exchange(0, 1, Ordering::Release, Ordering::Acquire).is_ok(); r } ``` On x86_64, there is no change. Here is the generated code before and after: ``` foo1: mov rcx, qword ptr [rip + example::X::h9e1b81da80078af7@GOTPCREL] mov edx, 1 xor eax, eax lock cmpxchg dword ptr [rcx], edx sete al ret example::X::h9e1b81da80078af7: .zero 4 ``` On AArch64, regardless of whether atomics are outlined or not, there is no change. Here is the generated code with inlined atomics: ``` foo1: adrp x8, :got:example::X::h40b04fb69d714de3 ldr x8, [x8, :got_lo12:example::X::h40b04fb69d714de3] .LBB0_1: ldaxr w9, [x8] cbnz w9, .LBB0_4 mov w0, matklad#1 stlxr w9, w0, [x8] cbnz w9, .LBB0_1 ret .LBB0_4: mov w0, wzr clrex ret example::X::h40b04fb69d714de3: .zero 4 ``` For 32-bit ARMv7, with inlined atomics, the resulting diff in the object code is: ```diff @@ -10,14 +10,13 @@ mov r0, matklad#1 strex r2, r0, [r1] cmp r2, #0 - beq .LBB0_5 + bxeq lr ldrex r0, [r1] cmp r0, #0 beq .LBB0_2 .LBB0_4: - mov r0, #0 clrex -.LBB0_5: + mov r0, #0 dmb ish bx lr .LCPI0_0: @@ -54,4 +53,3 @@ example::X::h47e2038445e1c648: .zero 4 ```
1 parent 0d6bc31 commit 059d0c1

File tree

1 file changed

+14
-6
lines changed

1 file changed

+14
-6
lines changed

src/race.rs

+14-6
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,14 @@
1919
//! `Acquire` and `Release` have very little performance overhead on most
2020
//! architectures versus `Relaxed`.
2121
22+
// The "atomic orderings" section of the documentation above promises
23+
// "happens-before" semantics. This drives the choice of orderings in the uses
24+
// of `compare_exchange` below. On success, the value was zero/null, so there
25+
// was nothing to acquire (there is never any `Ordering::Release` store of 0).
26+
// On failure, the value was nonzero, so it was initialized previously (perhaps
27+
// on another thread) using `Ordering::Release`, so we must use
28+
// `Ordering::Acquire` to ensure that store "happens-before" this load.
29+
2230
#[cfg(not(feature = "portable-atomic"))]
2331
use core::sync::atomic;
2432
#[cfg(feature = "portable-atomic")]
@@ -82,7 +90,7 @@ impl OnceNonZeroUsize {
8290
#[inline]
8391
pub fn set(&self, value: NonZeroUsize) -> Result<(), ()> {
8492
let exchange =
85-
self.inner.compare_exchange(0, value.get(), Ordering::AcqRel, Ordering::Acquire);
93+
self.inner.compare_exchange(0, value.get(), Ordering::Release, Ordering::Acquire);
8694
match exchange {
8795
Ok(_) => Ok(()),
8896
Err(_) => Err(()),
@@ -128,7 +136,7 @@ impl OnceNonZeroUsize {
128136
#[inline(never)]
129137
fn init<E>(&self, f: impl FnOnce() -> Result<NonZeroUsize, E>) -> Result<NonZeroUsize, E> {
130138
let mut val = f()?.get();
131-
let exchange = self.inner.compare_exchange(0, val, Ordering::AcqRel, Ordering::Acquire);
139+
let exchange = self.inner.compare_exchange(0, val, Ordering::Release, Ordering::Acquire);
132140
if let Err(old) = exchange {
133141
val = old;
134142
}
@@ -242,7 +250,7 @@ impl<'a, T> OnceRef<'a, T> {
242250
pub fn set(&self, value: &'a T) -> Result<(), ()> {
243251
let ptr = value as *const T as *mut T;
244252
let exchange =
245-
self.inner.compare_exchange(ptr::null_mut(), ptr, Ordering::AcqRel, Ordering::Acquire);
253+
self.inner.compare_exchange(ptr::null_mut(), ptr, Ordering::Release, Ordering::Acquire);
246254
match exchange {
247255
Ok(_) => Ok(()),
248256
Err(_) => Err(()),
@@ -285,7 +293,7 @@ impl<'a, T> OnceRef<'a, T> {
285293
let exchange = self.inner.compare_exchange(
286294
ptr::null_mut(),
287295
ptr,
288-
Ordering::AcqRel,
296+
Ordering::Release,
289297
Ordering::Acquire,
290298
);
291299
if let Err(old) = exchange {
@@ -380,7 +388,7 @@ mod once_box {
380388
let exchange = self.inner.compare_exchange(
381389
ptr::null_mut(),
382390
ptr,
383-
Ordering::AcqRel,
391+
Ordering::Release,
384392
Ordering::Acquire,
385393
);
386394
if exchange.is_err() {
@@ -426,7 +434,7 @@ mod once_box {
426434
let exchange = self.inner.compare_exchange(
427435
ptr::null_mut(),
428436
ptr,
429-
Ordering::AcqRel,
437+
Ordering::Release,
430438
Ordering::Acquire,
431439
);
432440
if let Err(old) = exchange {

0 commit comments

Comments
 (0)