fix division on SPARC

Author: AaronKutch

src/int/specialized_div_rem/delegate.rs

@@ -185,3 +185,133 @@ macro_rules! impl_delegate {
         }
     };
 }
+
+/// Returns `n / d` and sets `*rem = n % d`.
+///
+/// This specialization exists because:
+///  - The LLVM backend for 32-bit SPARC cannot compile functions that return `(u128, u128)`,
+///    so we have to use an old fashioned `&mut u128` argument to return the remainder.
+///  - 64-bit SPARC does not have u64 * u64 => u128 widening multiplication, which makes the
+///    delegate algorithm strategy the only reasonably fast way to perform `u128` division.
+#[doc(hidden)]
+pub fn u128_divide_sparc(duo: u128, div: u128, rem: &mut u128) -> u128 {
+    use super::*;
+    let duo_lo = duo as u64;
+    let duo_hi = (duo >> 64) as u64;
+    let div_lo = div as u64;
+    let div_hi = (div >> 64) as u64;
+
+    match (div_lo == 0, div_hi == 0, duo_hi == 0) {
+        (true, true, _) => zero_div_fn(),
+        (_, false, true) => {
+            *rem = duo;
+            return 0;
+        }
+        (false, true, true) => {
+            let tmp = u64_by_u64_div_rem(duo_lo, div_lo);
+            *rem = tmp.1 as u128;
+            return tmp.0 as u128;
+        }
+        (false, true, false) => {
+            if duo_hi < div_lo {
+                let norm_shift = u64_normalization_shift(div_lo, duo_hi, false);
+                let shl = if norm_shift == 0 {
+                    64 - 1
+                } else {
+                    64 - norm_shift
+                };
+
+                let mut div: u128 = div << shl;
+                let mut pow_lo: u64 = 1 << shl;
+                let mut quo_lo: u64 = 0;
+                let mut duo = duo;
+                loop {
+                    let sub = duo.wrapping_sub(div);
+                    if 0 <= (sub as i128) {
+                        duo = sub;
+                        quo_lo |= pow_lo;
+                        let duo_hi = (duo >> 64) as u64;
+                        if duo_hi == 0 {
+                            let tmp = u64_by_u64_div_rem(duo as u64, div_lo);
+                            *rem = tmp.1 as u128;
+                            return (quo_lo | tmp.0) as u128;
+                        }
+                    }
+                    div >>= 1;
+                    pow_lo >>= 1;
+                }
+            } else if duo_hi == div_lo {
+                let tmp = u64_by_u64_div_rem(duo as u64, div as u64);
+                *rem = tmp.1 as u128;
+                return (1 << 64) | (tmp.0 as u128);
+            } else {
+                if (div_lo >> 32) == 0 {
+                    let div_0 = div_lo as u32 as u64;
+                    let (quo_hi, rem_3) = u64_by_u64_div_rem(duo_hi, div_0);
+
+                    let duo_mid = ((duo >> 32) as u32 as u64) | (rem_3 << 32);
+                    let (quo_1, rem_2) = u64_by_u64_div_rem(duo_mid, div_0);
+
+                    let duo_lo = (duo as u32 as u64) | (rem_2 << 32);
+                    let (quo_0, rem_1) = u64_by_u64_div_rem(duo_lo, div_0);
+
+                    *rem = rem_1 as u128;
+                    return (quo_0 as u128) | ((quo_1 as u128) << 32) | ((quo_hi as u128) << 64);
+                }
+
+                let duo_lo = duo as u64;
+                let tmp = u64_by_u64_div_rem(duo_hi, div_lo);
+                let quo_hi = tmp.0;
+                let mut duo = (duo_lo as u128) | ((tmp.1 as u128) << 64);
+                if duo < div {
+                    *rem = duo;
+                    return (quo_hi as u128) << 64;
+                }
+
+                let mut div: u128 = div << (64 - 1);
+                let mut pow_lo: u64 = 1 << (64 - 1);
+                let mut quo_lo: u64 = 0;
+                loop {
+                    let sub = duo.wrapping_sub(div);
+                    if 0 <= (sub as i128) {
+                        duo = sub;
+                        quo_lo |= pow_lo;
+                        let duo_hi = (duo >> 64) as u64;
+                        if duo_hi == 0 {
+                            let tmp = u64_by_u64_div_rem(duo as u64, div_lo);
+                            *rem = tmp.1 as u128;
+                            return (tmp.0) as u128 | (quo_lo as u128) | ((quo_hi as u128) << 64);
+                        }
+                    }
+                    div >>= 1;
+                    pow_lo >>= 1;
+                }
+            }
+        }
+        (_, false, false) => {
+            if duo < div {
+                *rem = duo;
+                return 0;
+            }
+            let div_original = div;
+            let shl = u64_normalization_shift(duo_hi, div_hi, false);
+            let mut duo = duo;
+            let mut div: u128 = div << shl;
+            let mut pow_lo: u64 = 1 << shl;
+            let mut quo_lo: u64 = 0;
+            loop {
+                let sub = duo.wrapping_sub(div);
+                if 0 <= (sub as i128) {
+                    duo = sub;
+                    quo_lo |= pow_lo;
+                    if duo < div_original {
+                        *rem = duo;
+                        return quo_lo as u128;
+                    }
+                }
+                div >>= 1;
+                pow_lo >>= 1;
+            }
+        }
+    }
+}

src/int/specialized_div_rem/mod.rs

@@ -46,6 +46,7 @@ mod binary_long;
 
 #[macro_use]
 mod delegate;
+pub use self::delegate::u128_divide_sparc;
 
 #[macro_use]
 mod trifecta;
@@ -60,10 +61,6 @@ fn zero_div_fn() -> ! {
     unsafe { core::hint::unreachable_unchecked() }
 }
 
-// The `B` extension on RISC-V determines if a CLZ assembly instruction exists
-#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
-const USE_LZ: bool = cfg!(target_feature = "b");
-
 #[cfg(target_arch = "arm")]
 const USE_LZ: bool = if cfg!(target_feature = "thumb-mode") {
     // ARM thumb targets have CLZ instructions if the instruction set of ARMv6T2 is supported. This
@@ -77,8 +74,22 @@ const USE_LZ: bool = if cfg!(target_feature = "thumb-mode") {
     cfg!(target_feature = "v5te")
 };
 
-// All other targets Rust supports have CLZ instructions
-#[cfg(not(any(target_arch = "arm", target_arch = "riscv32", target_arch = "riscv64")))]
+// LZD or LZCNT on SPARC only exists for the VIS 3 extension and later.
+#[cfg(any(target_arch = "sparc", target_arch = "sparc64"))]
+const USE_LZ: bool = cfg!(target_feature = "vis3");
+
+// The `B` extension on RISC-V determines if a CLZ assembly instruction exists
+#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
+const USE_LZ: bool = cfg!(target_feature = "b");
+
+// All other common targets Rust supports should have CLZ instructions
+#[cfg(not(any(
+    target_arch = "arm",
+    target_arch = "sparc",
+    target_arch = "sparc64",
+    target_arch = "riscv32",
+    target_arch = "riscv64"
+)))]
 const USE_LZ: bool = true;
 
 impl_normalization_shift!(
@@ -115,8 +126,9 @@ fn u64_by_u64_div_rem(duo: u64, div: u64) -> (u64, u64) {
 // microarchitecture can multiply and divide. We decide to be optimistic and assume `trifecta` is
 // faster if the target pointer width is at least 64.
 #[cfg(all(
+    not(any(target_pointer_width = "16", target_pointer_width = "32")),
     not(all(not(feature = "no-asm"), target_arch = "x86_64")),
-    not(any(target_pointer_width = "16", target_pointer_width = "32"))
+    not(any(target_arch = "sparc", target_arch = "sparc64"))
 ))]
 impl_trifecta!(
     u128_div_rem,
@@ -131,8 +143,9 @@ impl_trifecta!(
 // If the pointer width less than 64, then the target architecture almost certainly does not have
 // the fast 64 to 128 bit widening multiplication needed for `trifecta` to be faster.
 #[cfg(all(
+    any(target_pointer_width = "16", target_pointer_width = "32"),
     not(all(not(feature = "no-asm"), target_arch = "x86_64")),
-    any(target_pointer_width = "16", target_pointer_width = "32")
+    not(any(target_arch = "sparc", target_arch = "sparc64"))
 ))]
 impl_delegate!(
     u128_div_rem,

src/int/udiv.rs

@@ -1,3 +1,4 @@
+pub use int::specialized_div_rem::u128_divide_sparc;
 use int::specialized_div_rem::*;
 
 intrinsics! {
@@ -45,7 +46,10 @@ intrinsics! {
         }
         quo_rem.0
     }
+}
 
+#[cfg(not(any(target_arch = "sparc", target_arch = "sparc64")))]
+intrinsics! {
     #[win64_128bit_abi_hack]
     /// Returns `n / d`
     pub extern "C" fn __udivti3(n: u128, d: u128) -> u128 {
@@ -68,3 +72,31 @@ intrinsics! {
         quo_rem.0
     }
 }
+
+#[cfg(any(target_arch = "sparc", target_arch = "sparc64"))]
+intrinsics! {
+    #[win64_128bit_abi_hack]
+    /// Returns `n / d`
+    pub extern "C" fn __udivti3(n: u128, d: u128) -> u128 {
+        u128_divide_sparc(n, d, &mut 0)
+    }
+
+    #[win64_128bit_abi_hack]
+    /// Returns `n % d`
+    pub extern "C" fn __umodti3(n: u128, d: u128) -> u128 {
+        let mut rem = 0;
+        u128_divide_sparc(n, d, &mut rem);
+        rem
+    }
+
+    #[win64_128bit_abi_hack]
+    /// Returns `n / d` and sets `*rem = n % d`
+    pub extern "C" fn __udivmodti4(n: u128, d: u128, rem: Option<&mut u128>) -> u128 {
+        let mut tmp = 0;
+        let quo = u128_divide_sparc(n, d, &mut tmp);
+        if let Some(rem) = rem {
+            *rem = tmp;
+        }
+        quo
+    }
+}