From 2ca628eb3cb7a4a5a4f46a5d6dedc48f621e1917 Mon Sep 17 00:00:00 2001 From: tangaac Date: Wed, 22 Jul 2026 18:36:47 +0800 Subject: [PATCH 01/55] loongarch: Add portable intrinsics::simd implementations for VAVG/VAVGR intrinsics --- .../src/loongarch64/lasx/generated.rs | 144 ------------------ .../src/loongarch64/lasx/portable.rs | 18 +++ .../src/loongarch64/lsx/generated.rs | 144 ------------------ .../core_arch/src/loongarch64/lsx/portable.rs | 18 +++ .../crates/core_arch/src/loongarch64/simd.rs | 38 +++++ .../crates/stdarch-gen-loongarch/lasx.spec | 16 ++ .../crates/stdarch-gen-loongarch/lsx.spec | 16 ++ .../src/portable-intrinsics.txt | 32 ++++ 8 files changed, 138 insertions(+), 288 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs b/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs index 711ade40391a9..fa5a8ccc1a5f2 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs @@ -91,38 +91,6 @@ unsafe extern "unadjusted" { fn __lasx_xvsat_wu(a: __v8u32, b: u32) -> __v8u32; #[link_name = "llvm.loongarch.lasx.xvsat.du"] fn __lasx_xvsat_du(a: __v4u64, b: u32) -> __v4u64; - #[link_name = "llvm.loongarch.lasx.xvavg.b"] - fn __lasx_xvavg_b(a: __v32i8, b: __v32i8) -> __v32i8; - #[link_name = "llvm.loongarch.lasx.xvavg.h"] - fn __lasx_xvavg_h(a: __v16i16, b: __v16i16) -> __v16i16; - #[link_name = "llvm.loongarch.lasx.xvavg.w"] - fn __lasx_xvavg_w(a: __v8i32, b: __v8i32) -> __v8i32; - #[link_name = "llvm.loongarch.lasx.xvavg.d"] - fn __lasx_xvavg_d(a: __v4i64, b: __v4i64) -> __v4i64; - #[link_name = "llvm.loongarch.lasx.xvavg.bu"] - fn __lasx_xvavg_bu(a: __v32u8, b: __v32u8) -> __v32u8; - #[link_name = "llvm.loongarch.lasx.xvavg.hu"] - fn __lasx_xvavg_hu(a: __v16u16, b: __v16u16) -> __v16u16; - #[link_name = "llvm.loongarch.lasx.xvavg.wu"] - fn __lasx_xvavg_wu(a: __v8u32, b: __v8u32) -> __v8u32; - #[link_name = "llvm.loongarch.lasx.xvavg.du"] - fn __lasx_xvavg_du(a: __v4u64, b: __v4u64) -> __v4u64; - #[link_name = "llvm.loongarch.lasx.xvavgr.b"] - fn __lasx_xvavgr_b(a: __v32i8, b: __v32i8) -> __v32i8; - #[link_name = "llvm.loongarch.lasx.xvavgr.h"] - fn __lasx_xvavgr_h(a: __v16i16, b: __v16i16) -> __v16i16; - #[link_name = "llvm.loongarch.lasx.xvavgr.w"] - fn __lasx_xvavgr_w(a: __v8i32, b: __v8i32) -> __v8i32; - #[link_name = "llvm.loongarch.lasx.xvavgr.d"] - fn __lasx_xvavgr_d(a: __v4i64, b: __v4i64) -> __v4i64; - #[link_name = "llvm.loongarch.lasx.xvavgr.bu"] - fn __lasx_xvavgr_bu(a: __v32u8, b: __v32u8) -> __v32u8; - #[link_name = "llvm.loongarch.lasx.xvavgr.hu"] - fn __lasx_xvavgr_hu(a: __v16u16, b: __v16u16) -> __v16u16; - #[link_name = "llvm.loongarch.lasx.xvavgr.wu"] - fn __lasx_xvavgr_wu(a: __v8u32, b: __v8u32) -> __v8u32; - #[link_name = "llvm.loongarch.lasx.xvavgr.du"] - fn __lasx_xvavgr_du(a: __v4u64, b: __v4u64) -> __v4u64; #[link_name = "llvm.loongarch.lasx.xvhaddw.h.b"] fn __lasx_xvhaddw_h_b(a: __v32i8, b: __v32i8) -> __v16i16; #[link_name = "llvm.loongarch.lasx.xvhaddw.w.h"] @@ -1283,118 +1251,6 @@ pub fn lasx_xvsat_du(a: m256i) -> m256i { unsafe { transmute(__lasx_xvsat_du(transmute(a), IMM6)) } } -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavg_b(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavg_b(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavg_h(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavg_h(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavg_w(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavg_w(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavg_d(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavg_d(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavg_bu(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavg_bu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavg_hu(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavg_hu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavg_wu(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavg_wu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavg_du(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavg_du(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavgr_b(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavgr_b(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavgr_h(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavgr_h(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavgr_w(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavgr_w(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavgr_d(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavgr_d(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavgr_bu(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavgr_bu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavgr_hu(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavgr_hu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavgr_wu(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavgr_wu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lasx_xvavgr_du(a: m256i, b: m256i) -> m256i { - unsafe { transmute(__lasx_xvavgr_du(transmute(a), transmute(b))) } -} - #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lasx/portable.rs b/library/stdarch/crates/core_arch/src/loongarch64/lasx/portable.rs index 73ea74d9dc3bb..d53f21c792626 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lasx/portable.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lasx/portable.rs @@ -896,6 +896,24 @@ impl_vvvv!("lasx", lasx_xvfnmsub_d, simd_ext_fnmsub, m256d, f64x4); impl_vugv!("lasx", lasx_xvinsgr2vr_w, simd_insert, m256i, i32x8, i32, 3); impl_vugv!("lasx", lasx_xvinsgr2vr_d, simd_insert, m256i, i64x4, i64, 2); +impl_vavg!("lasx", lasx_xvavg_b, m256i, i8x32, i16x32); +impl_vavg!("lasx", lasx_xvavg_h, m256i, i16x16, i32x16); +impl_vavg!("lasx", lasx_xvavg_w, m256i, i32x8, i64x8); +impl_vavg!("lasx", lasx_xvavg_d, m256i, i64x4, i128x4); +impl_vavg!("lasx", lasx_xvavg_bu, m256i, u8x32, u16x32); +impl_vavg!("lasx", lasx_xvavg_hu, m256i, u16x16, u32x16); +impl_vavg!("lasx", lasx_xvavg_wu, m256i, u32x8, u64x8); +impl_vavg!("lasx", lasx_xvavg_du, m256i, u64x4, u128x4); + +impl_vavgr!("lasx", lasx_xvavgr_b, m256i, i8x32, i16x32); +impl_vavgr!("lasx", lasx_xvavgr_h, m256i, i16x16, i32x16); +impl_vavgr!("lasx", lasx_xvavgr_w, m256i, i32x8, i64x8); +impl_vavgr!("lasx", lasx_xvavgr_d, m256i, i64x4, i128x4); +impl_vavgr!("lasx", lasx_xvavgr_bu, m256i, u8x32, u16x32); +impl_vavgr!("lasx", lasx_xvavgr_hu, m256i, u16x16, u32x16); +impl_vavgr!("lasx", lasx_xvavgr_wu, m256i, u32x8, u64x8); +impl_vavgr!("lasx", lasx_xvavgr_du, m256i, u64x4, u128x4); + #[cfg(test)] mod tests { use crate::{ diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs b/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs index 1542633a34b35..e39413c5fa0c0 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs @@ -91,38 +91,6 @@ unsafe extern "unadjusted" { fn __lsx_vsat_wu(a: __v4u32, b: u32) -> __v4u32; #[link_name = "llvm.loongarch.lsx.vsat.du"] fn __lsx_vsat_du(a: __v2u64, b: u32) -> __v2u64; - #[link_name = "llvm.loongarch.lsx.vavg.b"] - fn __lsx_vavg_b(a: __v16i8, b: __v16i8) -> __v16i8; - #[link_name = "llvm.loongarch.lsx.vavg.h"] - fn __lsx_vavg_h(a: __v8i16, b: __v8i16) -> __v8i16; - #[link_name = "llvm.loongarch.lsx.vavg.w"] - fn __lsx_vavg_w(a: __v4i32, b: __v4i32) -> __v4i32; - #[link_name = "llvm.loongarch.lsx.vavg.d"] - fn __lsx_vavg_d(a: __v2i64, b: __v2i64) -> __v2i64; - #[link_name = "llvm.loongarch.lsx.vavg.bu"] - fn __lsx_vavg_bu(a: __v16u8, b: __v16u8) -> __v16u8; - #[link_name = "llvm.loongarch.lsx.vavg.hu"] - fn __lsx_vavg_hu(a: __v8u16, b: __v8u16) -> __v8u16; - #[link_name = "llvm.loongarch.lsx.vavg.wu"] - fn __lsx_vavg_wu(a: __v4u32, b: __v4u32) -> __v4u32; - #[link_name = "llvm.loongarch.lsx.vavg.du"] - fn __lsx_vavg_du(a: __v2u64, b: __v2u64) -> __v2u64; - #[link_name = "llvm.loongarch.lsx.vavgr.b"] - fn __lsx_vavgr_b(a: __v16i8, b: __v16i8) -> __v16i8; - #[link_name = "llvm.loongarch.lsx.vavgr.h"] - fn __lsx_vavgr_h(a: __v8i16, b: __v8i16) -> __v8i16; - #[link_name = "llvm.loongarch.lsx.vavgr.w"] - fn __lsx_vavgr_w(a: __v4i32, b: __v4i32) -> __v4i32; - #[link_name = "llvm.loongarch.lsx.vavgr.d"] - fn __lsx_vavgr_d(a: __v2i64, b: __v2i64) -> __v2i64; - #[link_name = "llvm.loongarch.lsx.vavgr.bu"] - fn __lsx_vavgr_bu(a: __v16u8, b: __v16u8) -> __v16u8; - #[link_name = "llvm.loongarch.lsx.vavgr.hu"] - fn __lsx_vavgr_hu(a: __v8u16, b: __v8u16) -> __v8u16; - #[link_name = "llvm.loongarch.lsx.vavgr.wu"] - fn __lsx_vavgr_wu(a: __v4u32, b: __v4u32) -> __v4u32; - #[link_name = "llvm.loongarch.lsx.vavgr.du"] - fn __lsx_vavgr_du(a: __v2u64, b: __v2u64) -> __v2u64; #[link_name = "llvm.loongarch.lsx.vhaddw.h.b"] fn __lsx_vhaddw_h_b(a: __v16i8, b: __v16i8) -> __v8i16; #[link_name = "llvm.loongarch.lsx.vhaddw.w.h"] @@ -1203,118 +1171,6 @@ pub fn lsx_vsat_du(a: m128i) -> m128i { unsafe { transmute(__lsx_vsat_du(transmute(a), IMM6)) } } -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavg_b(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavg_b(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavg_h(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavg_h(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavg_w(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavg_w(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavg_d(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavg_d(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavg_bu(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavg_bu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavg_hu(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavg_hu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavg_wu(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavg_wu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavg_du(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavg_du(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavgr_b(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavgr_b(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavgr_h(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavgr_h(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavgr_w(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavgr_w(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavgr_d(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavgr_d(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavgr_bu(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavgr_bu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavgr_hu(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavgr_hu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavgr_wu(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavgr_wu(transmute(a), transmute(b))) } -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub fn lsx_vavgr_du(a: m128i, b: m128i) -> m128i { - unsafe { transmute(__lsx_vavgr_du(transmute(a), transmute(b))) } -} - #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lsx/portable.rs b/library/stdarch/crates/core_arch/src/loongarch64/lsx/portable.rs index 31467bf013e27..2b5bfe1ab4d2b 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lsx/portable.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lsx/portable.rs @@ -574,6 +574,24 @@ impl_vugv!("lsx", lsx_vinsgr2vr_h, simd_insert, m128i, i16x8, i32, 3); impl_vugv!("lsx", lsx_vinsgr2vr_w, simd_insert, m128i, i32x4, i32, 2); impl_vugv!("lsx", lsx_vinsgr2vr_d, simd_insert, m128i, i64x2, i64, 1); +impl_vavg!("lsx", lsx_vavg_b, m128i, i8x16, i16x16); +impl_vavg!("lsx", lsx_vavg_h, m128i, i16x8, i32x8); +impl_vavg!("lsx", lsx_vavg_w, m128i, i32x4, i64x4); +impl_vavg!("lsx", lsx_vavg_d, m128i, i64x2, i128x2); +impl_vavg!("lsx", lsx_vavg_bu, m128i, u8x16, u16x16); +impl_vavg!("lsx", lsx_vavg_hu, m128i, u16x8, u32x8); +impl_vavg!("lsx", lsx_vavg_wu, m128i, u32x4, u64x4); +impl_vavg!("lsx", lsx_vavg_du, m128i, u64x2, u128x2); + +impl_vavgr!("lsx", lsx_vavgr_b, m128i, i8x16, i16x16); +impl_vavgr!("lsx", lsx_vavgr_h, m128i, i16x8, i32x8); +impl_vavgr!("lsx", lsx_vavgr_w, m128i, i32x4, i64x4); +impl_vavgr!("lsx", lsx_vavgr_d, m128i, i64x2, i128x2); +impl_vavgr!("lsx", lsx_vavgr_bu, m128i, u8x16, u16x16); +impl_vavgr!("lsx", lsx_vavgr_hu, m128i, u16x8, u32x8); +impl_vavgr!("lsx", lsx_vavgr_wu, m128i, u32x4, u64x4); +impl_vavgr!("lsx", lsx_vavgr_du, m128i, u64x2, u128x2); + #[cfg(test)] mod tests { use crate::{ diff --git a/library/stdarch/crates/core_arch/src/loongarch64/simd.rs b/library/stdarch/crates/core_arch/src/loongarch64/simd.rs index 1d83333e2f532..7ce6071841460 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/simd.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/simd.rs @@ -304,6 +304,44 @@ pub(super) const unsafe fn simd_ext_stx(a: T, b: *mut i8, c: i64) { core::ptr::write_unaligned(b, a); } +macro_rules! impl_vavg { + ($ft:literal, $name:ident, $oty:ty, $ity:ty, $wty:ty) => { + #[inline] + #[target_feature(enable = $ft)] + #[unstable(feature = "stdarch_loongarch", issue = "117427")] + pub fn $name(a: $oty, b: $oty) -> $oty { + unsafe { + let a: $wty = simd_cast(transmute::<_, $ity>(a)); + let b: $wty = simd_cast(transmute::<_, $ity>(b)); + let r: $ity = simd_cast(simd_shr(simd_add(a, b), <$wty>::splat(1))); + transmute(r) + } + } + }; +} + +macro_rules! impl_vavgr { + ($ft:literal, $name:ident, $oty:ty, $ity:ty, $wty:ty) => { + #[inline] + #[target_feature(enable = $ft)] + #[unstable(feature = "stdarch_loongarch", issue = "117427")] + pub fn $name(a: $oty, b: $oty) -> $oty { + unsafe { + let a: $wty = simd_cast(transmute::<_, $ity>(a)); + let b: $wty = simd_cast(transmute::<_, $ity>(b)); + let r: $ity = simd_cast(simd_shr( + simd_add(simd_add(a, b), <$wty>::splat(1)), + <$wty>::splat(1), + )); + transmute(r) + } + } + }; +} + +pub(super) use impl_vavg; +pub(super) use impl_vavgr; + macro_rules! impl_vv { ($ft:literal, $name:ident, $op:ident, $oty:ty, $ity:ty) => { #[inline] diff --git a/library/stdarch/crates/stdarch-gen-loongarch/lasx.spec b/library/stdarch/crates/stdarch-gen-loongarch/lasx.spec index 9eff3d01fa1f3..eb3ff074a3978 100644 --- a/library/stdarch/crates/stdarch-gen-loongarch/lasx.spec +++ b/library/stdarch/crates/stdarch-gen-loongarch/lasx.spec @@ -996,81 +996,97 @@ asm-fmts = xd, xj, xk data-types = UV4DI, UV4DI, UV4DI /// lasx_xvavg_b +impl = portable name = lasx_xvavg_b asm-fmts = xd, xj, xk data-types = V32QI, V32QI, V32QI /// lasx_xvavg_h +impl = portable name = lasx_xvavg_h asm-fmts = xd, xj, xk data-types = V16HI, V16HI, V16HI /// lasx_xvavg_w +impl = portable name = lasx_xvavg_w asm-fmts = xd, xj, xk data-types = V8SI, V8SI, V8SI /// lasx_xvavg_d +impl = portable name = lasx_xvavg_d asm-fmts = xd, xj, xk data-types = V4DI, V4DI, V4DI /// lasx_xvavg_bu +impl = portable name = lasx_xvavg_bu asm-fmts = xd, xj, xk data-types = UV32QI, UV32QI, UV32QI /// lasx_xvavg_hu +impl = portable name = lasx_xvavg_hu asm-fmts = xd, xj, xk data-types = UV16HI, UV16HI, UV16HI /// lasx_xvavg_wu +impl = portable name = lasx_xvavg_wu asm-fmts = xd, xj, xk data-types = UV8SI, UV8SI, UV8SI /// lasx_xvavg_du +impl = portable name = lasx_xvavg_du asm-fmts = xd, xj, xk data-types = UV4DI, UV4DI, UV4DI /// lasx_xvavgr_b +impl = portable name = lasx_xvavgr_b asm-fmts = xd, xj, xk data-types = V32QI, V32QI, V32QI /// lasx_xvavgr_h +impl = portable name = lasx_xvavgr_h asm-fmts = xd, xj, xk data-types = V16HI, V16HI, V16HI /// lasx_xvavgr_w +impl = portable name = lasx_xvavgr_w asm-fmts = xd, xj, xk data-types = V8SI, V8SI, V8SI /// lasx_xvavgr_d +impl = portable name = lasx_xvavgr_d asm-fmts = xd, xj, xk data-types = V4DI, V4DI, V4DI /// lasx_xvavgr_bu +impl = portable name = lasx_xvavgr_bu asm-fmts = xd, xj, xk data-types = UV32QI, UV32QI, UV32QI /// lasx_xvavgr_hu +impl = portable name = lasx_xvavgr_hu asm-fmts = xd, xj, xk data-types = UV16HI, UV16HI, UV16HI /// lasx_xvavgr_wu +impl = portable name = lasx_xvavgr_wu asm-fmts = xd, xj, xk data-types = UV8SI, UV8SI, UV8SI /// lasx_xvavgr_du +impl = portable name = lasx_xvavgr_du asm-fmts = xd, xj, xk data-types = UV4DI, UV4DI, UV4DI diff --git a/library/stdarch/crates/stdarch-gen-loongarch/lsx.spec b/library/stdarch/crates/stdarch-gen-loongarch/lsx.spec index ba2554b0cf9ff..89970c3657048 100644 --- a/library/stdarch/crates/stdarch-gen-loongarch/lsx.spec +++ b/library/stdarch/crates/stdarch-gen-loongarch/lsx.spec @@ -996,81 +996,97 @@ asm-fmts = vd, vj, vk data-types = UV2DI, UV2DI, UV2DI /// lsx_vavg_b +impl = portable name = lsx_vavg_b asm-fmts = vd, vj, vk data-types = V16QI, V16QI, V16QI /// lsx_vavg_h +impl = portable name = lsx_vavg_h asm-fmts = vd, vj, vk data-types = V8HI, V8HI, V8HI /// lsx_vavg_w +impl = portable name = lsx_vavg_w asm-fmts = vd, vj, vk data-types = V4SI, V4SI, V4SI /// lsx_vavg_d +impl = portable name = lsx_vavg_d asm-fmts = vd, vj, vk data-types = V2DI, V2DI, V2DI /// lsx_vavg_bu +impl = portable name = lsx_vavg_bu asm-fmts = vd, vj, vk data-types = UV16QI, UV16QI, UV16QI /// lsx_vavg_hu +impl = portable name = lsx_vavg_hu asm-fmts = vd, vj, vk data-types = UV8HI, UV8HI, UV8HI /// lsx_vavg_wu +impl = portable name = lsx_vavg_wu asm-fmts = vd, vj, vk data-types = UV4SI, UV4SI, UV4SI /// lsx_vavg_du +impl = portable name = lsx_vavg_du asm-fmts = vd, vj, vk data-types = UV2DI, UV2DI, UV2DI /// lsx_vavgr_b +impl = portable name = lsx_vavgr_b asm-fmts = vd, vj, vk data-types = V16QI, V16QI, V16QI /// lsx_vavgr_h +impl = portable name = lsx_vavgr_h asm-fmts = vd, vj, vk data-types = V8HI, V8HI, V8HI /// lsx_vavgr_w +impl = portable name = lsx_vavgr_w asm-fmts = vd, vj, vk data-types = V4SI, V4SI, V4SI /// lsx_vavgr_d +impl = portable name = lsx_vavgr_d asm-fmts = vd, vj, vk data-types = V2DI, V2DI, V2DI /// lsx_vavgr_bu +impl = portable name = lsx_vavgr_bu asm-fmts = vd, vj, vk data-types = UV16QI, UV16QI, UV16QI /// lsx_vavgr_hu +impl = portable name = lsx_vavgr_hu asm-fmts = vd, vj, vk data-types = UV8HI, UV8HI, UV8HI /// lsx_vavgr_wu +impl = portable name = lsx_vavgr_wu asm-fmts = vd, vj, vk data-types = UV4SI, UV4SI, UV4SI /// lsx_vavgr_du +impl = portable name = lsx_vavgr_du asm-fmts = vd, vj, vk data-types = UV2DI, UV2DI, UV2DI diff --git a/library/stdarch/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt b/library/stdarch/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt index 8b8c82b3bb247..f47c17b23adea 100644 --- a/library/stdarch/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt +++ b/library/stdarch/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt @@ -219,6 +219,22 @@ lsx_vssub_bu lsx_vssub_hu lsx_vssub_wu lsx_vssub_du +lsx_vavg_b +lsx_vavg_h +lsx_vavg_w +lsx_vavg_d +lsx_vavg_bu +lsx_vavg_hu +lsx_vavg_wu +lsx_vavg_du +lsx_vavgr_b +lsx_vavgr_h +lsx_vavgr_w +lsx_vavgr_d +lsx_vavgr_bu +lsx_vavgr_hu +lsx_vavgr_wu +lsx_vavgr_du lsx_vadda_b lsx_vadda_h lsx_vadda_w @@ -512,6 +528,22 @@ lasx_xvssub_bu lasx_xvssub_hu lasx_xvssub_wu lasx_xvssub_du +lasx_xvavg_b +lasx_xvavg_h +lasx_xvavg_w +lasx_xvavg_d +lasx_xvavg_bu +lasx_xvavg_hu +lasx_xvavg_wu +lasx_xvavg_du +lasx_xvavgr_b +lasx_xvavgr_h +lasx_xvavgr_w +lasx_xvavgr_d +lasx_xvavgr_bu +lasx_xvavgr_hu +lasx_xvavgr_wu +lasx_xvavgr_du lasx_xvadda_b lasx_xvadda_h lasx_xvadda_w From fcf456259d86219325234f0bc22363e4534e8ba6 Mon Sep 17 00:00:00 2001 From: Valentyn Kit Date: Thu, 20 Aug 2026 19:21:22 +0300 Subject: [PATCH 02/55] neon: drop the align requirement on vld1/vst1 lane loads and stores vld1 and vst1 lane intrinsics were dereferencing pointer directly which requires it to be aligned to the element type. (Too strict alignment requirements) The instructions don't requires alignment, so calling these with unaligned pointer caused UB. Lane loads and stores were updated to use `read_unaligned()` and `write_unaligned()` instead. --- .../core_arch/src/aarch64/neon/generated.rs | 4 +- .../crates/core_arch/src/aarch64/neon/mod.rs | 4 +- .../src/arm_shared/neon/generated.rs | 116 +++++++++--------- .../src/arm_shared/neon/load_tests.rs | 29 +++++ .../src/arm_shared/neon/store_tests.rs | 24 ++++ .../spec/neon/aarch64.spec.yml | 10 +- .../spec/neon/arm_shared.spec.yml | 43 +++---- 7 files changed, 131 insertions(+), 99 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs index 1b5b17e538bde..c471880919eb6 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs @@ -25081,7 +25081,7 @@ pub unsafe fn vst1q_f64_x4(a: *mut f64, b: float64x2x4_t) { #[stable(feature = "neon_intrinsics", since = "1.59.0")] pub unsafe fn vst1_lane_f64(a: *mut f64, b: float64x1_t) { static_assert!(LANE == 0); - *a = simd_extract!(b, LANE as u32); + core::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_f64)"] @@ -25094,7 +25094,7 @@ pub unsafe fn vst1_lane_f64(a: *mut f64, b: float64x1_t) { #[stable(feature = "neon_intrinsics", since = "1.59.0")] pub unsafe fn vst1q_lane_f64(a: *mut f64, b: float64x2_t) { static_assert_uimm_bits!(LANE, 1); - *a = simd_extract!(b, LANE as u32); + core::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple 2-element structures from two registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f64)"] diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/mod.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/mod.rs index c66702814cfb2..e04c93ac1acae 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/neon/mod.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/neon/mod.rs @@ -120,7 +120,7 @@ pub unsafe fn vld1q_dup_f64(ptr: *const f64) -> float64x2_t { #[stable(feature = "neon_intrinsics", since = "1.59.0")] pub unsafe fn vld1_lane_f64(ptr: *const f64, src: float64x1_t) -> float64x1_t { static_assert!(LANE == 0); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } /// Load one single-element structure to one lane of one register. @@ -131,7 +131,7 @@ pub unsafe fn vld1_lane_f64(ptr: *const f64, src: float64x1_t) #[stable(feature = "neon_intrinsics", since = "1.59.0")] pub unsafe fn vld1q_lane_f64(ptr: *const f64, src: float64x2_t) -> float64x2_t { static_assert_uimm_bits!(LANE, 1); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } /// Bitwise Select instructions. This instruction sets each bit in the destination SIMD&FP register diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs index 3a47ede1acc88..2c6f663d6ed08 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs @@ -19307,7 +19307,7 @@ pub unsafe fn vld1q_f32_x4(a: *const f32) -> float32x4x4_t { #[cfg(not(target_arch = "arm64ec"))] pub unsafe fn vld1_lane_f16(ptr: *const f16, src: float16x4_t) -> float16x4_t { static_assert_uimm_bits!(LANE, 2); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_f16)"] @@ -19327,7 +19327,7 @@ pub unsafe fn vld1_lane_f16(ptr: *const f16, src: float16x4_t) #[cfg(not(target_arch = "arm64ec"))] pub unsafe fn vld1q_lane_f16(ptr: *const f16, src: float16x8_t) -> float16x8_t { static_assert_uimm_bits!(LANE, 3); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_f32)"] @@ -19352,7 +19352,7 @@ pub unsafe fn vld1q_lane_f16(ptr: *const f16, src: float16x8_t) )] pub unsafe fn vld1_lane_f32(ptr: *const f32, src: float32x2_t) -> float32x2_t { static_assert_uimm_bits!(LANE, 1); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_p16)"] @@ -19377,7 +19377,7 @@ pub unsafe fn vld1_lane_f32(ptr: *const f32, src: float32x2_t) )] pub unsafe fn vld1_lane_p16(ptr: *const p16, src: poly16x4_t) -> poly16x4_t { static_assert_uimm_bits!(LANE, 2); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_p8)"] @@ -19402,7 +19402,7 @@ pub unsafe fn vld1_lane_p16(ptr: *const p16, src: poly16x4_t) - )] pub unsafe fn vld1_lane_p8(ptr: *const p8, src: poly8x8_t) -> poly8x8_t { static_assert_uimm_bits!(LANE, 3); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_s16)"] @@ -19427,7 +19427,7 @@ pub unsafe fn vld1_lane_p8(ptr: *const p8, src: poly8x8_t) -> p )] pub unsafe fn vld1_lane_s16(ptr: *const i16, src: int16x4_t) -> int16x4_t { static_assert_uimm_bits!(LANE, 2); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_s32)"] @@ -19452,7 +19452,7 @@ pub unsafe fn vld1_lane_s16(ptr: *const i16, src: int16x4_t) -> )] pub unsafe fn vld1_lane_s32(ptr: *const i32, src: int32x2_t) -> int32x2_t { static_assert_uimm_bits!(LANE, 1); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_s64)"] @@ -19462,7 +19462,7 @@ pub unsafe fn vld1_lane_s32(ptr: *const i32, src: int32x2_t) -> #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] #[rustc_legacy_const_generics(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr, LANE = 0))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 0))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ldr, LANE = 0) @@ -19477,7 +19477,7 @@ pub unsafe fn vld1_lane_s32(ptr: *const i32, src: int32x2_t) -> )] pub unsafe fn vld1_lane_s64(ptr: *const i64, src: int64x1_t) -> int64x1_t { static_assert!(LANE == 0); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_s8)"] @@ -19502,7 +19502,7 @@ pub unsafe fn vld1_lane_s64(ptr: *const i64, src: int64x1_t) -> )] pub unsafe fn vld1_lane_s8(ptr: *const i8, src: int8x8_t) -> int8x8_t { static_assert_uimm_bits!(LANE, 3); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_u16)"] @@ -19527,7 +19527,7 @@ pub unsafe fn vld1_lane_s8(ptr: *const i8, src: int8x8_t) -> in )] pub unsafe fn vld1_lane_u16(ptr: *const u16, src: uint16x4_t) -> uint16x4_t { static_assert_uimm_bits!(LANE, 2); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_u32)"] @@ -19552,7 +19552,7 @@ pub unsafe fn vld1_lane_u16(ptr: *const u16, src: uint16x4_t) - )] pub unsafe fn vld1_lane_u32(ptr: *const u32, src: uint32x2_t) -> uint32x2_t { static_assert_uimm_bits!(LANE, 1); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_u64)"] @@ -19562,7 +19562,7 @@ pub unsafe fn vld1_lane_u32(ptr: *const u32, src: uint32x2_t) - #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] #[rustc_legacy_const_generics(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr, LANE = 0))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 0))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ldr, LANE = 0) @@ -19577,7 +19577,7 @@ pub unsafe fn vld1_lane_u32(ptr: *const u32, src: uint32x2_t) - )] pub unsafe fn vld1_lane_u64(ptr: *const u64, src: uint64x1_t) -> uint64x1_t { static_assert!(LANE == 0); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_u8)"] @@ -19602,7 +19602,7 @@ pub unsafe fn vld1_lane_u64(ptr: *const u64, src: uint64x1_t) - )] pub unsafe fn vld1_lane_u8(ptr: *const u8, src: uint8x8_t) -> uint8x8_t { static_assert_uimm_bits!(LANE, 3); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_f32)"] @@ -19627,7 +19627,7 @@ pub unsafe fn vld1_lane_u8(ptr: *const u8, src: uint8x8_t) -> u )] pub unsafe fn vld1q_lane_f32(ptr: *const f32, src: float32x4_t) -> float32x4_t { static_assert_uimm_bits!(LANE, 2); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_p16)"] @@ -19652,7 +19652,7 @@ pub unsafe fn vld1q_lane_f32(ptr: *const f32, src: float32x4_t) )] pub unsafe fn vld1q_lane_p16(ptr: *const p16, src: poly16x8_t) -> poly16x8_t { static_assert_uimm_bits!(LANE, 3); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_p8)"] @@ -19677,7 +19677,7 @@ pub unsafe fn vld1q_lane_p16(ptr: *const p16, src: poly16x8_t) )] pub unsafe fn vld1q_lane_p8(ptr: *const p8, src: poly8x16_t) -> poly8x16_t { static_assert_uimm_bits!(LANE, 4); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_s16)"] @@ -19702,7 +19702,7 @@ pub unsafe fn vld1q_lane_p8(ptr: *const p8, src: poly8x16_t) -> )] pub unsafe fn vld1q_lane_s16(ptr: *const i16, src: int16x8_t) -> int16x8_t { static_assert_uimm_bits!(LANE, 3); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_s32)"] @@ -19727,7 +19727,7 @@ pub unsafe fn vld1q_lane_s16(ptr: *const i16, src: int16x8_t) - )] pub unsafe fn vld1q_lane_s32(ptr: *const i32, src: int32x4_t) -> int32x4_t { static_assert_uimm_bits!(LANE, 2); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_s64)"] @@ -19737,7 +19737,7 @@ pub unsafe fn vld1q_lane_s32(ptr: *const i32, src: int32x4_t) - #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] #[rustc_legacy_const_generics(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr, LANE = 1))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 1))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1, LANE = 1) @@ -19752,7 +19752,7 @@ pub unsafe fn vld1q_lane_s32(ptr: *const i32, src: int32x4_t) - )] pub unsafe fn vld1q_lane_s64(ptr: *const i64, src: int64x2_t) -> int64x2_t { static_assert_uimm_bits!(LANE, 1); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_s8)"] @@ -19777,7 +19777,7 @@ pub unsafe fn vld1q_lane_s64(ptr: *const i64, src: int64x2_t) - )] pub unsafe fn vld1q_lane_s8(ptr: *const i8, src: int8x16_t) -> int8x16_t { static_assert_uimm_bits!(LANE, 4); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_u16)"] @@ -19802,7 +19802,7 @@ pub unsafe fn vld1q_lane_s8(ptr: *const i8, src: int8x16_t) -> )] pub unsafe fn vld1q_lane_u16(ptr: *const u16, src: uint16x8_t) -> uint16x8_t { static_assert_uimm_bits!(LANE, 3); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_u32)"] @@ -19827,7 +19827,7 @@ pub unsafe fn vld1q_lane_u16(ptr: *const u16, src: uint16x8_t) )] pub unsafe fn vld1q_lane_u32(ptr: *const u32, src: uint32x4_t) -> uint32x4_t { static_assert_uimm_bits!(LANE, 2); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_u64)"] @@ -19837,7 +19837,7 @@ pub unsafe fn vld1q_lane_u32(ptr: *const u32, src: uint32x4_t) #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] #[rustc_legacy_const_generics(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr, LANE = 1))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 1))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1, LANE = 1) @@ -19852,7 +19852,7 @@ pub unsafe fn vld1q_lane_u32(ptr: *const u32, src: uint32x4_t) )] pub unsafe fn vld1q_lane_u64(ptr: *const u64, src: uint64x2_t) -> uint64x2_t { static_assert_uimm_bits!(LANE, 1); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_u8)"] @@ -19877,7 +19877,7 @@ pub unsafe fn vld1q_lane_u64(ptr: *const u64, src: uint64x2_t) )] pub unsafe fn vld1q_lane_u8(ptr: *const u8, src: uint8x16_t) -> uint8x16_t { static_assert_uimm_bits!(LANE, 4); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_lane_p64)"] @@ -19887,7 +19887,7 @@ pub unsafe fn vld1q_lane_u8(ptr: *const u8, src: uint8x16_t) -> #[target_feature(enable = "neon,aes")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] #[rustc_legacy_const_generics(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr, LANE = 0))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 0))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ldr, LANE = 0) @@ -19902,7 +19902,7 @@ pub unsafe fn vld1q_lane_u8(ptr: *const u8, src: uint8x16_t) -> )] pub unsafe fn vld1_lane_p64(ptr: *const p64, src: poly64x1_t) -> poly64x1_t { static_assert!(LANE == 0); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load one single-element structure to one lane of one register."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_lane_p64)"] @@ -19912,7 +19912,7 @@ pub unsafe fn vld1_lane_p64(ptr: *const p64, src: poly64x1_t) - #[target_feature(enable = "neon,aes")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] #[rustc_legacy_const_generics(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr, LANE = 1))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8", LANE = 1))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1, LANE = 1) @@ -19927,7 +19927,7 @@ pub unsafe fn vld1_lane_p64(ptr: *const p64, src: poly64x1_t) - )] pub unsafe fn vld1q_lane_p64(ptr: *const p64, src: poly64x2_t) -> poly64x2_t { static_assert_uimm_bits!(LANE, 1); - simd_insert!(src, LANE as u32, *ptr) + simd_insert!(src, LANE as u32, crate::ptr::read_unaligned(ptr)) } #[doc = "Load multiple single-element structures to one, two, three, or four registers."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_p64)"] @@ -58579,7 +58579,7 @@ pub unsafe fn vst1q_f32_x4(a: *mut f32, b: float32x4x4_t) { #[cfg(not(target_arch = "arm64ec"))] pub unsafe fn vst1_lane_f16(a: *mut f16, b: float16x4_t) { static_assert_uimm_bits!(LANE, 2); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_f16)"] @@ -58599,7 +58599,7 @@ pub unsafe fn vst1_lane_f16(a: *mut f16, b: float16x4_t) { #[cfg(not(target_arch = "arm64ec"))] pub unsafe fn vst1q_lane_f16(a: *mut f16, b: float16x8_t) { static_assert_uimm_bits!(LANE, 3); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_f32)"] @@ -58624,7 +58624,7 @@ pub unsafe fn vst1q_lane_f16(a: *mut f16, b: float16x8_t) { )] pub unsafe fn vst1_lane_f32(a: *mut f32, b: float32x2_t) { static_assert_uimm_bits!(LANE, 1); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_f32)"] @@ -58649,7 +58649,7 @@ pub unsafe fn vst1_lane_f32(a: *mut f32, b: float32x2_t) { )] pub unsafe fn vst1q_lane_f32(a: *mut f32, b: float32x4_t) { static_assert_uimm_bits!(LANE, 2); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s8)"] @@ -58674,7 +58674,7 @@ pub unsafe fn vst1q_lane_f32(a: *mut f32, b: float32x4_t) { )] pub unsafe fn vst1_lane_s8(a: *mut i8, b: int8x8_t) { static_assert_uimm_bits!(LANE, 3); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s8)"] @@ -58699,7 +58699,7 @@ pub unsafe fn vst1_lane_s8(a: *mut i8, b: int8x8_t) { )] pub unsafe fn vst1q_lane_s8(a: *mut i8, b: int8x16_t) { static_assert_uimm_bits!(LANE, 4); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s16)"] @@ -58724,7 +58724,7 @@ pub unsafe fn vst1q_lane_s8(a: *mut i8, b: int8x16_t) { )] pub unsafe fn vst1_lane_s16(a: *mut i16, b: int16x4_t) { static_assert_uimm_bits!(LANE, 2); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s16)"] @@ -58749,7 +58749,7 @@ pub unsafe fn vst1_lane_s16(a: *mut i16, b: int16x4_t) { )] pub unsafe fn vst1q_lane_s16(a: *mut i16, b: int16x8_t) { static_assert_uimm_bits!(LANE, 3); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s32)"] @@ -58774,7 +58774,7 @@ pub unsafe fn vst1q_lane_s16(a: *mut i16, b: int16x8_t) { )] pub unsafe fn vst1_lane_s32(a: *mut i32, b: int32x2_t) { static_assert_uimm_bits!(LANE, 1); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s32)"] @@ -58799,7 +58799,7 @@ pub unsafe fn vst1_lane_s32(a: *mut i32, b: int32x2_t) { )] pub unsafe fn vst1q_lane_s32(a: *mut i32, b: int32x4_t) { static_assert_uimm_bits!(LANE, 2); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_s64)"] @@ -58824,7 +58824,7 @@ pub unsafe fn vst1q_lane_s32(a: *mut i32, b: int32x4_t) { )] pub unsafe fn vst1q_lane_s64(a: *mut i64, b: int64x2_t) { static_assert_uimm_bits!(LANE, 1); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u8)"] @@ -58849,7 +58849,7 @@ pub unsafe fn vst1q_lane_s64(a: *mut i64, b: int64x2_t) { )] pub unsafe fn vst1_lane_u8(a: *mut u8, b: uint8x8_t) { static_assert_uimm_bits!(LANE, 3); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u8)"] @@ -58874,7 +58874,7 @@ pub unsafe fn vst1_lane_u8(a: *mut u8, b: uint8x8_t) { )] pub unsafe fn vst1q_lane_u8(a: *mut u8, b: uint8x16_t) { static_assert_uimm_bits!(LANE, 4); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u16)"] @@ -58899,7 +58899,7 @@ pub unsafe fn vst1q_lane_u8(a: *mut u8, b: uint8x16_t) { )] pub unsafe fn vst1_lane_u16(a: *mut u16, b: uint16x4_t) { static_assert_uimm_bits!(LANE, 2); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u16)"] @@ -58924,7 +58924,7 @@ pub unsafe fn vst1_lane_u16(a: *mut u16, b: uint16x4_t) { )] pub unsafe fn vst1q_lane_u16(a: *mut u16, b: uint16x8_t) { static_assert_uimm_bits!(LANE, 3); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u32)"] @@ -58949,7 +58949,7 @@ pub unsafe fn vst1q_lane_u16(a: *mut u16, b: uint16x8_t) { )] pub unsafe fn vst1_lane_u32(a: *mut u32, b: uint32x2_t) { static_assert_uimm_bits!(LANE, 1); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u32)"] @@ -58974,7 +58974,7 @@ pub unsafe fn vst1_lane_u32(a: *mut u32, b: uint32x2_t) { )] pub unsafe fn vst1q_lane_u32(a: *mut u32, b: uint32x4_t) { static_assert_uimm_bits!(LANE, 2); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_u64)"] @@ -58999,7 +58999,7 @@ pub unsafe fn vst1q_lane_u32(a: *mut u32, b: uint32x4_t) { )] pub unsafe fn vst1q_lane_u64(a: *mut u64, b: uint64x2_t) { static_assert_uimm_bits!(LANE, 1); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p8)"] @@ -59024,7 +59024,7 @@ pub unsafe fn vst1q_lane_u64(a: *mut u64, b: uint64x2_t) { )] pub unsafe fn vst1_lane_p8(a: *mut p8, b: poly8x8_t) { static_assert_uimm_bits!(LANE, 3); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_p8)"] @@ -59049,7 +59049,7 @@ pub unsafe fn vst1_lane_p8(a: *mut p8, b: poly8x8_t) { )] pub unsafe fn vst1q_lane_p8(a: *mut p8, b: poly8x16_t) { static_assert_uimm_bits!(LANE, 4); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p16)"] @@ -59074,7 +59074,7 @@ pub unsafe fn vst1q_lane_p8(a: *mut p8, b: poly8x16_t) { )] pub unsafe fn vst1_lane_p16(a: *mut p16, b: poly16x4_t) { static_assert_uimm_bits!(LANE, 2); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1q_lane_p16)"] @@ -59099,7 +59099,7 @@ pub unsafe fn vst1_lane_p16(a: *mut p16, b: poly16x4_t) { )] pub unsafe fn vst1q_lane_p16(a: *mut p16, b: poly16x8_t) { static_assert_uimm_bits!(LANE, 3); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_p64)"] @@ -59124,7 +59124,7 @@ pub unsafe fn vst1q_lane_p16(a: *mut p16, b: poly16x8_t) { )] pub unsafe fn vst1_lane_p64(a: *mut p64, b: poly64x1_t) { static_assert!(LANE == 0); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_s64)"] @@ -59149,7 +59149,7 @@ pub unsafe fn vst1_lane_p64(a: *mut p64, b: poly64x1_t) { )] pub unsafe fn vst1_lane_s64(a: *mut i64, b: int64x1_t) { static_assert!(LANE == 0); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures from one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_lane_u64)"] @@ -59174,7 +59174,7 @@ pub unsafe fn vst1_lane_s64(a: *mut i64, b: int64x1_t) { )] pub unsafe fn vst1_lane_u64(a: *mut u64, b: uint64x1_t) { static_assert!(LANE == 0); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple single-element structures to one, two, three, or four registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst1_p64_x2)"] @@ -61181,7 +61181,7 @@ unsafe fn vst1q_v8f16(addr: *const i8, val: float16x8_t, align: i32) { )] pub unsafe fn vst1q_lane_p64(a: *mut p64, b: poly64x2_t) { static_assert_uimm_bits!(LANE, 1); - *a = simd_extract!(b, LANE as u32); + crate::ptr::write_unaligned(a, simd_extract!(b, LANE as u32)) } #[doc = "Store multiple 2-element structures from two registers"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vst2_f16)"] diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs index 70a37f7c05dad..ecb5cd53f4575 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs @@ -223,3 +223,32 @@ fn test_vld1q_f32() { let r = unsafe { f32x4::from(vld1q_f32(a[1..].as_ptr())) }; assert_eq!(r, e) } + +#[simd_test(enable = "neon")] +fn test_vld1q_lane_u32_unaligned() { + // ld1 single-lane loads impose no alignment requirement: read from an odd byte offset. + let a: [u8; 5] = [0, 1, 2, 3, 4]; + let e = u32::from_ne_bytes([1, 2, 3, 4]); + let src = u32x4::new(10, 11, 12, 13); + let r = unsafe { + u32x4::from(vld1q_lane_u32::<2>( + a.as_ptr().add(1) as *const u32, + src.into(), + )) + }; + assert_eq!(r, u32x4::new(10, 11, e, 13)); +} + +#[simd_test(enable = "neon")] +fn test_vld1q_lane_u64_unaligned() { + let a: [u8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e = u64::from_ne_bytes([1, 2, 3, 4, 5, 6, 7, 8]); + let src = u64x2::new(10, 11); + let r = unsafe { + u64x2::from(vld1q_lane_u64::<1>( + a.as_ptr().add(1) as *const u64, + src.into(), + )) + }; + assert_eq!(r, u64x2::new(10, e)); +} diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs index 6eb60e4c78bc8..68df3ec944abd 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/store_tests.rs @@ -473,3 +473,27 @@ fn test_vst1q_f32() { assert_eq!(vals[3], 3.); assert_eq!(vals[4], 4.); } + +#[simd_test(enable = "neon")] +fn test_vst1q_lane_u32_unaligned() { + // st1 single-lane stores impose no alignment requirement: write to an odd byte offset. + let mut vals = [0_u8; 5]; + let a = u32x4::new(1, 2, 3, 4); + unsafe { + vst1q_lane_u32::<2>(vals.as_mut_ptr().add(1) as *mut u32, a.into()); + } + assert_eq!(vals[0], 0); + assert_eq!(u32::from_ne_bytes([vals[1], vals[2], vals[3], vals[4]]), 3); +} + +#[simd_test(enable = "neon")] +fn test_vst1q_lane_u64_unaligned() { + let mut vals = [0_u8; 9]; + let a = u64x2::new(1, 2); + unsafe { + vst1q_lane_u64::<1>(vals.as_mut_ptr().add(1) as *mut u64, a.into()); + } + assert_eq!(vals[0], 0); + let stored: [u8; 8] = vals[1..9].try_into().unwrap(); + assert_eq!(u64::from_ne_bytes(stored), 2); +} diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml index e5ce77ed8b33f..01942246a0997 100644 --- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml @@ -4321,10 +4321,7 @@ intrinsics: - ['*mut f64', float64x1_t] compose: - FnCall: [static_assert!, ['LANE == 0']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] + - FnCall: [core::ptr::write_unaligned, [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] - name: "vst1{neon_type[1].lane_nox}" doc: "Store multiple single-element structures from one, two, three, or four registers" @@ -4340,10 +4337,7 @@ intrinsics: - ['*mut f64', float64x2_t] compose: - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] + - FnCall: [core::ptr::write_unaligned, [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] - name: "vst2{neon_type[1].nox}" doc: "Store multiple 2-element structures from two registers" diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml index 7a7656e4c14b3..a91bfb2eb8a67 100644 --- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml @@ -2868,7 +2868,7 @@ intrinsics: - ["*const f16", float16x8_t, 'q_lane', '3'] compose: - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: [simd_insert!, [src, "LANE as u32", "*ptr"]] + - FnCall: [simd_insert!, [src, "LANE as u32", {FnCall: ["crate::ptr::read_unaligned", [ptr]]}]] - name: "vld1{type[2]}_{neon_type[1]}" doc: "Load one single-element structure and replicate to all lanes of one register" @@ -4683,10 +4683,7 @@ intrinsics: - ['*mut u64', uint64x1_t] compose: - FnCall: [static_assert!, ['LANE == 0']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] + - FnCall: ['crate::ptr::write_unaligned', [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] - name: "vst1{neon_type[1].lane_nox}" doc: "Store multiple single-element structures from one, two, three, or four registers" @@ -4708,10 +4705,7 @@ intrinsics: - ['*mut p64', poly64x1_t] compose: - FnCall: [static_assert!, ['LANE == 0']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] + - FnCall: ['crate::ptr::write_unaligned', [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] - name: "vst1{neon_type[1].lane_nox}" doc: "Store multiple single-element structures from one, two, three, or four registers" @@ -4733,10 +4727,7 @@ intrinsics: - ['*mut p64', poly64x2_t] compose: - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] + - FnCall: ['crate::ptr::write_unaligned', [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] - name: "vst1{neon_type[1].lane_nox}" doc: "Store multiple single-element structures from one, two, three, or four registers" @@ -4774,10 +4765,7 @@ intrinsics: - ['*mut f32', float32x4_t, '2'] compose: - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] + - FnCall: ['crate::ptr::write_unaligned', [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] - name: "vst1{neon_type[1].lane_nox}" @@ -4799,10 +4787,7 @@ intrinsics: - ['*mut f16', float16x8_t, '3'] compose: - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] + - FnCall: ['crate::ptr::write_unaligned', [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] - name: 'vst1{neon_type[1].no}' @@ -14314,13 +14299,13 @@ intrinsics: - ['vld1q_lane_s32', '*const i32', 'int32x4_t', '"vld1.32"', '3', 'ld1', 'static_assert_uimm_bits!', 'LANE, 2'] - ['vld1q_lane_u32', '*const u32', 'uint32x4_t', '"vld1.32"', '3', 'ld1', 'static_assert_uimm_bits!', 'LANE, 2'] - ['vld1q_lane_f32', '*const f32', 'float32x4_t', '"vld1.32"', '3', 'ld1', 'static_assert_uimm_bits!', 'LANE, 2'] - - ['vld1_lane_s64', '*const i64', 'int64x1_t', 'vldr', '0', 'ldr', 'static_assert!', 'LANE == 0'] - - ['vld1_lane_u64', '*const u64', 'uint64x1_t', 'vldr', '0', 'ldr', 'static_assert!', 'LANE == 0'] - - ['vld1q_lane_s64', '*const i64', 'int64x2_t', 'vldr', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] - - ['vld1q_lane_u64', '*const u64', 'uint64x2_t', 'vldr', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] + - ['vld1_lane_s64', '*const i64', 'int64x1_t', '"vld1.8"', '0', 'ldr', 'static_assert!', 'LANE == 0'] + - ['vld1_lane_u64', '*const u64', 'uint64x1_t', '"vld1.8"', '0', 'ldr', 'static_assert!', 'LANE == 0'] + - ['vld1q_lane_s64', '*const i64', 'int64x2_t', '"vld1.8"', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] + - ['vld1q_lane_u64', '*const u64', 'uint64x2_t', '"vld1.8"', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] compose: - FnCall: ["{type[6]}", ["{type[7]}"]] - - FnCall: [simd_insert!, [src, 'LANE as u32', '*ptr']] + - FnCall: [simd_insert!, [src, 'LANE as u32', {FnCall: ['crate::ptr::read_unaligned', [ptr]]}]] - name: "{type[0]}" doc: "Load one single-element structure to one lane of one register." @@ -14338,11 +14323,11 @@ intrinsics: safety: unsafe: [neon] types: - - ['vld1_lane_p64', '*const p64', 'poly64x1_t', 'vldr', '0', 'ldr', 'static_assert!', 'LANE == 0'] - - ['vld1q_lane_p64', '*const p64', 'poly64x2_t', 'vldr', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] + - ['vld1_lane_p64', '*const p64', 'poly64x1_t', '"vld1.8"', '0', 'ldr', 'static_assert!', 'LANE == 0'] + - ['vld1q_lane_p64', '*const p64', 'poly64x2_t', '"vld1.8"', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] compose: - FnCall: ["{type[6]}", ["{type[7]}"]] - - FnCall: [simd_insert!, [src, 'LANE as u32', '*ptr']] + - FnCall: [simd_insert!, [src, 'LANE as u32', {FnCall: ['crate::ptr::read_unaligned', [ptr]]}]] - name: "{type[0]}" doc: "Load one single-element structure and Replicate to all lanes (of one register)." From e9bc995100c0360b83e611d5b65540a3bf0820a6 Mon Sep 17 00:00:00 2001 From: Valentyn Kit Date: Thu, 20 Aug 2026 20:32:59 +0300 Subject: [PATCH 03/55] neon: drop the align requirement on vld1 dup loads --- .../src/arm_shared/neon/generated.rs | 46 +++++++++---------- .../src/arm_shared/neon/load_tests.rs | 17 +++++++ .../spec/neon/arm_shared.spec.yml | 8 ++-- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs index 2c6f663d6ed08..b7cc4057cec29 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs @@ -18279,7 +18279,7 @@ pub unsafe fn vld1q_dup_f16(ptr: *const f16) -> float16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_f32(ptr: *const f32) -> float32x2_t { - transmute(f32x2::splat(*ptr)) + transmute(f32x2::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_p16)"] @@ -18302,7 +18302,7 @@ pub unsafe fn vld1_dup_f32(ptr: *const f32) -> float32x2_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_p16(ptr: *const p16) -> poly16x4_t { - transmute(u16x4::splat(*ptr)) + transmute(u16x4::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_p8)"] @@ -18325,7 +18325,7 @@ pub unsafe fn vld1_dup_p16(ptr: *const p16) -> poly16x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_p8(ptr: *const p8) -> poly8x8_t { - transmute(u8x8::splat(*ptr)) + transmute(u8x8::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_s16)"] @@ -18348,7 +18348,7 @@ pub unsafe fn vld1_dup_p8(ptr: *const p8) -> poly8x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_s16(ptr: *const i16) -> int16x4_t { - transmute(i16x4::splat(*ptr)) + transmute(i16x4::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_s32)"] @@ -18371,7 +18371,7 @@ pub unsafe fn vld1_dup_s16(ptr: *const i16) -> int16x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_s32(ptr: *const i32) -> int32x2_t { - transmute(i32x2::splat(*ptr)) + transmute(i32x2::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_s8)"] @@ -18394,7 +18394,7 @@ pub unsafe fn vld1_dup_s32(ptr: *const i32) -> int32x2_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_s8(ptr: *const i8) -> int8x8_t { - transmute(i8x8::splat(*ptr)) + transmute(i8x8::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_u16)"] @@ -18417,7 +18417,7 @@ pub unsafe fn vld1_dup_s8(ptr: *const i8) -> int8x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_u16(ptr: *const u16) -> uint16x4_t { - transmute(u16x4::splat(*ptr)) + transmute(u16x4::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_u32)"] @@ -18440,7 +18440,7 @@ pub unsafe fn vld1_dup_u16(ptr: *const u16) -> uint16x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_u32(ptr: *const u32) -> uint32x2_t { - transmute(u32x2::splat(*ptr)) + transmute(u32x2::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_u8)"] @@ -18463,7 +18463,7 @@ pub unsafe fn vld1_dup_u32(ptr: *const u32) -> uint32x2_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1_dup_u8(ptr: *const u8) -> uint8x8_t { - transmute(u8x8::splat(*ptr)) + transmute(u8x8::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_f32)"] @@ -18486,7 +18486,7 @@ pub unsafe fn vld1_dup_u8(ptr: *const u8) -> uint8x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_f32(ptr: *const f32) -> float32x4_t { - transmute(f32x4::splat(*ptr)) + transmute(f32x4::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_p16)"] @@ -18509,7 +18509,7 @@ pub unsafe fn vld1q_dup_f32(ptr: *const f32) -> float32x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_p16(ptr: *const p16) -> poly16x8_t { - transmute(u16x8::splat(*ptr)) + transmute(u16x8::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_p8)"] @@ -18532,7 +18532,7 @@ pub unsafe fn vld1q_dup_p16(ptr: *const p16) -> poly16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_p8(ptr: *const p8) -> poly8x16_t { - transmute(u8x16::splat(*ptr)) + transmute(u8x16::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_s16)"] @@ -18555,7 +18555,7 @@ pub unsafe fn vld1q_dup_p8(ptr: *const p8) -> poly8x16_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_s16(ptr: *const i16) -> int16x8_t { - transmute(i16x8::splat(*ptr)) + transmute(i16x8::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_s32)"] @@ -18578,7 +18578,7 @@ pub unsafe fn vld1q_dup_s16(ptr: *const i16) -> int16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_s32(ptr: *const i32) -> int32x4_t { - transmute(i32x4::splat(*ptr)) + transmute(i32x4::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_s64)"] @@ -18587,7 +18587,7 @@ pub unsafe fn vld1q_dup_s32(ptr: *const i32) -> int32x4_t { #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1r) @@ -18601,7 +18601,7 @@ pub unsafe fn vld1q_dup_s32(ptr: *const i32) -> int32x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_s64(ptr: *const i64) -> int64x2_t { - transmute(i64x2::splat(*ptr)) + transmute(i64x2::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_s8)"] @@ -18624,7 +18624,7 @@ pub unsafe fn vld1q_dup_s64(ptr: *const i64) -> int64x2_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_s8(ptr: *const i8) -> int8x16_t { - transmute(i8x16::splat(*ptr)) + transmute(i8x16::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_u16)"] @@ -18647,7 +18647,7 @@ pub unsafe fn vld1q_dup_s8(ptr: *const i8) -> int8x16_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_u16(ptr: *const u16) -> uint16x8_t { - transmute(u16x8::splat(*ptr)) + transmute(u16x8::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_u32)"] @@ -18670,7 +18670,7 @@ pub unsafe fn vld1q_dup_u16(ptr: *const u16) -> uint16x8_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_u32(ptr: *const u32) -> uint32x4_t { - transmute(u32x4::splat(*ptr)) + transmute(u32x4::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_u64)"] @@ -18679,7 +18679,7 @@ pub unsafe fn vld1q_dup_u32(ptr: *const u32) -> uint32x4_t { #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vldr"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1r) @@ -18693,7 +18693,7 @@ pub unsafe fn vld1q_dup_u32(ptr: *const u32) -> uint32x4_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_u64(ptr: *const u64) -> uint64x2_t { - transmute(u64x2::splat(*ptr)) + transmute(u64x2::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1q_dup_u8)"] @@ -18716,7 +18716,7 @@ pub unsafe fn vld1q_dup_u64(ptr: *const u64) -> uint64x2_t { unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") )] pub unsafe fn vld1q_dup_u8(ptr: *const u8) -> uint8x16_t { - transmute(u8x16::splat(*ptr)) + transmute(u8x16::splat(crate::ptr::read_unaligned(ptr))) } #[doc = "Load one single-element structure and Replicate to all lanes (of one register)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vld1_dup_p64)"] @@ -21734,7 +21734,7 @@ unsafe fn vld1q_v8f16(a: *const i8, b: i32) -> float16x8_t { #[inline] #[target_feature(enable = "neon,aes")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vldr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.8"))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1r) diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs index ecb5cd53f4575..863fe87f6130a 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs @@ -252,3 +252,20 @@ fn test_vld1q_lane_u64_unaligned() { }; assert_eq!(r, u64x2::new(10, e)); } + +#[simd_test(enable = "neon")] +fn test_vld1q_dup_u32_unaligned() { + // ld1r replicate loads impose no alignment requirement either. + let a: [u8; 5] = [0, 1, 2, 3, 4]; + let e = u32::from_ne_bytes([1, 2, 3, 4]); + let r = unsafe { u32x4::from(vld1q_dup_u32(a.as_ptr().add(1) as *const u32)) }; + assert_eq!(r, u32x4::new(e, e, e, e)); +} + +#[simd_test(enable = "neon")] +fn test_vld1q_dup_u64_unaligned() { + let a: [u8; 9] = [0, 1, 2, 3, 4, 5, 6, 7, 8]; + let e = u64::from_ne_bytes([1, 2, 3, 4, 5, 6, 7, 8]); + let r = unsafe { u64x2::from(vld1q_dup_u64(a.as_ptr().add(1) as *const u64)) }; + assert_eq!(r, u64x2::new(e, e)); +} diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml index a91bfb2eb8a67..0f378dd9ef4ab 100644 --- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml @@ -14381,7 +14381,7 @@ intrinsics: safety: unsafe: [neon] types: - - ['vld1q_dup_p64', '*const p64', 'poly64x2_t', 'vldr', 'ld1r', 'vld1q_lane_p64::<0>', 'u64x2::splat(0)', '[0, 0]'] + - ['vld1q_dup_p64', '*const p64', 'poly64x2_t', '"vld1.8"', 'ld1r', 'vld1q_lane_p64::<0>', 'u64x2::splat(0)', '[0, 0]'] compose: - Let: - x @@ -14428,12 +14428,12 @@ intrinsics: - ['vld1q_dup_u32', '*const u32', 'uint32x4_t', 'vld1.32', 'ld1r', 'u32x4::splat'] - ['vld1q_dup_f32', '*const f32', 'float32x4_t', 'vld1.32', 'ld1r', 'f32x4::splat'] - - ['vld1q_dup_s64', '*const i64', 'int64x2_t', 'vldr', 'ld1r', 'i64x2::splat'] - - ['vld1q_dup_u64', '*const u64', 'uint64x2_t', 'vldr', 'ld1r', 'u64x2::splat'] + - ['vld1q_dup_s64', '*const i64', 'int64x2_t', 'vld1.8', 'ld1r', 'i64x2::splat'] + - ['vld1q_dup_u64', '*const u64', 'uint64x2_t', 'vld1.8', 'ld1r', 'u64x2::splat'] compose: - FnCall: - transmute - - - FnCall: ['{type[5]}', ["*ptr"]] + - - FnCall: ['{type[5]}', [{FnCall: ['crate::ptr::read_unaligned', [ptr]]}]] - name: "{type[0]}" doc: "Absolute difference and accumulate (64-bit)" From 2b059404ff57577e7ef8313837d47487ea26afcd Mon Sep 17 00:00:00 2001 From: Valentyn Kit Date: Fri, 21 Aug 2026 00:28:12 +0300 Subject: [PATCH 04/55] neon: assert ldr for the f32 lane and dup loads on arm --- .../core_arch/src/arm_shared/neon/generated.rs | 8 ++++---- .../core_arch/src/arm_shared/neon/load_tests.rs | 16 ++++++++++++++++ .../spec/neon/arm_shared.spec.yml | 8 ++++---- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs index b7cc4057cec29..12958f7b85ec0 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs @@ -18265,7 +18265,7 @@ pub unsafe fn vld1q_dup_f16(ptr: *const f16) -> float16x8_t { #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1r) @@ -18472,7 +18472,7 @@ pub unsafe fn vld1_dup_u8(ptr: *const u8) -> uint8x8_t { #[inline] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr("ldr"))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1r) @@ -19337,7 +19337,7 @@ pub unsafe fn vld1q_lane_f16(ptr: *const f16, src: float16x8_t) #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] #[rustc_legacy_const_generics(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 1))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(ldr, LANE = 1))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1, LANE = 1) @@ -19612,7 +19612,7 @@ pub unsafe fn vld1_lane_u8(ptr: *const u8, src: uint8x8_t) -> u #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] #[rustc_legacy_const_generics(2)] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr("vld1.32", LANE = 3))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(ldr, LANE = 3))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(ld1, LANE = 3) diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs index 863fe87f6130a..a4db10f7c1d26 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/load_tests.rs @@ -253,6 +253,22 @@ fn test_vld1q_lane_u64_unaligned() { assert_eq!(r, u64x2::new(10, e)); } +#[simd_test(enable = "neon")] +fn test_vld1q_lane_f32_unaligned() { + // Float loads legalize differently from integer ones under low alignment + // (armv7 uses ldr + vmov), so the type class needs its own coverage. + let a: [u8; 5] = [0, 1, 2, 3, 4]; + let e = f32::from_ne_bytes([1, 2, 3, 4]); + let src = f32x4::new(10., 11., 12., 13.); + let r = unsafe { + f32x4::from(vld1q_lane_f32::<2>( + a.as_ptr().add(1) as *const f32, + src.into(), + )) + }; + assert_eq!(r, f32x4::new(10., 11., e, 13.)); +} + #[simd_test(enable = "neon")] fn test_vld1q_dup_u32_unaligned() { // ld1r replicate loads impose no alignment requirement either. diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml index 0f378dd9ef4ab..de82b203c1f27 100644 --- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml @@ -14295,10 +14295,10 @@ intrinsics: - ['vld1q_lane_p16', '*const p16', 'poly16x8_t', '"vld1.16"', '7', 'ld1', 'static_assert_uimm_bits!', 'LANE, 3'] - ['vld1_lane_s32', '*const i32', 'int32x2_t', '"vld1.32"', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] - ['vld1_lane_u32', '*const u32', 'uint32x2_t', '"vld1.32"', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] - - ['vld1_lane_f32', '*const f32', 'float32x2_t', '"vld1.32"', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] + - ['vld1_lane_f32', '*const f32', 'float32x2_t', 'ldr', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] - ['vld1q_lane_s32', '*const i32', 'int32x4_t', '"vld1.32"', '3', 'ld1', 'static_assert_uimm_bits!', 'LANE, 2'] - ['vld1q_lane_u32', '*const u32', 'uint32x4_t', '"vld1.32"', '3', 'ld1', 'static_assert_uimm_bits!', 'LANE, 2'] - - ['vld1q_lane_f32', '*const f32', 'float32x4_t', '"vld1.32"', '3', 'ld1', 'static_assert_uimm_bits!', 'LANE, 2'] + - ['vld1q_lane_f32', '*const f32', 'float32x4_t', 'ldr', '3', 'ld1', 'static_assert_uimm_bits!', 'LANE, 2'] - ['vld1_lane_s64', '*const i64', 'int64x1_t', '"vld1.8"', '0', 'ldr', 'static_assert!', 'LANE == 0'] - ['vld1_lane_u64', '*const u64', 'uint64x1_t', '"vld1.8"', '0', 'ldr', 'static_assert!', 'LANE == 0'] - ['vld1q_lane_s64', '*const i64', 'int64x2_t', '"vld1.8"', '1', 'ld1', 'static_assert_uimm_bits!', 'LANE, 1'] @@ -14422,11 +14422,11 @@ intrinsics: - ['vld1_dup_s32', '*const i32', 'int32x2_t', 'vld1.32', 'ld1r', 'i32x2::splat'] - ['vld1_dup_u32', '*const u32', 'uint32x2_t', 'vld1.32', 'ld1r', 'u32x2::splat'] - - ['vld1_dup_f32', '*const f32', 'float32x2_t', 'vld1.32', 'ld1r', 'f32x2::splat'] + - ['vld1_dup_f32', '*const f32', 'float32x2_t', 'ldr', 'ld1r', 'f32x2::splat'] - ['vld1q_dup_s32', '*const i32', 'int32x4_t', 'vld1.32', 'ld1r', 'i32x4::splat'] - ['vld1q_dup_u32', '*const u32', 'uint32x4_t', 'vld1.32', 'ld1r', 'u32x4::splat'] - - ['vld1q_dup_f32', '*const f32', 'float32x4_t', 'vld1.32', 'ld1r', 'f32x4::splat'] + - ['vld1q_dup_f32', '*const f32', 'float32x4_t', 'ldr', 'ld1r', 'f32x4::splat'] - ['vld1q_dup_s64', '*const i64', 'int64x2_t', 'vld1.8', 'ld1r', 'i64x2::splat'] - ['vld1q_dup_u64', '*const u64', 'uint64x2_t', 'vld1.8', 'ld1r', 'u64x2::splat'] From 98f513b2376f13aa427a79dcedb9a10874c849cd Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 26 Jul 2026 10:08:43 +0100 Subject: [PATCH 05/55] Add missing avx512vl intrinsics for f32->u32 conversions --- .../crates/core_arch/src/x86/avx512f.rs | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index 76e7297bcda40..3588e146f0395 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -13091,6 +13091,104 @@ pub const fn _mm512_maskz_cvtepu32_ps(k: __mmask16, a: __m512i) -> __m512 { } } +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst. +/// +/// Due to an omission by Intel, these intrinsics are not documented in the Intrinsics Guide. +/// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] +pub const fn _mm256_cvtepu32_ps(a: __m256i) -> __m256 { + unsafe { + let a = a.as_u32x8(); + transmute::(simd_cast(a)) + } +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// +/// Due to an omission by Intel, these intrinsics are not documented in the Intrinsics Guide. +/// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] +pub const fn _mm256_mask_cvtepu32_ps(src: __m256, k: __mmask8, a: __m256i) -> __m256 { + unsafe { + let convert = _mm256_cvtepu32_ps(a).as_f32x8(); + transmute(simd_select_bitmask(k, convert, src.as_f32x8())) + } +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// +/// Due to an omission by Intel, these intrinsics are not documented in the Intrinsics Guide. +/// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] +pub const fn _mm256_maskz_cvtepu32_ps(k: __mmask8, a: __m256i) -> __m256 { + unsafe { + let convert = _mm256_cvtepu32_ps(a).as_f32x8(); + transmute(simd_select_bitmask(k, convert, f32x8::ZERO)) + } +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst. +/// +/// Due to an omission by Intel, these intrinsics are not documented in the Intrinsics Guide. +/// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] +pub const fn _mm_cvtepu32_ps(a: __m128i) -> __m128 { + unsafe { + let a = a.as_u32x4(); + transmute::(simd_cast(a)) + } +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). +/// Bits 4 through 7 of k are ignored. +/// +/// Due to an omission by Intel, these intrinsics are not documented in the Intrinsics Guide. +/// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] +pub const fn _mm_mask_cvtepu32_ps(src: __m128, k: __mmask8, a: __m128i) -> __m128 { + unsafe { + let convert = _mm_cvtepu32_ps(a).as_f32x4(); + transmute(simd_select_bitmask(k, convert, src.as_f32x4())) + } +} + +/// Convert packed unsigned 32-bit integers in a to packed single-precision (32-bit) floating-point elements, and store the results in dst using zeromask k (elements are zeroed out when the corresponding mask bit is not set). +/// Bits 4 through 7 of k are ignored. +/// +/// Due to an omission by Intel, these intrinsics are not documented in the Intrinsics Guide. +/// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). +#[inline] +#[target_feature(enable = "avx512f,avx512vl")] +#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[cfg_attr(test, assert_instr(vcvtudq2ps))] +#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] +pub const fn _mm_maskz_cvtepu32_ps(k: __mmask8, a: __m128i) -> __m128 { + unsafe { + let convert = _mm_cvtepu32_ps(a).as_f32x4(); + transmute(simd_select_bitmask(k, convert, f32x4::ZERO)) + } +} + /// Convert packed unsigned 32-bit integers in a to packed double-precision (64-bit) floating-point elements, and store the results in dst. /// /// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_cvtepu32_pd&expand=1580) @@ -50036,6 +50134,65 @@ mod tests { assert_eq_m512(r, e); } + #[simd_test(enable = "avx512f,avx512vl")] + const fn test_mm256_cvtepu32_ps() { + let a = _mm256_set_epi32(-1, i32::MIN, 16_777_217, 16_777_216, 4, 3, 2, 1); + let r = _mm256_cvtepu32_ps(a); + let e = _mm256_set_ps( + 4_294_967_296., + 2_147_483_648., + 16_777_216., + 16_777_216., + 4., + 3., + 2., + 1., + ); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + const fn test_mm256_mask_cvtepu32_ps() { + let a = _mm256_set_epi32(-1, i32::MIN, 6, 5, 4, 3, 2, 1); + let src = _mm256_set1_ps(-1.); + let r = _mm256_mask_cvtepu32_ps(src, 0b10101010, a); + let e = _mm256_set_ps(4_294_967_296., -1., 6., -1., 4., -1., 2., -1.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + const fn test_mm256_maskz_cvtepu32_ps() { + let a = _mm256_set_epi32(-1, i32::MIN, 6, 5, 4, 3, 2, 1); + let r = _mm256_maskz_cvtepu32_ps(0b01010101, a); + let e = _mm256_set_ps(0., 2_147_483_648., 0., 5., 0., 3., 0., 1.); + assert_eq_m256(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + const fn test_mm_cvtepu32_ps() { + let a = _mm_set_epi32(-1, i32::MIN, 1, 0); + let r = _mm_cvtepu32_ps(a); + let e = _mm_set_ps(4_294_967_296., 2_147_483_648., 1., 0.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + const fn test_mm_mask_cvtepu32_ps() { + let a = _mm_set_epi32(-1, i32::MIN, 2, 1); + let src = _mm_set1_ps(-1.); + let r = _mm_mask_cvtepu32_ps(src, 0b11110101, a); + let e = _mm_set_ps(-1., 2_147_483_648., -1., 1.); + assert_eq_m128(r, e); + } + + #[simd_test(enable = "avx512f,avx512vl")] + const fn test_mm_maskz_cvtepu32_ps() { + let a = _mm_set_epi32(-1, i32::MIN, 2, 1); + let r = _mm_maskz_cvtepu32_ps(0b11111010, a); + let e = _mm_set_ps(4_294_967_296., 0., 2., 0.); + assert_eq_m128(r, e); + } + #[simd_test(enable = "avx512f")] const fn test_mm512_cvtepi32_epi16() { let a = _mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); From 773242f1d39b66f58fa6cc6f883892ebd8440023 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 23 Aug 2026 11:36:59 +0100 Subject: [PATCH 06/55] Add manual exception for the intrinsics forgotten in the intel intrinsics guide xml --- .../stdarch/crates/stdarch-verify/tests/x86-intel.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/library/stdarch/crates/stdarch-verify/tests/x86-intel.rs b/library/stdarch/crates/stdarch-verify/tests/x86-intel.rs index be948df541b79..d2839bb300c2d 100644 --- a/library/stdarch/crates/stdarch-verify/tests/x86-intel.rs +++ b/library/stdarch/crates/stdarch-verify/tests/x86-intel.rs @@ -293,7 +293,16 @@ fn verify_all_signatures() { "_MM_SHUFFLE" | "_xabort_code" | // Not listed with intel, but manually verified - "cmpxchg16b" + "cmpxchg16b" | + // Apparently forgotten in the Intel Intrinsics Guide + // but present in other Intel documentation and clang, + // see https://github.com/rust-lang/rust/issues/158196 + "_mm_cvtepu32_ps" | + "_mm_mask_cvtepu32_ps" | + "_mm_maskz_cvtepu32_ps" | + "_mm256_cvtepu32_ps" | + "_mm256_mask_cvtepu32_ps" | + "_mm256_maskz_cvtepu32_ps" => continue, _ => {} } From d78d2069074fe3da4125f8a3417dd73cb7adc461 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 23 Aug 2026 11:54:11 +0100 Subject: [PATCH 07/55] Use the proper tracking issue --- library/stdarch/crates/core_arch/src/x86/avx512f.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index 3588e146f0395..10a0d370df7ba 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -13097,7 +13097,7 @@ pub const fn _mm512_maskz_cvtepu32_ps(k: __mmask16, a: __m512i) -> __m512 { /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm256_cvtepu32_ps(a: __m256i) -> __m256 { @@ -13113,7 +13113,7 @@ pub const fn _mm256_cvtepu32_ps(a: __m256i) -> __m256 { /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm256_mask_cvtepu32_ps(src: __m256, k: __mmask8, a: __m256i) -> __m256 { @@ -13129,7 +13129,7 @@ pub const fn _mm256_mask_cvtepu32_ps(src: __m256, k: __mmask8, a: __m256i) -> __ /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm256_maskz_cvtepu32_ps(k: __mmask8, a: __m256i) -> __m256 { @@ -13145,7 +13145,7 @@ pub const fn _mm256_maskz_cvtepu32_ps(k: __mmask8, a: __m256i) -> __m256 { /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm_cvtepu32_ps(a: __m128i) -> __m128 { @@ -13162,7 +13162,7 @@ pub const fn _mm_cvtepu32_ps(a: __m128i) -> __m128 { /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm_mask_cvtepu32_ps(src: __m128, k: __mmask8, a: __m128i) -> __m128 { @@ -13179,7 +13179,7 @@ pub const fn _mm_mask_cvtepu32_ps(src: __m128, k: __mmask8, a: __m128i) -> __m12 /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_updates", issue = "158196")] +#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm_maskz_cvtepu32_ps(k: __mmask8, a: __m128i) -> __m128 { From 9f02c537bbd73a47de67659e9c812e73496c2a44 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 23 Aug 2026 11:55:04 +0100 Subject: [PATCH 08/55] cargo fmt --- .../crates/core_arch/src/x86/avx512f.rs | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index 10a0d370df7ba..50f8a13ac98d4 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -13097,7 +13097,10 @@ pub const fn _mm512_maskz_cvtepu32_ps(k: __mmask16, a: __m512i) -> __m512 { /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] +#[unstable( + feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", + issue = "161585" +)] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm256_cvtepu32_ps(a: __m256i) -> __m256 { @@ -13113,7 +13116,10 @@ pub const fn _mm256_cvtepu32_ps(a: __m256i) -> __m256 { /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] +#[unstable( + feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", + issue = "161585" +)] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm256_mask_cvtepu32_ps(src: __m256, k: __mmask8, a: __m256i) -> __m256 { @@ -13129,7 +13135,10 @@ pub const fn _mm256_mask_cvtepu32_ps(src: __m256, k: __mmask8, a: __m256i) -> __ /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] +#[unstable( + feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", + issue = "161585" +)] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm256_maskz_cvtepu32_ps(k: __mmask8, a: __m256i) -> __m256 { @@ -13145,7 +13154,10 @@ pub const fn _mm256_maskz_cvtepu32_ps(k: __mmask8, a: __m256i) -> __m256 { /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] +#[unstable( + feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", + issue = "161585" +)] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm_cvtepu32_ps(a: __m128i) -> __m128 { @@ -13162,7 +13174,10 @@ pub const fn _mm_cvtepu32_ps(a: __m128i) -> __m128 { /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] +#[unstable( + feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", + issue = "161585" +)] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm_mask_cvtepu32_ps(src: __m128, k: __mmask8, a: __m128i) -> __m128 { @@ -13179,7 +13194,10 @@ pub const fn _mm_mask_cvtepu32_ps(src: __m128, k: __mmask8, a: __m128i) -> __m12 /// Documentation on them can be found in the [Intel Software Development Manual](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html). #[inline] #[target_feature(enable = "avx512f,avx512vl")] -#[unstable(feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", issue = "161585")] +#[unstable( + feature = "stdarch_x86_avx512_vl_u32_to_f32_conversions", + issue = "161585" +)] #[cfg_attr(test, assert_instr(vcvtudq2ps))] #[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] pub const fn _mm_maskz_cvtepu32_ps(k: __mmask8, a: __m128i) -> __m128 { From 2e36890c46be699fc43e7357220068813b6a8d15 Mon Sep 17 00:00:00 2001 From: The rustc-josh-sync Cronjob Bot Date: Mon, 24 Aug 2026 04:22:38 +0000 Subject: [PATCH 09/55] Prepare for merging from rust-lang/rust This updates the rust-version file to da5114692c9ebe46b869488c5f34f92eb10b98c1. --- library/stdarch/rust-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/stdarch/rust-version b/library/stdarch/rust-version index 61a4b2d8c095f..7b170626619e4 100644 --- a/library/stdarch/rust-version +++ b/library/stdarch/rust-version @@ -1 +1 @@ -1e5ee356374211706221b71b6106d297a646ee57 +da5114692c9ebe46b869488c5f34f92eb10b98c1 From 2cc2b5791e0fb3ea7b086b48a8e0ae380611d09e Mon Sep 17 00:00:00 2001 From: The rustc-josh-sync Cronjob Bot Date: Mon, 7 Sep 2026 04:17:44 +0000 Subject: [PATCH 10/55] Prepare for merging from rust-lang/rust This updates the rust-version file to 32d94cc9be3f6e6c3fa1deaea9e0ab93c4980dba. --- library/stdarch/rust-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/stdarch/rust-version b/library/stdarch/rust-version index 7b170626619e4..18fea436747c7 100644 --- a/library/stdarch/rust-version +++ b/library/stdarch/rust-version @@ -1 +1 @@ -da5114692c9ebe46b869488c5f34f92eb10b98c1 +32d94cc9be3f6e6c3fa1deaea9e0ab93c4980dba From 0016f59536f3ca4050a621342470fba5983e1a0f Mon Sep 17 00:00:00 2001 From: Adam Gemmell Date: Mon, 7 Sep 2026 19:00:53 +0100 Subject: [PATCH 11/55] Run rustfmt --- library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs b/library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs index e5722a2d9f49f..537c59bbce861 100644 --- a/library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs +++ b/library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs @@ -213,7 +213,7 @@ fn parse_compound_expr(expr: &str) -> Option { if let Some(paren_pos) = after_prefix.find(')') { let builtin_name = &after_prefix[..paren_pos]; let rest = &after_prefix[paren_pos + 1..]; // Skip the closing ) of the WRAP - // rest should now be "(args)" + // rest should now be "(args)" if rest.starts_with('(') && rest.ends_with(')') { let args_str = &rest[1..rest.len() - 1]; let args = parse_compound_args(args_str)?; From 7cbced258c8d0891a2a794fbfdb7253a9dbcd0e6 Mon Sep 17 00:00:00 2001 From: Adam Gemmell Date: Tue, 8 Sep 2026 14:38:52 +0100 Subject: [PATCH 12/55] Update vzipq arm instruction assertions --- .../src/arm_shared/neon/generated.rs | 36 +++++++++---------- .../spec/neon/arm_shared.spec.yml | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs index 4fc46a3ba39e8..094b13ca8023a 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs @@ -70649,7 +70649,7 @@ pub fn vzip_p16(a: poly16x4_t, b: poly16x4_t) -> poly16x4x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70679,7 +70679,7 @@ pub fn vzipq_f32(a: float32x4_t, b: float32x4_t) -> float32x4x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70714,7 +70714,7 @@ pub fn vzipq_f32(a: float32x4_t, b: float32x4_t) -> float32x4x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70752,7 +70752,7 @@ pub fn vzipq_s8(a: int8x16_t, b: int8x16_t) -> int8x16x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70805,7 +70805,7 @@ pub fn vzipq_s8(a: int8x16_t, b: int8x16_t) -> int8x16x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70835,7 +70835,7 @@ pub fn vzipq_s16(a: int16x8_t, b: int16x8_t) -> int16x8x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70870,7 +70870,7 @@ pub fn vzipq_s16(a: int16x8_t, b: int16x8_t) -> int16x8x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70900,7 +70900,7 @@ pub fn vzipq_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70935,7 +70935,7 @@ pub fn vzipq_s32(a: int32x4_t, b: int32x4_t) -> int32x4x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -70973,7 +70973,7 @@ pub fn vzipq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -71026,7 +71026,7 @@ pub fn vzipq_u8(a: uint8x16_t, b: uint8x16_t) -> uint8x16x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -71056,7 +71056,7 @@ pub fn vzipq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -71091,7 +71091,7 @@ pub fn vzipq_u16(a: uint16x8_t, b: uint16x8_t) -> uint16x8x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -71121,7 +71121,7 @@ pub fn vzipq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -71156,7 +71156,7 @@ pub fn vzipq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -71194,7 +71194,7 @@ pub fn vzipq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -71247,7 +71247,7 @@ pub fn vzipq_p8(a: poly8x16_t, b: poly8x16_t) -> poly8x16x2_t { #[cfg(target_endian = "little")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) @@ -71277,7 +71277,7 @@ pub fn vzipq_p16(a: poly16x8_t, b: poly16x8_t) -> poly16x8x2_t { #[cfg(target_endian = "big")] #[target_feature(enable = "neon")] #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vorr))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vzip))] #[cfg_attr( all(test, any(target_arch = "aarch64", target_arch = "arm64ec")), assert_instr(zip1) diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml index 195243ac65d3a..a4ade26e45c37 100644 --- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml @@ -9824,7 +9824,7 @@ intrinsics: return_type: "{neon_type[1]}" attr: - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vorr]]}]] + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vzip]]}]] - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip1]]}]] - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip2]]}]] - *neon-not-arm-stable From 0c4077db7693376f1c6ff85b470b9c7e04ed3d6d Mon Sep 17 00:00:00 2001 From: Adam Gemmell Date: Tue, 8 Sep 2026 15:19:05 +0100 Subject: [PATCH 13/55] Run cargo fmt on JOSH syncs --- library/stdarch/josh-sync.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/library/stdarch/josh-sync.toml b/library/stdarch/josh-sync.toml index ebdb4576287c8..eeeb82659a0a6 100644 --- a/library/stdarch/josh-sync.toml +++ b/library/stdarch/josh-sync.toml @@ -1,3 +1,7 @@ org = "rust-lang" repo = "stdarch" path = "library/stdarch" + +[[post-pull]] +cmd = ["cargo", "fmt"] +commit-message = "Run `cargo fmt`" From dd13995fa28b60860cfc166de754376fd4d101f0 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 5 Sep 2026 16:36:31 +0200 Subject: [PATCH 14/55] Revert "Use SIMD intrinsics for vector shifts" This reverts commit 102f03d2c410660a45d0231fdafdd347e0d4e77f. --- .../stdarch/crates/core_arch/src/x86/avx2.rs | 120 ++++++------------ .../crates/core_arch/src/x86/avx512bw.rs | 111 ++++++---------- .../crates/core_arch/src/x86/avx512f.rs | 99 +++++---------- 3 files changed, 114 insertions(+), 216 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/x86/avx2.rs b/library/stdarch/crates/core_arch/src/x86/avx2.rs index eb636a4fa0397..e3b6054568f11 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx2.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx2.rs @@ -2871,14 +2871,8 @@ pub const fn _mm256_bslli_epi128(a: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsllvd))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_sllv_epi32(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u32x4(); - let no_overflow: u32x4 = simd_lt(count, u32x4::splat(u32::BITS)); - let count = simd_select(no_overflow, count, u32x4::ZERO); - simd_select(no_overflow, simd_shl(a.as_u32x4(), count), u32x4::ZERO).as_m128i() - } +pub fn _mm_sllv_epi32(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(psllvd(a.as_i32x4(), count.as_i32x4())) } } /// Shifts packed 32-bit integers in `a` left by the amount @@ -2890,14 +2884,8 @@ pub const fn _mm_sllv_epi32(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsllvd))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_sllv_epi32(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u32x8(); - let no_overflow: u32x8 = simd_lt(count, u32x8::splat(u32::BITS)); - let count = simd_select(no_overflow, count, u32x8::ZERO); - simd_select(no_overflow, simd_shl(a.as_u32x8(), count), u32x8::ZERO).as_m256i() - } +pub fn _mm256_sllv_epi32(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(psllvd256(a.as_i32x8(), count.as_i32x8())) } } /// Shifts packed 64-bit integers in `a` left by the amount @@ -2909,14 +2897,8 @@ pub const fn _mm256_sllv_epi32(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsllvq))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_sllv_epi64(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u64x2(); - let no_overflow: u64x2 = simd_lt(count, u64x2::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, count, u64x2::ZERO); - simd_select(no_overflow, simd_shl(a.as_u64x2(), count), u64x2::ZERO).as_m128i() - } +pub fn _mm_sllv_epi64(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(psllvq(a.as_i64x2(), count.as_i64x2())) } } /// Shifts packed 64-bit integers in `a` left by the amount @@ -2928,14 +2910,8 @@ pub const fn _mm_sllv_epi64(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsllvq))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_sllv_epi64(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u64x4(); - let no_overflow: u64x4 = simd_lt(count, u64x4::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, count, u64x4::ZERO); - simd_select(no_overflow, simd_shl(a.as_u64x4(), count), u64x4::ZERO).as_m256i() - } +pub fn _mm256_sllv_epi64(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(psllvq256(a.as_i64x4(), count.as_i64x4())) } } /// Shifts packed 16-bit integers in `a` right by `count` while @@ -3000,14 +2976,8 @@ pub const fn _mm256_srai_epi32(a: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsravd))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_srav_epi32(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u32x4(); - let no_overflow: u32x4 = simd_lt(count, u32x4::splat(u32::BITS)); - let count = simd_select(no_overflow, transmute(count), i32x4::splat(31)); - simd_shr(a.as_i32x4(), count).as_m128i() - } +pub fn _mm_srav_epi32(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(psravd(a.as_i32x4(), count.as_i32x4())) } } /// Shifts packed 32-bit integers in `a` right by the amount specified by the @@ -3018,14 +2988,8 @@ pub const fn _mm_srav_epi32(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsravd))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_srav_epi32(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u32x8(); - let no_overflow: u32x8 = simd_lt(count, u32x8::splat(u32::BITS)); - let count = simd_select(no_overflow, transmute(count), i32x8::splat(31)); - simd_shr(a.as_i32x8(), count).as_m256i() - } +pub fn _mm256_srav_epi32(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(psravd256(a.as_i32x8(), count.as_i32x8())) } } /// Shifts 128-bit lanes in `a` right by `imm8` bytes while shifting in zeros. @@ -3212,14 +3176,8 @@ pub const fn _mm256_srli_epi64(a: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsrlvd))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_srlv_epi32(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u32x4(); - let no_overflow: u32x4 = simd_lt(count, u32x4::splat(u32::BITS)); - let count = simd_select(no_overflow, count, u32x4::ZERO); - simd_select(no_overflow, simd_shr(a.as_u32x4(), count), u32x4::ZERO).as_m128i() - } +pub fn _mm_srlv_epi32(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(psrlvd(a.as_i32x4(), count.as_i32x4())) } } /// Shifts packed 32-bit integers in `a` right by the amount specified by @@ -3230,14 +3188,8 @@ pub const fn _mm_srlv_epi32(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsrlvd))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_srlv_epi32(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u32x8(); - let no_overflow: u32x8 = simd_lt(count, u32x8::splat(u32::BITS)); - let count = simd_select(no_overflow, count, u32x8::ZERO); - simd_select(no_overflow, simd_shr(a.as_u32x8(), count), u32x8::ZERO).as_m256i() - } +pub fn _mm256_srlv_epi32(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(psrlvd256(a.as_i32x8(), count.as_i32x8())) } } /// Shifts packed 64-bit integers in `a` right by the amount specified by @@ -3248,14 +3200,8 @@ pub const fn _mm256_srlv_epi32(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsrlvq))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_srlv_epi64(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u64x2(); - let no_overflow: u64x2 = simd_lt(count, u64x2::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, count, u64x2::ZERO); - simd_select(no_overflow, simd_shr(a.as_u64x2(), count), u64x2::ZERO).as_m128i() - } +pub fn _mm_srlv_epi64(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(psrlvq(a.as_i64x2(), count.as_i64x2())) } } /// Shifts packed 64-bit integers in `a` right by the amount specified by @@ -3266,14 +3212,8 @@ pub const fn _mm_srlv_epi64(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpsrlvq))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_srlv_epi64(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u64x4(); - let no_overflow: u64x4 = simd_lt(count, u64x4::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, count, u64x4::ZERO); - simd_select(no_overflow, simd_shr(a.as_u64x4(), count), u64x4::ZERO).as_m256i() - } +pub fn _mm256_srlv_epi64(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(psrlvq256(a.as_i64x4(), count.as_i64x4())) } } /// Load 256-bits of integer data from memory into dst using a non-temporal memory hint. mem_addr @@ -3849,16 +3789,36 @@ unsafe extern "llvm-intrinsic" { fn pslld(a: i32x8, count: i32x4) -> i32x8; #[link_name = "llvm.x86.avx2.psll.q"] fn psllq(a: i64x4, count: i64x2) -> i64x4; + #[link_name = "llvm.x86.avx2.psllv.d"] + fn psllvd(a: i32x4, count: i32x4) -> i32x4; + #[link_name = "llvm.x86.avx2.psllv.d.256"] + fn psllvd256(a: i32x8, count: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx2.psllv.q"] + fn psllvq(a: i64x2, count: i64x2) -> i64x2; + #[link_name = "llvm.x86.avx2.psllv.q.256"] + fn psllvq256(a: i64x4, count: i64x4) -> i64x4; #[link_name = "llvm.x86.avx2.psra.w"] fn psraw(a: i16x16, count: i16x8) -> i16x16; #[link_name = "llvm.x86.avx2.psra.d"] fn psrad(a: i32x8, count: i32x4) -> i32x8; + #[link_name = "llvm.x86.avx2.psrav.d"] + fn psravd(a: i32x4, count: i32x4) -> i32x4; + #[link_name = "llvm.x86.avx2.psrav.d.256"] + fn psravd256(a: i32x8, count: i32x8) -> i32x8; #[link_name = "llvm.x86.avx2.psrl.w"] fn psrlw(a: i16x16, count: i16x8) -> i16x16; #[link_name = "llvm.x86.avx2.psrl.d"] fn psrld(a: i32x8, count: i32x4) -> i32x8; #[link_name = "llvm.x86.avx2.psrl.q"] fn psrlq(a: i64x4, count: i64x2) -> i64x4; + #[link_name = "llvm.x86.avx2.psrlv.d"] + fn psrlvd(a: i32x4, count: i32x4) -> i32x4; + #[link_name = "llvm.x86.avx2.psrlv.d.256"] + fn psrlvd256(a: i32x8, count: i32x8) -> i32x8; + #[link_name = "llvm.x86.avx2.psrlv.q"] + fn psrlvq(a: i64x2, count: i64x2) -> i64x2; + #[link_name = "llvm.x86.avx2.psrlv.q.256"] + fn psrlvq256(a: i64x4, count: i64x4) -> i64x4; #[link_name = "llvm.x86.avx2.pshuf.b"] fn pshufb(a: u8x32, b: u8x32) -> u8x32; #[link_name = "llvm.x86.avx2.permd"] diff --git a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs index cda2fad4ef2de..054ca9816379f 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs @@ -7370,14 +7370,8 @@ pub const fn _mm_maskz_slli_epi16(k: __mmask8, a: __m128i) -> _ #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_sllv_epi16(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u16x32(); - let no_overflow: u16x32 = simd_lt(count, u16x32::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, count, u16x32::ZERO); - simd_select(no_overflow, simd_shl(a.as_u16x32(), count), u16x32::ZERO).as_m512i() - } +pub fn _mm512_sllv_epi16(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsllvw(a.as_i16x32(), count.as_i16x32())) } } /// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -7422,14 +7416,8 @@ pub const fn _mm512_maskz_sllv_epi16(k: __mmask32, a: __m512i, count: __m512i) - #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_sllv_epi16(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u16x16(); - let no_overflow: u16x16 = simd_lt(count, u16x16::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, count, u16x16::ZERO); - simd_select(no_overflow, simd_shl(a.as_u16x16(), count), u16x16::ZERO).as_m256i() - } +pub fn _mm256_sllv_epi16(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(vpsllvw256(a.as_i16x16(), count.as_i16x16())) } } /// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -7474,14 +7462,8 @@ pub const fn _mm256_maskz_sllv_epi16(k: __mmask16, a: __m256i, count: __m256i) - #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_sllv_epi16(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u16x8(); - let no_overflow: u16x8 = simd_lt(count, u16x8::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, count, u16x8::ZERO); - simd_select(no_overflow, simd_shl(a.as_u16x8(), count), u16x8::ZERO).as_m128i() - } +pub fn _mm_sllv_epi16(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(vpsllvw128(a.as_i16x8(), count.as_i16x8())) } } /// Shift packed 16-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -7759,14 +7741,8 @@ pub const fn _mm_maskz_srli_epi16(k: __mmask8, a: __m128i) -> _ #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_srlv_epi16(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u16x32(); - let no_overflow: u16x32 = simd_lt(count, u16x32::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, count, u16x32::ZERO); - simd_select(no_overflow, simd_shr(a.as_u16x32(), count), u16x32::ZERO).as_m512i() - } +pub fn _mm512_srlv_epi16(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsrlvw(a.as_i16x32(), count.as_i16x32())) } } /// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -7811,14 +7787,8 @@ pub const fn _mm512_maskz_srlv_epi16(k: __mmask32, a: __m512i, count: __m512i) - #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_srlv_epi16(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u16x16(); - let no_overflow: u16x16 = simd_lt(count, u16x16::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, count, u16x16::ZERO); - simd_select(no_overflow, simd_shr(a.as_u16x16(), count), u16x16::ZERO).as_m256i() - } +pub fn _mm256_srlv_epi16(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(vpsrlvw256(a.as_i16x16(), count.as_i16x16())) } } /// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -7863,14 +7833,8 @@ pub const fn _mm256_maskz_srlv_epi16(k: __mmask16, a: __m256i, count: __m256i) - #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_srlv_epi16(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u16x8(); - let no_overflow: u16x8 = simd_lt(count, u16x8::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, count, u16x8::ZERO); - simd_select(no_overflow, simd_shr(a.as_u16x8(), count), u16x8::ZERO).as_m128i() - } +pub fn _mm_srlv_epi16(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(vpsrlvw128(a.as_i16x8(), count.as_i16x8())) } } /// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -8135,14 +8099,8 @@ pub const fn _mm_maskz_srai_epi16(k: __mmask8, a: __m128i) -> _ #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_srav_epi16(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u16x32(); - let no_overflow: u16x32 = simd_lt(count, u16x32::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, transmute(count), i16x32::splat(15)); - simd_shr(a.as_i16x32(), count).as_m512i() - } +pub fn _mm512_srav_epi16(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsravw(a.as_i16x32(), count.as_i16x32())) } } /// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -8187,14 +8145,8 @@ pub const fn _mm512_maskz_srav_epi16(k: __mmask32, a: __m512i, count: __m512i) - #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_srav_epi16(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u16x16(); - let no_overflow: u16x16 = simd_lt(count, u16x16::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, transmute(count), i16x16::splat(15)); - simd_shr(a.as_i16x16(), count).as_m256i() - } +pub fn _mm256_srav_epi16(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(vpsravw256(a.as_i16x16(), count.as_i16x16())) } } /// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -8239,14 +8191,8 @@ pub const fn _mm256_maskz_srav_epi16(k: __mmask16, a: __m256i, count: __m256i) - #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_srav_epi16(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u16x8(); - let no_overflow: u16x8 = simd_lt(count, u16x8::splat(u16::BITS as u16)); - let count = simd_select(no_overflow, transmute(count), i16x8::splat(15)); - simd_shr(a.as_i16x8(), count).as_m128i() - } +pub fn _mm_srav_epi16(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(vpsravw128(a.as_i16x8(), count.as_i16x8())) } } /// Shift packed 16-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -12618,12 +12564,33 @@ unsafe extern "llvm-intrinsic" { #[link_name = "llvm.x86.avx512.psll.w.512"] fn vpsllw(a: i16x32, count: i16x8) -> i16x32; + #[link_name = "llvm.x86.avx512.psllv.w.512"] + fn vpsllvw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.psllv.w.256"] + fn vpsllvw256(a: i16x16, b: i16x16) -> i16x16; + #[link_name = "llvm.x86.avx512.psllv.w.128"] + fn vpsllvw128(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.x86.avx512.psrl.w.512"] fn vpsrlw(a: i16x32, count: i16x8) -> i16x32; + #[link_name = "llvm.x86.avx512.psrlv.w.512"] + fn vpsrlvw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.psrlv.w.256"] + fn vpsrlvw256(a: i16x16, b: i16x16) -> i16x16; + #[link_name = "llvm.x86.avx512.psrlv.w.128"] + fn vpsrlvw128(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.x86.avx512.psra.w.512"] fn vpsraw(a: i16x32, count: i16x8) -> i16x32; + #[link_name = "llvm.x86.avx512.psrav.w.512"] + fn vpsravw(a: i16x32, count: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.psrav.w.256"] + fn vpsravw256(a: i16x16, count: i16x16) -> i16x16; + #[link_name = "llvm.x86.avx512.psrav.w.128"] + fn vpsravw128(a: i16x8, count: i16x8) -> i16x8; + #[link_name = "llvm.x86.avx512.vpermi2var.hi.512"] fn vpermi2w(a: i16x32, idx: i16x32, b: i16x32) -> i16x32; #[link_name = "llvm.x86.avx512.vpermi2var.hi.256"] diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index f0103df1be79e..2b833e247b1bc 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -21648,14 +21648,8 @@ pub const fn _mm_maskz_srai_epi64(k: __mmask8, a: __m128i) -> _ #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_srav_epi32(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u32x16(); - let no_overflow: u32x16 = simd_lt(count, u32x16::splat(u32::BITS)); - let count = simd_select(no_overflow, transmute(count), i32x16::splat(31)); - simd_shr(a.as_i32x16(), count).as_m512i() - } +pub fn _mm512_srav_epi32(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsravd(a.as_i32x16(), count.as_i32x16())) } } /// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -21765,14 +21759,8 @@ pub const fn _mm_maskz_srav_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __ #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_srav_epi64(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u64x8(); - let no_overflow: u64x8 = simd_lt(count, u64x8::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, transmute(count), i64x8::splat(63)); - simd_shr(a.as_i64x8(), count).as_m512i() - } +pub fn _mm512_srav_epi64(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsravq(a.as_i64x8(), count.as_i64x8())) } } /// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -21817,14 +21805,8 @@ pub const fn _mm512_maskz_srav_epi64(k: __mmask8, a: __m512i, count: __m512i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_srav_epi64(a: __m256i, count: __m256i) -> __m256i { - unsafe { - let count = count.as_u64x4(); - let no_overflow: u64x4 = simd_lt(count, u64x4::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, transmute(count), i64x4::splat(63)); - simd_shr(a.as_i64x4(), count).as_m256i() - } +pub fn _mm256_srav_epi64(a: __m256i, count: __m256i) -> __m256i { + unsafe { transmute(vpsravq256(a.as_i64x4(), count.as_i64x4())) } } /// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -21869,14 +21851,8 @@ pub const fn _mm256_maskz_srav_epi64(k: __mmask8, a: __m256i, count: __m256i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_srav_epi64(a: __m128i, count: __m128i) -> __m128i { - unsafe { - let count = count.as_u64x2(); - let no_overflow: u64x2 = simd_lt(count, u64x2::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, transmute(count), i64x2::splat(63)); - simd_shr(a.as_i64x2(), count).as_m128i() - } +pub fn _mm_srav_epi64(a: __m128i, count: __m128i) -> __m128i { + unsafe { transmute(vpsravq128(a.as_i64x2(), count.as_i64x2())) } } /// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in sign bits, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -22492,14 +22468,8 @@ pub const fn _mm_maskz_rorv_epi64(k: __mmask8, a: __m128i, b: __m128i) -> __m128 #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_sllv_epi32(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u32x16(); - let no_overflow: u32x16 = simd_lt(count, u32x16::splat(u32::BITS)); - let count = simd_select(no_overflow, count, u32x16::ZERO); - simd_select(no_overflow, simd_shl(a.as_u32x16(), count), u32x16::ZERO).as_m512i() - } +pub fn _mm512_sllv_epi32(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsllvd(a.as_i32x16(), count.as_i32x16())) } } /// Shift packed 32-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -22609,14 +22579,8 @@ pub const fn _mm_maskz_sllv_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __ #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_srlv_epi32(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u32x16(); - let no_overflow: u32x16 = simd_lt(count, u32x16::splat(u32::BITS)); - let count = simd_select(no_overflow, count, u32x16::ZERO); - simd_select(no_overflow, simd_shr(a.as_u32x16(), count), u32x16::ZERO).as_m512i() - } +pub fn _mm512_srlv_epi32(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsrlvd(a.as_i32x16(), count.as_i32x16())) } } /// Shift packed 32-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -22726,14 +22690,8 @@ pub const fn _mm_maskz_srlv_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __ #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_sllv_epi64(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u64x8(); - let no_overflow: u64x8 = simd_lt(count, u64x8::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, count, u64x8::ZERO); - simd_select(no_overflow, simd_shl(a.as_u64x8(), count), u64x8::ZERO).as_m512i() - } +pub fn _mm512_sllv_epi64(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsllvq(a.as_i64x8(), count.as_i64x8())) } } /// Shift packed 64-bit integers in a left by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -22843,14 +22801,8 @@ pub const fn _mm_maskz_sllv_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __ #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_srlv_epi64(a: __m512i, count: __m512i) -> __m512i { - unsafe { - let count = count.as_u64x8(); - let no_overflow: u64x8 = simd_lt(count, u64x8::splat(u64::BITS as u64)); - let count = simd_select(no_overflow, count, u64x8::ZERO); - simd_select(no_overflow, simd_shr(a.as_u64x8(), count), u64x8::ZERO).as_m512i() - } +pub fn _mm512_srlv_epi64(a: __m512i, count: __m512i) -> __m512i { + unsafe { transmute(vpsrlvq(a.as_i64x8(), count.as_i64x8())) } } /// Shift packed 64-bit integers in a right by the amount specified by the corresponding element in count while shifting in zeros, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -44884,6 +44836,15 @@ unsafe extern "llvm-intrinsic" { #[link_name = "llvm.x86.avx512.mask.cmp.pd.128"] fn vcmppd128(a: f64x2, b: f64x2, op: i32, m: i8) -> i8; + #[link_name = "llvm.x86.avx512.psllv.d.512"] + fn vpsllvd(a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.psrlv.d.512"] + fn vpsrlvd(a: i32x16, b: i32x16) -> i32x16; + #[link_name = "llvm.x86.avx512.psllv.q.512"] + fn vpsllvq(a: i64x8, b: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.psrlv.q.512"] + fn vpsrlvq(a: i64x8, b: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.psll.d.512"] fn vpslld(a: i32x16, count: i32x4) -> i32x16; #[link_name = "llvm.x86.avx512.psrl.d.512"] @@ -44903,6 +44864,16 @@ unsafe extern "llvm-intrinsic" { #[link_name = "llvm.x86.avx512.psra.q.128"] fn vpsraq128(a: i64x2, count: i64x2) -> i64x2; + #[link_name = "llvm.x86.avx512.psrav.d.512"] + fn vpsravd(a: i32x16, count: i32x16) -> i32x16; + + #[link_name = "llvm.x86.avx512.psrav.q.512"] + fn vpsravq(a: i64x8, count: i64x8) -> i64x8; + #[link_name = "llvm.x86.avx512.psrav.q.256"] + fn vpsravq256(a: i64x4, count: i64x4) -> i64x4; + #[link_name = "llvm.x86.avx512.psrav.q.128"] + fn vpsravq128(a: i64x2, count: i64x2) -> i64x2; + #[link_name = "llvm.x86.avx512.vpermilvar.ps.512"] fn vpermilps(a: f32x16, b: i32x16) -> f32x16; #[link_name = "llvm.x86.avx512.vpermilvar.pd.512"] From 39333b94ddfb4457bc8a6dba64a9f542830e7ae0 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 5 Sep 2026 16:37:30 +0200 Subject: [PATCH 15/55] de-constify methods that depended on the vector shift ones --- .../stdarch/crates/core_arch/src/x86/avx2.rs | 20 +-- .../crates/core_arch/src/x86/avx512bw.rs | 108 ++++++------- .../crates/core_arch/src/x86/avx512f.rs | 150 +++++++----------- .../crates/core_arch/src/x86_64/avx512f.rs | 46 +++--- 4 files changed, 135 insertions(+), 189 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/x86/avx2.rs b/library/stdarch/crates/core_arch/src/x86/avx2.rs index e3b6054568f11..333f38e5474cc 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx2.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx2.rs @@ -5123,7 +5123,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm_sllv_epi32() { + fn test_mm_sllv_epi32() { let a = _mm_set1_epi32(2); let b = _mm_set1_epi32(1); let r = _mm_sllv_epi32(a, b); @@ -5132,7 +5132,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm256_sllv_epi32() { + fn test_mm256_sllv_epi32() { let a = _mm256_set1_epi32(2); let b = _mm256_set1_epi32(1); let r = _mm256_sllv_epi32(a, b); @@ -5141,7 +5141,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm_sllv_epi64() { + fn test_mm_sllv_epi64() { let a = _mm_set1_epi64x(2); let b = _mm_set1_epi64x(1); let r = _mm_sllv_epi64(a, b); @@ -5150,7 +5150,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm256_sllv_epi64() { + fn test_mm256_sllv_epi64() { let a = _mm256_set1_epi64x(2); let b = _mm256_set1_epi64x(1); let r = _mm256_sllv_epi64(a, b); @@ -5191,7 +5191,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm_srav_epi32() { + fn test_mm_srav_epi32() { let a = _mm_set1_epi32(4); let count = _mm_set1_epi32(1); let r = _mm_srav_epi32(a, count); @@ -5200,7 +5200,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm256_srav_epi32() { + fn test_mm256_srav_epi32() { let a = _mm256_set1_epi32(4); let count = _mm256_set1_epi32(1); let r = _mm256_srav_epi32(a, count); @@ -5277,7 +5277,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm_srlv_epi32() { + fn test_mm_srlv_epi32() { let a = _mm_set1_epi32(2); let count = _mm_set1_epi32(1); let r = _mm_srlv_epi32(a, count); @@ -5286,7 +5286,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm256_srlv_epi32() { + fn test_mm256_srlv_epi32() { let a = _mm256_set1_epi32(2); let count = _mm256_set1_epi32(1); let r = _mm256_srlv_epi32(a, count); @@ -5295,7 +5295,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm_srlv_epi64() { + fn test_mm_srlv_epi64() { let a = _mm_set1_epi64x(2); let count = _mm_set1_epi64x(1); let r = _mm_srlv_epi64(a, count); @@ -5304,7 +5304,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm256_srlv_epi64() { + fn test_mm256_srlv_epi64() { let a = _mm256_set1_epi64x(2); let count = _mm256_set1_epi64x(1); let r = _mm256_srlv_epi64(a, count); diff --git a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs index 054ca9816379f..8e642432151ea 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs @@ -7381,8 +7381,7 @@ pub fn _mm512_sllv_epi16(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_sllv_epi16( +pub fn _mm512_mask_sllv_epi16( src: __m512i, k: __mmask32, a: __m512i, @@ -7401,8 +7400,7 @@ pub const fn _mm512_mask_sllv_epi16( #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_sllv_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_sllv_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_sllv_epi16(a, count).as_i16x32(); transmute(simd_select_bitmask(k, shf, i16x32::ZERO)) @@ -7427,8 +7425,7 @@ pub fn _mm256_sllv_epi16(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_sllv_epi16( +pub fn _mm256_mask_sllv_epi16( src: __m256i, k: __mmask16, a: __m256i, @@ -7447,8 +7444,7 @@ pub const fn _mm256_mask_sllv_epi16( #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_sllv_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_sllv_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_sllv_epi16(a, count).as_i16x16(); transmute(simd_select_bitmask(k, shf, i16x16::ZERO)) @@ -7473,8 +7469,7 @@ pub fn _mm_sllv_epi16(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_sllv_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_sllv_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_sllv_epi16(a, count).as_i16x8(); transmute(simd_select_bitmask(k, shf, src.as_i16x8())) @@ -7488,8 +7483,7 @@ pub const fn _mm_mask_sllv_epi16(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_sllv_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_sllv_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_sllv_epi16(a, count).as_i16x8(); transmute(simd_select_bitmask(k, shf, i16x8::ZERO)) @@ -7752,8 +7746,7 @@ pub fn _mm512_srlv_epi16(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_srlv_epi16( +pub fn _mm512_mask_srlv_epi16( src: __m512i, k: __mmask32, a: __m512i, @@ -7772,8 +7765,7 @@ pub const fn _mm512_mask_srlv_epi16( #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_srlv_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_srlv_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srlv_epi16(a, count).as_i16x32(); transmute(simd_select_bitmask(k, shf, i16x32::ZERO)) @@ -7798,8 +7790,7 @@ pub fn _mm256_srlv_epi16(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_srlv_epi16( +pub fn _mm256_mask_srlv_epi16( src: __m256i, k: __mmask16, a: __m256i, @@ -7818,8 +7809,7 @@ pub const fn _mm256_mask_srlv_epi16( #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_srlv_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_srlv_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srlv_epi16(a, count).as_i16x16(); transmute(simd_select_bitmask(k, shf, i16x16::ZERO)) @@ -7844,8 +7834,7 @@ pub fn _mm_srlv_epi16(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_srlv_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_srlv_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srlv_epi16(a, count).as_i16x8(); transmute(simd_select_bitmask(k, shf, src.as_i16x8())) @@ -7859,8 +7848,7 @@ pub const fn _mm_mask_srlv_epi16(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_srlv_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_srlv_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srlv_epi16(a, count).as_i16x8(); transmute(simd_select_bitmask(k, shf, i16x8::ZERO)) @@ -8110,8 +8098,7 @@ pub fn _mm512_srav_epi16(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_srav_epi16( +pub fn _mm512_mask_srav_epi16( src: __m512i, k: __mmask32, a: __m512i, @@ -8130,8 +8117,7 @@ pub const fn _mm512_mask_srav_epi16( #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_srav_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_srav_epi16(k: __mmask32, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srav_epi16(a, count).as_i16x32(); transmute(simd_select_bitmask(k, shf, i16x32::ZERO)) @@ -8156,8 +8142,7 @@ pub fn _mm256_srav_epi16(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_srav_epi16( +pub fn _mm256_mask_srav_epi16( src: __m256i, k: __mmask16, a: __m256i, @@ -8176,8 +8161,7 @@ pub const fn _mm256_mask_srav_epi16( #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_srav_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_srav_epi16(k: __mmask16, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srav_epi16(a, count).as_i16x16(); transmute(simd_select_bitmask(k, shf, i16x16::ZERO)) @@ -8202,8 +8186,7 @@ pub fn _mm_srav_epi16(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_srav_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_srav_epi16(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srav_epi16(a, count).as_i16x8(); transmute(simd_select_bitmask(k, shf, src.as_i16x8())) @@ -8217,8 +8200,7 @@ pub const fn _mm_mask_srav_epi16(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_srav_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_srav_epi16(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srav_epi16(a, count).as_i16x8(); transmute(simd_select_bitmask(k, shf, i16x8::ZERO)) @@ -18353,7 +18335,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_sllv_epi16() { + fn test_mm512_sllv_epi16() { let a = _mm512_set1_epi16(1 << 15); let count = _mm512_set1_epi16(2); let r = _mm512_sllv_epi16(a, count); @@ -18362,7 +18344,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mask_sllv_epi16() { + fn test_mm512_mask_sllv_epi16() { let a = _mm512_set1_epi16(1 << 15); let count = _mm512_set1_epi16(2); let r = _mm512_mask_sllv_epi16(a, 0, a, count); @@ -18373,7 +18355,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_maskz_sllv_epi16() { + fn test_mm512_maskz_sllv_epi16() { let a = _mm512_set1_epi16(1 << 15); let count = _mm512_set1_epi16(2); let r = _mm512_maskz_sllv_epi16(0, a, count); @@ -18384,7 +18366,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_sllv_epi16() { + fn test_mm256_sllv_epi16() { let a = _mm256_set1_epi16(1 << 15); let count = _mm256_set1_epi16(2); let r = _mm256_sllv_epi16(a, count); @@ -18393,7 +18375,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_mask_sllv_epi16() { + fn test_mm256_mask_sllv_epi16() { let a = _mm256_set1_epi16(1 << 15); let count = _mm256_set1_epi16(2); let r = _mm256_mask_sllv_epi16(a, 0, a, count); @@ -18404,7 +18386,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_maskz_sllv_epi16() { + fn test_mm256_maskz_sllv_epi16() { let a = _mm256_set1_epi16(1 << 15); let count = _mm256_set1_epi16(2); let r = _mm256_maskz_sllv_epi16(0, a, count); @@ -18415,7 +18397,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_sllv_epi16() { + fn test_mm_sllv_epi16() { let a = _mm_set1_epi16(1 << 15); let count = _mm_set1_epi16(2); let r = _mm_sllv_epi16(a, count); @@ -18424,7 +18406,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_mask_sllv_epi16() { + fn test_mm_mask_sllv_epi16() { let a = _mm_set1_epi16(1 << 15); let count = _mm_set1_epi16(2); let r = _mm_mask_sllv_epi16(a, 0, a, count); @@ -18435,7 +18417,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_maskz_sllv_epi16() { + fn test_mm_maskz_sllv_epi16() { let a = _mm_set1_epi16(1 << 15); let count = _mm_set1_epi16(2); let r = _mm_maskz_sllv_epi16(0, a, count); @@ -18589,7 +18571,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_srlv_epi16() { + fn test_mm512_srlv_epi16() { let a = _mm512_set1_epi16(1 << 1); let count = _mm512_set1_epi16(2); let r = _mm512_srlv_epi16(a, count); @@ -18598,7 +18580,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mask_srlv_epi16() { + fn test_mm512_mask_srlv_epi16() { let a = _mm512_set1_epi16(1 << 1); let count = _mm512_set1_epi16(2); let r = _mm512_mask_srlv_epi16(a, 0, a, count); @@ -18609,7 +18591,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_maskz_srlv_epi16() { + fn test_mm512_maskz_srlv_epi16() { let a = _mm512_set1_epi16(1 << 1); let count = _mm512_set1_epi16(2); let r = _mm512_maskz_srlv_epi16(0, a, count); @@ -18620,7 +18602,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_srlv_epi16() { + fn test_mm256_srlv_epi16() { let a = _mm256_set1_epi16(1 << 1); let count = _mm256_set1_epi16(2); let r = _mm256_srlv_epi16(a, count); @@ -18629,7 +18611,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_mask_srlv_epi16() { + fn test_mm256_mask_srlv_epi16() { let a = _mm256_set1_epi16(1 << 1); let count = _mm256_set1_epi16(2); let r = _mm256_mask_srlv_epi16(a, 0, a, count); @@ -18640,7 +18622,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_maskz_srlv_epi16() { + fn test_mm256_maskz_srlv_epi16() { let a = _mm256_set1_epi16(1 << 1); let count = _mm256_set1_epi16(2); let r = _mm256_maskz_srlv_epi16(0, a, count); @@ -18651,7 +18633,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_srlv_epi16() { + fn test_mm_srlv_epi16() { let a = _mm_set1_epi16(1 << 1); let count = _mm_set1_epi16(2); let r = _mm_srlv_epi16(a, count); @@ -18660,7 +18642,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_mask_srlv_epi16() { + fn test_mm_mask_srlv_epi16() { let a = _mm_set1_epi16(1 << 1); let count = _mm_set1_epi16(2); let r = _mm_mask_srlv_epi16(a, 0, a, count); @@ -18671,7 +18653,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_maskz_srlv_epi16() { + fn test_mm_maskz_srlv_epi16() { let a = _mm_set1_epi16(1 << 1); let count = _mm_set1_epi16(2); let r = _mm_maskz_srlv_epi16(0, a, count); @@ -18825,7 +18807,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_srav_epi16() { + fn test_mm512_srav_epi16() { let a = _mm512_set1_epi16(8); let count = _mm512_set1_epi16(2); let r = _mm512_srav_epi16(a, count); @@ -18834,7 +18816,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mask_srav_epi16() { + fn test_mm512_mask_srav_epi16() { let a = _mm512_set1_epi16(8); let count = _mm512_set1_epi16(2); let r = _mm512_mask_srav_epi16(a, 0, a, count); @@ -18845,7 +18827,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_maskz_srav_epi16() { + fn test_mm512_maskz_srav_epi16() { let a = _mm512_set1_epi16(8); let count = _mm512_set1_epi16(2); let r = _mm512_maskz_srav_epi16(0, a, count); @@ -18856,7 +18838,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_srav_epi16() { + fn test_mm256_srav_epi16() { let a = _mm256_set1_epi16(8); let count = _mm256_set1_epi16(2); let r = _mm256_srav_epi16(a, count); @@ -18865,7 +18847,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_mask_srav_epi16() { + fn test_mm256_mask_srav_epi16() { let a = _mm256_set1_epi16(8); let count = _mm256_set1_epi16(2); let r = _mm256_mask_srav_epi16(a, 0, a, count); @@ -18876,7 +18858,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_maskz_srav_epi16() { + fn test_mm256_maskz_srav_epi16() { let a = _mm256_set1_epi16(8); let count = _mm256_set1_epi16(2); let r = _mm256_maskz_srav_epi16(0, a, count); @@ -18887,7 +18869,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_srav_epi16() { + fn test_mm_srav_epi16() { let a = _mm_set1_epi16(8); let count = _mm_set1_epi16(2); let r = _mm_srav_epi16(a, count); @@ -18896,7 +18878,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_mask_srav_epi16() { + fn test_mm_mask_srav_epi16() { let a = _mm_set1_epi16(8); let count = _mm_set1_epi16(2); let r = _mm_mask_srav_epi16(a, 0, a, count); @@ -18907,7 +18889,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_maskz_srav_epi16() { + fn test_mm_maskz_srav_epi16() { let a = _mm_set1_epi16(8); let count = _mm_set1_epi16(2); let r = _mm_maskz_srav_epi16(0, a, count); diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index 2b833e247b1bc..27c80312d6ac6 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -21659,8 +21659,7 @@ pub fn _mm512_srav_epi32(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_srav_epi32( +pub fn _mm512_mask_srav_epi32( src: __m512i, k: __mmask16, a: __m512i, @@ -21679,8 +21678,7 @@ pub const fn _mm512_mask_srav_epi32( #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_srav_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_srav_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srav_epi32(a, count).as_i32x16(); transmute(simd_select_bitmask(k, shf, i32x16::ZERO)) @@ -21694,8 +21692,7 @@ pub const fn _mm512_maskz_srav_epi32(k: __mmask16, a: __m512i, count: __m512i) - #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_srav_epi32( +pub fn _mm256_mask_srav_epi32( src: __m256i, k: __mmask8, a: __m256i, @@ -21714,8 +21711,7 @@ pub const fn _mm256_mask_srav_epi32( #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_srav_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_srav_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srav_epi32(a, count).as_i32x8(); transmute(simd_select_bitmask(k, shf, i32x8::ZERO)) @@ -21729,8 +21725,7 @@ pub const fn _mm256_maskz_srav_epi32(k: __mmask8, a: __m256i, count: __m256i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_srav_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_srav_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srav_epi32(a, count).as_i32x4(); transmute(simd_select_bitmask(k, shf, src.as_i32x4())) @@ -21744,8 +21739,7 @@ pub const fn _mm_mask_srav_epi32(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_srav_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_srav_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srav_epi32(a, count).as_i32x4(); transmute(simd_select_bitmask(k, shf, i32x4::ZERO)) @@ -21770,8 +21764,7 @@ pub fn _mm512_srav_epi64(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_srav_epi64( +pub fn _mm512_mask_srav_epi64( src: __m512i, k: __mmask8, a: __m512i, @@ -21790,8 +21783,7 @@ pub const fn _mm512_mask_srav_epi64( #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_srav_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_srav_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srav_epi64(a, count).as_i64x8(); transmute(simd_select_bitmask(k, shf, i64x8::ZERO)) @@ -21816,8 +21808,7 @@ pub fn _mm256_srav_epi64(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_srav_epi64( +pub fn _mm256_mask_srav_epi64( src: __m256i, k: __mmask8, a: __m256i, @@ -21836,8 +21827,7 @@ pub const fn _mm256_mask_srav_epi64( #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_srav_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_srav_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srav_epi64(a, count).as_i64x4(); transmute(simd_select_bitmask(k, shf, i64x4::ZERO)) @@ -21862,8 +21852,7 @@ pub fn _mm_srav_epi64(a: __m128i, count: __m128i) -> __m128i { #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_srav_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_srav_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srav_epi64(a, count).as_i64x2(); transmute(simd_select_bitmask(k, shf, src.as_i64x2())) @@ -21877,8 +21866,7 @@ pub const fn _mm_mask_srav_epi64(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_srav_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_srav_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srav_epi64(a, count).as_i64x2(); transmute(simd_select_bitmask(k, shf, i64x2::ZERO)) @@ -22479,8 +22467,7 @@ pub fn _mm512_sllv_epi32(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_sllv_epi32( +pub fn _mm512_mask_sllv_epi32( src: __m512i, k: __mmask16, a: __m512i, @@ -22499,8 +22486,7 @@ pub const fn _mm512_mask_sllv_epi32( #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_sllv_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_sllv_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_sllv_epi32(a, count).as_i32x16(); transmute(simd_select_bitmask(k, shf, i32x16::ZERO)) @@ -22514,8 +22500,7 @@ pub const fn _mm512_maskz_sllv_epi32(k: __mmask16, a: __m512i, count: __m512i) - #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_sllv_epi32( +pub fn _mm256_mask_sllv_epi32( src: __m256i, k: __mmask8, a: __m256i, @@ -22534,8 +22519,7 @@ pub const fn _mm256_mask_sllv_epi32( #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_sllv_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_sllv_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_sllv_epi32(a, count).as_i32x8(); transmute(simd_select_bitmask(k, shf, i32x8::ZERO)) @@ -22549,8 +22533,7 @@ pub const fn _mm256_maskz_sllv_epi32(k: __mmask8, a: __m256i, count: __m256i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_sllv_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_sllv_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_sllv_epi32(a, count).as_i32x4(); transmute(simd_select_bitmask(k, shf, src.as_i32x4())) @@ -22564,8 +22547,7 @@ pub const fn _mm_mask_sllv_epi32(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_sllv_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_sllv_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_sllv_epi32(a, count).as_i32x4(); transmute(simd_select_bitmask(k, shf, i32x4::ZERO)) @@ -22590,8 +22572,7 @@ pub fn _mm512_srlv_epi32(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_srlv_epi32( +pub fn _mm512_mask_srlv_epi32( src: __m512i, k: __mmask16, a: __m512i, @@ -22610,8 +22591,7 @@ pub const fn _mm512_mask_srlv_epi32( #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_srlv_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_srlv_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srlv_epi32(a, count).as_i32x16(); transmute(simd_select_bitmask(k, shf, i32x16::ZERO)) @@ -22625,8 +22605,7 @@ pub const fn _mm512_maskz_srlv_epi32(k: __mmask16, a: __m512i, count: __m512i) - #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_srlv_epi32( +pub fn _mm256_mask_srlv_epi32( src: __m256i, k: __mmask8, a: __m256i, @@ -22645,8 +22624,7 @@ pub const fn _mm256_mask_srlv_epi32( #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_srlv_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_srlv_epi32(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srlv_epi32(a, count).as_i32x8(); transmute(simd_select_bitmask(k, shf, i32x8::ZERO)) @@ -22660,8 +22638,7 @@ pub const fn _mm256_maskz_srlv_epi32(k: __mmask8, a: __m256i, count: __m256i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_srlv_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_srlv_epi32(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srlv_epi32(a, count).as_i32x4(); transmute(simd_select_bitmask(k, shf, src.as_i32x4())) @@ -22675,8 +22652,7 @@ pub const fn _mm_mask_srlv_epi32(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_srlv_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_srlv_epi32(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srlv_epi32(a, count).as_i32x4(); transmute(simd_select_bitmask(k, shf, i32x4::ZERO)) @@ -22701,8 +22677,7 @@ pub fn _mm512_sllv_epi64(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_sllv_epi64( +pub fn _mm512_mask_sllv_epi64( src: __m512i, k: __mmask8, a: __m512i, @@ -22721,8 +22696,7 @@ pub const fn _mm512_mask_sllv_epi64( #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_sllv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_sllv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_sllv_epi64(a, count).as_i64x8(); transmute(simd_select_bitmask(k, shf, i64x8::ZERO)) @@ -22736,8 +22710,7 @@ pub const fn _mm512_maskz_sllv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_sllv_epi64( +pub fn _mm256_mask_sllv_epi64( src: __m256i, k: __mmask8, a: __m256i, @@ -22756,8 +22729,7 @@ pub const fn _mm256_mask_sllv_epi64( #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_sllv_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_sllv_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_sllv_epi64(a, count).as_i64x4(); transmute(simd_select_bitmask(k, shf, i64x4::ZERO)) @@ -22771,8 +22743,7 @@ pub const fn _mm256_maskz_sllv_epi64(k: __mmask8, a: __m256i, count: __m256i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_sllv_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_sllv_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_sllv_epi64(a, count).as_i64x2(); transmute(simd_select_bitmask(k, shf, src.as_i64x2())) @@ -22786,8 +22757,7 @@ pub const fn _mm_mask_sllv_epi64(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_sllv_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_sllv_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_sllv_epi64(a, count).as_i64x2(); transmute(simd_select_bitmask(k, shf, i64x2::ZERO)) @@ -22812,8 +22782,7 @@ pub fn _mm512_srlv_epi64(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_srlv_epi64( +pub fn _mm512_mask_srlv_epi64( src: __m512i, k: __mmask8, a: __m512i, @@ -22832,8 +22801,7 @@ pub const fn _mm512_mask_srlv_epi64( #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_srlv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { +pub fn _mm512_maskz_srlv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srlv_epi64(a, count).as_i64x8(); transmute(simd_select_bitmask(k, shf, i64x8::ZERO)) @@ -22847,8 +22815,7 @@ pub const fn _mm512_maskz_srlv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_srlv_epi64( +pub fn _mm256_mask_srlv_epi64( src: __m256i, k: __mmask8, a: __m256i, @@ -22867,8 +22834,7 @@ pub const fn _mm256_mask_srlv_epi64( #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_srlv_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { +pub fn _mm256_maskz_srlv_epi64(k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srlv_epi64(a, count).as_i64x4(); transmute(simd_select_bitmask(k, shf, i64x4::ZERO)) @@ -22882,8 +22848,7 @@ pub const fn _mm256_maskz_srlv_epi64(k: __mmask8, a: __m256i, count: __m256i) -> #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_srlv_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_mask_srlv_epi64(src: __m128i, k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srlv_epi64(a, count).as_i64x2(); transmute(simd_select_bitmask(k, shf, src.as_i64x2())) @@ -22897,8 +22862,7 @@ pub const fn _mm_mask_srlv_epi64(src: __m128i, k: __mmask8, a: __m128i, count: _ #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_srlv_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { +pub fn _mm_maskz_srlv_epi64(k: __mmask8, a: __m128i, count: __m128i) -> __m128i { unsafe { let shf = _mm_srlv_epi64(a, count).as_i64x2(); transmute(simd_select_bitmask(k, shf, i64x2::ZERO)) @@ -54639,7 +54603,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_sllv_epi32() { + fn test_mm512_sllv_epi32() { let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); let count = _mm512_set1_epi32(1); let r = _mm512_sllv_epi32(a, count); @@ -54648,7 +54612,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_mask_sllv_epi32() { + fn test_mm512_mask_sllv_epi32() { let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); let count = _mm512_set1_epi32(1); let r = _mm512_mask_sllv_epi32(a, 0, a, count); @@ -54659,7 +54623,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_maskz_sllv_epi32() { + fn test_mm512_maskz_sllv_epi32() { let a = _mm512_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 << 31); let count = _mm512_set_epi32(0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); let r = _mm512_maskz_sllv_epi32(0, a, count); @@ -54670,7 +54634,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_mask_sllv_epi32() { + fn test_mm256_mask_sllv_epi32() { let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); let count = _mm256_set1_epi32(1); let r = _mm256_mask_sllv_epi32(a, 0, a, count); @@ -54681,7 +54645,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_maskz_sllv_epi32() { + fn test_mm256_maskz_sllv_epi32() { let a = _mm256_set_epi32(1 << 31, 1, 1, 1, 1, 1, 1, 1); let count = _mm256_set1_epi32(1); let r = _mm256_maskz_sllv_epi32(0, a, count); @@ -54692,7 +54656,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_mask_sllv_epi32() { + fn test_mm_mask_sllv_epi32() { let a = _mm_set_epi32(1 << 31, 1, 1, 1); let count = _mm_set1_epi32(1); let r = _mm_mask_sllv_epi32(a, 0, a, count); @@ -54703,7 +54667,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_maskz_sllv_epi32() { + fn test_mm_maskz_sllv_epi32() { let a = _mm_set_epi32(1 << 31, 1, 1, 1); let count = _mm_set1_epi32(1); let r = _mm_maskz_sllv_epi32(0, a, count); @@ -54714,7 +54678,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_srlv_epi32() { + fn test_mm512_srlv_epi32() { let a = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); let count = _mm512_set1_epi32(1); let r = _mm512_srlv_epi32(a, count); @@ -54723,7 +54687,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_mask_srlv_epi32() { + fn test_mm512_mask_srlv_epi32() { let a = _mm512_set_epi32(0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2); let count = _mm512_set1_epi32(1); let r = _mm512_mask_srlv_epi32(a, 0, a, count); @@ -54734,7 +54698,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_maskz_srlv_epi32() { + fn test_mm512_maskz_srlv_epi32() { let a = _mm512_set_epi32(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0); let count = _mm512_set_epi32(0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1); let r = _mm512_maskz_srlv_epi32(0, a, count); @@ -54745,7 +54709,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_mask_srlv_epi32() { + fn test_mm256_mask_srlv_epi32() { let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); let count = _mm256_set1_epi32(1); let r = _mm256_mask_srlv_epi32(a, 0, a, count); @@ -54756,7 +54720,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_maskz_srlv_epi32() { + fn test_mm256_maskz_srlv_epi32() { let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); let count = _mm256_set1_epi32(1); let r = _mm256_maskz_srlv_epi32(0, a, count); @@ -54767,7 +54731,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_mask_srlv_epi32() { + fn test_mm_mask_srlv_epi32() { let a = _mm_set_epi32(1 << 5, 0, 0, 0); let count = _mm_set1_epi32(1); let r = _mm_mask_srlv_epi32(a, 0, a, count); @@ -54778,7 +54742,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_maskz_srlv_epi32() { + fn test_mm_maskz_srlv_epi32() { let a = _mm_set_epi32(1 << 5, 0, 0, 0); let count = _mm_set1_epi32(1); let r = _mm_maskz_srlv_epi32(0, a, count); @@ -55062,7 +55026,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_srav_epi32() { + fn test_mm512_srav_epi32() { let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); let count = _mm512_set_epi32(2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); let r = _mm512_srav_epi32(a, count); @@ -55071,7 +55035,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_mask_srav_epi32() { + fn test_mm512_mask_srav_epi32() { let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16); let count = _mm512_set_epi32(2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); let r = _mm512_mask_srav_epi32(a, 0, a, count); @@ -55082,7 +55046,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_maskz_srav_epi32() { + fn test_mm512_maskz_srav_epi32() { let a = _mm512_set_epi32(8, -8, 16, -15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -15, -14); let count = _mm512_set_epi32(2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2); let r = _mm512_maskz_srav_epi32(0, a, count); @@ -55093,7 +55057,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_mask_srav_epi32() { + fn test_mm256_mask_srav_epi32() { let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); let count = _mm256_set1_epi32(1); let r = _mm256_mask_srav_epi32(a, 0, a, count); @@ -55104,7 +55068,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_maskz_srav_epi32() { + fn test_mm256_maskz_srav_epi32() { let a = _mm256_set_epi32(1 << 5, 0, 0, 0, 0, 0, 0, 0); let count = _mm256_set1_epi32(1); let r = _mm256_maskz_srav_epi32(0, a, count); @@ -55115,7 +55079,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_mask_srav_epi32() { + fn test_mm_mask_srav_epi32() { let a = _mm_set_epi32(1 << 5, 0, 0, 0); let count = _mm_set1_epi32(1); let r = _mm_mask_srav_epi32(a, 0, a, count); @@ -55126,7 +55090,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_maskz_srav_epi32() { + fn test_mm_maskz_srav_epi32() { let a = _mm_set_epi32(1 << 5, 0, 0, 0); let count = _mm_set1_epi32(1); let r = _mm_maskz_srav_epi32(0, a, count); diff --git a/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs b/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs index 28e68d798b1c5..e45a79718209e 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs @@ -9035,7 +9035,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_sllv_epi64() { + fn test_mm512_sllv_epi64() { #[rustfmt::skip] let a = _mm512_set_epi64( 1 << 32, 1 << 63, 1 << 32, 1 << 32, @@ -9052,7 +9052,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_mask_sllv_epi64() { + fn test_mm512_mask_sllv_epi64() { #[rustfmt::skip] let a = _mm512_set_epi64( 1 << 32, 1 << 32, 1 << 63, 1 << 32, @@ -9071,7 +9071,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_maskz_sllv_epi64() { + fn test_mm512_maskz_sllv_epi64() { #[rustfmt::skip] let a = _mm512_set_epi64( 1 << 32, 1 << 32, 1 << 32, 1 << 32, @@ -9086,7 +9086,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_mask_sllv_epi64() { + fn test_mm256_mask_sllv_epi64() { let a = _mm256_set_epi64x(1 << 32, 1 << 32, 1 << 63, 1 << 32); let count = _mm256_set_epi64x(0, 1, 2, 3); let r = _mm256_mask_sllv_epi64(a, 0, a, count); @@ -9097,7 +9097,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_maskz_sllv_epi64() { + fn test_mm256_maskz_sllv_epi64() { let a = _mm256_set_epi64x(1 << 32, 1 << 32, 1 << 63, 1 << 32); let count = _mm256_set_epi64x(0, 1, 2, 3); let r = _mm256_maskz_sllv_epi64(0, a, count); @@ -9108,7 +9108,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_mask_sllv_epi64() { + fn test_mm_mask_sllv_epi64() { let a = _mm_set_epi64x(1 << 63, 1 << 32); let count = _mm_set_epi64x(2, 3); let r = _mm_mask_sllv_epi64(a, 0, a, count); @@ -9119,7 +9119,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_maskz_sllv_epi64() { + fn test_mm_maskz_sllv_epi64() { let a = _mm_set_epi64x(1 << 63, 1 << 32); let count = _mm_set_epi64x(2, 3); let r = _mm_maskz_sllv_epi64(0, a, count); @@ -9130,7 +9130,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_srlv_epi64() { + fn test_mm512_srlv_epi64() { #[rustfmt::skip] let a = _mm512_set_epi64( 1 << 32, 1 << 0, 1 << 32, 1 << 32, @@ -9147,7 +9147,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_mask_srlv_epi64() { + fn test_mm512_mask_srlv_epi64() { #[rustfmt::skip] let a = _mm512_set_epi64( 1 << 32, 1 << 0, 1 << 32, 1 << 32, @@ -9166,7 +9166,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_maskz_srlv_epi64() { + fn test_mm512_maskz_srlv_epi64() { #[rustfmt::skip] let a = _mm512_set_epi64( 1 << 32, 1 << 32, 1 << 32, 1 << 32, @@ -9181,7 +9181,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_mask_srlv_epi64() { + fn test_mm256_mask_srlv_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); let count = _mm256_set1_epi64x(1); let r = _mm256_mask_srlv_epi64(a, 0, a, count); @@ -9192,7 +9192,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_maskz_srlv_epi64() { + fn test_mm256_maskz_srlv_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); let count = _mm256_set1_epi64x(1); let r = _mm256_maskz_srlv_epi64(0, a, count); @@ -9203,7 +9203,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_mask_srlv_epi64() { + fn test_mm_mask_srlv_epi64() { let a = _mm_set_epi64x(1 << 5, 0); let count = _mm_set1_epi64x(1); let r = _mm_mask_srlv_epi64(a, 0, a, count); @@ -9214,7 +9214,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_maskz_srlv_epi64() { + fn test_mm_maskz_srlv_epi64() { let a = _mm_set_epi64x(1 << 5, 0); let count = _mm_set1_epi64x(1); let r = _mm_maskz_srlv_epi64(0, a, count); @@ -9511,7 +9511,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_srav_epi64() { + fn test_mm512_srav_epi64() { let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); let count = _mm512_set_epi64(2, 2, 0, 0, 0, 0, 2, 1); let r = _mm512_srav_epi64(a, count); @@ -9520,7 +9520,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_mask_srav_epi64() { + fn test_mm512_mask_srav_epi64() { let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); let count = _mm512_set_epi64(2, 2, 0, 0, 0, 0, 2, 1); let r = _mm512_mask_srav_epi64(a, 0, a, count); @@ -9531,7 +9531,7 @@ mod tests { } #[simd_test(enable = "avx512f")] - const fn test_mm512_maskz_srav_epi64() { + fn test_mm512_maskz_srav_epi64() { let a = _mm512_set_epi64(1, -8, 0, 0, 0, 0, 15, -16); let count = _mm512_set_epi64(2, 2, 0, 0, 0, 0, 2, 1); let r = _mm512_maskz_srav_epi64(0, a, count); @@ -9542,7 +9542,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_srav_epi64() { + fn test_mm256_srav_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); let count = _mm256_set1_epi64x(1); let r = _mm256_srav_epi64(a, count); @@ -9551,7 +9551,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_mask_srav_epi64() { + fn test_mm256_mask_srav_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); let count = _mm256_set1_epi64x(1); let r = _mm256_mask_srav_epi64(a, 0, a, count); @@ -9562,7 +9562,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm256_maskz_srav_epi64() { + fn test_mm256_maskz_srav_epi64() { let a = _mm256_set_epi64x(1 << 5, 0, 0, 0); let count = _mm256_set1_epi64x(1); let r = _mm256_maskz_srav_epi64(0, a, count); @@ -9573,7 +9573,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_srav_epi64() { + fn test_mm_srav_epi64() { let a = _mm_set_epi64x(1 << 5, 0); let count = _mm_set1_epi64x(1); let r = _mm_srav_epi64(a, count); @@ -9582,7 +9582,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_mask_srav_epi64() { + fn test_mm_mask_srav_epi64() { let a = _mm_set_epi64x(1 << 5, 0); let count = _mm_set1_epi64x(1); let r = _mm_mask_srav_epi64(a, 0, a, count); @@ -9593,7 +9593,7 @@ mod tests { } #[simd_test(enable = "avx512f,avx512vl")] - const fn test_mm_maskz_srav_epi64() { + fn test_mm_maskz_srav_epi64() { let a = _mm_set_epi64x(1 << 5, 0); let count = _mm_set1_epi64x(1); let r = _mm_maskz_srav_epi64(0, a, count); From aa62febfad931562420cdd90d7e790eeea7288ee Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 5 Sep 2026 16:48:34 +0200 Subject: [PATCH 16/55] fmt --- .../crates/core_arch/src/x86/avx512bw.rs | 42 ++-------- .../crates/core_arch/src/x86/avx512f.rs | 84 +++---------------- 2 files changed, 18 insertions(+), 108 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs index 8e642432151ea..e400453f627bc 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs @@ -7381,12 +7381,7 @@ pub fn _mm512_sllv_epi16(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -pub fn _mm512_mask_sllv_epi16( - src: __m512i, - k: __mmask32, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_sllv_epi16(src: __m512i, k: __mmask32, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_sllv_epi16(a, count).as_i16x32(); transmute(simd_select_bitmask(k, shf, src.as_i16x32())) @@ -7425,12 +7420,7 @@ pub fn _mm256_sllv_epi16(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvw))] -pub fn _mm256_mask_sllv_epi16( - src: __m256i, - k: __mmask16, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_sllv_epi16(src: __m256i, k: __mmask16, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_sllv_epi16(a, count).as_i16x16(); transmute(simd_select_bitmask(k, shf, src.as_i16x16())) @@ -7746,12 +7736,7 @@ pub fn _mm512_srlv_epi16(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -pub fn _mm512_mask_srlv_epi16( - src: __m512i, - k: __mmask32, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_srlv_epi16(src: __m512i, k: __mmask32, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srlv_epi16(a, count).as_i16x32(); transmute(simd_select_bitmask(k, shf, src.as_i16x32())) @@ -7790,12 +7775,7 @@ pub fn _mm256_srlv_epi16(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvw))] -pub fn _mm256_mask_srlv_epi16( - src: __m256i, - k: __mmask16, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_srlv_epi16(src: __m256i, k: __mmask16, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srlv_epi16(a, count).as_i16x16(); transmute(simd_select_bitmask(k, shf, src.as_i16x16())) @@ -8098,12 +8078,7 @@ pub fn _mm512_srav_epi16(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -pub fn _mm512_mask_srav_epi16( - src: __m512i, - k: __mmask32, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_srav_epi16(src: __m512i, k: __mmask32, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srav_epi16(a, count).as_i16x32(); transmute(simd_select_bitmask(k, shf, src.as_i16x32())) @@ -8142,12 +8117,7 @@ pub fn _mm256_srav_epi16(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravw))] -pub fn _mm256_mask_srav_epi16( - src: __m256i, - k: __mmask16, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_srav_epi16(src: __m256i, k: __mmask16, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srav_epi16(a, count).as_i16x16(); transmute(simd_select_bitmask(k, shf, src.as_i16x16())) diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index 27c80312d6ac6..5d9aecff64d84 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -21659,12 +21659,7 @@ pub fn _mm512_srav_epi32(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -pub fn _mm512_mask_srav_epi32( - src: __m512i, - k: __mmask16, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_srav_epi32(src: __m512i, k: __mmask16, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srav_epi32(a, count).as_i32x16(); transmute(simd_select_bitmask(k, shf, src.as_i32x16())) @@ -21692,12 +21687,7 @@ pub fn _mm512_maskz_srav_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m5 #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravd))] -pub fn _mm256_mask_srav_epi32( - src: __m256i, - k: __mmask8, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_srav_epi32(src: __m256i, k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srav_epi32(a, count).as_i32x8(); transmute(simd_select_bitmask(k, shf, src.as_i32x8())) @@ -21764,12 +21754,7 @@ pub fn _mm512_srav_epi64(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -pub fn _mm512_mask_srav_epi64( - src: __m512i, - k: __mmask8, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_srav_epi64(src: __m512i, k: __mmask8, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srav_epi64(a, count).as_i64x8(); transmute(simd_select_bitmask(k, shf, src.as_i64x8())) @@ -21808,12 +21793,7 @@ pub fn _mm256_srav_epi64(a: __m256i, count: __m256i) -> __m256i { #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsravq))] -pub fn _mm256_mask_srav_epi64( - src: __m256i, - k: __mmask8, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_srav_epi64(src: __m256i, k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srav_epi64(a, count).as_i64x4(); transmute(simd_select_bitmask(k, shf, src.as_i64x4())) @@ -22467,12 +22447,7 @@ pub fn _mm512_sllv_epi32(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -pub fn _mm512_mask_sllv_epi32( - src: __m512i, - k: __mmask16, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_sllv_epi32(src: __m512i, k: __mmask16, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_sllv_epi32(a, count).as_i32x16(); transmute(simd_select_bitmask(k, shf, src.as_i32x16())) @@ -22500,12 +22475,7 @@ pub fn _mm512_maskz_sllv_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m5 #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvd))] -pub fn _mm256_mask_sllv_epi32( - src: __m256i, - k: __mmask8, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_sllv_epi32(src: __m256i, k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_sllv_epi32(a, count).as_i32x8(); transmute(simd_select_bitmask(k, shf, src.as_i32x8())) @@ -22572,12 +22542,7 @@ pub fn _mm512_srlv_epi32(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -pub fn _mm512_mask_srlv_epi32( - src: __m512i, - k: __mmask16, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_srlv_epi32(src: __m512i, k: __mmask16, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srlv_epi32(a, count).as_i32x16(); transmute(simd_select_bitmask(k, shf, src.as_i32x16())) @@ -22605,12 +22570,7 @@ pub fn _mm512_maskz_srlv_epi32(k: __mmask16, a: __m512i, count: __m512i) -> __m5 #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvd))] -pub fn _mm256_mask_srlv_epi32( - src: __m256i, - k: __mmask8, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_srlv_epi32(src: __m256i, k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srlv_epi32(a, count).as_i32x8(); transmute(simd_select_bitmask(k, shf, src.as_i32x8())) @@ -22677,12 +22637,7 @@ pub fn _mm512_sllv_epi64(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -pub fn _mm512_mask_sllv_epi64( - src: __m512i, - k: __mmask8, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_sllv_epi64(src: __m512i, k: __mmask8, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_sllv_epi64(a, count).as_i64x8(); transmute(simd_select_bitmask(k, shf, src.as_i64x8())) @@ -22710,12 +22665,7 @@ pub fn _mm512_maskz_sllv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m51 #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsllvq))] -pub fn _mm256_mask_sllv_epi64( - src: __m256i, - k: __mmask8, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_sllv_epi64(src: __m256i, k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_sllv_epi64(a, count).as_i64x4(); transmute(simd_select_bitmask(k, shf, src.as_i64x4())) @@ -22782,12 +22732,7 @@ pub fn _mm512_srlv_epi64(a: __m512i, count: __m512i) -> __m512i { #[target_feature(enable = "avx512f")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -pub fn _mm512_mask_srlv_epi64( - src: __m512i, - k: __mmask8, - a: __m512i, - count: __m512i, -) -> __m512i { +pub fn _mm512_mask_srlv_epi64(src: __m512i, k: __mmask8, a: __m512i, count: __m512i) -> __m512i { unsafe { let shf = _mm512_srlv_epi64(a, count).as_i64x8(); transmute(simd_select_bitmask(k, shf, src.as_i64x8())) @@ -22815,12 +22760,7 @@ pub fn _mm512_maskz_srlv_epi64(k: __mmask8, a: __m512i, count: __m512i) -> __m51 #[target_feature(enable = "avx512f,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpsrlvq))] -pub fn _mm256_mask_srlv_epi64( - src: __m256i, - k: __mmask8, - a: __m256i, - count: __m256i, -) -> __m256i { +pub fn _mm256_mask_srlv_epi64(src: __m256i, k: __mmask8, a: __m256i, count: __m256i) -> __m256i { unsafe { let shf = _mm256_srlv_epi64(a, count).as_i64x4(); transmute(simd_select_bitmask(k, shf, src.as_i64x4())) From cfc12ac04fe5606aa90f31059ecbc809883fc371 Mon Sep 17 00:00:00 2001 From: Adam Gemmell Date: Wed, 9 Sep 2026 13:56:18 +0100 Subject: [PATCH 17/55] Fix "explicit `package.readme` can be inferred" --- library/stdarch/crates/core_arch/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/library/stdarch/crates/core_arch/Cargo.toml b/library/stdarch/crates/core_arch/Cargo.toml index 670447a2d5a8b..c7b8527f4d9fc 100644 --- a/library/stdarch/crates/core_arch/Cargo.toml +++ b/library/stdarch/crates/core_arch/Cargo.toml @@ -9,7 +9,6 @@ authors = [ description = "`core::arch` - Rust's core library architecture-specific intrinsics." homepage = "https://github.com/rust-lang/stdarch" repository = "https://github.com/rust-lang/stdarch" -readme = "README.md" keywords = ["core", "simd", "arch", "intrinsics"] categories = ["hardware-support", "no-std"] license = "MIT OR Apache-2.0" From 89e9fe135e343ed4117489a82f2a32f1da9606c3 Mon Sep 17 00:00:00 2001 From: Adam Gemmell Date: Wed, 9 Sep 2026 14:01:30 +0100 Subject: [PATCH 18/55] Fix "`package.homepage` is redundant with `package.repository`" --- library/stdarch/crates/core_arch/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/library/stdarch/crates/core_arch/Cargo.toml b/library/stdarch/crates/core_arch/Cargo.toml index c7b8527f4d9fc..ab8b9bcd9fcef 100644 --- a/library/stdarch/crates/core_arch/Cargo.toml +++ b/library/stdarch/crates/core_arch/Cargo.toml @@ -7,7 +7,6 @@ authors = [ "Gonzalo Brito Gadeschi ", ] description = "`core::arch` - Rust's core library architecture-specific intrinsics." -homepage = "https://github.com/rust-lang/stdarch" repository = "https://github.com/rust-lang/stdarch" keywords = ["core", "simd", "arch", "intrinsics"] categories = ["hardware-support", "no-std"] From dce8dd2fe293a941940ef618b5c5a85cb3b7a43d Mon Sep 17 00:00:00 2001 From: Adam Gemmell Date: Wed, 9 Sep 2026 14:19:10 +0100 Subject: [PATCH 19/55] Fix "unused dependency" --- library/stdarch/Cargo.lock | 46 ------------------- .../stdarch/crates/intrinsic-test/Cargo.toml | 2 - library/stdarch/examples/Cargo.toml | 4 +- 3 files changed, 3 insertions(+), 49 deletions(-) diff --git a/library/stdarch/Cargo.lock b/library/stdarch/Cargo.lock index 9923b630cd5bb..9ec3227f49e7d 100644 --- a/library/stdarch/Cargo.lock +++ b/library/stdarch/Cargo.lock @@ -217,12 +217,6 @@ dependencies = [ "syn", ] -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "either" version = "1.15.0" @@ -400,7 +394,6 @@ name = "intrinsic-test" version = "0.1.0" dependencies = [ "clap", - "diff", "itertools", "log", "pretty_env_logger", @@ -408,7 +401,6 @@ dependencies = [ "rayon", "regex", "serde", - "serde-xml-rs", "serde_json", ] @@ -726,18 +718,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-xml-rs" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2215ce3e6a77550b80a1c37251b7d294febaf42e36e21b7b411e0bf54d540d" -dependencies = [ - "log", - "serde", - "thiserror", - "xml", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -935,26 +915,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "thiserror" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "unicode-ident" version = "1.0.24" @@ -1169,12 +1129,6 @@ dependencies = [ "wasmparser 0.244.0", ] -[[package]] -name = "xml" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" - [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/library/stdarch/crates/intrinsic-test/Cargo.toml b/library/stdarch/crates/intrinsic-test/Cargo.toml index e5c9e44e6d32a..2e11591fe7153 100644 --- a/library/stdarch/crates/intrinsic-test/Cargo.toml +++ b/library/stdarch/crates/intrinsic-test/Cargo.toml @@ -17,8 +17,6 @@ clap = { version = "4.4", features = ["derive"] } log = "0.4.11" pretty_env_logger = "0.5.0" rayon = "1.5.0" -diff = "0.1.12" itertools = "0.15.0" quick-xml = { version = "0.37.5", features = ["serialize", "overlapped-lists"] } -serde-xml-rs = "0.8.0" regex = "1.11.1" diff --git a/library/stdarch/examples/Cargo.toml b/library/stdarch/examples/Cargo.toml index 8752f206526c7..677407cf25206 100644 --- a/library/stdarch/examples/Cargo.toml +++ b/library/stdarch/examples/Cargo.toml @@ -12,9 +12,11 @@ default-run = "hex" [dependencies] core_arch = { path = "../crates/core_arch" } -quickcheck = "1.0" rand = "0.9.3" +[dev-dependencies] +quickcheck = "1.0" + [[bin]] name = "hex" path = "hex.rs" From d3d4a643b95748ceeb9b3a3a212c2ce3800421cc Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Mon, 14 Sep 2026 01:00:39 +0200 Subject: [PATCH 20/55] only run `f16` tests when `target_has_reliable_f16` --- library/stdarch/crates/core_arch/Cargo.toml | 3 +++ library/stdarch/crates/core_arch/src/lib.rs | 10 +++++++++- library/stdarch/crates/core_arch/src/x86/avx512fp16.rs | 1 + .../stdarch/crates/core_arch/src/x86_64/avx512fp16.rs | 1 + 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/library/stdarch/crates/core_arch/Cargo.toml b/library/stdarch/crates/core_arch/Cargo.toml index 670447a2d5a8b..3aa2e8de1043a 100644 --- a/library/stdarch/crates/core_arch/Cargo.toml +++ b/library/stdarch/crates/core_arch/Cargo.toml @@ -26,6 +26,9 @@ stdarch-test = { version = "0.*", path = "../stdarch-test" } [target.'cfg(all(target_arch = "x86_64", target_os = "linux"))'.dev-dependencies] syscalls = { version = "0.6.18", default-features = false } +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(target_has_reliable_f16)'] } + [lints.clippy] too_long_first_doc_paragraph = "allow" missing_transmute_annotations = "allow" diff --git a/library/stdarch/crates/core_arch/src/lib.rs b/library/stdarch/crates/core_arch/src/lib.rs index 55163fffedd78..513ecc5e5489d 100644 --- a/library/stdarch/crates/core_arch/src/lib.rs +++ b/library/stdarch/crates/core_arch/src/lib.rs @@ -42,7 +42,15 @@ clflushopt_target_feature, min_adt_const_params )] -#![cfg_attr(test, feature(test, abi_vectorcall, stdarch_internal))] +#![cfg_attr( + test, + feature( + test, + abi_vectorcall, + stdarch_internal, + cfg_target_has_reliable_f16_f128 + ) +)] #![deny(clippy::missing_inline_in_public_items)] #![allow( clippy::identity_op, diff --git a/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs b/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs index 869f577a55905..d54c222eb7ccf 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs @@ -16837,6 +16837,7 @@ unsafe extern "llvm-intrinsic" { } #[cfg(test)] +#[cfg(target_has_reliable_f16)] mod tests { use crate::core_arch::assert_eq_const as assert_eq; use crate::core_arch::x86::*; diff --git a/library/stdarch/crates/core_arch/src/x86_64/avx512fp16.rs b/library/stdarch/crates/core_arch/src/x86_64/avx512fp16.rs index f8d21e3f9d7fd..5b26fc9f45426 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/avx512fp16.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/avx512fp16.rs @@ -227,6 +227,7 @@ unsafe extern "llvm-intrinsic" { } #[cfg(test)] +#[cfg(target_has_reliable_f16)] mod tests { use crate::core_arch::{x86::*, x86_64::*}; use stdarch_test::simd_test; From 5bf69a211c2d6d93fb7fa74c87ea39edd7f93238 Mon Sep 17 00:00:00 2001 From: ltdk Date: Sun, 13 Sep 2026 14:46:47 -0400 Subject: [PATCH 21/55] Enable triagebot relabelling --- library/stdarch/triagebot.toml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/library/stdarch/triagebot.toml b/library/stdarch/triagebot.toml index 5b178f0cdf456..149646aa65f01 100644 --- a/library/stdarch/triagebot.toml +++ b/library/stdarch/triagebot.toml @@ -1,3 +1,35 @@ +[relabel] +allow-unauthenticated = [ + "A-*", + "B-*", + "C-*", + "D-*", + "E-*", + "F-*", + "I-*", + "L-*", + "NLL-*", + "O-*", + "PG-*", + "S-*", + "T-*", + "WG-*", + "-Z*", + "beta-nominated", + "CI-spurious-*", + "const-hack", + "llvm-*", + "needs-fcp", + "relnotes", + "release-blog-post", + "requires-*", + "regression-*", + "rla-*", + "perf-*", + "AsyncAwait-OnDeck", + "needs-triage", + "has-merge-commits",] + [assign] [assign.owners] From 8b691497dd8e0702b2f0c60c37ee3210c2418a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 14 Sep 2026 11:04:03 +0200 Subject: [PATCH 22/55] Use clap to parse arguments in `stdarch-gen-hexagon` --- library/stdarch/Cargo.lock | 1 + .../crates/stdarch-gen-common/src/lib.rs | 3 ++- .../crates/stdarch-gen-hexagon/Cargo.toml | 1 + .../crates/stdarch-gen-hexagon/src/main.rs | 27 ++++++++++++------- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/library/stdarch/Cargo.lock b/library/stdarch/Cargo.lock index 9ec3227f49e7d..5a1f6148ecc71 100644 --- a/library/stdarch/Cargo.lock +++ b/library/stdarch/Cargo.lock @@ -826,6 +826,7 @@ dependencies = [ name = "stdarch-gen-hexagon" version = "0.1.0" dependencies = [ + "clap", "regex", "stdarch-gen-common", ] diff --git a/library/stdarch/crates/stdarch-gen-common/src/lib.rs b/library/stdarch/crates/stdarch-gen-common/src/lib.rs index 13d788594ca55..f825712fcef6e 100644 --- a/library/stdarch/crates/stdarch-gen-common/src/lib.rs +++ b/library/stdarch/crates/stdarch-gen-common/src/lib.rs @@ -13,7 +13,7 @@ use std::path::{Path, PathBuf}; pub const GENERATED_MARKER: &str = "// This code is automatically generated. DO NOT MODIFY."; /// Controls what `run_generator` does with the generator's output. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum Mode { /// Verify that the `committed` matches the generator's output for owned files. /// @@ -26,6 +26,7 @@ pub enum Mode { /// into `committed`. If the generator no longer produces an owned file, the /// committed copy is deleted. Files in `committed` that are not owned /// are left untouched. + #[default] Bless, } diff --git a/library/stdarch/crates/stdarch-gen-hexagon/Cargo.toml b/library/stdarch/crates/stdarch-gen-hexagon/Cargo.toml index c7dfce2c0fd7b..7d457868398ac 100644 --- a/library/stdarch/crates/stdarch-gen-hexagon/Cargo.toml +++ b/library/stdarch/crates/stdarch-gen-hexagon/Cargo.toml @@ -6,5 +6,6 @@ license = "MIT OR Apache-2.0" edition = "2021" [dependencies] +clap = { version = "4", features = ["derive", "env"] } regex = "1.10" stdarch-gen-common = { path = "../stdarch-gen-common" } diff --git a/library/stdarch/crates/stdarch-gen-hexagon/src/main.rs b/library/stdarch/crates/stdarch-gen-hexagon/src/main.rs index 88ef4fe11f332..98da03b4fcfb5 100644 --- a/library/stdarch/crates/stdarch-gen-hexagon/src/main.rs +++ b/library/stdarch/crates/stdarch-gen-hexagon/src/main.rs @@ -1,18 +1,27 @@ -//! Hexagon code generator. -//! -//! Single binary that produces every generated file under -//! `core_arch/src/hexagon/`: scalar.rs (scalar intrinsics) and -//! v64.rs / v128.rs (HVX intrinsics). -//! -//! Run in check or bless mode via `STDARCH_GEN_MODE`. - mod hvx; mod scalar; +use clap::Parser; use std::path::PathBuf; use stdarch_gen_common::{run_generator, Mode}; +/// Hexagon code generator. +/// +/// Produces every generated file under +/// `core_arch/src/hexagon/`: scalar.rs (scalar intrinsics) and +/// v64.rs / v128.rs (HVX intrinsics). +/// +/// Run in check or bless mode via `STDARCH_GEN_MODE`. +#[derive(clap::Parser)] +struct Args { + /// Generation mode. + #[arg(long, env = "STDARCH_GEN_MODE")] + mode: Option, +} + fn main() -> Result<(), String> { + let args = Args::parse(); + let crate_dir = std::env::var("CARGO_MANIFEST_DIR") .map(PathBuf::from) .unwrap_or_else(|_| std::env::current_dir().unwrap()); @@ -20,7 +29,7 @@ fn main() -> Result<(), String> { let hexagon_dir = crate_dir.join("../core_arch/src/hexagon"); // Either "check" to check the output versus the committed output, or "bless" // to update the output. - let mode = Mode::from_env(); + let mode = args.mode.unwrap_or_default(); run_generator(&hexagon_dir, mode, |out_dir| -> Result<(), String> { // Here scalar::generate writes scalar.rs . From d4056727a00dcdba290303cb33eb2ae7415db5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 14 Sep 2026 11:19:12 +0200 Subject: [PATCH 23/55] Automatically reformat all generated files with a unified rustfmt config --- .../crates/stdarch-gen-common/src/lib.rs | 71 ++++++++++++++++++- .../crates/stdarch-gen-hexagon/src/hvx.rs | 37 ++++------ .../crates/stdarch-gen-hexagon/src/main.rs | 9 ++- .../crates/stdarch-gen-hexagon/src/scalar.rs | 36 +++------- library/stdarch/rustfmt.toml | 2 + 5 files changed, 102 insertions(+), 53 deletions(-) diff --git a/library/stdarch/crates/stdarch-gen-common/src/lib.rs b/library/stdarch/crates/stdarch-gen-common/src/lib.rs index f825712fcef6e..5308296d8588a 100644 --- a/library/stdarch/crates/stdarch-gen-common/src/lib.rs +++ b/library/stdarch/crates/stdarch-gen-common/src/lib.rs @@ -6,6 +6,8 @@ use std::fs; use std::io; use std::io::Read; use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str::FromStr; /// First-line marker identifying an auto-generated file. Generators emit this /// as the first line of every file they produce; the harness uses it to @@ -48,6 +50,20 @@ impl Mode { } } +impl FromStr for Mode { + type Err = String; + + fn from_str(s: &str) -> std::result::Result { + match s { + "check" => Ok(Mode::Check), + "bless" => Ok(Mode::Bless), + other => Err(format!( + "unknown stdarch generation mode: {other:?}. Possible values are `check` or `bless`." + )), + } + } +} + #[derive(Debug)] pub enum Error { Io(io::Error), @@ -103,6 +119,18 @@ impl From for Error { pub type Result = std::result::Result; +pub struct GeneratorCtx { + rustfmt_path: PathBuf, +} + +impl GeneratorCtx { + pub fn new(rustfmt_path: Option) -> Self { + Self { + rustfmt_path: rustfmt_path.unwrap_or_else(|| PathBuf::from("rustfmt")), + } + } +} + /// Run a generator under the chosen `mode`, reconciling its output with `committed`. /// /// Arguments: @@ -121,7 +149,12 @@ pub type Result = std::result::Result; /// - [`Mode::Bless`]: runs the generator into a temp dir and copies owned /// files into `committed`, or removes `committed`'s copy if the generator no /// longer produces them. -pub fn run_generator(committed: &Path, mode: Mode, generate: F) -> Result<()> +pub fn run_generator( + ctx: &GeneratorCtx, + committed: &Path, + mode: Mode, + generate: F, +) -> Result<()> where F: FnOnce(&Path) -> std::result::Result<(), E>, E: Into>, @@ -132,6 +165,14 @@ where let owned = discover_owned(committed)?; let produced = discover_all(scratch.path())?; + // Format all generated Rust files + for file in &produced { + let fullpath = scratch.path().join(file); + if fullpath.extension().and_then(|s| s.to_str()) == Some("rs") { + reformat_file(ctx, &fullpath)?; + } + } + let mut names: Vec<&String> = owned.iter().chain(produced.iter()).collect(); names.sort(); names.dedup(); @@ -145,6 +186,34 @@ where Ok(()) } +fn reformat_file(ctx: &GeneratorCtx, path: &Path) -> std::io::Result<()> { + let file = std::fs::File::open(path)?; + let proc = Command::new(&ctx.rustfmt_path) + // Ensure that rustfmt config files in other directories won't interfere with the formatting + // This is important for usage within the rust-lang/rust repository + .arg("--config-path") + .arg( + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("rustfmt.toml"), + ) + .stdin(Stdio::from(file)) + .stdout(Stdio::piped()) + .spawn()?; + + let output = proc.wait_with_output()?; + if !output.status.success() { + panic!( + "Running {:?} on {path:?} failed with exit code {:?}", + ctx.rustfmt_path, output.status + ); + } + std::fs::write(path, output.stdout) +} + /// Returns the names of files in `dir` whose first line begins with /// [`GENERATED_MARKER`]. Files without the marker are skipped. fn discover_owned(dir: &Path) -> Result> { diff --git a/library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs b/library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs index 537c59bbce861..95c77ae4700a4 100644 --- a/library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs +++ b/library/stdarch/crates/stdarch-gen-hexagon/src/hvx.rs @@ -19,7 +19,7 @@ use regex::Regex; use std::collections::{HashMap, HashSet}; use std::fs::File; -use std::io::Write; +use std::io::{BufWriter, Write}; use std::path::Path; use stdarch_gen_common::GENERATED_MARKER; @@ -1606,30 +1606,16 @@ fn generate_module_file( intrinsics: &[IntrinsicInfo], output_path: &Path, mode: VectorMode, -) -> Result<(), String> { - let mut output = - File::create(output_path).map_err(|e| format!("Failed to create output: {}", e))?; - - writeln!(output, "{}", GENERATED_MARKER).map_err(|e| e.to_string())?; - writeln!(output, "{}", generate_module_doc(mode)).map_err(|e| e.to_string())?; - writeln!(output, "{}", generate_types(mode)).map_err(|e| e.to_string())?; - writeln!(output, "{}", generate_extern_block(intrinsics, mode)).map_err(|e| e.to_string())?; - writeln!(output, "{}", generate_functions(intrinsics)).map_err(|e| e.to_string())?; - - // Ensure file is flushed before running rustfmt - drop(output); - - // Run rustfmt on the generated file - let status = std::process::Command::new("rustfmt") - .arg(output_path) - .status() - .map_err(|e| format!("Failed to run rustfmt: {}", e))?; - - if !status.success() { - return Err("rustfmt failed".to_string()); - } +) -> std::io::Result<()> { + let mut output = BufWriter::new(File::create(output_path)?); - Ok(()) + writeln!(output, "{}", GENERATED_MARKER)?; + writeln!(output, "{}", generate_module_doc(mode))?; + writeln!(output, "{}", generate_types(mode))?; + writeln!(output, "{}", generate_extern_block(intrinsics, mode))?; + writeln!(output, "{}", generate_functions(intrinsics))?; + + output.flush() } /// Parse the HVX header in `crate_dir` and write `v64.rs` and `v128.rs` into `out_dir`. @@ -1642,7 +1628,8 @@ pub fn generate(crate_dir: &std::path::Path, out_dir: &std::path::Path) -> Resul .collect(); for (filename, vmode) in [("v64.rs", VectorMode::V64), ("v128.rs", VectorMode::V128)] { let path = out_dir.join(filename); - generate_module_file(&intrinsics, &path, vmode)?; + generate_module_file(&intrinsics, &path, vmode) + .map_err(|e| format!("Cannot generate {path:?}: {e:?}"))?; } Ok(()) } diff --git a/library/stdarch/crates/stdarch-gen-hexagon/src/main.rs b/library/stdarch/crates/stdarch-gen-hexagon/src/main.rs index 98da03b4fcfb5..1e23e993ef918 100644 --- a/library/stdarch/crates/stdarch-gen-hexagon/src/main.rs +++ b/library/stdarch/crates/stdarch-gen-hexagon/src/main.rs @@ -3,7 +3,7 @@ mod scalar; use clap::Parser; use std::path::PathBuf; -use stdarch_gen_common::{run_generator, Mode}; +use stdarch_gen_common::{run_generator, GeneratorCtx, Mode}; /// Hexagon code generator. /// @@ -17,6 +17,10 @@ struct Args { /// Generation mode. #[arg(long, env = "STDARCH_GEN_MODE")] mode: Option, + /// Path to a rustfmt binary that will be used to reformat the generated code. + /// If unset, it will just use "rustfmt" from the environment. + #[arg(long)] + rustfmt_path: Option, } fn main() -> Result<(), String> { @@ -30,8 +34,9 @@ fn main() -> Result<(), String> { // Either "check" to check the output versus the committed output, or "bless" // to update the output. let mode = args.mode.unwrap_or_default(); + let ctx = GeneratorCtx::new(args.rustfmt_path); - run_generator(&hexagon_dir, mode, |out_dir| -> Result<(), String> { + run_generator(&ctx, &hexagon_dir, mode, |out_dir| -> Result<(), String> { // Here scalar::generate writes scalar.rs . scalar::generate(&crate_dir, out_dir)?; // Here hvx::generate writes v64.rs and v128.rs . diff --git a/library/stdarch/crates/stdarch-gen-hexagon/src/scalar.rs b/library/stdarch/crates/stdarch-gen-hexagon/src/scalar.rs index 2a64798d097d1..8c7880b58a023 100644 --- a/library/stdarch/crates/stdarch-gen-hexagon/src/scalar.rs +++ b/library/stdarch/crates/stdarch-gen-hexagon/src/scalar.rs @@ -19,7 +19,7 @@ use regex::Regex; use std::collections::HashMap; use std::fs::File; -use std::io::Write; +use std::io::{BufWriter, Write}; use std::path::Path; use stdarch_gen_common::GENERATED_MARKER; @@ -627,30 +627,16 @@ fn generate_functions(intrinsics: &[ScalarIntrinsic]) -> String { } /// Generate the complete scalar.rs file -fn generate_scalar_file(intrinsics: &[ScalarIntrinsic], output_path: &Path) -> Result<(), String> { - let mut output = - File::create(output_path).map_err(|e| format!("Failed to create output: {}", e))?; - - writeln!(output, "{}", GENERATED_MARKER).map_err(|e| e.to_string())?; - writeln!(output, "{}", generate_module_doc()).map_err(|e| e.to_string())?; - writeln!(output, "").map_err(|e| e.to_string())?; - writeln!(output, "{}", generate_extern_block(intrinsics)).map_err(|e| e.to_string())?; - writeln!(output, "{}", generate_functions(intrinsics)).map_err(|e| e.to_string())?; - - // Flush before running rustfmt - drop(output); - - // Run rustfmt on the generated file - let status = std::process::Command::new("rustfmt") - .arg(output_path) - .status() - .map_err(|e| format!("Failed to run rustfmt: {}", e))?; - - if !status.success() { - return Err("rustfmt failed".to_string()); - } +fn generate_scalar_file(intrinsics: &[ScalarIntrinsic], output_path: &Path) -> std::io::Result<()> { + let mut output = BufWriter::new(File::create(output_path)?); - Ok(()) + writeln!(output, "{}", GENERATED_MARKER)?; + writeln!(output, "{}", generate_module_doc())?; + writeln!(output, "")?; + writeln!(output, "{}", generate_extern_block(intrinsics))?; + writeln!(output, "{}", generate_functions(intrinsics))?; + + output.flush() } /// Parse the scalar header in `crate_dir` and write `scalar.rs` into `out_dir`. @@ -659,6 +645,6 @@ pub fn generate(crate_dir: &std::path::Path, out_dir: &std::path::Path) -> Resul let intrinsics = parse_header(&header_content); std::fs::create_dir_all(out_dir).map_err(|e| e.to_string())?; let scalar_path = out_dir.join("scalar.rs"); - generate_scalar_file(&intrinsics, &scalar_path)?; + generate_scalar_file(&intrinsics, &scalar_path).map_err(|e| e.to_string())?; Ok(()) } diff --git a/library/stdarch/rustfmt.toml b/library/stdarch/rustfmt.toml index e69de29bb2d1d..3dd85bb2defbf 100644 --- a/library/stdarch/rustfmt.toml +++ b/library/stdarch/rustfmt.toml @@ -0,0 +1,2 @@ +# Ensure that we generate the same file contents on both Linux and Windows. +newline_style = "Unix" From d0f689751bccaea01f3aeab772ba97150a853cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 14 Sep 2026 11:40:35 +0200 Subject: [PATCH 24/55] Use rustfmt formatting in stdarch-gen-loongarch --- library/stdarch/.github/workflows/main.yml | 537 +++++++++--------- library/stdarch/Cargo.lock | 1 + .../src/loongarch64/lasx/generated.rs | 194 ++++++- .../src/loongarch64/lsx/generated.rs | 130 ++++- .../crates/stdarch-gen-loongarch/Cargo.toml | 1 + .../crates/stdarch-gen-loongarch/src/main.rs | 102 ++-- 6 files changed, 609 insertions(+), 356 deletions(-) diff --git a/library/stdarch/.github/workflows/main.yml b/library/stdarch/.github/workflows/main.yml index 5c2e2ae4a8e21..754f01d399117 100644 --- a/library/stdarch/.github/workflows/main.yml +++ b/library/stdarch/.github/workflows/main.yml @@ -8,208 +8,208 @@ jobs: name: Check Style runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - name: Install Rust - run: rustup update nightly --no-self-update && rustup default nightly - - run: ci/style.sh + - uses: actions/checkout@v6 + - name: Install Rust + run: rustup update nightly --no-self-update && rustup default nightly + - run: ci/style.sh docs: name: Build Documentation - needs: [style] + needs: [ style ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - name: Install Rust - run: rustup update nightly --no-self-update && rustup default nightly - - run: ci/dox.sh - env: - CI: 1 + - uses: actions/checkout@v6 + - name: Install Rust + run: rustup update nightly --no-self-update && rustup default nightly + - run: ci/dox.sh + env: + CI: 1 verify: name: Automatic intrinsic verification - needs: [style] + needs: [ style ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - name: Install Rust - run: rustup update nightly --no-self-update && rustup default nightly - - run: cargo test --manifest-path crates/stdarch-verify/Cargo.toml + - uses: actions/checkout@v6 + - name: Install Rust + run: rustup update nightly --no-self-update && rustup default nightly + - run: cargo test --manifest-path crates/stdarch-verify/Cargo.toml test: - needs: [style] + needs: [ style ] name: Test runs-on: ${{ matrix.target.os }} strategy: matrix: profile: - - dev - - release + - dev + - release target: - # Dockers that are run through docker on linux - - tuple: i686-unknown-linux-gnu - os: ubuntu-latest - - tuple: x86_64-unknown-linux-gnu - os: ubuntu-latest - - tuple: arm-unknown-linux-gnueabihf - os: ubuntu-latest - - tuple: armv7-unknown-linux-gnueabihf - os: ubuntu-latest - - tuple: aarch64-unknown-linux-gnu - os: ubuntu-latest - - tuple: aarch64_be-unknown-linux-gnu - os: ubuntu-latest - - tuple: riscv32gc-unknown-linux-gnu - os: ubuntu-latest - - tuple: riscv64gc-unknown-linux-gnu - os: ubuntu-latest - - tuple: powerpc-unknown-linux-gnu - os: ubuntu-latest - - tuple: powerpc64-unknown-linux-gnu - os: ubuntu-latest - - tuple: powerpc64le-unknown-linux-gnu - os: ubuntu-latest - # MIPS targets disabled since they are dropped to tier 3. - # See https://github.com/rust-lang/compiler-team/issues/648 - #- tuple: mips-unknown-linux-gnu - # os: ubuntu-latest - #- tuple: mips64-unknown-linux-gnuabi64 - # os: ubuntu-latest - #- tuple: mips64el-unknown-linux-gnuabi64 - # os: ubuntu-latest - #- tuple: mipsel-unknown-linux-musl - # os: ubuntu-latest - - tuple: s390x-unknown-linux-gnu - os: ubuntu-latest - - tuple: i586-unknown-linux-gnu - os: ubuntu-latest - - tuple: nvptx64-nvidia-cuda - os: ubuntu-latest - - tuple: amdgcn-amd-amdhsa - os: ubuntu-latest - - tuple: thumbv6m-none-eabi - os: ubuntu-latest - - tuple: thumbv7m-none-eabi - os: ubuntu-latest - - tuple: thumbv7em-none-eabi - os: ubuntu-latest - - tuple: thumbv7em-none-eabihf - os: ubuntu-latest - - tuple: loongarch64-unknown-linux-gnu - os: ubuntu-latest - # hexagon doesn't build at the moment due to a libc issue. - # - tuple: hexagon-unknown-linux-musl - # os: ubuntu-latest - - tuple: wasm32-wasip1 - os: ubuntu-latest - - # macOS targets - - tuple: x86_64-apple-darwin - os: macos-15-intel - - tuple: x86_64-apple-ios-macabi - os: macos-15-intel - - tuple: aarch64-apple-darwin - os: macos-15 - - tuple: aarch64-apple-ios-macabi - os: macos-15 - # FIXME: gh-actions build environment doesn't have linker support - # - tuple: i686-apple-darwin - # os: macos-13 - - # Windows targets - - tuple: x86_64-pc-windows-msvc - os: windows-2025 - - tuple: i686-pc-windows-msvc - os: windows-2025 - - tuple: aarch64-pc-windows-msvc - os: windows-11-arm - - tuple: arm64ec-pc-windows-msvc - os: windows-11-arm - - tuple: x86_64-pc-windows-gnu - os: windows-2025 - # - tuple: i686-pc-windows-gnu - # os: windows-latest - - # Add additional variables to the matrix variations generated above using `include`: - include: - # `TEST_EVERYTHING` setups - there should be at least 1 for each architecture - - target: - tuple: aarch64-unknown-linux-gnu + # Dockers that are run through docker on linux + - tuple: i686-unknown-linux-gnu + os: ubuntu-latest + - tuple: x86_64-unknown-linux-gnu + os: ubuntu-latest + - tuple: arm-unknown-linux-gnueabihf + os: ubuntu-latest + - tuple: armv7-unknown-linux-gnueabihf + os: ubuntu-latest + - tuple: aarch64-unknown-linux-gnu + os: ubuntu-latest + - tuple: aarch64_be-unknown-linux-gnu + os: ubuntu-latest + - tuple: riscv32gc-unknown-linux-gnu + os: ubuntu-latest + - tuple: riscv64gc-unknown-linux-gnu + os: ubuntu-latest + - tuple: powerpc-unknown-linux-gnu + os: ubuntu-latest + - tuple: powerpc64-unknown-linux-gnu + os: ubuntu-latest + - tuple: powerpc64le-unknown-linux-gnu os: ubuntu-latest - test_everything: true - - target: - tuple: aarch64_be-unknown-linux-gnu + # MIPS targets disabled since they are dropped to tier 3. + # See https://github.com/rust-lang/compiler-team/issues/648 + #- tuple: mips-unknown-linux-gnu + # os: ubuntu-latest + #- tuple: mips64-unknown-linux-gnuabi64 + # os: ubuntu-latest + #- tuple: mips64el-unknown-linux-gnuabi64 + # os: ubuntu-latest + #- tuple: mipsel-unknown-linux-musl + # os: ubuntu-latest + - tuple: s390x-unknown-linux-gnu os: ubuntu-latest - test_everything: true - build_std: true - - target: - tuple: armv7-unknown-linux-gnueabihf + - tuple: i586-unknown-linux-gnu os: ubuntu-latest - test_everything: true - - target: - tuple: loongarch64-unknown-linux-gnu + - tuple: nvptx64-nvidia-cuda os: ubuntu-latest - test_everything: true - - target: - tuple: powerpc-unknown-linux-gnu + - tuple: amdgcn-amd-amdhsa os: ubuntu-latest - disable_assert_instr: true - test_everything: true - - target: - tuple: powerpc64-unknown-linux-gnu + - tuple: thumbv6m-none-eabi os: ubuntu-latest - disable_assert_instr: true - test_everything: true - - target: - tuple: powerpc64le-unknown-linux-gnu + - tuple: thumbv7m-none-eabi os: ubuntu-latest - test_everything: true - - target: - tuple: riscv32gc-unknown-linux-gnu + - tuple: thumbv7em-none-eabi os: ubuntu-latest - test_everything: true - build_std: true - - target: - tuple: riscv64gc-unknown-linux-gnu + - tuple: thumbv7em-none-eabihf os: ubuntu-latest - test_everything: true - - target: - tuple: s390x-unknown-linux-gnu + - tuple: loongarch64-unknown-linux-gnu os: ubuntu-latest - test_everything: true - - target: - tuple: x86_64-unknown-linux-gnu + # hexagon doesn't build at the moment due to a libc issue. + # - tuple: hexagon-unknown-linux-musl + # os: ubuntu-latest + - tuple: wasm32-wasip1 os: ubuntu-latest - test_everything: true - # MIPS targets disabled since they are dropped to tier 3. - # See https://github.com/rust-lang/compiler-team/issues/648 - #- target: - # tuple: mips-unknown-linux-gnu - # os: ubuntu-latest - # norun: true - #- target: - # tuple: mips64-unknown-linux-gnuabi64 - # os: ubuntu-latest - # norun: true - #- target: - # tuple: mips64el-unknown-linux-gnuabi64 - # os: ubuntu-latest - # norun: true - #- target: - # tuple: mipsel-unknown-linux-musl - # os: ubuntu-latest - # norun: true - - target: - tuple: aarch64-apple-darwin + + # macOS targets + - tuple: x86_64-apple-darwin + os: macos-15-intel + - tuple: x86_64-apple-ios-macabi + os: macos-15-intel + - tuple: aarch64-apple-darwin os: macos-15 - norun: true # https://github.com/rust-lang/stdarch/issues/1206 - - target: - tuple: aarch64-apple-ios-macabi + - tuple: aarch64-apple-ios-macabi os: macos-15 - norun: true # https://github.com/rust-lang/stdarch/issues/1206 - - target: - tuple: amdgcn-amd-amdhsa - os: ubuntu-latest - norun: true + # FIXME: gh-actions build environment doesn't have linker support + # - tuple: i686-apple-darwin + # os: macos-13 + + # Windows targets + - tuple: x86_64-pc-windows-msvc + os: windows-2025 + - tuple: i686-pc-windows-msvc + os: windows-2025 + - tuple: aarch64-pc-windows-msvc + os: windows-11-arm + - tuple: arm64ec-pc-windows-msvc + os: windows-11-arm + - tuple: x86_64-pc-windows-gnu + os: windows-2025 + # - tuple: i686-pc-windows-gnu + # os: windows-latest + + # Add additional variables to the matrix variations generated above using `include`: + include: + # `TEST_EVERYTHING` setups - there should be at least 1 for each architecture + - target: + tuple: aarch64-unknown-linux-gnu + os: ubuntu-latest + test_everything: true + - target: + tuple: aarch64_be-unknown-linux-gnu + os: ubuntu-latest + test_everything: true + build_std: true + - target: + tuple: armv7-unknown-linux-gnueabihf + os: ubuntu-latest + test_everything: true + - target: + tuple: loongarch64-unknown-linux-gnu + os: ubuntu-latest + test_everything: true + - target: + tuple: powerpc-unknown-linux-gnu + os: ubuntu-latest + disable_assert_instr: true + test_everything: true + - target: + tuple: powerpc64-unknown-linux-gnu + os: ubuntu-latest + disable_assert_instr: true + test_everything: true + - target: + tuple: powerpc64le-unknown-linux-gnu + os: ubuntu-latest + test_everything: true + - target: + tuple: riscv32gc-unknown-linux-gnu + os: ubuntu-latest + test_everything: true + build_std: true + - target: + tuple: riscv64gc-unknown-linux-gnu + os: ubuntu-latest + test_everything: true + - target: + tuple: s390x-unknown-linux-gnu + os: ubuntu-latest + test_everything: true + - target: + tuple: x86_64-unknown-linux-gnu + os: ubuntu-latest + test_everything: true + # MIPS targets disabled since they are dropped to tier 3. + # See https://github.com/rust-lang/compiler-team/issues/648 + #- target: + # tuple: mips-unknown-linux-gnu + # os: ubuntu-latest + # norun: true + #- target: + # tuple: mips64-unknown-linux-gnuabi64 + # os: ubuntu-latest + # norun: true + #- target: + # tuple: mips64el-unknown-linux-gnuabi64 + # os: ubuntu-latest + # norun: true + #- target: + # tuple: mipsel-unknown-linux-musl + # os: ubuntu-latest + # norun: true + - target: + tuple: aarch64-apple-darwin + os: macos-15 + norun: true # https://github.com/rust-lang/stdarch/issues/1206 + - target: + tuple: aarch64-apple-ios-macabi + os: macos-15 + norun: true # https://github.com/rust-lang/stdarch/issues/1206 + - target: + tuple: amdgcn-amd-amdhsa + os: ubuntu-latest + norun: true # hexagon doesn't build at the moment due to a libc issue. # - target: # tuple: hexagon-unknown-linux-musl @@ -218,59 +218,59 @@ jobs: # build_std: true steps: - - uses: actions/checkout@v6 - - name: Install Rust - run: | - rustup update nightly --no-self-update - rustup default nightly - shell: bash + - uses: actions/checkout@v6 + - name: Install Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + shell: bash - - run: rustup target add ${{ matrix.target.tuple }} - shell: bash - if: matrix.build_std == '' && matrix.target.tuple != 'amdgcn-amd-amdhsa' - - run: | - rustup component add rust-src - echo "CARGO_UNSTABLE_BUILD_STD=std" >> $GITHUB_ENV - shell: bash - if: matrix.build_std != '' - - run: | - rustup component add rust-src - echo "CARGO_UNSTABLE_BUILD_STD=core,alloc" >> $GITHUB_ENV - shell: bash - if: matrix.target.tuple == 'amdgcn-amd-amdhsa' + - run: rustup target add ${{ matrix.target.tuple }} + shell: bash + if: matrix.build_std == '' && matrix.target.tuple != 'amdgcn-amd-amdhsa' + - run: | + rustup component add rust-src + echo "CARGO_UNSTABLE_BUILD_STD=std" >> $GITHUB_ENV + shell: bash + if: matrix.build_std != '' + - run: | + rustup component add rust-src + echo "CARGO_UNSTABLE_BUILD_STD=core,alloc" >> $GITHUB_ENV + shell: bash + if: matrix.target.tuple == 'amdgcn-amd-amdhsa' - # Configure some env vars based on matrix configuration - - run: echo "PROFILE=${{matrix.profile}}" >> $GITHUB_ENV - shell: bash - - run: echo "NORUN=1" >> $GITHUB_ENV - shell: bash - if: matrix.norun != '' || startsWith(matrix.target.tuple, 'thumb') || matrix.target.tuple == 'nvptx64-nvidia-cuda' - - run: echo "STDARCH_TEST_EVERYTHING=1" >> $GITHUB_ENV - shell: bash - if: matrix.test_everything != '' - - run: echo "STDARCH_DISABLE_ASSERT_INSTR=1" >> $GITHUB_ENV - shell: bash - if: matrix.disable_assert_instr != '' - - run: echo "NOSTD=1" >> $GITHUB_ENV - shell: bash - if: startsWith(matrix.target.tuple, 'thumb') || matrix.target.tuple == 'nvptx64-nvidia-cuda' || matrix.target.tuple == 'amdgcn-amd-amdhsa' + # Configure some env vars based on matrix configuration + - run: echo "PROFILE=${{matrix.profile}}" >> $GITHUB_ENV + shell: bash + - run: echo "NORUN=1" >> $GITHUB_ENV + shell: bash + if: matrix.norun != '' || startsWith(matrix.target.tuple, 'thumb') || matrix.target.tuple == 'nvptx64-nvidia-cuda' + - run: echo "STDARCH_TEST_EVERYTHING=1" >> $GITHUB_ENV + shell: bash + if: matrix.test_everything != '' + - run: echo "STDARCH_DISABLE_ASSERT_INSTR=1" >> $GITHUB_ENV + shell: bash + if: matrix.disable_assert_instr != '' + - run: echo "NOSTD=1" >> $GITHUB_ENV + shell: bash + if: startsWith(matrix.target.tuple, 'thumb') || matrix.target.tuple == 'nvptx64-nvidia-cuda' || matrix.target.tuple == 'amdgcn-amd-amdhsa' - # Windows & OSX go straight to `run.sh` ... - - run: ./ci/run.sh - shell: bash - if: matrix.target.os != 'ubuntu-latest' || startsWith(matrix.target.tuple, 'thumb') - env: - TARGET: ${{ matrix.target.tuple }} + # Windows & OSX go straight to `run.sh` ... + - run: ./ci/run.sh + shell: bash + if: matrix.target.os != 'ubuntu-latest' || startsWith(matrix.target.tuple, 'thumb') + env: + TARGET: ${{ matrix.target.tuple }} - # ... while Linux goes to `run-docker.sh` - - run: ./ci/run-docker.sh ${{ matrix.target.tuple }} - shell: bash - if: matrix.target.os == 'ubuntu-latest' && !startsWith(matrix.target.tuple, 'thumb') - env: - TARGET: ${{ matrix.target.tuple }} + # ... while Linux goes to `run-docker.sh` + - run: ./ci/run-docker.sh ${{ matrix.target.tuple }} + shell: bash + if: matrix.target.os == 'ubuntu-latest' && !startsWith(matrix.target.tuple, 'thumb') + env: + TARGET: ${{ matrix.target.tuple }} intrinsic-test: - needs: [style] + needs: [ style ] name: Intrinsic Test runs-on: ubuntu-latest strategy: @@ -280,8 +280,8 @@ jobs: - aarch64_be-unknown-linux-gnu - armv7-unknown-linux-gnueabihf - x86_64-unknown-linux-gnu - profile: [dev, release] - cc: [clang, gcc] + profile: [ dev, release ] + cc: [ clang, gcc ] include: - target: aarch64_be-unknown-linux-gnu build_std: true @@ -295,74 +295,71 @@ jobs: - target: armv7-unknown-linux-gnueabihf cc: gcc steps: - - uses: actions/checkout@v6 - - name: Install Rust - run: | - rustup update nightly --no-self-update - rustup default nightly - - run: rustup target add ${{ matrix.target }} - if: ${{ (matrix.build_std || false) == false }} - - run: | - rustup component add rust-src - echo "CARGO_UNSTABLE_BUILD_STD=std" >> $GITHUB_ENV - if: ${{ matrix.build_std }} - - run: rustup component add rustfmt + - uses: actions/checkout@v6 + - name: Install Rust + run: | + rustup update nightly --no-self-update + rustup default nightly + - run: rustup target add ${{ matrix.target }} + if: ${{ (matrix.build_std || false) == false }} + - run: | + rustup component add rust-src + echo "CARGO_UNSTABLE_BUILD_STD=std" >> $GITHUB_ENV + if: ${{ matrix.build_std }} + - run: rustup component add rustfmt - # Configure some env vars based on matrix configuration - - run: echo "PROFILE=${{ matrix.profile }}" >> $GITHUB_ENV - - run: ./ci/intrinsic-test-docker.sh ${{ matrix.target }} ${{ matrix.cc }} - if: ${{ !startsWith(matrix.target, 'thumb') }} - env: - TARGET: ${{ matrix.target }} + # Configure some env vars based on matrix configuration + - run: echo "PROFILE=${{ matrix.profile }}" >> $GITHUB_ENV + - run: ./ci/intrinsic-test-docker.sh ${{ matrix.target }} ${{ matrix.cc }} + if: ${{ !startsWith(matrix.target, 'thumb') }} + env: + TARGET: ${{ matrix.target }} # Check that the generated files agree with the checked-in versions. check-stdarch-gen: - needs: [style] + needs: [ style ] name: Check stdarch-gen-{arm, loongarch, hexagon} output runs-on: ubuntu-latest env: STDARCH_GEN_MODE: check steps: - - uses: actions/checkout@v6 - - name: Install Rust - run: rustup update nightly && rustup default nightly && rustup component add rustfmt - - name: Check arm spec - run: | - cargo run --bin=stdarch-gen-arm --release -- crates/stdarch-gen-arm/spec - - name: Check loongarch lsx - run: | - cargo run -p stdarch-gen-loongarch --release -- lsx - git diff --exit-code - - name: Check loongarch lasx - run: | - cargo run -p stdarch-gen-loongarch --release -- lasx - git diff --exit-code - - name: Check hexagon - run: | - cargo run -p stdarch-gen-hexagon --release - git diff --exit-code - + - uses: actions/checkout@v6 + - name: Install Rust + run: rustup update nightly && rustup default nightly && rustup component add rustfmt + - name: Check arm spec + run: | + cargo run --bin=stdarch-gen-arm --release -- crates/stdarch-gen-arm/spec + - name: Check loongarch lsx + run: | + cargo run -p stdarch-gen-loongarch --release -- lsx + - name: Check loongarch lasx + run: | + cargo run -p stdarch-gen-loongarch --release -- lasx + - name: Check hexagon + run: | + cargo run -p stdarch-gen-hexagon --release + # Run some tests with Miri. Most stdarch functions use platform-specific intrinsics # that Miri does not support. Also Miri is reltively slow. # # Below we run some tests where Miri might catch UB, for instance on intrinsics that read from # or write to pointers. miri: - needs: [style] + needs: [ style ] name: Run some tests with miri runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - name: Install Rust - run: rustup update nightly && rustup default nightly && rustup component add miri - - name: Run miri tests - env: - TARGET: "aarch64-unknown-linux-gnu" - RUSTFLAGS: "-Ctarget-cpu=neoverse-v3" - run: | - # read filters and join them with a space. - FILTERS=$(cat aarch64-miri-tests.txt | tr '\n' ' ') - cargo miri test -p core_arch --target aarch64-unknown-linux-gnu -- $FILTERS + - uses: actions/checkout@v6 + - name: Install Rust + run: rustup update nightly && rustup default nightly && rustup component add miri + - name: Run miri tests + env: + TARGET: "aarch64-unknown-linux-gnu" + RUSTFLAGS: "-Ctarget-cpu=neoverse-v3" + run: | + # read filters and join them with a space. + FILTERS=$(cat aarch64-miri-tests.txt | tr '\n' ' ') + cargo miri test -p core_arch --target aarch64-unknown-linux-gnu -- $FILTERS conclusion: needs: diff --git a/library/stdarch/Cargo.lock b/library/stdarch/Cargo.lock index 5a1f6148ecc71..6510b00da9b52 100644 --- a/library/stdarch/Cargo.lock +++ b/library/stdarch/Cargo.lock @@ -835,6 +835,7 @@ dependencies = [ name = "stdarch-gen-loongarch" version = "0.1.0" dependencies = [ + "clap", "rand 0.9.4", "stdarch-gen-common", ] diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs b/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs index f481a159eb632..cfc4aced34173 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs @@ -6,8 +6,8 @@ // OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen-loongarch -- crates/stdarch-gen-loongarch/lasx.spec // ``` -use crate::mem::transmute; use super::super::*; +use crate::mem::transmute; #[allow(improper_ctypes)] unsafe extern "llvm-intrinsic" { @@ -3033,168 +3033,312 @@ pub fn lasx_xvhsubw_qu_du(a: m256i, b: m256i) -> m256i { #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_q_d(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_q_d(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_q_d( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_d_w(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_d_w(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_d_w( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_w_h(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_w_h(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_w_h( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_h_b(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_h_b(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_h_b( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_q_du(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_q_du(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_q_du( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_d_wu(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_d_wu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_d_wu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_w_hu(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_w_hu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_w_hu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_h_bu(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_h_bu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_h_bu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_q_d(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_q_d(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_q_d( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_d_w(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_d_w(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_d_w( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_w_h(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_w_h(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_w_h( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_h_b(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_h_b(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_h_b( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_q_du(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_q_du(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_q_du( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_d_wu(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_d_wu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_d_wu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_w_hu(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_w_hu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_w_hu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_h_bu(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_h_bu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_h_bu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_q_du_d(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_q_du_d(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_q_du_d( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_d_wu_w(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_d_wu_w(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_d_wu_w( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_w_hu_h(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_w_hu_h(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_w_hu_h( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwev_h_bu_b(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwev_h_bu_b(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwev_h_bu_b( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_q_du_d(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_q_du_d(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_q_du_d( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_d_wu_w(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_d_wu_w(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_d_wu_w( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_w_hu_h(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_w_hu_h(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_w_hu_h( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lasx_xvmaddwod_h_bu_b(a: m256i, b: m256i, c: m256i) -> m256i { - unsafe { transmute(__lasx_xvmaddwod_h_bu_b(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lasx_xvmaddwod_h_bu_b( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs b/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs index 7915ef07d68e7..aa4e31ce8da6c 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs @@ -6,8 +6,8 @@ // OUT_DIR=`pwd`/crates/core_arch cargo run -p stdarch-gen-loongarch -- crates/stdarch-gen-loongarch/lsx.spec // ``` -use crate::mem::transmute; use super::super::*; +use crate::mem::transmute; #[allow(improper_ctypes)] unsafe extern "llvm-intrinsic" { @@ -2747,21 +2747,39 @@ pub fn lsx_vmaddwev_h_b(a: m128i, b: m128i, c: m128i) -> m128i { #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwev_d_wu(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwev_d_wu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwev_d_wu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwev_w_hu(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwev_w_hu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwev_w_hu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwev_h_bu(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwev_h_bu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwev_h_bu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] @@ -2789,63 +2807,117 @@ pub fn lsx_vmaddwod_h_b(a: m128i, b: m128i, c: m128i) -> m128i { #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwod_d_wu(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwod_d_wu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwod_d_wu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwod_w_hu(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwod_w_hu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwod_w_hu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwod_h_bu(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwod_h_bu(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwod_h_bu( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwev_d_wu_w(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwev_d_wu_w(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwev_d_wu_w( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwev_w_hu_h(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwev_w_hu_h(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwev_w_hu_h( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwev_h_bu_b(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwev_h_bu_b(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwev_h_bu_b( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwod_d_wu_w(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwod_d_wu_w(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwod_d_wu_w( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwod_w_hu_h(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwod_w_hu_h(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwod_w_hu_h( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwod_h_bu_b(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwod_h_bu_b(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwod_h_bu_b( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] @@ -2866,28 +2938,52 @@ pub fn lsx_vmaddwod_q_d(a: m128i, b: m128i, c: m128i) -> m128i { #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwev_q_du(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwev_q_du(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwev_q_du( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwod_q_du(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwod_q_du(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwod_q_du( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwev_q_du_d(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwev_q_du_d(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwev_q_du_d( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] pub fn lsx_vmaddwod_q_du_d(a: m128i, b: m128i, c: m128i) -> m128i { - unsafe { transmute(__lsx_vmaddwod_q_du_d(transmute(a), transmute(b), transmute(c))) } + unsafe { + transmute(__lsx_vmaddwod_q_du_d( + transmute(a), + transmute(b), + transmute(c), + )) + } } #[inline] diff --git a/library/stdarch/crates/stdarch-gen-loongarch/Cargo.toml b/library/stdarch/crates/stdarch-gen-loongarch/Cargo.toml index a9ddc123b3688..d174b844b430a 100644 --- a/library/stdarch/crates/stdarch-gen-loongarch/Cargo.toml +++ b/library/stdarch/crates/stdarch-gen-loongarch/Cargo.toml @@ -7,5 +7,6 @@ edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +clap = { version = "4", features = ["derive", "env"] } rand = "0.9.3" stdarch-gen-common = { path = "../stdarch-gen-common" } diff --git a/library/stdarch/crates/stdarch-gen-loongarch/src/main.rs b/library/stdarch/crates/stdarch-gen-loongarch/src/main.rs index 86fa8956e6ed1..1fed0b96f9426 100644 --- a/library/stdarch/crates/stdarch-gen-loongarch/src/main.rs +++ b/library/stdarch/crates/stdarch-gen-loongarch/src/main.rs @@ -1,3 +1,4 @@ +use clap::Parser; use std::collections::HashSet; use std::env; use std::fmt; @@ -6,7 +7,7 @@ use std::io::prelude::*; use std::io::{self, BufReader}; use std::path::Path; use std::path::PathBuf; -use stdarch_gen_common::{Mode, run_generator}; +use stdarch_gen_common::{GeneratorCtx, Mode, run_generator}; /// Complete lines of generated source. /// @@ -1602,51 +1603,64 @@ static void {current_name}(void) /// Runs the check/bless harness for `lsx`/`lasx` when invoked with /// no args or a bare ext name. +#[derive(clap::Parser, Debug)] +struct Args { + /// Either: + /// - The extension (lsx/lasx) to generate, or: + /// - A path to a intrin.h file to generate the spec file from. Optionally followed + /// by "test". + arguments: Vec, + /// Generation mode. + #[arg(long, env = "STDARCH_GEN_MODE")] + mode: Option, + /// Path to a rustfmt binary that will be used to reformat the generated code. + /// If unset, it will just use "rustfmt" from the environment. + #[arg(long)] + rustfmt_path: Option, +} + pub fn main() -> Result<(), String> { - let args: Vec = env::args().collect(); - let arg_strs: Vec<&str> = args.iter().map(String::as_str).collect(); - let harness_exts: Option<&[&str]> = match arg_strs.as_slice() { - [_] => Some(&["lsx", "lasx"]), - [_, "lsx"] => Some(&["lsx"]), - [_, "lasx"] => Some(&["lasx"]), - _ => None, - }; - if let Some(exts) = harness_exts { - let crate_dir = - PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")); - let core_arch_src = crate_dir.join("../core_arch/src"); - let mode = Mode::from_env(); - for ext in exts { - let spec_rel = format!("crates/stdarch-gen-loongarch/{ext}.spec"); - let committed = core_arch_src.join("loongarch64").join(ext); - run_generator(&committed, mode, |out_dir| { - gen_bind(&spec_rel, ext, out_dir) - }) - .map_err(|e| e.to_string())?; - } - return Ok(()); - } + let args = Args::parse(); + + let crate_dir = + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")); + let core_arch_src = crate_dir.join("../core_arch/src"); + let mode = args.mode.unwrap_or_default(); + let ctx = GeneratorCtx::new(args.rustfmt_path); - let in_file = args[1].clone(); - let in_file_name = PathBuf::from(&in_file) - .file_name() - .unwrap() - .to_string_lossy() - .into_owned(); - let ext_name = if in_file_name.starts_with("lasx") { - "lasx" + let arguments = &args.arguments; + if arguments.len() == 1 && (arguments[0] == "lsx" || arguments[0] == "lasx") { + let extension = arguments[0].as_str(); + let spec_rel = format!("crates/stdarch-gen-loongarch/{extension}.spec"); + let committed = core_arch_src.join("loongarch64").join(extension); + run_generator(&ctx, &committed, mode, |out_dir| { + gen_bind(&spec_rel, extension, out_dir) + }) + .map_err(|e| e.to_string())?; } else { - "lsx" - }; - if in_file_name.ends_with(".h") { - return gen_spec(in_file, ext_name).map_err(|e| e.to_string()); - } - if let [_, _lsx_or_lasx, "test"] = arg_strs.as_slice() { - return gen_test(in_file, ext_name).map_err(|e| e.to_string()); + let in_file = arguments[0].clone(); + let in_file_name = PathBuf::from(&in_file) + .file_name() + .unwrap() + .to_string_lossy() + .into_owned(); + let ext_name = if in_file_name.starts_with("lasx") { + "lasx" + } else { + "lsx" + }; + if in_file_name.ends_with(".h") { + return gen_spec(in_file, ext_name).map_err(|e| e.to_string()); + } + if arguments.last().map(|s| s.as_str()) == Some("test") { + return gen_test(in_file, ext_name).map_err(|e| e.to_string()); + } + // Note: this does not apply rustfmt formatting + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap_or("crates/core_arch".to_string())) + .join("src") + .join("loongarch64") + .join(ext_name); + gen_bind(&in_file, ext_name, &out_path).map_err(|e| e.to_string())?; } - let out_path = PathBuf::from(env::var("OUT_DIR").unwrap_or("crates/core_arch".to_string())) - .join("src") - .join("loongarch64") - .join(ext_name); - gen_bind(&in_file, ext_name, &out_path).map_err(|e| e.to_string()) + Ok(()) } From 2b0ee0b195bcd2adffe18296a73d9901cbbc78c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 14 Sep 2026 11:48:14 +0200 Subject: [PATCH 25/55] Use unified rustfmt formatting in stdarch-gen-arm --- library/stdarch/Cargo.lock | 1 + .../stdarch/crates/stdarch-gen-arm/Cargo.toml | 1 + .../stdarch-gen-arm/src/load_store_tests.rs | 25 ++--- .../crates/stdarch-gen-arm/src/main.rs | 99 ++++++++----------- .../crates/stdarch-gen-common/src/lib.rs | 18 ---- 5 files changed, 50 insertions(+), 94 deletions(-) diff --git a/library/stdarch/Cargo.lock b/library/stdarch/Cargo.lock index 6510b00da9b52..b902e966dc2ea 100644 --- a/library/stdarch/Cargo.lock +++ b/library/stdarch/Cargo.lock @@ -804,6 +804,7 @@ dependencies = [ name = "stdarch-gen-arm" version = "0.1.0" dependencies = [ + "clap", "itertools", "proc-macro2", "quote", diff --git a/library/stdarch/crates/stdarch-gen-arm/Cargo.toml b/library/stdarch/crates/stdarch-gen-arm/Cargo.toml index cb284d5f2c57b..9d803538474c7 100644 --- a/library/stdarch/crates/stdarch-gen-arm/Cargo.toml +++ b/library/stdarch/crates/stdarch-gen-arm/Cargo.toml @@ -12,6 +12,7 @@ edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +clap = { version = "4", features = ["derive", "env"] } itertools = "0.15.0" proc-macro2 = "1.0" quote = "1.0" diff --git a/library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs b/library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs index 672047fc057b8..3e3c20c592be0 100644 --- a/library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs +++ b/library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs @@ -1,10 +1,9 @@ use std::fs::File; -use std::io::Write; -use std::path::PathBuf; +use std::io::{BufWriter, Write}; +use std::path::Path; use std::str::FromStr; use std::sync::LazyLock; -use crate::format_code; use crate::input::InputType; use crate::intrinsic::Intrinsic; use crate::typekinds::BaseType; @@ -38,15 +37,11 @@ const LEN_U64: usize = VL_MAX_BYTES / core::mem::size_of::(); pub fn generate_load_store_tests( load_intrinsics: Vec, store_intrinsics: Vec, - out_path: Option<&PathBuf>, + out_path: &Path, ) -> Result<(), String> { - let output = match out_path { - Some(out) => { - Box::new(File::create(out).map_err(|e| format!("couldn't create tests file: {e}"))?) - as Box - } - None => Box::new(std::io::stdout()) as Box, - }; + let mut output = BufWriter::new( + File::create(out_path).map_err(|e| format!("couldn't create tests file: {e}"))?, + ); let mut used_stores = vec![false; store_intrinsics.len()]; let tests: Vec<_> = load_intrinsics .iter() @@ -89,10 +84,9 @@ pub fn generate_load_store_tests( .map_err(|e| format!("Manual tests are invalid: {e}"))?, _ => quote!(), }; - format_code( + write!( output, - format!( - "// This code is automatically generated. DO NOT MODIFY. + "// This code is automatically generated. DO NOT MODIFY. // // Instead, modify `crates/stdarch-gen-arm/spec/sve` and run the following command to re-generate // this file: @@ -101,8 +95,7 @@ pub fn generate_load_store_tests( // cargo run --bin=stdarch-gen-arm -- crates/stdarch-gen-arm/spec // ``` {}", - quote! { #preamble #(#tests)* #manual_tests } - ), + quote! { #preamble #(#tests)* #manual_tests } ) .map_err(|e| format!("couldn't write tests: {e}")) } diff --git a/library/stdarch/crates/stdarch-gen-arm/src/main.rs b/library/stdarch/crates/stdarch-gen-arm/src/main.rs index 276cc5ba41b01..db1cfb2bb92b2 100644 --- a/library/stdarch/crates/stdarch-gen-arm/src/main.rs +++ b/library/stdarch/crates/stdarch-gen-arm/src/main.rs @@ -14,19 +14,51 @@ mod typekinds; mod wildcards; mod wildstring; +use clap::Parser; use intrinsic::Test; use itertools::Itertools; use quote::quote; use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use stdarch_gen_common::{Mode, run_generator}; +use stdarch_gen_common::{GeneratorCtx, Mode, run_generator}; use walkdir::WalkDir; +#[derive(clap::Parser)] +struct Args { + /// Directory with spec files: //.spec.yml + input_dir: PathBuf, + /// Output directory to generate the files into, such as crates/core_arch/src + output_dir: Option, + /// Generation mode. + #[arg(long, env = "STDARCH_GEN_MODE")] + mode: Option, + /// Path to a rustfmt binary that will be used to reformat the generated code. + /// If unset, it will just use "rustfmt" from the environment. + #[arg(long)] + rustfmt_path: Option, +} + fn main() -> Result<(), String> { - let (in_path, out_base) = parse_args(); - let mode = Mode::from_env(); + let args = Args::parse(); + + let in_path = args.input_dir; + let out_base = args.output_dir.unwrap_or_else(|| { + std::env::current_exe() + .ok() + .map(|mut f| { + f.pop(); + f.push("../../crates/core_arch/src/"); + f + }) + .filter(|f| f.exists()) + .expect("could not locate crates/core_arch/src; pass OUTPUT_DIR command-line argument explicitly") + }); + assert!(in_path.exists()); + assert!(out_base.exists()); + + let mode = args.mode.unwrap_or_default(); + let ctx = GeneratorCtx::new(args.rustfmt_path); for filepath in WalkDir::new(&in_path) .into_iter() @@ -42,7 +74,7 @@ fn main() -> Result<(), String> { .expect("generated output path must have a parent directory") .to_path_buf(); - run_generator(&committed, mode, |scratch: &Path| { + run_generator(&ctx, &committed, mode, |scratch: &Path| { generate_spec(&filepath, scratch) }) .map_err(|e| e.to_string())?; @@ -92,7 +124,7 @@ fn generate_spec(filepath: &Path, out_dir: &Path) -> Result<(), String> { .expect("load/store test path must have a file name") .to_owned(); let tests_path = out_dir.join(tests_name); - load_store_tests::generate_load_store_tests(loads, stores, Some(&tests_path))?; + load_store_tests::generate_load_store_tests(loads, stores, &tests_path)?; } let generated = input::GeneratorInput { @@ -105,44 +137,6 @@ fn generate_spec(filepath: &Path, out_dir: &Path) -> Result<(), String> { .map_err(|e| format!("could not generate output file: {e}")) } -fn parse_args() -> (PathBuf, PathBuf) { - let mut args_it = std::env::args().skip(1); - assert!( - 1 <= args_it.len() && args_it.len() <= 2, - "Usage: cargo run -p stdarch-gen-arm -- INPUT_DIR [OUTPUT_DIR]\n\ - where:\n\ - - INPUT_DIR contains a tree like: INPUT_DIR//.spec.yml\n\ - - OUTPUT_DIR is a directory like: crates/core_arch/src/" - ); - - let in_path = Path::new(args_it.next().unwrap().as_str()).to_path_buf(); - assert!( - in_path.exists() && in_path.is_dir(), - "invalid path {in_path:#?} given" - ); - - let out_base = if let Some(dir) = args_it.next() { - let out_path = Path::new(dir.as_str()).to_path_buf(); - assert!( - out_path.exists() && out_path.is_dir(), - "invalid path {out_path:#?} given" - ); - out_path - } else { - std::env::current_exe() - .ok() - .map(|mut f| { - f.pop(); - f.push("../../crates/core_arch/src/"); - f - }) - .filter(|f| f.exists()) - .expect("could not locate crates/core_arch/src; pass OUTPUT_DIR command-line argument explicitly") - }; - - (in_path, out_base) -} - fn generate_file( generated_input: input::GeneratorInput, mut out: Box, @@ -171,25 +165,10 @@ use super::*;{uses_neon} }, )?; let intrinsics = generated_input.intrinsics; - format_code(out, quote! { #(#intrinsics)* })?; + write!(out, "{}", quote! { #(#intrinsics)* })?; Ok(()) } -pub fn format_code( - mut output: impl std::io::Write, - input: impl std::fmt::Display, -) -> std::io::Result<()> { - let proc = Command::new("rustfmt") - // Ensure that we generate the same file contents on both Linux and Windows. - .arg("--config") - .arg("newline_style=Unix") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn()?; - write!(proc.stdin.as_ref().unwrap(), "{input}")?; - output.write_all(proc.wait_with_output()?.stdout.as_slice()) -} - /// Derive an output file path from an input file path and an output directory. /// /// `in_filepath` is expected to have a structure like: diff --git a/library/stdarch/crates/stdarch-gen-common/src/lib.rs b/library/stdarch/crates/stdarch-gen-common/src/lib.rs index 5308296d8588a..92393eb37885d 100644 --- a/library/stdarch/crates/stdarch-gen-common/src/lib.rs +++ b/library/stdarch/crates/stdarch-gen-common/src/lib.rs @@ -32,24 +32,6 @@ pub enum Mode { Bless, } -impl Mode { - /// Read the mode from the `STDARCH_GEN_MODE` environment variable. - /// - /// Recognized values: - /// - `"check"` → [`Mode::Check`] - /// - `"bless"` → [`Mode::Bless`] - /// - unset → [`Mode::Bless`] - /// - any other value → panic - pub fn from_env() -> Self { - match std::env::var("STDARCH_GEN_MODE").as_deref() { - Ok("check") => Mode::Check, - Ok("bless") => Mode::Bless, - Ok(other) => panic!("unknown STDARCH_GEN_MODE value: {other:?}"), - Err(_) => Mode::Bless, - } - } -} - impl FromStr for Mode { type Err = String; From 34bda05490f632c106ee25f5a553519fb1bd3616 Mon Sep 17 00:00:00 2001 From: xonx <119700621+xonx4l@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:42:56 +0200 Subject: [PATCH 26/55] Show diff when generated files do not match expectations --- library/stdarch/Cargo.lock | 20 +++++++++++++++++++ .../crates/stdarch-gen-common/Cargo.toml | 3 ++- .../crates/stdarch-gen-common/src/lib.rs | 19 +++++++++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/library/stdarch/Cargo.lock b/library/stdarch/Cargo.lock index b902e966dc2ea..490161626c164 100644 --- a/library/stdarch/Cargo.lock +++ b/library/stdarch/Cargo.lock @@ -88,6 +88,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + [[package]] name = "cc" version = "1.2.59" @@ -800,6 +810,15 @@ dependencies = [ "syn", ] +[[package]] +name = "similar" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f66ca1f7aca2474dc10c942eb22feffc897735f54cd1db90138c2fddb490987" +dependencies = [ + "bstr", +] + [[package]] name = "stdarch-gen-arm" version = "0.1.0" @@ -820,6 +839,7 @@ dependencies = [ name = "stdarch-gen-common" version = "0.1.0" dependencies = [ + "similar", "tempfile", ] diff --git a/library/stdarch/crates/stdarch-gen-common/Cargo.toml b/library/stdarch/crates/stdarch-gen-common/Cargo.toml index 691be14971a0d..fda09cb4c5994 100644 --- a/library/stdarch/crates/stdarch-gen-common/Cargo.toml +++ b/library/stdarch/crates/stdarch-gen-common/Cargo.toml @@ -4,4 +4,5 @@ version = "0.1.0" edition = "2024" [dependencies] -tempfile = "3" \ No newline at end of file +tempfile = "3" +similar = "3" diff --git a/library/stdarch/crates/stdarch-gen-common/src/lib.rs b/library/stdarch/crates/stdarch-gen-common/src/lib.rs index 92393eb37885d..535e0bc11cc0f 100644 --- a/library/stdarch/crates/stdarch-gen-common/src/lib.rs +++ b/library/stdarch/crates/stdarch-gen-common/src/lib.rs @@ -1,5 +1,6 @@ //! Shared check/bless harness for stdarch generators. +use similar::TextDiff; use std::error::Error as StdError; use std::fmt; use std::fs; @@ -251,7 +252,23 @@ fn compare(generated_dir: &Path, committed_dir: &Path, filename: &str) -> Result }), (false, false) => Ok(()), (true, true) => { - if fs::read(&gen_path)? != fs::read(&comm_path)? { + let generated = fs::read(&gen_path)?; + let committed = fs::read(&comm_path)?; + if generated != committed { + if let (Ok(committed), Ok(generated)) = + (str::from_utf8(&committed), str::from_utf8(&generated)) + { + eprintln!( + "{}", + TextDiff::from_lines(committed, generated) + .unified_diff() + .context_radius(3) + .header( + &format!("committed/{filename}"), + &format!("generated/{filename}"), + ) + ); + } Err(Error::Mismatch { path: rel_path, kind: MismatchKind::ContentsDiffer, From 988cc4b43b88a14303a3a40ff0252b99a6a623df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 14 Sep 2026 12:44:18 +0200 Subject: [PATCH 27/55] Bless arm output --- library/stdarch/crates/core_arch/src/aarch64/sve/generated.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/stdarch/crates/core_arch/src/aarch64/sve/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/sve/generated.rs index 324b5bccf3777..75393f594ad64 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/sve/generated.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/sve/generated.rs @@ -11,8 +11,8 @@ use stdarch_test::assert_instr; use super::*; -use crate::core_arch::arch::aarch64::*; use super::{AsSigned, AsUnsigned}; +use crate::core_arch::arch::aarch64::*; #[doc = "Absolute difference"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/svabd[_f32]_m)"] From 1bda26edef6ac407370ca297e9ef1d061b9f2b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 14 Sep 2026 13:01:30 +0200 Subject: [PATCH 28/55] Fix tests --- .../crates/stdarch-gen-common/src/lib.rs | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/library/stdarch/crates/stdarch-gen-common/src/lib.rs b/library/stdarch/crates/stdarch-gen-common/src/lib.rs index 535e0bc11cc0f..5dbfc76f27dfe 100644 --- a/library/stdarch/crates/stdarch-gen-common/src/lib.rs +++ b/library/stdarch/crates/stdarch-gen-common/src/lib.rs @@ -318,7 +318,10 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let committed = tmp.path().join("c"); write_marker(&committed.join("a.txt"), b"hi"); + + let ctx = GeneratorCtx::new(None); let e = run_generator( + &ctx, &committed, Mode::Check, |out| -> std::result::Result<(), io::Error> { @@ -341,7 +344,10 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let committed = tmp.path().join("c"); write_marker(&committed.join("a.txt"), b"hi"); + + let ctx = GeneratorCtx::new(None); let e = run_generator( + &ctx, &committed, Mode::Check, |_| -> std::result::Result<(), io::Error> { Ok(()) }, @@ -361,7 +367,10 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let committed = tmp.path().join("c"); fs::create_dir_all(&committed).unwrap(); + + let ctx = GeneratorCtx::new(None); let e = run_generator( + &ctx, &committed, Mode::Check, |out| -> std::result::Result<(), io::Error> { @@ -385,7 +394,10 @@ mod tests { let committed = tmp.path().join("c"); write_marker(&committed.join("keep.txt"), b""); write_marker(&committed.join("stale.txt"), b""); + + let ctx = GeneratorCtx::new(None); run_generator( + &ctx, &committed, Mode::Bless, |out| -> std::result::Result<(), io::Error> { @@ -405,7 +417,9 @@ mod tests { fs::create_dir_all(&committed).unwrap(); fs::write(committed.join("mod.rs"), b"hand-written").unwrap(); fs::write(committed.join("old.txt"), b"old").unwrap(); + let ctx = GeneratorCtx::new(None); run_generator( + &ctx, &committed, Mode::Bless, |out| -> std::result::Result<(), io::Error> { @@ -418,4 +432,32 @@ mod tests { assert_eq!(fs::read(committed.join("old.txt")).unwrap(), b"old"); assert!(committed.join("new.txt").exists()); } + + #[test] + fn generation_reformats_files() { + let tmp = tempfile::tempdir().unwrap(); + let committed = tmp.path().join("c"); + let file = committed.join("a.rs"); + write_marker(&file, b"foo"); + + let ctx = GeneratorCtx::new(None); + run_generator( + &ctx, + &committed, + Mode::Bless, + |out| -> std::result::Result<(), io::Error> { + write_marker(&out.join("a.rs"), b"fn main() {}"); + Ok(()) + }, + ) + .unwrap(); + assert_eq!( + std::fs::read_to_string(file).unwrap(), + format!( + r#"{GENERATED_MARKER} +fn main() {{}} +"# + ) + ); + } } From 1d6ecb3d7e5b9c1135c9aead64d003d58822204d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 14 Sep 2026 13:45:15 +0200 Subject: [PATCH 29/55] Install rustfmt on CI --- library/stdarch/.github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/stdarch/.github/workflows/main.yml b/library/stdarch/.github/workflows/main.yml index 754f01d399117..182da81ac2371 100644 --- a/library/stdarch/.github/workflows/main.yml +++ b/library/stdarch/.github/workflows/main.yml @@ -225,6 +225,8 @@ jobs: rustup default nightly shell: bash + - run: rustup component add rustfmt + - run: rustup target add ${{ matrix.target.tuple }} shell: bash if: matrix.build_std == '' && matrix.target.tuple != 'amdgcn-amd-amdhsa' From 4ff0ef647aca9489660f0b48487dc42dfc200980 Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Tue, 15 Sep 2026 00:08:33 +0800 Subject: [PATCH 30/55] intrinsic-test: Fix intrinsic test names for negative immediate values --- .../crates/intrinsic-test/src/common/gen_c.rs | 11 ++++++++--- .../crates/intrinsic-test/src/common/gen_rust.rs | 13 +++++++++---- .../stdarch/crates/intrinsic-test/src/common/mod.rs | 5 +++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs b/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs index bbff7a91b6c41..30e7bb1d375a3 100644 --- a/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs +++ b/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs @@ -1,5 +1,6 @@ use itertools::Itertools; +use crate::common::imm_value_to_ident; use crate::common::{SupportedArchitecture, intrinsic::Intrinsic}; use super::intrinsic_helpers::TypeDefinition; @@ -41,9 +42,13 @@ void {name}_wrapper{imm_arglist}({return_ty}* __dst{arglist}) {{ }}", return_ty = intrinsic.results.c_type(), name = intrinsic.name, - imm_arglist = imm_values - .iter() - .format_with("", |i, fmt| fmt(&format_args!("_{i}"))), + imm_arglist = + imm_values + .iter() + .format_with("", |i, fmt| fmt(&format_args!( + "_{}", + imm_value_to_ident(i) + ))), arglist = intrinsic.arguments.as_non_imm_arglist_c(), params = intrinsic.arguments.as_call_params_c(&imm_values) )) diff --git a/library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs b/library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs index bfe37edbcfe43..f319418b49c7f 100644 --- a/library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs +++ b/library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs @@ -4,6 +4,7 @@ use itertools::Itertools; use super::intrinsic_helpers::TypeDefinition; use crate::common::cli::{CcArgStyle, ProcessedCli}; +use crate::common::imm_value_to_ident; use crate::common::intrinsic::Intrinsic; use crate::common::intrinsic_helpers::TypeKind; use crate::common::values::{test_values_array_name, test_values_array_static}; @@ -276,7 +277,7 @@ for (id, rust, c) in specializations {{ } }) .join(","), - c_const_args = imm_values.iter().join("_"), + c_const_args = imm_values.iter().map(imm_value_to_ident).join("_"), )) } }), @@ -343,9 +344,13 @@ unsafe extern "C" {{ "fn {name}_wrapper{imm_arglist}(__dst: *mut {return_ty}{arglist});", return_ty = intrinsic.results.rust_type(), name = intrinsic.name, - imm_arglist = imm_values - .iter() - .format_with("", |i, fmt| fmt(&format_args!("_{i}"))), + imm_arglist = + imm_values + .iter() + .format_with("", |i, fmt| fmt(&format_args!( + "_{}", + imm_value_to_ident(i) + ))), arglist = intrinsic.arguments.as_non_imm_arglist_rust(), )) })) diff --git a/library/stdarch/crates/intrinsic-test/src/common/mod.rs b/library/stdarch/crates/intrinsic-test/src/common/mod.rs index b476ad477ca36..e89599d06ea77 100644 --- a/library/stdarch/crates/intrinsic-test/src/common/mod.rs +++ b/library/stdarch/crates/intrinsic-test/src/common/mod.rs @@ -138,3 +138,8 @@ pub fn manual_chunk(intrinsic_count: usize) -> (usize, usize) { let number_of_chunks = intrinsic_count.div_ceil(max_intrinsics_per_chunk); (max_intrinsics_per_chunk, number_of_chunks) } + +pub fn imm_value_to_ident(value: impl std::fmt::Display) -> String { + let value = value.to_string(); + value.replace('-', "neg") +} From 5a0103421261759da77fb801e2cba0dead819695 Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Tue, 15 Sep 2026 00:08:34 +0800 Subject: [PATCH 31/55] intrinsic-test: Add C intrinsic name prefix support Add a per-architecture C intrinsic name prefix to support architectures whose C intrinsic names differ from their Rust intrinsic names. --- library/stdarch/crates/intrinsic-test/src/arm/mod.rs | 2 ++ library/stdarch/crates/intrinsic-test/src/common/gen_c.rs | 3 ++- library/stdarch/crates/intrinsic-test/src/common/mod.rs | 4 ++++ library/stdarch/crates/intrinsic-test/src/x86/mod.rs | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/library/stdarch/crates/intrinsic-test/src/arm/mod.rs b/library/stdarch/crates/intrinsic-test/src/arm/mod.rs index 9c2d0f336a7aa..5e892c613464a 100644 --- a/library/stdarch/crates/intrinsic-test/src/arm/mod.rs +++ b/library/stdarch/crates/intrinsic-test/src/arm/mod.rs @@ -37,6 +37,8 @@ impl SupportedArchitecture for Arm { "#; const RUST_PRELUDE: &str = RUST_PRELUDE; + const C_NAME_PREFIX: &str = ""; + fn c_compiler_flags(&self, cli_options: &ProcessedCli) -> Vec<&str> { // GCC uses an extra `-` in the arch name let big_endian = cli_options.target.starts_with("aarch64_be"); diff --git a/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs b/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs index 30e7bb1d375a3..a749afd17d367 100644 --- a/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs +++ b/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs @@ -38,9 +38,10 @@ pub fn write_wrapper_c( fmt(&format_args!( " void {name}_wrapper{imm_arglist}({return_ty}* __dst{arglist}) {{ - *__dst = {name}({params}); + *__dst = {prefix}{name}({params}); }}", return_ty = intrinsic.results.c_type(), + prefix = A::C_NAME_PREFIX, name = intrinsic.name, imm_arglist = imm_values diff --git a/library/stdarch/crates/intrinsic-test/src/common/mod.rs b/library/stdarch/crates/intrinsic-test/src/common/mod.rs index e89599d06ea77..9cde80daff76c 100644 --- a/library/stdarch/crates/intrinsic-test/src/common/mod.rs +++ b/library/stdarch/crates/intrinsic-test/src/common/mod.rs @@ -49,6 +49,10 @@ pub trait SupportedArchitecture: Sized { const C_PRELUDE: &str; const RUST_PRELUDE: &str; + /// Per-architecture prefix used to convert a Rust intrinsic name to the + /// corresponding C intrinsic name by prepending it to the Rust name. + const C_NAME_PREFIX: &str; + fn c_compiler_flags(&self, cli_options: &ProcessedCli) -> Vec<&str>; fn generate_c_file(&self) { diff --git a/library/stdarch/crates/intrinsic-test/src/x86/mod.rs b/library/stdarch/crates/intrinsic-test/src/x86/mod.rs index 36f4fee43741b..9eafcf1fc7b96 100644 --- a/library/stdarch/crates/intrinsic-test/src/x86/mod.rs +++ b/library/stdarch/crates/intrinsic-test/src/x86/mod.rs @@ -32,6 +32,8 @@ impl SupportedArchitecture for X86 { "#; const RUST_PRELUDE: &str = RUST_PRELUDE; + const C_NAME_PREFIX: &str = ""; + fn c_compiler_flags(&self, _cli_options: &ProcessedCli) -> Vec<&str> { vec![ "-maes", From bb2b679c54f314b73736ffc4e98418afb94849c8 Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Mon, 14 Sep 2026 11:32:27 +0800 Subject: [PATCH 32/55] intrinsic-test: Support testing LoongArch64 SIMD intrinsics --- library/stdarch/.github/workflows/main.yml | 1 + .../loongarch64-unknown-linux-gnu/Dockerfile | 9 +- library/stdarch/ci/intrinsic-test.sh | 14 + .../missing_loongarch64_clang.txt | 0 .../missing_loongarch64_common.txt | 31 ++ .../missing_loongarch64_gcc.txt | 0 .../intrinsic-test/src/loongarch/intrinsic.rs | 19 ++ .../intrinsic-test/src/loongarch/mod.rs | 139 ++++++++ .../intrinsic-test/src/loongarch/parser.rs | 179 ++++++++++ .../intrinsic-test/src/loongarch/types.rs | 309 ++++++++++++++++++ .../stdarch/crates/intrinsic-test/src/main.rs | 7 + 11 files changed, 707 insertions(+), 1 deletion(-) create mode 100644 library/stdarch/crates/intrinsic-test/missing_loongarch64_clang.txt create mode 100644 library/stdarch/crates/intrinsic-test/missing_loongarch64_common.txt create mode 100644 library/stdarch/crates/intrinsic-test/missing_loongarch64_gcc.txt create mode 100644 library/stdarch/crates/intrinsic-test/src/loongarch/intrinsic.rs create mode 100644 library/stdarch/crates/intrinsic-test/src/loongarch/mod.rs create mode 100644 library/stdarch/crates/intrinsic-test/src/loongarch/parser.rs create mode 100644 library/stdarch/crates/intrinsic-test/src/loongarch/types.rs diff --git a/library/stdarch/.github/workflows/main.yml b/library/stdarch/.github/workflows/main.yml index 5c2e2ae4a8e21..51a37375d12f5 100644 --- a/library/stdarch/.github/workflows/main.yml +++ b/library/stdarch/.github/workflows/main.yml @@ -279,6 +279,7 @@ jobs: - aarch64-unknown-linux-gnu - aarch64_be-unknown-linux-gnu - armv7-unknown-linux-gnueabihf + - loongarch64-unknown-linux-gnu - x86_64-unknown-linux-gnu profile: [dev, release] cc: [clang, gcc] diff --git a/library/stdarch/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile b/library/stdarch/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile index e803dae1006a0..8cbd2d8080002 100644 --- a/library/stdarch/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile +++ b/library/stdarch/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile @@ -3,8 +3,15 @@ FROM ubuntu:25.10 RUN apt-get update && \ apt-get install -y --no-install-recommends \ gcc libc6-dev qemu-user ca-certificates \ - gcc-loongarch64-linux-gnu libc6-dev-loong64-cross + gcc-loongarch64-linux-gnu libc6-dev-loong64-cross \ + wget +RUN wget https://ci-mirrors.rust-lang.org/llvm/llvm-22.1.4-x86_64.tar.gz -O llvm.tar.xz +RUN mkdir llvm +RUN tar -xvf llvm.tar.xz --strip-components=1 -C llvm + +ENV CLANG_PATH="/llvm/bin/clang" +ENV GCC_PATH=loongarch64-linux-gnu-gcc ENV CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_LINKER=loongarch64-linux-gnu-gcc \ CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-loongarch64 -cpu max -L /usr/loongarch64-linux-gnu" \ diff --git a/library/stdarch/ci/intrinsic-test.sh b/library/stdarch/ci/intrinsic-test.sh index 3966aa88d8f41..4b7a3d27ea4e2 100755 --- a/library/stdarch/ci/intrinsic-test.sh +++ b/library/stdarch/ci/intrinsic-test.sh @@ -65,6 +65,12 @@ case ${TARGET} in ARCH=x86 RUNTIME_RUSTFLAGS= ;; + + loongarch64*) + export CFLAGS="-I/usr/loongarch64-linux-gnu/include/" + ARCH=loongarch64 + RUNTIME_RUSTFLAGS=-Ctarget-feature=+lsx,+lasx,+frecipe + ;; *) ;; @@ -80,6 +86,14 @@ case "${TARGET}" in --target "${TARGET}" \ --cc-arg-style "${CC_ARG_STYLE}" ;; + loongarch64*) + cargo run "${INTRINSIC_TEST}" --release \ + --bin intrinsic-test -- crates/stdarch-gen-loongarch \ + --skip "crates/intrinsic-test/missing_${ARCH}_common.txt" \ + --skip "crates/intrinsic-test/missing_${ARCH}_${CC_KIND}.txt" \ + --target "${TARGET}" \ + --cc-arg-style "${CC_ARG_STYLE}" + ;; *) cargo run "${INTRINSIC_TEST}" --release \ --bin intrinsic-test -- intrinsics_data/arm_intrinsics.json \ diff --git a/library/stdarch/crates/intrinsic-test/missing_loongarch64_clang.txt b/library/stdarch/crates/intrinsic-test/missing_loongarch64_clang.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/library/stdarch/crates/intrinsic-test/missing_loongarch64_common.txt b/library/stdarch/crates/intrinsic-test/missing_loongarch64_common.txt new file mode 100644 index 0000000000000..5817247c2f4d2 --- /dev/null +++ b/library/stdarch/crates/intrinsic-test/missing_loongarch64_common.txt @@ -0,0 +1,31 @@ +# Missing in old c compiler +lasx_concat_128 +lasx_concat_128_d +lasx_concat_128_s +lasx_extract_128_hi +lasx_extract_128_lo +lasx_extract_128_hi_d +lasx_extract_128_lo_d +lasx_extract_128_hi_s +lasx_extract_128_lo_s +lasx_insert_128_hi +lasx_insert_128_lo +lasx_insert_128_hi_d +lasx_insert_128_lo_d +lasx_insert_128_hi_s +lasx_insert_128_lo_s + +# Missing in old qemu +lasx_xvfrecipe_d +lasx_xvfrecipe_s +lasx_xvfrsqrte_d +lasx_xvfrsqrte_s +lsx_vfrecipe_d +lsx_vfrecipe_s +lsx_vfrsqrte_d +lsx_vfrsqrte_s + +# Top bits are undefined, unclear how to test these +lasx_cast_128 +lasx_cast_128_d +lasx_cast_128_s diff --git a/library/stdarch/crates/intrinsic-test/missing_loongarch64_gcc.txt b/library/stdarch/crates/intrinsic-test/missing_loongarch64_gcc.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/library/stdarch/crates/intrinsic-test/src/loongarch/intrinsic.rs b/library/stdarch/crates/intrinsic-test/src/loongarch/intrinsic.rs new file mode 100644 index 0000000000000..f8ebb3e7ab34c --- /dev/null +++ b/library/stdarch/crates/intrinsic-test/src/loongarch/intrinsic.rs @@ -0,0 +1,19 @@ +use crate::common::intrinsic_helpers::IntrinsicType; +use std::ops::{Deref, DerefMut}; + +#[derive(Debug, Clone, PartialEq)] +pub struct LoongArchType(pub IntrinsicType); + +impl Deref for LoongArchType { + type Target = IntrinsicType; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for LoongArchType { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/library/stdarch/crates/intrinsic-test/src/loongarch/mod.rs b/library/stdarch/crates/intrinsic-test/src/loongarch/mod.rs new file mode 100644 index 0000000000000..00cff20a0f083 --- /dev/null +++ b/library/stdarch/crates/intrinsic-test/src/loongarch/mod.rs @@ -0,0 +1,139 @@ +mod intrinsic; +mod parser; +mod types; + +use std::path::Path; + +use crate::common::SupportedArchitecture; +use crate::common::cli::ProcessedCli; +use crate::common::intrinsic::Intrinsic; +use crate::common::intrinsic_helpers::TypeKind; +use intrinsic::LoongArchType; +use parser::get_intrinsics; + +#[derive(PartialEq)] +pub struct LoongArch { + intrinsics: Vec>, +} + +impl SupportedArchitecture for LoongArch { + type Type = LoongArchType; + + fn intrinsics(&self) -> &[Intrinsic] { + &self.intrinsics + } + + const NOTICE: &str = r#" +// This is a transient test file, not intended for distribution. Some aspects of the +// test are derived from LoongArch specification files, published under the same license as the +// `intrinsic-test` crate. +"#; + + const C_PRELUDE: &str = r#" +#include +#include +"#; + const RUST_PRELUDE: &str = RUST_PRELUDE; + + const C_NAME_PREFIX: &str = "__"; + + fn c_compiler_flags(&self, _cli_options: &ProcessedCli) -> Vec<&str> { + let mut flags = vec!["-mlsx"]; + if self + .intrinsics + .iter() + .any(|intrinsic| intrinsic.extension == "LASX") + { + flags.push("-mlasx"); + } + if self.intrinsics.iter().any(|intrinsic| { + intrinsic.name.contains("frecipe") || intrinsic.name.contains("frsqrte") + }) { + flags.push("-mfrecipe"); + } + flags + } + + fn create(cli_options: &ProcessedCli) -> Self { + let mut intrinsics = + load_intrinsics(&cli_options.filename).expect("Error parsing input file"); + + intrinsics.sort_by(|a, b| a.name.cmp(&b.name)); + intrinsics.dedup_by(|a, b| { + a.name == b.name && a.results == b.results && a.arguments == b.arguments + }); + + let intrinsics = intrinsics + .into_iter() + // Skip intrinsics that don't return a value. + .filter(|intrinsic| intrinsic.results.kind() != TypeKind::Void) + .filter(|intrinsic| !intrinsic.arguments.args.is_empty()) + // Skip pointers for now, we would probably need to look at the return + // type to work out how many elements we need to point to. + .filter(|intrinsic| !intrinsic.arguments.iter().any(|arg| arg.is_ptr())) + // Skip intrinsics from `--skip` + .filter(|intrinsic| !cli_options.skip.contains(&intrinsic.name)) + .collect::>(); + + let sample_percentage: usize = cli_options.sample_percentage as usize; + let sample_size = (intrinsics.len() * sample_percentage) / 100; + let intrinsics = intrinsics.into_iter().take(sample_size).collect(); + + Self { intrinsics } + } + + fn predicate_function(_: u32) -> String { + unimplemented!("no scalable vectors on LoongArch") + } +} + +fn load_intrinsics(path: &Path) -> Result>, Box> { + if path.is_dir() { + let mut intrinsics = Vec::new(); + for spec in ["lsx.spec", "lasx.spec"] { + let spec_path = path.join(spec); + if spec_path.exists() { + intrinsics.extend(get_intrinsics(&spec_path)?); + } + } + return Ok(intrinsics); + } + + get_intrinsics(path) +} + +const RUST_PRELUDE: &str = r#" +#![feature(stdarch_loongarch)] + +use core_arch::arch::loongarch64::*; + +#[inline] +unsafe fn lsx_vld_to_m128i(mem_addr: *const i8) -> m128i { + lsx_vld::<0>(mem_addr) +} + +#[inline] +unsafe fn lsx_vld_to_m128(mem_addr: *const i8) -> m128 { + core::mem::transmute(lsx_vld::<0>(mem_addr)) +} + +#[inline] +unsafe fn lsx_vld_to_m128d(mem_addr: *const i8) -> m128d { + core::mem::transmute(lsx_vld::<0>(mem_addr)) +} + +#[inline] +unsafe fn lasx_xvld_to_m256i(mem_addr: *const i8) -> m256i { + lasx_xvld::<0>(mem_addr) +} + +#[inline] +unsafe fn lasx_xvld_to_m256(mem_addr: *const i8) -> m256 { + core::mem::transmute(lasx_xvld::<0>(mem_addr)) +} + +#[inline] +unsafe fn lasx_xvld_to_m256d(mem_addr: *const i8) -> m256d { + core::mem::transmute(lasx_xvld::<0>(mem_addr)) +} +"#; diff --git a/library/stdarch/crates/intrinsic-test/src/loongarch/parser.rs b/library/stdarch/crates/intrinsic-test/src/loongarch/parser.rs new file mode 100644 index 0000000000000..ecb5d1ab3e8df --- /dev/null +++ b/library/stdarch/crates/intrinsic-test/src/loongarch/parser.rs @@ -0,0 +1,179 @@ +use std::path::Path; + +use super::intrinsic::LoongArchType; +use super::types::parse_intrinsic_type; +use crate::common::argument::{Argument, ArgumentList}; +use crate::common::constraint::Constraint; +use crate::common::intrinsic::Intrinsic; +use crate::loongarch::LoongArch; + +pub fn get_intrinsics( + filename: &Path, +) -> Result>, Box> { + parse_spec_file(filename) +} + +fn parse_spec_file( + filename: &Path, +) -> Result>, Box> { + let contents = std::fs::read_to_string(filename)?; + parse_spec_contents(&contents) +} + +fn parse_spec_contents( + contents: &str, +) -> Result>, Box> { + let mut intrinsics = Vec::new(); + let mut record = Vec::new(); + + for line in contents.lines().chain(std::iter::once("")) { + let line = line.trim(); + if line.is_empty() { + if !record.is_empty() { + if let Some(intrinsic) = parse_record(&record)? { + intrinsics.push(intrinsic); + } + record.clear(); + } + continue; + } + record.push(line); + } + + Ok(intrinsics) +} + +fn parse_record( + record: &[&str], +) -> Result>, Box> { + let mut name = None; + let mut asm_formats = Vec::new(); + let mut data_types = None; + + for line in record { + if let Some(value) = line.strip_prefix("name = ") { + name = Some(value.to_string()); + } else if let Some(value) = line.strip_prefix("asm-fmts = ") { + asm_formats = value + .split(',') + .map(|part| part.trim().to_string()) + .collect(); + } else if let Some(value) = line.strip_prefix("data-types = ") { + data_types = Some(value); + } + } + + let Some(data_types) = data_types else { + return Ok(None); + }; + + let name = name.ok_or("missing name before data-types")?; + Ok(Some(parse_intrinsic(&name, &asm_formats, data_types)?)) +} + +fn parse_intrinsic( + name: &str, + asm_formats: &[String], + data_types: &str, +) -> Result, Box> { + let data_types = data_types + .split(',') + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + .collect::>(); + let Some((result_type, argument_types)) = data_types.split_first() else { + return Err("missing data-types for intrinsic".into()); + }; + let result = LoongArchType(parse_intrinsic_type(result_type)?); + let asm_offset = asm_formats + .len() + .checked_sub(argument_types.len()) + .ok_or_else(|| format!("{name}: fewer asm formats than arguments"))?; + let arguments = argument_types + .iter() + .enumerate() + .map(|(pos, data_type)| { + let constraint = asm_formats + .get(pos + asm_offset) + .and_then(|format| parse_constraint(name, format)); + let mut ty = LoongArchType(parse_intrinsic_type(data_type)?); + if constraint.is_some() { + ty.constant = true; + } + Ok(Argument::new( + pos, + format!("arg_{pos}"), + ty, + constraint, + false, + )) + }) + .collect::, String>>()?; + + Ok(Intrinsic { + name: name.to_string(), + arguments: ArgumentList { args: arguments }, + results: result, + arch_tags: Vec::new(), + extension: extension_for(name)?.to_string(), + }) +} + +fn parse_constraint(name: &str, asm_format: &str) -> Option { + if let Some(cons) = special_constraint(name) { + return Some(cons); + } + if let Some(bits) = asm_format.strip_prefix("ui") { + return Some(unsigned_constraint(bits.parse::().ok()?)); + } + if let Some(bits) = asm_format + .strip_prefix("si") + .or_else(|| asm_format.strip_prefix('i')) + { + return Some(signed_constraint(bits.parse::().ok()?)); + } + None +} + +fn special_constraint(name: &str) -> Option { + match name { + "lsx_vldi" | "lasx_xvldi" => { + // CC: imm13 only support 0000 ~ 1100 in bits 9 ~ 12 when bit ‘13’ is 1 + let values: Vec = (0i64..8192i64) + .filter(|&x| x < 4096 || ((x >> 8) & 0xf) < 13) + // sign extend + .map(|x| ((x << 51) as i64) >> 51) + .collect(); + Some(Constraint::Set(values)) + } + _ => None, + } +} + +fn unsigned_constraint(bits: u32) -> Constraint { + Constraint::Range(0..(1i64 << bits.min(63))) +} + +fn signed_constraint(bits: u32) -> Constraint { + let min = if bits == 64 { + i64::MIN + } else { + -(1i64 << (bits - 1)) + }; + let max = if bits == 64 { + i64::MAX + } else { + 1i64 << (bits - 1) + }; + Constraint::Range(min..max) +} + +fn extension_for(name: &str) -> Result<&'static str, Box> { + if name.starts_with("lasx_") { + Ok("LASX") + } else if name.starts_with("lsx_") { + Ok("LSX") + } else { + Err(format!("unsupported LoongArch intrinsic name {name}").into()) + } +} diff --git a/library/stdarch/crates/intrinsic-test/src/loongarch/types.rs b/library/stdarch/crates/intrinsic-test/src/loongarch/types.rs new file mode 100644 index 0000000000000..dd4e578e67a41 --- /dev/null +++ b/library/stdarch/crates/intrinsic-test/src/loongarch/types.rs @@ -0,0 +1,309 @@ +use super::intrinsic::LoongArchType; +use crate::common::intrinsic_helpers::{IntrinsicType, Sign, SimdLen, TypeDefinition, TypeKind}; + +impl TypeDefinition for LoongArchType { + fn c_type(&self) -> String { + if self.ptr { + return if self.ptr_constant { + "const void*".to_string() + } else { + "void*".to_string() + }; + } + + match (self.kind(), self.simd_len) { + (_, Some(SimdLen::Fixed(lanes))) => { + format!( + "__{}", + vector_type_name(lanes, self.inner_size(), self.kind()) + ) + } + (TypeKind::Int(Sign::Signed), None) => { + scalar_type_name(true, scalar_signature_bits(self.inner_size())).to_string() + } + (TypeKind::Int(Sign::Unsigned), None) => { + scalar_type_name(false, scalar_signature_bits(self.inner_size())).to_string() + } + (TypeKind::Float, None) => match self.inner_size() { + 32 => "float".to_string(), + 64 => "double".to_string(), + bits => unreachable!("unsupported scalar float width {bits}"), + }, + (TypeKind::Void, None) => "void".to_string(), + _ => unreachable!("unsupported LoongArch type {self:#?}"), + } + } + + fn rust_type(&self) -> String { + if self.ptr { + return format!( + "*{} core::ffi::c_void", + if self.ptr_constant { "const" } else { "mut" }, + ); + } + + match (self.kind(), self.simd_len) { + (_, Some(SimdLen::Fixed(lanes))) => { + vector_type_name(lanes, self.inner_size(), self.kind()).to_string() + } + (TypeKind::Int(Sign::Signed), None) => { + semantic_rust_scalar_type(true, scalar_signature_bits(self.inner_size())) + .to_string() + } + (TypeKind::Int(Sign::Unsigned), None) => { + semantic_rust_scalar_type(false, scalar_signature_bits(self.inner_size())) + .to_string() + } + (TypeKind::Float, None) => match self.inner_size() { + 32 => "f32".to_string(), + 64 => "f64".to_string(), + bits => unreachable!("unsupported scalar float width {bits}"), + }, + (TypeKind::Void, None) => "()".to_string(), + _ => unreachable!("unsupported LoongArch type {self:#?}"), + } + } + + fn rust_scalar_type(&self) -> String { + match self.kind() { + TypeKind::Int(Sign::Signed) => { + semantic_rust_scalar_type(true, self.inner_size()).to_string() + } + TypeKind::Int(Sign::Unsigned) => { + semantic_rust_scalar_type(false, self.inner_size()).to_string() + } + TypeKind::Float => match self.inner_size() { + 32 => "f32".to_string(), + 64 => "f64".to_string(), + bits => unreachable!("unsupported scalar float width {bits}"), + }, + _ => unreachable!("unsupported LoongArch scalar type {self:#?}"), + } + } + + fn load_function(&self) -> String { + let Some(SimdLen::Fixed(lanes)) = self.simd_len else { + unreachable!("LoongArch loads are only used for SIMD types") + }; + + match (lanes * self.inner_size(), self.kind()) { + (128, TypeKind::Float) if self.inner_size() == 64 => "lsx_vld_to_m128d".to_string(), + (128, TypeKind::Float) => "lsx_vld_to_m128".to_string(), + (128, _) => "lsx_vld_to_m128i".to_string(), + (256, TypeKind::Float) if self.inner_size() == 64 => "lasx_xvld_to_m256d".to_string(), + (256, TypeKind::Float) => "lasx_xvld_to_m256".to_string(), + (256, _) => "lasx_xvld_to_m256i".to_string(), + bits => unreachable!("unsupported LoongArch vector width {bits:?}"), + } + } +} + +fn vector_type_name(lanes: u32, bit_len: u32, kind: TypeKind) -> &'static str { + match (lanes * bit_len, kind, bit_len) { + (128, TypeKind::Float, 32) => "m128", + (128, TypeKind::Float, 64) => "m128d", + (128, _, _) => "m128i", + (256, TypeKind::Float, 32) => "m256", + (256, TypeKind::Float, 64) => "m256d", + (256, _, _) => "m256i", + _ => unreachable!("unsupported LoongArch vector shape {kind:?}x{bit_len}x{lanes}"), + } +} + +fn scalar_type_name(signed: bool, bit_len: u32) -> &'static str { + match (signed, bit_len) { + (true, 32) => "int32_t", + (true, 64) => "int64_t", + (false, 32) => "uint32_t", + (false, 64) => "uint64_t", + _ => unreachable!("unsupported LoongArch scalar width {bit_len}"), + } +} + +fn scalar_signature_bits(bit_len: u32) -> u32 { + match bit_len { + 8 | 16 | 32 => 32, + 64 => 64, + _ => unreachable!("unsupported LoongArch scalar width {bit_len}"), + } +} + +fn semantic_rust_scalar_type(signed: bool, bit_len: u32) -> &'static str { + match (signed, bit_len) { + (true, 8) => "i8", + (true, 16) => "i16", + (true, 32) => "i32", + (true, 64) => "i64", + (false, 8) => "u8", + (false, 16) => "u16", + (false, 32) => "u32", + (false, 64) => "u64", + _ => unreachable!("unsupported LoongArch scalar width {bit_len}"), + } +} + +pub fn parse_intrinsic_type(s: &str) -> Result { + let (kind, bit_len, simd_len, ptr, ptr_constant) = match s { + "V16QI" => ( + TypeKind::Int(Sign::Signed), + Some(8), + Some(SimdLen::Fixed(16)), + false, + false, + ), + "V32QI" => ( + TypeKind::Int(Sign::Signed), + Some(8), + Some(SimdLen::Fixed(32)), + false, + false, + ), + "V8HI" => ( + TypeKind::Int(Sign::Signed), + Some(16), + Some(SimdLen::Fixed(8)), + false, + false, + ), + "V16HI" => ( + TypeKind::Int(Sign::Signed), + Some(16), + Some(SimdLen::Fixed(16)), + false, + false, + ), + "V4SI" => ( + TypeKind::Int(Sign::Signed), + Some(32), + Some(SimdLen::Fixed(4)), + false, + false, + ), + "V8SI" => ( + TypeKind::Int(Sign::Signed), + Some(32), + Some(SimdLen::Fixed(8)), + false, + false, + ), + "V2DI" => ( + TypeKind::Int(Sign::Signed), + Some(64), + Some(SimdLen::Fixed(2)), + false, + false, + ), + "V4DI" => ( + TypeKind::Int(Sign::Signed), + Some(64), + Some(SimdLen::Fixed(4)), + false, + false, + ), + "UV16QI" => ( + TypeKind::Int(Sign::Unsigned), + Some(8), + Some(SimdLen::Fixed(16)), + false, + false, + ), + "UV32QI" => ( + TypeKind::Int(Sign::Unsigned), + Some(8), + Some(SimdLen::Fixed(32)), + false, + false, + ), + "UV8HI" => ( + TypeKind::Int(Sign::Unsigned), + Some(16), + Some(SimdLen::Fixed(8)), + false, + false, + ), + "UV16HI" => ( + TypeKind::Int(Sign::Unsigned), + Some(16), + Some(SimdLen::Fixed(16)), + false, + false, + ), + "UV4SI" => ( + TypeKind::Int(Sign::Unsigned), + Some(32), + Some(SimdLen::Fixed(4)), + false, + false, + ), + "UV8SI" => ( + TypeKind::Int(Sign::Unsigned), + Some(32), + Some(SimdLen::Fixed(8)), + false, + false, + ), + "UV2DI" => ( + TypeKind::Int(Sign::Unsigned), + Some(64), + Some(SimdLen::Fixed(2)), + false, + false, + ), + "UV4DI" => ( + TypeKind::Int(Sign::Unsigned), + Some(64), + Some(SimdLen::Fixed(4)), + false, + false, + ), + "V4SF" => ( + TypeKind::Float, + Some(32), + Some(SimdLen::Fixed(4)), + false, + false, + ), + "V8SF" => ( + TypeKind::Float, + Some(32), + Some(SimdLen::Fixed(8)), + false, + false, + ), + "V2DF" => ( + TypeKind::Float, + Some(64), + Some(SimdLen::Fixed(2)), + false, + false, + ), + "V4DF" => ( + TypeKind::Float, + Some(64), + Some(SimdLen::Fixed(4)), + false, + false, + ), + "QI" => (TypeKind::Int(Sign::Signed), Some(8), None, false, false), + "HI" => (TypeKind::Int(Sign::Signed), Some(16), None, false, false), + "SI" => (TypeKind::Int(Sign::Signed), Some(32), None, false, false), + "UQI" => (TypeKind::Int(Sign::Unsigned), Some(8), None, false, false), + "UHI" => (TypeKind::Int(Sign::Unsigned), Some(16), None, false, false), + "USI" => (TypeKind::Int(Sign::Unsigned), Some(32), None, false, false), + "DI" => (TypeKind::Int(Sign::Signed), Some(64), None, false, false), + "UDI" => (TypeKind::Int(Sign::Unsigned), Some(64), None, false, false), + "CVPOINTER" => (TypeKind::Int(Sign::Signed), Some(8), None, true, true), + "VOID" => (TypeKind::Void, None, None, false, false), + _ => return Err(format!("unsupported LoongArch type {s}")), + }; + + Ok(IntrinsicType { + constant: false, + ptr_constant, + ptr, + kind, + bit_len, + simd_len, + vec_len: None, + }) +} diff --git a/library/stdarch/crates/intrinsic-test/src/main.rs b/library/stdarch/crates/intrinsic-test/src/main.rs index e25eb48a456d1..91e76d8f5fa50 100644 --- a/library/stdarch/crates/intrinsic-test/src/main.rs +++ b/library/stdarch/crates/intrinsic-test/src/main.rs @@ -3,11 +3,13 @@ extern crate log; mod arm; mod common; +mod loongarch; mod x86; use arm::Arm; use common::SupportedArchitecture; use common::cli::{Cli, ProcessedCli}; +use loongarch::LoongArch; use x86::X86; fn main() { @@ -21,6 +23,11 @@ fn main() { run(Arm::create(&processed_cli_options), processed_cli_options) } else if processed_cli_options.target.starts_with("x86") { run(X86::create(&processed_cli_options), processed_cli_options) + } else if processed_cli_options.target.starts_with("loongarch64") { + run( + LoongArch::create(&processed_cli_options), + processed_cli_options, + ) } else { unimplemented!("Unsupported target {}", processed_cli_options.target) } From a998f8ad99af7991547c2d399cc2f014f58cfdc7 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 17 Sep 2026 21:34:15 +0200 Subject: [PATCH 33/55] improve simd bitshift tests --- .../stdarch/crates/core_arch/src/x86/avx2.rs | 67 ++++++++++++++++--- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/x86/avx2.rs b/library/stdarch/crates/core_arch/src/x86/avx2.rs index 333f38e5474cc..7ad45421cf89f 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx2.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx2.rs @@ -5142,10 +5142,24 @@ mod tests { #[simd_test(enable = "avx2")] fn test_mm_sllv_epi64() { - let a = _mm_set1_epi64x(2); - let b = _mm_set1_epi64x(1); + let a = _mm_set_epi64x(2, 3); + let b = _mm_set_epi64x(1, 2); let r = _mm_sllv_epi64(a, b); - let e = _mm_set1_epi64x(4); + // Compare with the scalar version of the same computation. + let e = _mm_set_epi64x(2i64.unbounded_shl(1), 3i64.unbounded_shl(2)); + assert_eq_m128i(r, e); + // Compare with hardcoded output. + let e = _mm_set_epi64x(4, 12); + assert_eq_m128i(r, e); + + // The shift has unbounded semantics: if the shift amount + // is >= the number of bits the result is 0. + let a = _mm_set_epi64x(1, 2); + let b = _mm_set_epi64x(64, 65); + let r = _mm_sllv_epi64(a, b); + let e = _mm_set_epi64x(1i64.unbounded_shl(64), 2i64.unbounded_shl(65)); + assert_eq_m128i(r, e); + let e = _mm_set_epi64x(0, 0); assert_eq_m128i(r, e); } @@ -5192,10 +5206,27 @@ mod tests { #[simd_test(enable = "avx2")] fn test_mm_srav_epi32() { - let a = _mm_set1_epi32(4); - let count = _mm_set1_epi32(1); - let r = _mm_srav_epi32(a, count); - let e = _mm_set1_epi32(2); + let a = _mm_set_epi32(16, -32, 64, -128); + let b = _mm_set_epi32(4, 3, 2, 1); + let r = _mm_srav_epi32(a, b); + let e = _mm_set_epi32(1, -4, 16, -64); + assert_eq_m128i(r, e); + + // The shift has unbounded semantics: if the shift amount + // is >= the number of bits the result is -1. + let a = _mm_set_epi32(-16, -32, -64, -128); + let b = _mm_set_epi32(31, 32, 33, 0); + let r = _mm_srav_epi32(a, b); + // Compare with the scalar version of the same computation. + let e = _mm_set_epi32( + (-16i32).unbounded_shr(31), + (-32i32).unbounded_shr(32), + (-64i32).unbounded_shr(33), + (-128i32).unbounded_shr(0), + ); + assert_eq_m128i(r, e); + // Compare with hardcoded output. + let e = _mm_set_epi32(-1, -1, -1, -128); assert_eq_m128i(r, e); } @@ -5296,10 +5327,24 @@ mod tests { #[simd_test(enable = "avx2")] fn test_mm_srlv_epi64() { - let a = _mm_set1_epi64x(2); - let count = _mm_set1_epi64x(1); - let r = _mm_srlv_epi64(a, count); - let e = _mm_set1_epi64x(1); + let a = _mm_set_epi64x(4, 8); + let b = _mm_set_epi64x(2, 1); + let r = _mm_srlv_epi64(a, b); + // Compare with the scalar version of the same computation. + let e = _mm_set_epi64x(4i64.unbounded_shr(2), 8i64.unbounded_shr(1)); + assert_eq_m128i(r, e); + // Compare with hardcoded output. + let e = _mm_set_epi64x(1, 4); + assert_eq_m128i(r, e); + + // The shift has unbounded semantics: if the shift amount + // is >= the number of bits the result is 0. + let a = _mm_set_epi64x(i64::MAX, i64::MAX); + let b = _mm_set_epi64x(64, 65); + let r = _mm_sllv_epi64(a, b); + let e = _mm_set_epi64x(i64::MAX.unbounded_shr(64), i64::MAX.unbounded_shr(65)); + assert_eq_m128i(r, e); + let e = _mm_set_epi64x(0, 0); assert_eq_m128i(r, e); } From 74a45daaa7cd974bd44fb09b1e00dd2b306967b8 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 18 Sep 2026 20:51:24 +0200 Subject: [PATCH 34/55] Merge commit '39f626bc72757517b75d8e2dde50fa832b8d6c03' --- .cspell.json | 28 -- .github/workflows/ci.yml | 18 +- .github/workflows/failures.yml | 18 +- .gitignore | 3 +- Cargo.lock | 8 +- Cargo.toml | 4 +- Readme.md | 2 +- build_system/src/build.rs | 51 +- build_system/src/main.rs | 5 +- build_system/src/test.rs | 614 +++++++++++++++++------ clippy.toml | 3 + doc/tips.md | 2 +- libgccjit.version | 2 +- rust-toolchain | 2 +- src/abi.rs | 2 +- src/asm.rs | 1 + src/attributes.rs | 12 + src/back/lto.rs | 3 +- src/base.rs | 88 +++- src/builder.rs | 208 +++++--- src/common.rs | 113 +++-- src/consts.rs | 219 +++++--- src/context.rs | 66 ++- src/declare.rs | 6 +- src/gcc_util.rs | 168 ++++++- src/int.rs | 62 ++- src/intrinsic/llvm.rs | 13 +- src/intrinsic/mod.rs | 26 +- src/intrinsic/simd.rs | 88 +++- src/lib.rs | 52 +- src/mono_item.rs | 161 +++++- src/type_.rs | 106 +++- src/type_of.rs | 17 +- tests/asm/bulk_memory_alignment.rs | 45 ++ tests/asm/volatile_bulk_memory.rs | 37 ++ tests/c/import_linkage.c | 17 + tests/c/overaligned_byval_abi.c | 54 ++ tests/c/static_linkage.c | 37 ++ tests/c/weak_function_linkage.c | 54 ++ tests/compile/asm_noreturn_call.rs | 15 + tests/compile/recursive_types.rs | 22 + tests/failing-ice-tests.txt | 17 +- tests/failing-lto-tests.txt | 9 +- tests/failing-run-make-tests.txt | 3 +- tests/failing-ui-tests.txt | 100 +--- tests/lang_tests.rs | 102 +++- tests/run/catch_unwind.rs | 25 + tests/run/custom_abort.rs | 27 + tests/run/import_linkage.rs | 84 ++++ tests/run/nonzero_div_ceil.rs | 17 + tests/run/overaligned_byval_abi.rs | 89 ++++ tests/run/overaligned_byval_arg.rs | 41 ++ tests/run/ptr_to_int_div.rs | 19 + tests/run/simd.rs | 81 +++ tests/run/static_alloc_shapes.rs | 50 ++ tests/run/static_linkage.rs | 79 +++ tests/run/weak_function_linkage.rs | 106 ++++ tools/cspell_dicts/rust.txt | 2 - tools/cspell_dicts/rustc_codegen_gcc.txt | 78 --- 59 files changed, 2635 insertions(+), 746 deletions(-) delete mode 100644 .cspell.json create mode 100644 clippy.toml create mode 100644 tests/asm/bulk_memory_alignment.rs create mode 100644 tests/asm/volatile_bulk_memory.rs create mode 100644 tests/c/import_linkage.c create mode 100644 tests/c/overaligned_byval_abi.c create mode 100644 tests/c/static_linkage.c create mode 100644 tests/c/weak_function_linkage.c create mode 100644 tests/compile/asm_noreturn_call.rs create mode 100644 tests/compile/recursive_types.rs create mode 100644 tests/run/catch_unwind.rs create mode 100644 tests/run/custom_abort.rs create mode 100644 tests/run/import_linkage.rs create mode 100644 tests/run/nonzero_div_ceil.rs create mode 100644 tests/run/overaligned_byval_abi.rs create mode 100644 tests/run/overaligned_byval_arg.rs create mode 100644 tests/run/ptr_to_int_div.rs create mode 100644 tests/run/simd.rs create mode 100644 tests/run/static_alloc_shapes.rs create mode 100644 tests/run/static_linkage.rs create mode 100644 tests/run/weak_function_linkage.rs delete mode 100644 tools/cspell_dicts/rust.txt delete mode 100644 tools/cspell_dicts/rustc_codegen_gcc.txt diff --git a/.cspell.json b/.cspell.json deleted file mode 100644 index 556432d69a41b..0000000000000 --- a/.cspell.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "allowCompoundWords": true, - "dictionaries": ["cpp", "rust-extra", "rustc_codegen_gcc"], - "dictionaryDefinitions": [ - { - "name": "rust-extra", - "path": "tools/cspell_dicts/rust.txt", - "addWords": true - }, - { - "name": "rustc_codegen_gcc", - "path": "tools/cspell_dicts/rustc_codegen_gcc.txt", - "addWords": true - } - ], - "files": [ - "src/**/*.rs" - ], - "ignorePaths": [ - "src/intrinsic/archs.rs", - "src/intrinsic/old_archs.rs", - "src/intrinsic/llvm.rs" - ], - "ignoreRegExpList": [ - "/(FIXME|NOTE|TODO)\\([^)]+\\)/", - "__builtin_\\w*" - ] -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa9535a3729c3..8888fdc3ee093 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,11 +31,13 @@ jobs: # "--asm-tests", "--test-libcore", "--extended-rand-tests", - "--extended-regex-example-tests", + "--extended-regex-example-tests --test-libcore-doctests", "--extended-regex-tests", "--test-successful-rustc --nb-parts 2 --current-part 0", "--test-successful-rustc --nb-parts 2 --current-part 1", - "--projects", + "--projects --nb-parts 2 --current-part 0", + "--projects --nb-parts 2 --current-part 1", + "--gcc-asm-tests --test-release-libcore", ] steps: @@ -52,8 +54,9 @@ jobs: # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests. run: sudo apt-get install ninja-build ripgrep llvm-14-tools llvm - - name: Install rustfmt & clippy - run: rustup component add rustfmt clippy + - name: Install the libraries needed to build librsvg + if: ${{ contains(matrix.commands, '--projects') }} + run: sudo apt-get install libcairo2-dev libpango1.0-dev libfontconfig1-dev libfreetype-dev libharfbuzz-dev libxml2-dev libglib2.0-dev - name: Download artifact run: curl -LO https://github.com/rust-lang/gcc/releases/latest/download/${{ matrix.libgccjit_version.gcc }} @@ -127,13 +130,6 @@ jobs: - uses: actions/checkout@v4 - run: python tools/check_intrinsics_duplicates.py - spell_check: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1.32.0 - - uses: streetsidesoftware/cspell-action@v7 - build_system: runs-on: ubuntu-24.04 steps: diff --git a/.github/workflows/failures.yml b/.github/workflows/failures.yml index 2c9e4950706b2..52e96726129b3 100644 --- a/.github/workflows/failures.yml +++ b/.github/workflows/failures.yml @@ -98,7 +98,14 @@ jobs: if: matrix.libgccjit_version.gcc != 'libgccjit12.so' id: tests run: | - ${{ matrix.libgccjit_version.env_extra }} ./y.sh test --release --clean --build-sysroot --test-failing-rustc ${{ matrix.libgccjit_version.extra }} 2>&1 | tee output_log + # Without this, `tee` masks the exit status of `y.sh test`. + set -o pipefail + status=0 + ${{ matrix.libgccjit_version.env_extra }} ./y.sh test --release --clean --build-sysroot --test-failing-rustc ${{ matrix.libgccjit_version.extra }} 2>&1 | tee output_log || status=$? + # This suite runs the tests known to fail, so only a build system error must fail the job. + if [ "$status" -ne 0 ] && [ "$status" -ne 2 ]; then + exit "$status" + fi rg --text "test result" output_log >> $GITHUB_STEP_SUMMARY - name: Run failing ui pattern tests for ICE @@ -106,7 +113,14 @@ jobs: if: matrix.libgccjit_version.gcc != 'libgccjit12.so' id: ui-tests run: | - ${{ matrix.libgccjit_version.env_extra }} ./y.sh test --release --test-failing-ui-pattern-tests ${{ matrix.libgccjit_version.extra }} 2>&1 | tee output_log_ui + # Without this, `tee` masks the exit status of `y.sh test`. + set -o pipefail + status=0 + ${{ matrix.libgccjit_version.env_extra }} ./y.sh test --release --test-failing-ui-pattern-tests ${{ matrix.libgccjit_version.extra }} 2>&1 | tee output_log_ui || status=$? + # This suite runs tests that fail, so only a build system error must fail the job here. + if [ "$status" -ne 0 ] && [ "$status" -ne 2 ]; then + exit "$status" + fi if grep -q "the compiler unexpectedly panicked" output_log_ui; then echo "Error: 'the compiler unexpectedly panicked' found in output logs. CI Error!!" exit 1 diff --git a/.gitignore b/.gitignore index 8f73d3eb972a0..2a8fdcda0b483 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ llvm build_system/target config.toml build -rustlantis \ No newline at end of file +rustlantis +stuff/ diff --git a/Cargo.lock b/Cargo.lock index 7ce94b58c0516..c7e979a1112e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,18 +56,18 @@ dependencies = [ [[package]] name = "gccjit" -version = "3.3.0" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73d18b642ce16378af78f89664841d7eeafa113682ff5d14573424eb0232a" +checksum = "6d85b5754389edaad832ba320709a25086b3081a8c6c0fab2322965e5fb512b3" dependencies = [ "gccjit_sys", ] [[package]] name = "gccjit_sys" -version = "1.3.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee689456c013616942d5aef9a84d613cefcc3b335340d036f3650fc1a7459e15" +checksum = "e081669728b490723537f9def7eb674b7c9acd8de0b92ad4f4abf5f5cc75ea4b" dependencies = [ "libc", ] diff --git a/Cargo.toml b/Cargo.toml index ac5e94b9454e2..02be6d56c2310 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,11 +20,11 @@ default = ["master"] [dependencies] object = { version = "0.39.0", default-features = false, features = ["std", "read"] } tempfile = "3.20" -gccjit = { version = "3.3.0", features = ["dlopen"] } +gccjit = { version = "6.1.0", features = ["dlopen"] } #gccjit = { git = "https://github.com/rust-lang/gccjit.rs", branch = "error-dlopen", features = ["dlopen"] } # Local copy. -#gccjit = { path = "../gccjit.rs", features = ["dlopen"] } +# gccjit = { path = "../gccjit.rs", features = ["dlopen"] } [dev-dependencies] boml = "0.3.1" diff --git a/Readme.md b/Readme.md index ce5ee1e4adee6..6b1f90b918855 100644 --- a/Readme.md +++ b/Readme.md @@ -176,7 +176,7 @@ $ LIBRARY_PATH="[gcc-path value]" LD_LIBRARY_PATH="[gcc-path value]" rustc +$(ca More specific documentation is available in the [`doc`](./doc) folder: * [Common errors](./doc/errors.md) - * [Debugging GCC LTO](./doc/debugging-gcc-lto.md) + * [Debugging](./doc/debugging.md) * [Debugging libgccjit](./doc/debugging-libgccjit.md) * [Git subtree sync](./doc/subtree.md) * [List of useful commands](./doc/tips.md) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 839c762fed742..2f2900af5c88a 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -132,6 +132,24 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu // Builds libs let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + + // Record the sysroot sources under the path the `rust-src` component uses, which is where + // rustc looks for them to turn a sysroot span into `/rustc/$hash`. Without this, ui tests + // print the build path where they expect `$SRC_DIR`. + let sysroot_source_dir = lib_path.join("rustlib/src/rust/library"); + rustflags.push_str(&format!( + " --remap-path-prefix={library_dir}={sysroot_source_dir}", + library_dir = std::path::absolute(&library_dir) + .map_err(|error| format!( + "Failed to get the absolute path of the sysroot sources: {error:?}" + ))? + .display(), + sysroot_source_dir = std::path::absolute(&sysroot_source_dir) + .map_err(|error| format!( + "Failed to get the absolute path of the sysroot sources: {error:?}" + ))? + .display(), + )); if config.sysroot_panic_abort { rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests"); } @@ -188,12 +206,33 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu // FIXME: should not use shell command! run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ()) }; - walk_dir( - library_dir.join(format!("target/{}/{}/deps", config.target_triple, channel)), - &mut copier.clone(), - &mut copier, - false, - )?; + let target_dir = library_dir.join(format!("target/{}/{}", config.target_triple, channel)); + let deps_dir = target_dir.join("deps"); + if deps_dir.is_dir() { + // Keep copying in the old directory just in case. + walk_dir(&deps_dir, &mut copier.clone(), &mut copier, false)?; + } else { + let build_dir = target_dir.join("build"); + walk_dir( + &build_dir, + &mut |package_dir: &Path| { + walk_dir( + package_dir, + &mut |unit_dir: &Path| { + let out_dir = unit_dir.join("out"); + if out_dir.is_dir() { + walk_dir(&out_dir, &mut copier.clone(), &mut copier.clone(), false)?; + } + Ok(()) + }, + &mut |_| Ok(()), + false, + ) + }, + &mut |_| Ok(()), + false, + )?; + } // Copy the source files to the sysroot (Rust for Linux needs this). let sysroot_src_path = start_dir.join("sysroot/lib/rustlib/src/rust"); diff --git a/build_system/src/main.rs b/build_system/src/main.rs index ae975c94fff25..d0b9811ac8ceb 100644 --- a/build_system/src/main.rs +++ b/build_system/src/main.rs @@ -108,6 +108,9 @@ fn main() { Command::AbiTest => abi_test::run(), } { eprintln!("Command failed to run: {e}"); - process::exit(1); + // CI needs to tell a build system error apart from the test failures some suites expect. + let exit_code = + if e == test::TESTS_FAILED_ERROR { test::TESTS_FAILED_EXIT_CODE } else { 1 }; + process::exit(exit_code); } } diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 2475a3a6a7155..a4bdb80c94f28 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1,10 +1,12 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ffi::OsStr; -use std::fs::{File, remove_dir_all}; +use std::fs::{File, read_to_string, remove_dir_all}; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use std::str::FromStr; +use boml::Toml; + use crate::build; use crate::config::{Channel, ConfigInfo}; use crate::utils::{ @@ -13,6 +15,15 @@ use crate::utils::{ split_args, walk_dir, }; +/// Exit code of `y.sh test` when the tests ran and reported failures, as opposed to the build +/// system failing to run them at all. CI relies on the distinction: the suites of known-failing +/// tests are expected to report failures, but a broken build system must never pass silently. +pub const TESTS_FAILED_EXIT_CODE: i32 = 2; + +/// The error returned for that case. `main` compares against it to pick the exit code, so no other +/// error may use this message. +pub const TESTS_FAILED_ERROR: &str = "the test suite reported failures"; + type Env = HashMap; type Runner = fn(&Env, &TestArg) -> Result<(), String>; type Runners = HashMap<&'static str, (&'static str, Runner)>; @@ -30,6 +41,9 @@ fn get_runners() -> Runners { runners.insert("--test-failing-rustc", ("Run failing rustc tests", test_failing_rustc)); runners.insert("--projects", ("Run the tests of popular crates", test_projects)); runners.insert("--test-libcore", ("Run libcore tests", test_libcore)); + runners.insert("--test-release-libcore", ("Run libcore tests", test_release_libcore)); + runners.insert("--test-libcore-doctests", ("Run libcore doc-tests", test_libcore_doctests)); + runners.insert("--alloc-tests", ("Run alloc tests", test_alloc)); runners.insert("--clean", ("Empty cargo target directory", clean)); runners.insert("--build-sysroot", ("Build sysroot", build_sysroot)); runners.insert("--std-tests", ("Run std tests", std_tests)); @@ -700,40 +714,109 @@ where // echo "[BUILD] sysroot in release mode" // ./build_sysroot/build_sysroot.sh --release +struct Project { + url: &'static str, + /// Arguments added to both the `cargo build` and the `cargo test` invocations. + cargo_arguments: &'static [&'static str], + /// Arguments forwarded to the test harness by `cargo test`. + test_harness_arguments: &'static [&'static str], + /// Variables added to the environment of both invocations. + environment_variables: &'static [(&'static str, &'static str)], +} + +impl Project { + const fn new(url: &'static str) -> Self { + Self { url, cargo_arguments: &[], test_harness_arguments: &[], environment_variables: &[] } + } + + const fn cargo_arguments(mut self, arguments: &'static [&'static str]) -> Self { + self.cargo_arguments = arguments; + self + } + + const fn test_harness_arguments(mut self, arguments: &'static [&'static str]) -> Self { + self.test_harness_arguments = arguments; + self + } + + const fn environment_variables( + mut self, + variables: &'static [(&'static str, &'static str)], + ) -> Self { + self.environment_variables = variables; + self + } +} + fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> { let projects = [ - //"https://gitlab.gnome.org/GNOME/librsvg", // FIXME: doesn't compile in the CI since the - // version of cairo and other libraries is too old. - "https://github.com/rust-random/getrandom", - "https://github.com/BurntSushi/memchr", - "https://github.com/dtolnay/itoa", - "https://github.com/rust-lang/cfg-if", - //"https://github.com/rust-lang-nursery/lazy-static.rs", // FIXME: re-enable when the - //failing test is fixed upstream. - //"https://github.com/marshallpierce/rust-base64", // FIXME: one test is OOM-killed. - // FIXME: ignore the base64 test that is OOM-killed. - //"https://github.com/time-rs/time", // FIXME: one test fails (https://github.com/time-rs/time/issues/719). - "https://github.com/rust-lang/log", - "https://github.com/bitflags/bitflags", - //"https://github.com/serde-rs/serde", // FIXME: one test fails. - //"https://github.com/rayon-rs/rayon", // FIXME: very slow, only run on master? - //"https://github.com/rust-lang/cargo", // FIXME: very slow, only run on master? + // The reference images assume the exact cairo, pango and freetype that librsvg pins in its + // own CI; this one renders text decorations a pixel off with the versions Ubuntu ships. + Project::new("https://gitlab.gnome.org/GNOME/librsvg") + .test_harness_arguments(&["--skip", "tests::svg1_1_text_text_03_b_svg", "--exact"]) + // A debug build of librsvg needs about 5 MB of stack per `cargo test` thread to reach + // its maximum layer nesting depth; librsvg's own CI sets the same value. + .environment_variables(&[("RUST_MIN_STACK", "8388608")]), + Project::new("https://github.com/rust-random/getrandom"), + Project::new("https://github.com/BurntSushi/memchr"), + Project::new("https://github.com/dtolnay/itoa"), + Project::new("https://github.com/rust-lang/cfg-if"), + // The `ui` test compares against the diagnostics of the compiler it was blessed with, so it + // fails on the nightly we use no matter which backend produces the code. + Project::new("https://github.com/rust-lang-nursery/lazy-static.rs") + .test_harness_arguments(&["--skip", "ui", "--exact"]), + Project::new("https://github.com/marshallpierce/rust-base64"), + // The test suite refuses to build unless every feature is enabled; it otherwise spawns a + // nested `cargo test --all-features` which would not use this backend. + Project::new("https://github.com/time-rs/time").cargo_arguments(&["--all-features"]), + Project::new("https://github.com/rust-lang/log"), + Project::new("https://github.com/bitflags/bitflags"), + Project::new("https://github.com/serde-rs/serde"), + Project::new("https://github.com/rayon-rs/rayon"), + // FIXME: too slow to run in the CI: the release build alone takes 46 minutes and the + // `cargo` crate itself needs 5.4 GB of memory in a single rustc process. + //Project::new("https://github.com/rust-lang/cargo"), ]; let mut env = env.clone(); let rustflags = format!("{} --cap-lints allow", env.get("RUSTFLAGS").cloned().unwrap_or_default()); env.insert("RUSTFLAGS".to_string(), rustflags); - let run_tests = |projects_path, iter: &mut dyn Iterator| -> Result<(), String> { - for project in iter { - let clone_result = git_clone_root_dir(project, projects_path, true)?; - let repo_path = Path::new(&clone_result.repo_dir); - run_cargo_command(&[&"build", &"--release"], Some(repo_path), &env, args)?; - run_cargo_command(&[&"test"], Some(repo_path), &env, args)?; - } + let run_tests = + |projects_path, iter: &mut dyn Iterator| -> Result<(), String> { + for project in iter { + let clone_result = git_clone_root_dir(project.url, projects_path, true)?; + let repo_path = Path::new(&clone_result.repo_dir); + + let mut project_environment = env.clone(); + for (name, value) in project.environment_variables { + project_environment.insert(name.to_string(), value.to_string()); + } - Ok(()) - }; + let mut build_command: Vec<&dyn AsRef> = vec![&"build", &"--release"]; + build_command.extend( + project.cargo_arguments.iter().map(|argument| argument as &dyn AsRef), + ); + run_cargo_command(&build_command, Some(repo_path), &project_environment, args)?; + + let mut test_command: Vec<&dyn AsRef> = vec![&"test"]; + test_command.extend( + project.cargo_arguments.iter().map(|argument| argument as &dyn AsRef), + ); + if !project.test_harness_arguments.is_empty() { + test_command.push(&"--"); + test_command.extend( + project + .test_harness_arguments + .iter() + .map(|argument| argument as &dyn AsRef), + ); + } + run_cargo_command(&test_command, Some(repo_path), &project_environment, args)?; + } + + Ok(()) + }; let projects_path = Path::new("projects"); create_dir(projects_path)?; @@ -755,12 +838,86 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> { } fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> { + test_libcore_inner(env, args, false) +} + +fn test_release_libcore(env: &Env, args: &TestArg) -> Result<(), String> { + test_libcore_inner(env, args, true) +} + +fn test_libcore_inner(env: &Env, args: &TestArg, release: bool) -> Result<(), String> { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] libcore"); let path = get_sysroot_dir().join("sysroot_src/library/coretests"); let _ = remove_dir_all(path.join("target")); - // FIXME(antoyo): run in release mode when we fix the failures. - run_cargo_command(&[&"test"], Some(&path), env, args)?; + let mut command: Vec<&dyn AsRef> = vec![&"test"]; + if release { + command.push(&"--release"); + } + run_cargo_command(&command, Some(&path), env, args)?; + Ok(()) +} + +/// Returns the edition declared in the manifest of the given library crate, so that the doctests +/// are run with the same edition as the crate they are extracted from. +fn get_crate_edition(crate_dir: &Path) -> Result { + let manifest_path = crate_dir.join("Cargo.toml"); + let content = read_to_string(&manifest_path) + .map_err(|error| format!("Failed to read `{}`: {error:?}", manifest_path.display()))?; + let manifest = Toml::parse(&content) + .map_err(|error| format!("Failed to parse `{}`: {error:?}", manifest_path.display()))?; + manifest + .get_table("package") + .and_then(|package| package.get_string("edition")) + .map(|edition| edition.to_string()) + .map_err(|error| { + format!("Failed to get `package.edition` from `{}`: {error:?}", manifest_path.display()) + }) +} + +fn test_libcore_doctests(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] libcore doctests"); + + let library_dir = get_sysroot_dir().join("sysroot_src/library"); + let edition = get_crate_edition(&library_dir.join("core"))?; + // `rustdoc` is called directly instead of through `cargo test --doc` because `cargo` builds its + // own `core` and passes it with `--extern`, which then conflicts with the `core` of the sysroot + // the doctests are linked against ("duplicate lang item" errors). + let toolchain = get_toolchain()?; + let toolchain_arg = format!("+{toolchain}"); + let rustflags = split_args(&env.get("RUSTFLAGS").cloned().unwrap_or_default())?; + // `-Zunstable-options` is needed for `--test-args`. + let mut command: Vec<&dyn AsRef> = vec![ + &"rustdoc", + &toolchain_arg, + &"--test", + &"core/src/lib.rs", + &"--crate-name", + &"core", + &"--crate-type", + &"lib", + &"--edition", + &edition, + &"-Zunstable-options", + // FIXME: remove `-Zforce-unstable-if-unmarked` once the doctest of + // `core::io::ErrorKind`'s `Display` impl declares `#![feature(core_io)]` upstream: without + // it, that doctest fails to compile with `E0658` on any backend. + &"-Zforce-unstable-if-unmarked", + // FIXME: one test cannot compile due to an upstream bug in the new trait solver. + &"-Znext-solver=coherence", + ]; + for flag in &rustflags { + command.push(flag); + } + // Additional arguments are forwarded to the test harness, so that a subset of the doctests can + // be run. + let test_args = + args.test_args.iter().map(|test_arg| format!("--test-args={test_arg}")).collect::>(); + for test_arg in &test_args { + command.push(test_arg); + } + run_command_with_output_and_env(&command, Some(&library_dir), Some(env))?; Ok(()) } @@ -934,10 +1091,7 @@ fn contains_ui_error_patterns(file_path: &Path, keep_lto_tests: bool) -> Result< eprintln!("nothing found for {file_path:?}"); } // The files in this directory contain errors. - if file_path.contains("/error-emitter/") { - return Ok(true); - } - Ok(false) + Ok(file_path.contains("/error-emitter/")) } // # Parameters @@ -947,6 +1101,8 @@ fn contains_ui_error_patterns(file_path: &Path, keep_lto_tests: bool) -> Result< // * `prepare_files_callback`: A callback function that prepares the files needed for the test. Its used to remove/retain tests giving Error to run various rust test suits. // * `run_error_pattern_test`: A boolean that determines whether to run only error pattern tests. // * `test_type`: A string that indicates the type of the test being run. +// * `retained_tests_list_path`: The list of tests that `prepare_files_callback` retained, if any. +// It is checked against the tests remaining after the filtering to report dead lines. // fn test_rustc_inner( env: &Env, @@ -954,7 +1110,7 @@ fn test_rustc_inner( prepare_files_callback: F, run_error_pattern_test: bool, test_type: &str, - run_ignored_tests: bool, + retained_tests_list_path: Option<&str>, ) -> Result<(), String> where F: Fn(&Path) -> Result, @@ -970,74 +1126,57 @@ where } if test_type == "ui" { - if run_error_pattern_test { - // After we removed the error tests that are known to panic with rustc_codegen_gcc, we now remove the passing tests since this runs the error tests. - walk_dir( - rust_path.join("tests/ui"), - &mut |_dir| Ok(()), - &mut |file_path| { - if contains_ui_error_patterns(file_path, args.keep_lto_tests)? { - Ok(()) - } else { - remove_file(file_path).map_err(|e| e.to_string()) - } - }, - true, - )?; - } else { - walk_dir( - rust_path.join("tests/ui"), - &mut |dir| { - let dir_name = dir.file_name().and_then(|name| name.to_str()).unwrap_or(""); - if ["abi", "extern", "proc-macro", "threads-sendsync"].contains(&dir_name) { - remove_dir_all(dir).map_err(|error| { - format!("Failed to remove folder `{}`: {:?}", dir.display(), error) - })?; - } - Ok(()) - }, - &mut |_| Ok(()), - false, - )?; - - // These two functions are used to remove files that are known to not be working currently - // with the GCC backend to reduce noise. - fn dir_handling(keep_lto_tests: bool) -> impl Fn(&Path) -> Result<(), String> { - move |dir| { - if dir.file_name().map(|name| name == "auxiliary").unwrap_or(true) { - return Ok(()); - } - - walk_dir( - dir, - &mut dir_handling(keep_lto_tests), - &mut file_handling(keep_lto_tests), - false, - ) + // Each mode runs one half of the ui tests and removes the other: `run_error_pattern_test` + // runs the tests expected to error, the other mode runs the rest. Only `.rs` files outside + // `auxiliary` are tests, so the expected output and the auxiliary crates are left alone. + fn dir_handling( + keep_lto_tests: bool, + remove_error_pattern_tests: bool, + ) -> impl Fn(&Path) -> Result<(), String> { + move |dir| { + if dir.file_name().map(|name| name == "auxiliary").unwrap_or(true) { + return Ok(()); } + + walk_dir( + dir, + &mut dir_handling(keep_lto_tests, remove_error_pattern_tests), + &mut file_handling(keep_lto_tests, remove_error_pattern_tests), + false, + ) } + } - fn file_handling(keep_lto_tests: bool) -> impl Fn(&Path) -> Result<(), String> { - move |file_path| { - if !file_path.extension().map(|extension| extension == "rs").unwrap_or(false) { - return Ok(()); - } - let path_str = file_path.display().to_string().replace("\\", "/"); - if valid_ui_error_pattern_test(&path_str) { - return Ok(()); - } else if contains_ui_error_patterns(file_path, keep_lto_tests)? { - return remove_file(&file_path); - } - Ok(()) + fn file_handling( + keep_lto_tests: bool, + remove_error_pattern_tests: bool, + ) -> impl Fn(&Path) -> Result<(), String> { + move |file_path| { + if !file_path.extension().map(|extension| extension == "rs").unwrap_or(false) { + return Ok(()); + } + let path_str = file_path.display().to_string().replace("\\", "/"); + if valid_ui_error_pattern_test(&path_str) { + return Ok(()); } + if contains_ui_error_patterns(file_path, keep_lto_tests)? + == remove_error_pattern_tests + { + return remove_file(file_path); + } + Ok(()) } + } - walk_dir( - rust_path.join("tests/ui"), - &mut dir_handling(args.keep_lto_tests), - &mut file_handling(args.keep_lto_tests), - false, - )?; + let remove_error_pattern_tests = !run_error_pattern_test; + walk_dir( + rust_path.join("tests/ui"), + &mut dir_handling(args.keep_lto_tests, remove_error_pattern_tests), + &mut file_handling(args.keep_lto_tests, remove_error_pattern_tests), + false, + )?; + if let Some(retained_tests_list_path) = retained_tests_list_path { + check_for_dead_listed_tests(&rust_path, retained_tests_list_path)?; } let nb_parts = args.nb_parts.unwrap_or(0); if nb_parts > 0 { @@ -1097,7 +1236,7 @@ where env.get_mut("RUSTFLAGS").unwrap().clear(); let test_dir = format!("tests/{test_type}"); - let mut command: Vec<&dyn AsRef> = vec![ + let command: Vec<&dyn AsRef> = vec![ &"./x.py", &"test", &"--run", @@ -1112,19 +1251,85 @@ where &"--bypass-ignore-backends", ]; - if run_ignored_tests { - command.push(&"--"); - command.push(&"--ignored"); + run_test_command(&command, &rust_path, &env) +} + +/// Reads the list of tests at `list_path`, checking that each of them still exists in the rust +/// checkout at `rust_path` and that none is listed twice. +/// +/// Both problems make a line a no-op: the test it names is neither kept nor removed, so the test +/// suite silently drifts away from what the list claims to describe. +fn read_test_list(rust_path: &Path, list_path: &str) -> Result, String> { + let content = std::fs::read_to_string(list_path) + .map_err(|error| format!("Failed to read `{list_path}`: {error:?}"))?; + + let mut tests = Vec::new(); + let mut seen = HashSet::new(); + let mut missing = Vec::new(); + let mut duplicated = Vec::new(); + + for line in content.lines().map(|line| line.trim()).filter(|line| !line.is_empty()) { + if !seen.insert(line) { + duplicated.push(line); + continue; + } + if !rust_path.join(line.trim_end_matches('/')).exists() { + missing.push(line); + } + tests.push(line.to_string()); } - run_command_with_output_and_env(&command, Some(&rust_path), Some(&env))?; - Ok(()) + if missing.is_empty() && duplicated.is_empty() { + return Ok(tests); + } + + let mut error = format!("`{list_path}` is out of date:\n"); + if !missing.is_empty() { + error.push_str(&format!( + "\nThese tests no longer exist in `{rust_path}`:\n{missing}\n", + rust_path = rust_path.display(), + missing = missing.join("\n"), + )); + } + if !duplicated.is_empty() { + error.push_str(&format!( + "\nThese tests are listed more than once:\n{}\n", + duplicated.join("\n") + )); + } + error.push_str( + "\nEvery line must name a test that exists, exactly once, otherwise the line filters \ + nothing. Delete the stale lines, or update them to the test's current path.", + ); + Err(error) +} + +/// Checks that every test listed in `list_path` survived the filtering done by +/// `contains_ui_error_patterns`. +fn check_for_dead_listed_tests(rust_path: &Path, list_path: &str) -> Result<(), String> { + let listed_tests = std::fs::read_to_string(list_path) + .map_err(|error| format!("Failed to read `{list_path}`: {error:?}"))?; + let dead_tests = listed_tests + .lines() + .map(|line| line.trim()) + .filter(|line| !line.is_empty() && !rust_path.join(line).exists()) + .collect::>(); + if dead_tests.is_empty() { + return Ok(()); + } + Err(format!( + "The following tests listed in `{list_path}` are filtered out before the tests are run, \ + so listing them has no effect:\n{}\n\nThis happens when a test contains an error pattern \ + (like `//~` or `//@ known-bug`), in which case it should be removed from `{list_path}`, \ + or when it uses LTO, in which case it should be moved to `tests/failing-lto-tests.txt`.", + dead_tests.join("\n") + )) } fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> { - test_rustc_inner(env, args, |_| Ok(false), false, "run-make", false)?; - test_rustc_inner(env, args, |_| Ok(false), false, "run-make-cargo", false)?; - test_rustc_inner(env, args, |_| Ok(false), false, "ui", false) + test_rustc_inner(env, args, |_| Ok(false), false, "run-make", None)?; + test_rustc_inner(env, args, |_| Ok(false), false, "run-make-cargo", None)?; + test_rustc_inner(env, args, |_| Ok(false), false, "ui", None) } fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { @@ -1134,7 +1339,7 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { retain_files_callback("tests/failing-run-make-tests.txt", "run-make"), false, "run-make", - true, + None, ); let run_make_cargo_result = test_rustc_inner( @@ -1142,8 +1347,8 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { args, retain_files_callback("tests/failing-run-make-tests.txt", "run-make-cargo"), false, - "run-make", - true, + "run-make-cargo", + None, ); let ui_result = test_rustc_inner( @@ -1152,36 +1357,51 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { retain_files_callback("tests/failing-ui-tests.txt", "ui"), false, "ui", - true, + Some("tests/failing-ui-tests.txt"), ); - run_make_result.and(run_make_cargo_result).and(ui_result) + combine_test_results([run_make_result, run_make_cargo_result, ui_result]) +} + +/// Combines the results of several test suites, letting a build system error win over a test +/// failure so that a broken build system is never reported to CI as the failures those suites +/// expect. +fn combine_test_results(results: [Result<(), String>; N]) -> Result<(), String> { + let mut tests_failed = false; + for result in results { + match result { + Ok(()) => {} + Err(error) if error == TESTS_FAILED_ERROR => tests_failed = true, + Err(error) => return Err(error), + } + } + if tests_failed { Err(TESTS_FAILED_ERROR.to_string()) } else { Ok(()) } } fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { test_rustc_inner( env, args, - remove_files_callback("tests/failing-ui-tests.txt", "ui"), + remove_files_callback("tests/failing-ui-tests.txt"), false, "ui", - false, + None, )?; test_rustc_inner( env, args, - remove_files_callback("tests/failing-run-make-tests.txt", "run-make"), + remove_files_callback("tests/failing-run-make-tests.txt"), false, "run-make", - false, + None, )?; test_rustc_inner( env, args, - remove_files_callback("tests/failing-run-make-tests.txt", "run-make-cargo"), + remove_files_callback("tests/failing-run-make-tests.txt"), false, "run-make-cargo", - false, + None, ) } @@ -1189,20 +1409,73 @@ fn test_failing_ui_pattern_tests(env: &Env, args: &TestArg) -> Result<(), String test_rustc_inner( env, args, - remove_files_callback("tests/failing-ice-tests.txt", "ui"), + remove_files_callback("tests/failing-ice-tests.txt"), true, "ui", - false, + None, ) } +fn run_ui_tests(env: &Env, args: &TestArg) -> Result<(), String> { + let mut env = env.clone(); + let rust_path = setup_rustc(&mut env, args)?; + + let extra = + if args.is_using_gcc_master_branch() { "" } else { " -Csymbol-mangling-version=v0" }; + + let rustc_args = format!( + "{test_flags} -Zcodegen-backend={backend} --sysroot {sysroot}{extra}", + test_flags = env.get("TEST_FLAGS").unwrap_or(&String::new()), + backend = args.config_info.cg_backend_path, + sysroot = args.config_info.sysroot_path, + extra = extra, + ); + + env.get_mut("RUSTFLAGS").unwrap().clear(); + + let mut command: Vec<&dyn AsRef> = vec![ + &"./x.py", + &"test", + &"--run", + &"always", + &"--stage", + &"0", + &"--set", + &"build.compiletest-allow-stage0=true", + &"--compiletest-rustc-args", + &rustc_args, + &"--bypass-ignore-backends", + &"--force-rerun", + ]; + + for test_name in &args.test_args { + command.push(test_name); + } + + run_test_command(&command, &rust_path, &env) +} + +/// Runs the command that actually runs a test suite, mapping its failure to `TESTS_FAILED_ERROR`. +fn run_test_command( + command: &[&dyn AsRef], + rust_path: &Path, + env: &Env, +) -> Result<(), String> { + if let Err(error) = run_command_with_output_and_env(command, Some(rust_path), Some(env)) { + // The failures themselves were already streamed to the console. + eprintln!("{error}"); + return Err(TESTS_FAILED_ERROR.to_string()); + } + Ok(()) +} + fn retain_files_callback<'a>( file_path: &'a str, test_type: &'a str, ) -> impl Fn(&Path) -> Result + 'a { move |rust_path| { - let files = std::fs::read_to_string(file_path).unwrap_or_default(); - let first_file_name = files.lines().next().unwrap_or(""); + let tests = read_test_list(rust_path, file_path)?; + let first_file_name = tests.first().map(String::as_str).unwrap_or(""); // If the first line ends with a `/`, we treat all lines in the file as a directory. if first_file_name.ends_with('/') { // Treat as directory @@ -1244,53 +1517,31 @@ fn retain_files_callback<'a>( } // Putting back only the failing ones. - if let Ok(files) = std::fs::read_to_string(file_path) { - for file in files.split('\n').map(|line| line.trim()).filter(|line| !line.is_empty()) { - run_command(&[&"git", &"checkout", &"--", &file], Some(rust_path))?; - } - } else { - println!("Failed to read `{file_path}`, not putting back failing {test_type} tests"); + for test in &tests { + run_command(&[&"git", &"checkout", &"--", test], Some(rust_path))?; } Ok(true) } } -fn remove_files_callback<'a>( - file_path: &'a str, - test_type: &'a str, -) -> impl Fn(&Path) -> Result + 'a { +fn remove_files_callback(file_path: &str) -> impl Fn(&Path) -> Result + '_ { move |rust_path| { - let files = std::fs::read_to_string(file_path).unwrap_or_default(); - let first_file_name = files.lines().next().unwrap_or(""); + let tests = read_test_list(rust_path, file_path)?; + let first_file_name = tests.first().map(String::as_str).unwrap_or(""); // If the first line ends with a `/`, we treat all lines in the file as a directory. if first_file_name.ends_with('/') { // Removing the failing tests. - if let Ok(files) = std::fs::read_to_string(file_path) { - for file in - files.split('\n').map(|line| line.trim()).filter(|line| !line.is_empty()) - { - let path = rust_path.join(file); - if let Err(e) = remove_dir_all(&path) { - println!("Failed to remove directory `{}`: {}", path.display(), e); - } - } - } else { - println!( - "Failed to read `{file_path}`, not putting back failing {test_type} tests" - ); + for test in &tests { + let path = rust_path.join(test); + remove_dir_all(&path).map_err(|error| { + format!("Failed to remove directory `{}`: {error}", path.display()) + })?; } } else { // Removing the failing tests. - if let Ok(files) = std::fs::read_to_string(file_path) { - for file in - files.split('\n').map(|line| line.trim()).filter(|line| !line.is_empty()) - { - let path = rust_path.join(file); - remove_file(&path)?; - } - } else { - println!("Failed to read `{file_path}`, not putting back failing ui tests"); + for test in &tests { + remove_file(&rust_path.join(test))?; } } Ok(true) @@ -1342,3 +1593,58 @@ pub fn run() -> Result<(), String> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn write_test_list(directory: &Path, content: &str) -> PathBuf { + let list_path = directory.join("failing-tests.txt"); + std::fs::write(&list_path, content).unwrap(); + list_path + } + + #[test] + fn test_combine_test_results() { + let tests_failed = || Err(TESTS_FAILED_ERROR.to_string()); + let build_error = || Err("could not clone rust".to_string()); + + assert_eq!(combine_test_results([Ok(()), Ok(())]), Ok(())); + assert_eq!(combine_test_results([Ok(()), tests_failed()]), tests_failed()); + assert_eq!(combine_test_results([Ok(()), build_error()]), build_error()); + // A build system error wins, whichever suite reported it. + assert_eq!(combine_test_results([tests_failed(), build_error()]), build_error()); + assert_eq!(combine_test_results([build_error(), tests_failed()]), build_error()); + } + + #[test] + fn test_read_test_list() { + let rust_path = std::env::temp_dir().join("cg_gcc_read_test_list"); + let _ = remove_dir_all(&rust_path); + create_dir(rust_path.join("tests/ui")).unwrap(); + std::fs::write(rust_path.join("tests/ui/alive.rs"), "").unwrap(); + + let list_path = write_test_list(&rust_path, "\ntests/ui/alive.rs\n \n"); + let list_path = list_path.display().to_string(); + assert_eq!( + read_test_list(&rust_path, &list_path), + Ok(vec!["tests/ui/alive.rs".to_string()]) + ); + + write_test_list(&rust_path, "tests/ui/alive.rs\ntests/ui/gone.rs\n"); + let error = read_test_list(&rust_path, &list_path).unwrap_err(); + assert!(error.contains("no longer exist"), "{error}"); + assert!(error.contains("tests/ui/gone.rs"), "{error}"); + + write_test_list(&rust_path, "tests/ui/alive.rs\ntests/ui/alive.rs\n"); + let error = read_test_list(&rust_path, &list_path).unwrap_err(); + assert!(error.contains("listed more than once"), "{error}"); + assert!(error.contains("tests/ui/alive.rs"), "{error}"); + + // Directories are listed with a trailing `/`. + write_test_list(&rust_path, "tests/ui/\n"); + assert_eq!(read_test_list(&rust_path, &list_path), Ok(vec!["tests/ui/".to_string()])); + + remove_dir_all(&rust_path).unwrap(); + } +} diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000000000..cf1593c691734 --- /dev/null +++ b/clippy.toml @@ -0,0 +1,3 @@ +disallowed-methods = [ + { path = "gccjit::types::Type::add_attribute", reason = "go through `type_::apply_struct_attributes` instead: an attribute set directly on a type would not be part of the `CodegenCx::struct_types` cache key, so it would silently change every other use of that type" }, +] diff --git a/doc/tips.md b/doc/tips.md index ff92566d4a1ab..dc40ee4d39952 100644 --- a/doc/tips.md +++ b/doc/tips.md @@ -58,7 +58,7 @@ If you wish to build a custom sysroot, pass the path of your sysroot source to ` ### How to generate GIMPLE If you need to check what gccjit is generating (GIMPLE), then take a look at how to -generate it in [gimple.md](./doc/gimple.md). +generate it in [gimple.md](./gimple.md). ### How to build a cross-compiling libgccjit diff --git a/libgccjit.version b/libgccjit.version index 5eef70260466f..62417a80f827e 100644 --- a/libgccjit.version +++ b/libgccjit.version @@ -1 +1 @@ -6f155cc3f5a2dff33afe6cc3ed6c2e0e605ae6a3 +badf78d09d16e66f4ca07971c51aa6a227558d4f diff --git a/rust-toolchain b/rust-toolchain index 56fcfdff1c719..0c81f7c7c7398 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2026-04-29" +channel = "nightly-2026-09-18" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/src/abi.rs b/src/abi.rs index 6a05f1cbbeef1..09e009700a80c 100644 --- a/src/abi.rs +++ b/src/abi.rs @@ -73,7 +73,7 @@ impl GccType for CastTarget { args.push(cx.type_ix(rem_bytes * 8)); } - cx.type_struct(&args, false) + cx.type_struct(&args, &[]) } } diff --git a/src/asm.rs b/src/asm.rs index 420bf1e7a31c1..e84cf54cb1147 100644 --- a/src/asm.rs +++ b/src/asm.rs @@ -681,6 +681,7 @@ fn explicit_reg_to_gcc(reg: InlineAsmReg) -> &'static str { } InlineAsmReg::Arm(reg) => reg.name(), InlineAsmReg::AArch64(reg) => reg.name(), + InlineAsmReg::M68k(reg) => reg.name(), _ => unimplemented!(), } } diff --git a/src/attributes.rs b/src/attributes.rs index a5cc44a46e154..87c989031678d 100644 --- a/src/attributes.rs +++ b/src/attributes.rs @@ -12,6 +12,8 @@ use rustc_middle::ty; #[cfg(feature = "master")] use rustc_target::spec::Arch; +#[cfg(feature = "master")] +use crate::base; use crate::context::CodegenCx; use crate::gcc_util::to_gcc_features; @@ -102,6 +104,16 @@ pub fn from_fn_attrs<'gcc, 'tcx>( } else { codegen_fn_attrs.inline }; + // GCC drops `weak` from a function that is also `inline`, leaving the symbol strong, and + // the linkage is what has to survive. `inline(never)` does not conflict. + let inline = match inline { + InlineAttr::Always | InlineAttr::Hint | InlineAttr::Force { .. } + if codegen_fn_attrs.linkage.is_some_and(base::linkage_needs_weak_attribute) => + { + InlineAttr::None + } + inline => inline, + }; if let Some(attr) = inline_attr(cx, inline, instance) { if let FnAttribute::AlwaysInline = attr { func.add_attribute(FnAttribute::Inline); diff --git a/src/back/lto.rs b/src/back/lto.rs index 98f9abdb05c4c..b0de1ead56ed1 100644 --- a/src/back/lto.rs +++ b/src/back/lto.rs @@ -36,7 +36,8 @@ use tempfile::{TempDir, tempdir}; use crate::back::write::{codegen, save_temp_bitcode}; use crate::diagnostics::LtoBitcodeFromRlib; -use crate::{GccCodegenBackend, GccContext, LtoMode, to_gcc_opt_level}; +use crate::gcc_util::new_context; +use crate::{GccCodegenBackend, GccContext, LtoMode, SyncContext, to_gcc_opt_level}; struct LtoData { // FIXME(antoyo): use symbols_below_threshold. diff --git a/src/base.rs b/src/base.rs index 101af0bb0bff1..7cb734d523273 100644 --- a/src/base.rs +++ b/src/base.rs @@ -3,7 +3,9 @@ use std::env; use std::sync::Arc; use std::time::Instant; -use gccjit::{CType, Context, FunctionType, GlobalKind}; +#[cfg(feature = "master")] +use gccjit::VarAttribute; +use gccjit::{CType, FunctionType, GlobalKind}; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; @@ -21,7 +23,8 @@ use rustc_target::spec::{Arch, RelocModel}; use crate::builder::Builder; use crate::context::CodegenCx; -use crate::{GccContext, LtoMode, SharedTargetInfo, SyncContext, gcc_util, new_context}; +use crate::gcc_util::new_context; +use crate::{GccContext, LtoMode, SharedTargetInfo, SyncContext}; #[cfg(feature = "master")] pub fn visibility_to_gcc(visibility: Visibility) -> gccjit::Visibility { @@ -41,32 +44,72 @@ pub fn symbol_visibility_to_gcc(visibility: SymbolVisibility) -> gccjit::Visibil } } +/// The kind of a global *definition* with an explicit `#[linkage]`. +/// +/// The flavours that another object file is allowed to override also need +/// `global_linkage_attribute` from the caller: `GlobalKind` alone cannot express weakness. pub fn global_linkage_to_gcc(linkage: Linkage) -> GlobalKind { match linkage { - Linkage::External => GlobalKind::Imported, - Linkage::AvailableExternally => GlobalKind::Imported, - Linkage::LinkOnceAny => unimplemented!(), - Linkage::LinkOnceODR => unimplemented!(), - Linkage::WeakAny => unimplemented!(), - Linkage::WeakODR => unimplemented!(), - Linkage::Internal => GlobalKind::Internal, - Linkage::ExternalWeak => GlobalKind::Imported, // FIXME(antoyo): should be weak linkage. - Linkage::Common => unimplemented!(), + Linkage::External => GlobalKind::Exported, + // libgccjit cannot emit a definition that the linker discards in favour of the one in + // another object file, so emit a private copy of it instead. + Linkage::AvailableExternally | Linkage::Internal => GlobalKind::Internal, + // libgccjit exposes no comdat, so `weak` stands in for the linkonce flavours. + Linkage::LinkOnceAny + | Linkage::LinkOnceODR + | Linkage::WeakAny + | Linkage::WeakODR + | Linkage::ExternalWeak + | Linkage::Common => GlobalKind::Exported, + } +} + +/// The attribute a global *definition* needs on top of its [`GlobalKind`] to get this linkage. +#[cfg(feature = "master")] +pub fn global_linkage_attribute<'gcc>(linkage: Linkage) -> Option> { + match linkage { + Linkage::Common => Some(VarAttribute::Common), + _ if linkage_needs_weak_attribute(linkage) => Some(VarAttribute::Weak), + _ => None, } } +/// The type of a function *definition* with an explicit `#[linkage]`. +/// +/// The flavours that another object file is allowed to override also need +/// `linkage_needs_weak_attribute` from the caller: `FunctionType` alone cannot express weakness. pub fn linkage_to_gcc(linkage: Linkage) -> FunctionType { match linkage { Linkage::External => FunctionType::Exported, - // FIXME(antoyo): set the attribute externally_visible. - Linkage::AvailableExternally => FunctionType::Extern, - Linkage::LinkOnceAny => unimplemented!(), - Linkage::LinkOnceODR => unimplemented!(), - Linkage::WeakAny => FunctionType::Exported, // FIXME(antoyo): should be similar to linkonce. - Linkage::WeakODR => unimplemented!(), - Linkage::Internal => FunctionType::Internal, - Linkage::ExternalWeak => unimplemented!(), - Linkage::Common => unimplemented!(), + // libgccjit cannot emit a definition that the linker discards in favour of the one in + // another object file, so emit a private copy of it instead. + Linkage::AvailableExternally | Linkage::Internal => FunctionType::Internal, + // libgccjit exposes no comdat, so `weak` stands in for every overridable flavour. + Linkage::LinkOnceAny + | Linkage::LinkOnceODR + | Linkage::WeakAny + | Linkage::WeakODR + | Linkage::ExternalWeak + | Linkage::Common => FunctionType::Exported, + } +} + +/// Whether a definition with this linkage must carry the `weak` attribute, so that a strong +/// definition in another object file wins over it instead of clashing with it. +/// +/// `common` is in here for functions only: GCC honours that attribute on a variable, but drops it +/// on a function, so a common function falls back to weak. Globals go through +/// `global_linkage_attribute` instead. +#[cfg(feature = "master")] +pub fn linkage_needs_weak_attribute(linkage: Linkage) -> bool { + match linkage { + Linkage::LinkOnceAny + | Linkage::LinkOnceODR + | Linkage::WeakAny + | Linkage::WeakODR + | Linkage::ExternalWeak + | Linkage::Common => true, + Linkage::External | Linkage::AvailableExternally | Linkage::Internal => false, } } @@ -243,6 +286,11 @@ pub fn compile_codegen_unit( // ... and now that we have everything pre-defined, fill out those definitions. for &(mono_item, item_data) in &mono_items { mono_item.define::>(&mut cx, cgu_name.as_str(), item_data); + + // Now that this function's blocks all exist, fill in the cleanup + // regions reconstructed from MIR while lowering its `invoke`s. + #[cfg(feature = "master")] + cx.populate_cleanup_regions(); } // If this codegen unit contains the main function, also create the diff --git a/src/builder.rs b/src/builder.rs index 0a88085960e89..6de2827e63dee 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -33,9 +33,11 @@ use rustc_target::callconv::FnAbi; use rustc_target::spec::{HasTargetSpec, HasX86AbiOpt, Target, X86Abi}; use crate::abi::FnAbiGccExt; +use crate::builder; use crate::common::{SignType, TypeReflection, type_is_pointer}; use crate::context::CodegenCx; -use crate::diagnostics; +#[cfg(feature = "master")] +use crate::context::PendingCleanup; use crate::intrinsic::llvm; use crate::type_of::LayoutGccExt; @@ -64,6 +66,33 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.value_counter.get() } + /// Tell GCC that `pointer` is `align`-aligned, so that the bulk memory builtins can widen their + /// accesses: a pointer cast to an aligned type would be dropped as a useless conversion. + fn assume_aligned(&mut self, pointer: RValue<'gcc>, align: Align) -> RValue<'gcc> { + if align.bytes() <= 1 { + return pointer; + } + let assume_aligned = self.context.get_builtin_function("__builtin_assume_aligned"); + let alignment = self.context.new_rvalue_from_long(self.type_size_t(), align.bytes() as i64); + let pointer_type = pointer.get_type(); + let const_void_ptr_type = self.context.new_type::<()>().make_const().make_pointer(); + let pointer = self.context.new_cast(self.location, pointer, const_void_ptr_type); + let aligned = self.context.new_call(self.location, assume_aligned, &[pointer, alignment]); + self.context.new_cast(self.location, aligned, pointer_type) + } + + /// GCC ignores a volatile qualifier on the pointers given to `memcpy`/`memmove`/`memset` and + /// happily deletes the call, so a barrier is what keeps the operation observable. The pointers + /// are fed to it because a clobber alone does not reach memory GCC believes never escapes. + fn volatile_barrier(&mut self, pointers: &[RValue<'gcc>]) { + let barrier = self.block.add_extended_asm(self.location, ""); + for pointer in pointers { + barrier.add_input_operand(None, "r", *pointer); + } + barrier.add_clobber("memory"); + barrier.set_volatile_flag(true); + } + fn atomic_extremum( &mut self, operation: ExtremumOperation, @@ -316,6 +345,45 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.block.get_function() } + /// Shared implementation of `call` and `tail_call`. For tail call it is important that this + /// returns a bare call, and not the result assigned to a local, or the result of `add_eval`. + #[allow(clippy::too_many_arguments)] + fn build_call( + &mut self, + typ: Type<'gcc>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + func: RValue<'gcc>, + return_slot: ReturnSlot< as BackendTypes>::Value>, + args: &[RValue<'gcc>], + funclet: Option<&Funclet>, + must_tail: bool, + ) -> RValue<'gcc> { + // FIXME: change this in the `rustc_codegen_gcc` repo after the sync, to use the `libgccjit` indirect return suppport. + let args = match return_slot { + ReturnSlot::Direct => Cow::Borrowed(args), + ReturnSlot::Indirect(sret_ptr) => { + let mut args = args.to_vec(); + // Prepend the indirect return pointer + args.insert(0, sret_ptr); + Cow::Owned(args) + } + }; + // FIXME(antoyo): remove when having a proper API. + let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; + let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { + // FIXME(antoyo): remove when the API supports a different type for functions. + let func: Function<'gcc> = self.cx.rvalue_as_function(func); + self.function_call(func, &args, funclet, must_tail) + } else { + // If it's a not function that was defined, it's a function pointer. + self.function_ptr_call(typ, fn_abi, func, &args, funclet, must_tail) + }; + if let Some(_fn_abi) = fn_abi { + // FIXME(bjorn3): Apply function attributes + } + call + } + pub fn function_call( &mut self, func: Function<'gcc>, @@ -614,7 +682,9 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _funclet: Option<&Funclet>, instance: Option>, ) -> RValue<'gcc> { - let try_block = self.current_func().new_block("try"); + let current_func = self.current_func(); + let try_region = current_func.new_region(self.location); + let try_block = try_region.new_block("try"); let current_block = self.block; self.block = try_block; @@ -622,17 +692,25 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let call = self.call(typ, fn_attrs, fn_abi, func, return_slot, args, None, instance); self.block = current_block; - let return_value = - self.current_func().new_local(self.location, call.get_type(), "invokeResult"); + let return_value = self.new_temp(current_func, self.location, call.get_type()); try_block.add_assignment(self.location, return_value, call); try_block.end_with_jump(self.location, then); - if self.cleanup_blocks.borrow().contains(&catch) { - self.block.add_try_finally(self.location, try_block, catch); + if self.cx.landing_pads.borrow().contains(&catch) { + let cleanup_region = current_func.new_region(self.location); + self.block.add_cleanup(self.location, try_region, cleanup_region); + self.cx + .pending_cleanups + .borrow_mut() + .push(PendingCleanup { region: cleanup_region, landing_pad: catch }); } else { - self.block.add_try_catch(self.location, try_block, catch); + let catch_region = current_func.new_region(self.location); + for clone in gccjit::clone_blocks(&[catch]) { + catch_region.add_block(clone); + } + self.block.add_try_catch(self.location, try_region, catch_region); } self.block.end_with_jump(self.location, then); @@ -671,8 +749,9 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { if return_type == void_type { self.block.end_with_void_return(self.location) } else { - let return_value = - self.current_func().new_local(self.location, return_type, "unreachableReturn"); + let trap = self.context.get_builtin_function("__builtin_trap"); + self.block.add_eval(self.location, self.context.new_call(self.location, trap, &[])); + let return_value = self.new_temp(self.current_func(), self.location, return_type); self.block.end_with_return(self.location, return_value) } } @@ -1405,47 +1484,53 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn memcpy( &mut self, dst: RValue<'gcc>, - _dst_align: Align, + dst_align: Align, src: RValue<'gcc>, - _src_align: Align, + src_align: Align, size: RValue<'gcc>, flags: MemFlags, _tt: Option, // Autodiff TypeTrees are LLVM-only, ignored in GCC backend ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_size_t(), false); - let _is_volatile = flags.contains(MemFlags::VOLATILE); let dst = self.pointercast(dst, self.type_i8p()); + let dst = self.assume_aligned(dst, dst_align); let src = self.pointercast(src, self.type_ptr_to(self.type_void())); + let src = self.assume_aligned(src, src_align); let memcpy = self.context.get_builtin_function("memcpy"); - // FIXME(antoyo): handle aligns and is_volatile. self.block.add_eval( self.location, self.context.new_call(self.location, memcpy, &[dst, src, size]), ); + if flags.contains(MemFlags::VOLATILE) { + self.volatile_barrier(&[dst, src]); + } } fn memmove( &mut self, dst: RValue<'gcc>, - _dst_align: Align, + dst_align: Align, src: RValue<'gcc>, - _src_align: Align, + src_align: Align, size: RValue<'gcc>, flags: MemFlags, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported"); let size = self.intcast(size, self.type_size_t(), false); - let _is_volatile = flags.contains(MemFlags::VOLATILE); let dst = self.pointercast(dst, self.type_i8p()); + let dst = self.assume_aligned(dst, dst_align); let src = self.pointercast(src, self.type_ptr_to(self.type_void())); + let src = self.assume_aligned(src, src_align); let memmove = self.context.get_builtin_function("memmove"); - // FIXME(antoyo): handle is_volatile. self.block.add_eval( self.location, self.context.new_call(self.location, memmove, &[dst, src, size]), ); + if flags.contains(MemFlags::VOLATILE) { + self.volatile_barrier(&[dst, src]); + } } fn memset( @@ -1453,20 +1538,22 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { ptr: RValue<'gcc>, fill_byte: RValue<'gcc>, size: RValue<'gcc>, - _align: Align, + align: Align, flags: MemFlags, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported"); - let _is_volatile = flags.contains(MemFlags::VOLATILE); let ptr = self.pointercast(ptr, self.type_i8p()); + let ptr = self.assume_aligned(ptr, align); let memset = self.context.get_builtin_function("memset"); - // FIXME(antoyo): handle align and is_volatile. let fill_byte = self.context.new_cast(self.location, fill_byte, self.i32_type); let size = self.intcast(size, self.type_size_t(), false); self.block.add_eval( self.location, self.context.new_call(self.location, memset, &[ptr, fill_byte, size]), ); + if flags.contains(MemFlags::VOLATILE) { + self.volatile_barrier(&[ptr]); + } } fn vscale(&mut self, _: Self::Type) -> Self::Value { @@ -1606,18 +1693,12 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { // NOTE: insert the current block in a variable so that a later call to invoke knows to // generate a try/finally instead of a try/catch for this block. - self.cleanup_blocks.borrow_mut().insert(self.block); - - let eh_pointer_builtin = - self.cx.context.get_target_builtin_function("__builtin_eh_pointer"); - let zero = self.cx.context.new_rvalue_zero(self.int_type); - let ptr = self.cx.context.new_call(self.location, eh_pointer_builtin, &[zero]); - - let value1_type = self.u8_type.make_pointer(); - let ptr = self.cx.context.new_cast(self.location, ptr, value1_type); - let value1 = ptr; - let value2 = zero; // FIXME(antoyo): set the proper value here (the type of exception?). + self.cx.landing_pads.borrow_mut().insert(self.block); + // A cleanup resumes by falling through: it never inspects the exception + // object. + let value1 = self.context.new_null(self.u8_type.make_pointer()); + let value2 = self.context.new_rvalue_zero(self.i32_type); (value1, value2) } @@ -1633,18 +1714,13 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { } fn filter_landing_pad(&mut self, pers_fn: Function<'gcc>) { - // FIXME(antoyo): generate the correct landing pad - self.cleanup_landing_pad(pers_fn); + self.set_personality_fn(pers_fn); } #[cfg(feature = "master")] - fn resume(&mut self, exn0: RValue<'gcc>, _exn1: RValue<'gcc>) { - let exn_type = exn0.get_type(); - let exn = self.context.new_cast(self.location, exn0, exn_type); - let unwind_resume = self.context.get_target_builtin_function("__builtin_unwind_resume"); - self.llbb() - .add_eval(self.location, self.context.new_call(self.location, unwind_resume, &[exn])); - self.unreachable(); + fn resume(&mut self, _exn0: RValue<'gcc>, _exn1: RValue<'gcc>) { + // End the cleanup by falling off the end of its region body. + self.block.end_with_fallthrough(self.location); } #[cfg(not(feature = "master"))] @@ -1786,45 +1862,35 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { funclet: Option<&Funclet>, _instance: Option>, ) -> RValue<'gcc> { - // FIXME: change this in the `rustc_codegen_gcc` repo after the sync, to use the `libgccjit` indirect return suppport. - let args = match return_slot { - ReturnSlot::Direct => args.to_vec(), - ReturnSlot::Indirect(sret_ptr) => { - let mut args = args.to_vec(); - // Prepend the indirect return pointer - args.insert(0, sret_ptr); - args - } - }; - // FIXME(antoyo): remove when having a proper API. - let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; - let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { - // FIXME(antoyo): remove when the API supports a different type for functions. - let func: Function<'gcc> = self.cx.rvalue_as_function(func); - self.function_call(func, &args, funclet) - } else { - // If it's a not function that was defined, it's a function pointer. - self.function_ptr_call(typ, fn_abi, func, &args, funclet) - }; - if let Some(_fn_abi) = fn_abi { - // FIXME(bjorn3): Apply function attributes - } - call + self.build_call(typ, fn_abi, func, return_slot, args, funclet, false) } fn tail_call( &mut self, _llty: Self::Type, _fn_attrs: Option<&CodegenFnAttrs>, - _fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - _llfn: Self::Value, - _return_slot: ReturnSlot, - _args: &[Self::Value], - _funclet: Option<&Self::Funclet>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: Self::Value, + return_slot: ReturnSlot, + args: &[Self::Value], + funclet: Option<&Self::Funclet>, _instance: Option>, ) { - // FIXME: implement support for explicit tail calls like rustc_codegen_llvm. - self.tcx.dcx().emit_fatal(diagnostics::ExplicitTailCallsUnsupported); + // `emit_call` returns a bare call for here, it has not been assigned or passed to add_eval. + let call = self.build_call(llty, Some(fn_abi), llfn, return_slot, args, funclet, true); + call.set_require_tail_call(true); + + let return_type = self.current_func().get_return_type(); + let void_type = self.context.new_type::<()>(); + + if return_type == void_type { + // For a void return the call is emitted as its own statement, immediately + // followed by a void return, so the tail call sits in tail position. + self.llbb().add_eval(self.location, call); + self.ret_void(); + } else { + self.ret(call) + } } fn zext(&mut self, value: RValue<'gcc>, dest_typ: Type<'gcc>) -> RValue<'gcc> { diff --git a/src/common.rs b/src/common.rs index 6bd186f1121fc..21d92c6cc2936 100644 --- a/src/common.rs +++ b/src/common.rs @@ -12,6 +12,7 @@ use rustc_session::PointerAuthSchema; use crate::consts::const_alloc_to_gcc; use crate::context::{CodegenCx, new_array_type}; +use crate::type_::struct_attributes; use crate::type_of::LayoutGccExt; impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { @@ -125,78 +126,86 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { } } +/// The element type and element count of the array used to represent a run of `len` constant bytes. +/// +/// Larger integers are used where possible: this reduces the number of rvalues, which is a +/// significant memory saving on constant-heavy crates. +fn byte_run_shape<'gcc>(cx: &CodegenCx<'gcc, '_>, len: usize) -> (Type<'gcc>, u64) { + match len % 8 { + 0 => (cx.context.new_type::(), len as u64 / 8), + 4 => (cx.context.new_type::(), len as u64 / 4), + _ => (cx.context.new_type::(), len as u64), + } +} + +/// The type [`bytes_in_context`] gives a run of `len` constant bytes. +/// +/// Exposed separately so that the type of a constant allocation can be computed before any of its +/// rvalues exist; see [`crate::consts::const_alloc_type`]. +/// +/// The result is cached because `gcc_jit_context_new_array_type` mints a fresh type every call. +/// Two equal-but-distinct array types would key [`CodegenCx::type_struct`] differently and so +/// produce two distinct anonymous structs, and libgccjit compares struct types by identity. +pub fn bytes_type_in_context<'gcc>(cx: &CodegenCx<'gcc, '_>, len: usize) -> Type<'gcc> { + let (element_type, count) = byte_run_shape(cx, len); + if let Some(&typ) = cx.byte_array_types.borrow().get(&(element_type, count)) { + return typ; + } + let typ = new_array_type(cx.context, None, element_type, count); + cx.byte_array_types.borrow_mut().insert((element_type, count), typ); + typ +} + +// FIXME(FractalFir): Consider using `global_set_initializer` instead. Before this is done, we need to confirm that +// `global_set_initializer` is more memory efficient than the current solution. +// `global_set_initializer` calls `global_set_initializer_rvalue` under the hood - does it generate an array of rvalues, +// or is it using a more efficient representation? pub fn bytes_in_context<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, bytes: &[u8]) -> RValue<'gcc> { - // Instead of always using an array of bytes, use an array of larger integers of target endianness - // if possible. This reduces the amount of `rvalues` we use, which reduces memory usage significantly. - // - // FIXME(FractalFir): Consider using `global_set_initializer` instead. Before this is done, we need to confirm that - // `global_set_initializer` is more memory efficient than the current solution. - // `global_set_initializer` calls `global_set_initializer_rvalue` under the hood - does it generate an array of rvalues, - // or is it using a more efficient representation? - match bytes.len() % 8 { + let typ = bytes_type_in_context(cx, bytes.len()); + let (element_type, _) = byte_run_shape(cx, bytes.len()); + let context = &cx.context; + // Since we are representing arbitrary byte runs as integers, we need to follow the target + // endianness. + let endian = cx.sess().target.options.endian; + let elements: Vec<_> = match bytes.len() % 8 { 0 => { - debug_assert_eq!( - bytes.len() % 8, - 0, - "bytes length is not a multiple of 8, so bytes.as_chunks will have a remainder" - ); - let context = &cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64 / 8); - let elements: Vec<_> = bytes - .as_chunks::<8>() - .0 + let (arrays, remainder) = bytes.as_chunks::<8>(); + debug_assert!(remainder.is_empty()); + arrays .iter() .map(|&arr| { context.new_rvalue_from_long( - byte_type, - // Since we are representing arbitrary byte runs as integers, we need to follow the target - // endianness. - match cx.sess().target.options.endian { + element_type, + match endian { rustc_abi::Endian::Little => u64::from_le_bytes(arr) as i64, rustc_abi::Endian::Big => u64::from_be_bytes(arr) as i64, }, ) }) - .collect(); - context.new_array_constructor(None, typ, &elements) + .collect() } 4 => { - debug_assert_eq!( - bytes.len() % 4, - 0, - "bytes length is not a multiple of 4, so bytes.as_chunks will have a remainder" - ); - let context = &cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64 / 4); - let elements: Vec<_> = bytes - .as_chunks::<4>() - .0 + let (arrays, remainder) = bytes.as_chunks::<4>(); + debug_assert!(remainder.is_empty()); + arrays .iter() .map(|&arr| { context.new_rvalue_from_int( - byte_type, - match cx.sess().target.options.endian { + element_type, + match endian { rustc_abi::Endian::Little => u32::from_le_bytes(arr) as i32, rustc_abi::Endian::Big => u32::from_be_bytes(arr) as i32, }, ) }) - .collect(); - context.new_array_constructor(None, typ, &elements) - } - _ => { - let context = cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64); - let elements: Vec<_> = bytes - .iter() - .map(|&byte| context.new_rvalue_from_int(byte_type, byte as i32)) - .collect(); - context.new_array_constructor(None, typ, &elements) + .collect() } - } + _ => bytes + .iter() + .map(|&byte| context.new_rvalue_from_int(element_type, byte as i32)) + .collect(), + }; + context.new_array_constructor(None, typ, &elements) } pub fn type_is_pointer(typ: Type<'_>) -> bool { @@ -298,7 +307,7 @@ impl<'gcc, 'tcx> ConstCodegenMethods for CodegenCx<'gcc, 'tcx> { fn const_struct(&self, values: &[RValue<'gcc>], packed: bool) -> RValue<'gcc> { let fields: Vec<_> = values.iter().map(|value| value.get_type()).collect(); // FIXME(antoyo): cache the type? It's anonymous, so probably not. - let typ = self.type_struct(&fields, packed); + let typ = self.type_struct(&fields, &struct_attributes(packed, None)); let struct_type = typ.is_struct().expect("struct type"); self.context.new_struct_constructor(None, struct_type.as_type(), None, values) } diff --git a/src/consts.rs b/src/consts.rs index 6c3b404547cfc..e6acbc6625d44 100644 --- a/src/consts.rs +++ b/src/consts.rs @@ -1,3 +1,5 @@ +use std::ops::Range; + #[cfg(feature = "master")] use gccjit::{FnAttribute, VarAttribute, Visibility}; use gccjit::{Function, GlobalKind, LValue, RValue, ToRValue, Type}; @@ -11,15 +13,18 @@ use rustc_hir::def_id::LOCAL_CRATE; use rustc_log::tracing::trace; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ - self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint, + self, Allocation, ConstAllocation, CtfeProvenance, ErrorHandled, Scalar as InterpScalar, + read_target_uint, }; +use rustc_middle::mono::MonoItem; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance}; use rustc_span::def_id::DefId; use rustc_span::{bug, span_bug}; -use crate::base; +use crate::common::bytes_type_in_context; use crate::context::CodegenCx; +use crate::type_::struct_attributes; use crate::type_of::LayoutGccExt; pub(crate) fn const_alloc_to_gcc<'gcc, 'tcx>( @@ -99,15 +104,21 @@ impl<'gcc, 'tcx> StaticCodegenMethods for CodegenCx<'gcc, 'tcx> { let is_thread_local = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); let global = self.get_static_inner(def_id, val_llty); - #[cfg(feature = "master")] - if global.to_rvalue().get_type() != val_llty { - global.to_rvalue().set_type(val_llty); - } + debug_assert_eq!( + global.to_rvalue().get_type(), + val_llty, + "`predefine_static` declared this global with a type its initializer does not have" + ); // NOTE: Alignment from attributes has already been applied to the allocation. set_global_alignment(self, global, alloc.align); - global.global_set_initializer_rvalue(value); + // A common symbol is storage the linker allocates and zero-fills, so giving the definition + // an initializer — even an all-zero one — takes it back out of `.comm`. A non-zero one is + // kept: the symbol is then an ordinary definition, which is what GCC does with it too. + if attrs.linkage != Some(Linkage::Common) || !is_zero_initializer(alloc) { + global.global_set_initializer_rvalue(value); + } // As an optimization, all shared statics which do not have interior // mutability are placed into read-only memory. @@ -237,15 +248,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { return global; } - // FIXME: Once we stop removing globals in `codegen_static`, we can uncomment this code. - // let defined_in_current_codegen_unit = - // self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); - // assert!( - // !defined_in_current_codegen_unit, - // "consts::get_static() should always hit the cache for \ - // statics defined in the same CGU, but did not for `{:?}`", - // def_id - // ); + let defined_in_current_codegen_unit = + self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); + assert!( + !defined_in_current_codegen_unit, + "consts::get_static() should always hit the cache for \ + statics defined in the same CGU, but did not for `{:?}`", + def_id + ); let sym = self.tcx.symbol_name(instance).name; let fn_attrs = self.tcx.codegen_fn_attrs(def_id); @@ -309,75 +319,133 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { global } } -/// Converts a given const alloc to a gcc Rvalue, without any caching or deduplication. -/// YOU SHOULD NOT call this function directly - that may break the semantics of Rust. -/// Use `const_data_from_alloc` instead. -pub(crate) fn const_alloc_to_gcc_uncached<'gcc>( - cx: &CodegenCx<'gcc, '_>, - alloc: ConstAllocation<'_>, -) -> RValue<'gcc> { - let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); - let dl = cx.data_layout(); - let pointer_size = dl.pointer_size().bytes() as usize; +/// One field of the packed struct that a constant allocation is lowered to. +enum AllocField { + /// A run of bytes carrying no provenance. + Bytes { range: Range }, + /// A pointer with provenance, occupying one target pointer worth of bytes. + Pointer { offset: usize, prov: CtfeProvenance }, +} + +/// The field-by-field shape of `alloc`. +/// +/// [`const_alloc_to_gcc_uncached`] and [`const_alloc_type`] have to agree exactly on this, down to +/// the empty trailing run an allocation ending on a pointer produces, so both derive the shape here +/// instead of each walking the allocation on its own. +fn alloc_fields(cx: &CodegenCx<'_, '_>, alloc: &interpret::Allocation) -> Vec { + let pointer_size = cx.data_layout().pointer_size().bytes() as usize; + let mut fields = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); let mut next_offset = 0; for &(offset, prov) in alloc.provenance().ptrs().iter() { - let alloc_id = prov.alloc_id(); let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a pointer with provenance, it - // is within the bounds of the allocation, and it doesn't affect interpreter execution - // (we inspect the result after interpreter execution). Any undef byte is replaced with - // some arbitrary byte value. - // - // FIXME: relay undef bytes to codegen as undef const bytes - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(next_offset..offset); - llvals.push(cx.const_bytes(bytes)); + fields.push(AllocField::Bytes { range: next_offset..offset }); } - let ptr_offset = read_target_uint( - dl.endian, - // This `inspect` is okay since it is within the bounds of the allocation, it doesn't - // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the provenance as a relocation pointer offset. - alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), - ) - .expect("const_alloc_to_gcc_uncached: could not read relocation pointer") - as u64; - - let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx); - - llvals.push(cx.scalar_to_backend( - InterpScalar::from_pointer( - interpret::Pointer::new(prov, Size::from_bytes(ptr_offset)), - &cx.tcx, - ), - abi::Scalar::Initialized { - value: Primitive::Pointer(address_space), - valid_range: WrappingRange::full(dl.pointer_size()), - }, - cx.type_i8p_ext(address_space), - )); + fields.push(AllocField::Pointer { offset, prov }); next_offset = offset + pointer_size; } if alloc.len() >= next_offset { - let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all provenance, it is - // within the bounds of the allocation, and it doesn't affect interpreter execution (we - // inspect the result after interpreter execution). Any undef byte is replaced with some - // arbitrary byte value. - // - // FIXME: relay undef bytes to codegen as undef const bytes - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); - llvals.push(cx.const_bytes(bytes)); + fields.push(AllocField::Bytes { range: next_offset..alloc.len() }); } + fields +} + +/// The type [`const_alloc_to_gcc`] gives `alloc`, computed without building any rvalue. +/// +/// This lets `predefine_static` declare a static's global with the type its initializer will have, +/// so that the two never disagree. It must not reach for the rvalue of anything it points at: +/// during the predefine pass the pointee may not be declared yet, and `alloc_to_backend` would +/// declare it with the wrong type behind our back. +pub(crate) fn const_alloc_type<'gcc>( + cx: &CodegenCx<'gcc, '_>, + alloc: ConstAllocation<'_>, +) -> Type<'gcc> { + let fields: Vec<_> = alloc_fields(cx, alloc.inner()) + .into_iter() + .map(|field| match field { + AllocField::Bytes { range } => bytes_type_in_context(cx, range.len()), + AllocField::Pointer { prov, .. } => { + let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); + cx.type_i8p_ext(address_space) + } + }) + .collect(); + cx.type_struct(&fields, &struct_attributes(true, None)) +} + +/// Converts a given const alloc to a gcc Rvalue, without any caching or deduplication. +/// YOU SHOULD NOT call this function directly - that may break the semantics of Rust. +/// Use `const_data_from_alloc` instead. +pub(crate) fn const_alloc_to_gcc_uncached<'gcc>( + cx: &CodegenCx<'gcc, '_>, + alloc: ConstAllocation<'_>, +) -> RValue<'gcc> { + let alloc = alloc.inner(); + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size(); + + let llvals: Vec<_> = alloc_fields(cx, alloc) + .into_iter() + .map(|field| match field { + AllocField::Bytes { range } => { + // This `inspect` is okay since we have checked that it is not within a pointer with + // provenance, it is within the bounds of the allocation, and it doesn't affect + // interpreter execution (we inspect the result after interpreter execution). Any + // undef byte is replaced with some arbitrary byte value. + // + // FIXME: relay undef bytes to codegen as undef const bytes + cx.const_bytes(alloc.inspect_with_uninit_and_ptr_outside_interpreter(range)) + } + AllocField::Pointer { offset, prov } => { + let ptr_offset = read_target_uint( + dl.endian, + // This `inspect` is okay since it is within the bounds of the allocation, it + // doesn't affect interpreter execution (we inspect the result after interpreter + // execution), and we properly interpret the provenance as a relocation pointer + // offset. + alloc.inspect_with_uninit_and_ptr_outside_interpreter( + offset..(offset + pointer_size.bytes() as usize), + ), + ) + .expect("const_alloc_to_gcc_uncached: could not read relocation pointer") + as u64; + + let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); + + cx.scalar_to_backend( + InterpScalar::from_pointer( + interpret::Pointer::new(prov, Size::from_bytes(ptr_offset)), + &cx.tcx, + ), + abi::Scalar::Initialized { + value: Primitive::Pointer(address_space), + valid_range: WrappingRange::full(pointer_size), + }, + cx.type_i8p_ext(address_space), + ) + } + }) + .collect(); + // FIXME(bjorn3) avoid wrapping in a struct when there is only a single element. cx.const_struct(&llvals, true) } +/// Whether this allocation is all zeroes, and so needs no initializer to be spelled out. +fn is_zero_initializer(alloc: &Allocation) -> bool { + alloc.provenance().ptrs().is_empty() + // This `inspect` is okay: it is within the bounds of the allocation, there is no provenance + // to misread, and it does not affect interpreter execution. + && alloc + .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.size().bytes_usize()) + .iter() + .all(|&byte| byte == 0) +} + fn codegen_static_initializer<'gcc, 'tcx>( cx: &CodegenCx<'gcc, 'tcx>, def_id: DefId, @@ -394,10 +462,10 @@ fn check_and_apply_linkage<'gcc, 'tcx>( ) -> LValue<'gcc> { let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); if let Some(linkage) = attrs.import_linkage { - // Declare a symbol `foo` with the desired linkage. - let global1 = - cx.declare_global_with_linkage(sym, cx.type_i8(), base::global_linkage_to_gcc(linkage)); + // Whatever the flavour, an import is an undefined reference to a symbol defined elsewhere. + let global1 = cx.declare_global_with_linkage(sym, cx.type_i8(), GlobalKind::Imported); + // Only `extern_weak` lets the symbol stay unresolved, in which case it reads as null. if linkage == Linkage::ExternalWeak { #[cfg(feature = "master")] global1.add_attribute(VarAttribute::Weak); @@ -411,8 +479,13 @@ fn check_and_apply_linkage<'gcc, 'tcx>( // zero. let real_name = format!("_rust_extern_with_linkage_{:016x}_{sym}", cx.tcx.stable_crate_id(LOCAL_CRATE)); - let global2 = cx.define_global(&real_name, gcc_type, is_tls, attrs.link_section); - // FIXME(antoyo): set linkage. + let global2 = cx.define_global( + &real_name, + gcc_type, + GlobalKind::Internal, + is_tls, + attrs.link_section, + ); let value = cx.const_ptrcast(global1.get_address(None), gcc_type); global2.global_set_initializer_rvalue(value); global2 diff --git a/src/context.rs b/src/context.rs index 38e0e5f329f76..7a1931b4967ce 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,6 +1,8 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; +#[cfg(feature = "master")] +use gccjit::Region; use gccjit::{Block, CType, Context, Function, FunctionType, LValue, Location, RValue, Type}; use rustc_abi::{Align, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx}; use rustc_codegen_ssa::base::wants_msvc_seh; @@ -25,6 +27,13 @@ use rustc_target::spec::{HasTargetSpec, HasX86AbiOpt, Target, TlsModel, X86Abi}; use crate::abi::conv_to_fn_attribute; use crate::callee::get_fn; use crate::common::SignType; +use crate::type_::StructTypeKey; + +#[cfg(feature = "master")] +pub struct PendingCleanup<'gcc> { + pub region: Region<'gcc>, + pub landing_pad: Block<'gcc>, +} #[cfg_attr(not(feature = "master"), expect(dead_code))] pub struct CodegenCx<'gcc, 'tcx> { @@ -84,7 +93,14 @@ pub struct CodegenCx<'gcc, 'tcx> { pub types: RefCell, Option), Type<'gcc>>>, pub tcx: TyCtxt<'tcx>, - pub struct_types: RefCell>, Type<'gcc>>>, + /// Cache of the anonymous struct types. + pub struct_types: RefCell, Type<'gcc>>>, + + /// Cache of the array types used for runs of constant bytes, keyed by element type and count. + /// + /// libgccjit mints a fresh type on every `new_array_type`, and struct types are keyed on their + /// field types, so without this two equal byte runs would yield two distinct anonymous structs. + pub byte_array_types: RefCell, u64), Type<'gcc>>>, /// Cache instances of monomorphic and polymorphic items pub instances: RefCell, LValue<'gcc>>>, @@ -125,8 +141,14 @@ pub struct CodegenCx<'gcc, 'tcx> { pub pointee_infos: RefCell, Size), Option>>, + /// Blocks that are cleanup landing pads, so `invoke` can tell an unwind + /// edge into a cleanup from a catch/terminate. #[cfg(feature = "master")] - pub cleanup_blocks: RefCell>>, + pub landing_pads: RefCell>>, + /// Cleanup regions to be filled in once the function is fully codegened + /// (done in `populate_cleanup_regions`). + #[cfg(feature = "master")] + pub pending_cleanups: RefCell>>, /// The alignment of a u128/i128 type. // We cache this, since it is needed for alignment checks during loads. pub int128_align: Align, @@ -225,12 +247,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { let isize_type = usize_type; let bool_type = context.new_type::(); - let mut functions = FxHashMap::default(); - let builtins = ["abort"]; - - for builtin in builtins.iter() { - functions.insert(builtin.to_string(), context.get_builtin_function(builtin)); - } + let functions = FxHashMap::default(); let mut cx = Self { int128_align: tcx @@ -297,6 +314,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { types: Default::default(), tcx, struct_types: Default::default(), + byte_array_types: Default::default(), local_gen_sym_counter: Cell::new(0), global_gen_sym_counter: Cell::new(0), eh_personality: Cell::new(None), @@ -304,13 +322,43 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { rust_try_fn: Cell::new(None), pointee_infos: Default::default(), #[cfg(feature = "master")] - cleanup_blocks: Default::default(), + landing_pads: Default::default(), + #[cfg(feature = "master")] + pending_cleanups: Default::default(), }; // FIXME(antoyo): instead of doing this, add SsizeT to libgccjit. cx.isize_type = usize_type.to_signed(&cx); cx } + /// Fill in the member blocks of every pending cleanup region. + /// + /// Clone all blocks reachable from a cleanup block into the cleanup region. + #[cfg(feature = "master")] + pub fn populate_cleanup_regions(&self) { + let pending = std::mem::take(&mut *self.pending_cleanups.borrow_mut()); + + for cleanup in pending { + // The landing pad is the region's entry, so it must come first. + let mut blocks = vec![]; + let mut visited = FxHashSet::default(); + let mut stack = vec![cleanup.landing_pad]; + while let Some(block) = stack.pop() { + if !visited.insert(block) { + continue; + } + blocks.push(block); + stack.extend(block.get_successors()); + } + + for clone in gccjit::clone_blocks(&blocks) { + cleanup.region.add_block(clone); + } + } + + self.landing_pads.borrow_mut().clear(); + } + pub fn rvalue_as_function(&self, value: RValue<'gcc>) -> Function<'gcc> { let function: Function<'gcc> = unsafe { std::mem::transmute(value) }; debug_assert!( diff --git a/src/declare.rs b/src/declare.rs index 4174eebcf7b02..a9503574a03ed 100644 --- a/src/declare.rs +++ b/src/declare.rs @@ -14,6 +14,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { &self, name: &str, ty: Type<'gcc>, + global_kind: GlobalKind, is_tls: bool, link_section: Option, ) -> LValue<'gcc> { @@ -28,7 +29,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { } global } else { - self.declare_global(name, ty, GlobalKind::Exported, is_tls, link_section) + self.declare_global(name, ty, global_kind, is_tls, link_section) } } @@ -135,10 +136,11 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { &self, name: &str, ty: Type<'gcc>, + global_kind: GlobalKind, is_tls: bool, link_section: Option, ) -> LValue<'gcc> { - self.get_or_insert_global(name, ty, is_tls, link_section) + self.get_or_insert_global(name, ty, global_kind, is_tls, link_section) } pub fn get_declared_value(&self, name: &str) -> Option> { diff --git a/src/gcc_util.rs b/src/gcc_util.rs index 24f552fed32c6..4b0fef32cac03 100644 --- a/src/gcc_util.rs +++ b/src/gcc_util.rs @@ -1,10 +1,15 @@ +use std::borrow::Cow; +use std::collections::HashSet; +use std::env; + +use gccjit::Context; #[cfg(feature = "master")] use gccjit::Context; use rustc_codegen_ssa::target_features; use rustc_data_structures::smallvec::{SmallVec, smallvec}; -use rustc_session::EarlySession; use rustc_session::config::NATIVE_CPU; -use rustc_target::spec::{Arch, Target}; +use rustc_session::{EarlySession, Session}; +use rustc_target::spec::{Arch, RelocModel, StackProbeType, StackProtector, Target}; fn gcc_features_by_flags(sess: &EarlySession, features: &mut Vec) { target_features::retpoline_features_by_flags(sess, features); @@ -110,29 +115,180 @@ pub fn to_gcc_features<'a>(target: &Target, s: &'a str) -> SmallVec<[&'a str; 2] fn arch_to_gcc(name: &str) -> &str { match name { "M68000" => "68000", + "M68010" => "68010", "M68020" => "68020", + "M68030" => "68030", + "M68040" => "68040", + "M68060" => "68060", _ => name, } } -fn handle_native(name: &str) -> &str { +fn handle_native(name: &str) -> Cow<'_, str> { if name != NATIVE_CPU { - return arch_to_gcc(name); + return arch_to_gcc(name).into(); } #[cfg(feature = "master")] { // Get the native arch. let context = Context::default(); - context.get_target_info().arch().unwrap().to_str().unwrap() + Cow::Owned(context.get_target_info().arch().to_str().unwrap().to_string()) } #[cfg(not(feature = "master"))] unimplemented!(); } -pub fn target_cpu(sess: &EarlySession) -> &str { +pub fn target_cpu(sess: &EarlySession) -> Cow<'_, str> { match sess.opts.cg.target_cpu { Some(ref name) => handle_native(name), None => handle_native(sess.target.cpu.as_ref()), } } + +pub fn new_context<'gcc>(sess: &Session) -> Context<'gcc> { + let context = Context::default(); + if matches!(sess.target.arch, Arch::X86 | Arch::X86_64) { + context.add_command_line_option("-masm=intel"); + } + #[cfg(feature = "master")] + { + context.set_special_chars_allowed_in_func_names("$.*"); + let version = Version::get(); + let version = format!("{}.{}.{}", version.major, version.minor, version.patch); + context.set_output_ident(&format!( + "rustc version {} with libgccjit {}", + rustc_interface::util::rustc_version_str().unwrap_or("unknown version"), + version, + )); + } + if !sess.must_emit_unwind_tables() { + context.add_command_line_option("-fno-asynchronous-unwind-tables"); + } + + if sess.panic_strategy().unwinds() { + context.add_command_line_option("-fexceptions"); + context.add_driver_option("-fexceptions"); + } + + let disabled_features: HashSet<_> = sess + .opts + .cg + .target_feature + .split(',') + .filter(|feature| feature.starts_with('-')) + .map(|string| &string[1..]) + .collect(); + + if !disabled_features.contains("avx") && sess.target.arch == Arch::X86_64 { + // NOTE: we always enable AVX because the equivalent of llvm.x86.sse2.cmp.pd in GCC for + // SSE2 is multiple builtins, so we use the AVX __builtin_ia32_cmppd instead. + // FIXME(antoyo): use the proper builtins for llvm.x86.sse2.cmp.pd and similar. + context.add_command_line_option("-mavx"); + } + + for arg in &sess.opts.cg.llvm_args { + context.add_command_line_option(arg); + } + // NOTE: This is needed to compile the file src/intrinsic/archs.rs during a bootstrap of rustc. + context.add_command_line_option("-fno-var-tracking-assignments"); + // NOTE: an optimization (https://github.com/rust-lang/rustc_codegen_gcc/issues/53). + context.add_command_line_option("-fno-semantic-interposition"); + // NOTE: Rust relies on LLVM not doing TBAA (https://github.com/rust-lang/unsafe-code-guidelines/issues/292). + context.add_command_line_option("-fno-strict-aliasing"); + // NOTE: Rust relies on LLVM doing wrapping on overflow. + context.add_command_line_option("-fwrapv"); + // NOTE: This is needed to hide a warning caused by the alignment fix on byval arguments. + context.add_command_line_option("-Wno-psabi"); + + if let Some(model) = sess.code_model() { + use rustc_target::spec::CodeModel; + + context.add_command_line_option(match model { + CodeModel::Tiny => "-mcmodel=tiny", + CodeModel::Small => "-mcmodel=small", + CodeModel::Kernel => "-mcmodel=kernel", + CodeModel::Medium => "-mcmodel=medium", + CodeModel::Large => "-mcmodel=large", + }); + } + + match sess.stack_protector() { + StackProtector::All => context.add_command_line_option("-fstack-protector-all"), + StackProtector::Strong => context.add_command_line_option("-fstack-protector-strong"), + StackProtector::Basic => context.add_command_line_option("-fstack-protector"), + StackProtector::None => (), + } + + match sess.target.stack_probes { + StackProbeType::None => (), + StackProbeType::Inline | StackProbeType::InlineOrCall { .. } => { + context.add_command_line_option("-fstack-clash-protection") + } + // FIXME(antoyo): We should define the stack probe symbol to be __rust_probestack, but it seems GCC cannot do that. + StackProbeType::Call => (), + }; + + add_pic_option(&context, sess.relocation_model()); + + let target_cpu = target_cpu(sess); + if target_cpu != "generic" { + context.add_command_line_option(format!("-march={}", target_cpu)); + } + + if sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections) { + context.add_command_line_option("-ffunction-sections"); + context.add_command_line_option("-fdata-sections"); + } + + if env::var("CG_GCCJIT_DUMP_RTL").as_deref() == Ok("1") { + context.add_command_line_option("-fdump-rtl-vregs"); + } + if env::var("CG_GCCJIT_DUMP_RTL_ALL").as_deref() == Ok("1") { + context.add_command_line_option("-fdump-rtl-all"); + } + if env::var("CG_GCCJIT_DUMP_TREE_ALL").as_deref() == Ok("1") { + context.add_command_line_option("-fdump-tree-all-eh"); + } + if env::var("CG_GCCJIT_DUMP_IPA_ALL").as_deref() == Ok("1") { + context.add_command_line_option("-fdump-ipa-all-eh"); + } + if env::var("CG_GCCJIT_DUMP_CODE").as_deref() == Ok("1") { + context.set_dump_code_on_compile(true); + } + if env::var("CG_GCCJIT_DUMP_GIMPLE").as_deref() == Ok("1") { + context.set_dump_initial_gimple(true); + } + if env::var("CG_GCCJIT_DUMP_EVERYTHING").as_deref() == Ok("1") { + context.set_dump_everything(true); + } + if env::var("CG_GCCJIT_KEEP_INTERMEDIATES").as_deref() == Ok("1") { + context.set_keep_intermediates(true); + } + if env::var("CG_GCCJIT_VERBOSE").as_deref() == Ok("1") { + context.add_driver_option("-v"); + } + + context +} + +pub fn add_pic_option<'gcc>(context: &Context<'gcc>, relocation_model: RelocModel) { + match relocation_model { + rustc_target::spec::RelocModel::Static => { + context.add_command_line_option("-fno-pie"); + context.add_driver_option("-fno-pie"); + } + rustc_target::spec::RelocModel::Pic => { + context.add_command_line_option("-fPIC"); + // NOTE: we use both add_command_line_option and add_driver_option because the usage in + // base (compile_codegen_unit) requires add_command_line_option while the usage + // in the back::write module (codegen) requires add_driver_option. + context.add_driver_option("-fPIC"); + } + rustc_target::spec::RelocModel::Pie => { + context.add_command_line_option("-fPIE"); + context.add_driver_option("-fPIE"); + } + model => eprintln!("Unsupported relocation model: {:?}", model), + } +} diff --git a/src/int.rs b/src/int.rs index dfae4eceebe44..021f05c666d46 100644 --- a/src/int.rs +++ b/src/int.rs @@ -21,12 +21,12 @@ use crate::context::CodegenCx; impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { pub fn gcc_urem(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { // 128-bit unsigned %: __umodti3 - self.multiplicative_operation(BinaryOp::Modulo, "mod", false, a, b) + self.division_operation(BinaryOp::Modulo, "mod", false, a, b) } pub fn gcc_srem(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { // 128-bit signed %: __modti3 - self.multiplicative_operation(BinaryOp::Modulo, "mod", true, a, b) + self.division_operation(BinaryOp::Modulo, "mod", true, a, b) } pub fn gcc_not(&self, a: RValue<'gcc>) -> RValue<'gcc> { @@ -178,6 +178,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { } else { debug_assert!(a_type.dyncast_array().is_some()); debug_assert!(b_type.dyncast_array().is_some()); + if a_type != b_type { + b = self.gcc_int_cast(b, a_type); + } let signed = a_type.is_compatible_with(self.i128_type); let func_name = match (operation, signed) { (BinaryOp::Plus, true) => "__rust_i128_add", @@ -187,7 +190,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { _ => unreachable!("unexpected additive operation {:?}", operation), }; let param_a = self.context.new_parameter(self.location, a_type, "a"); - let param_b = self.context.new_parameter(self.location, b_type, "b"); + let param_b = self.context.new_parameter(self.location, a_type, "b"); let func = self.context.new_function( self.location, FunctionType::Extern, @@ -212,6 +215,27 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.additive_operation(BinaryOp::Minus, a, b) } + fn division_operation( + &self, + operation: BinaryOp, + operation_name: &str, + signed: bool, + mut a: RValue<'gcc>, + mut b: RValue<'gcc>, + ) -> RValue<'gcc> { + let a_type = a.get_type(); + if self.is_native_int_type(a_type) && self.is_native_int_type(b.get_type()) { + let typ = if signed { a_type.to_signed(self.cx) } else { a_type.to_unsigned(self.cx) }; + if !typ.is_compatible_with(a_type) { + a = self.context.new_cast(self.location, a, typ); + } + if !typ.is_compatible_with(b.get_type()) { + b = self.context.new_cast(self.location, b, typ); + } + } + self.multiplicative_operation(operation, operation_name, signed, a, b) + } + fn multiplicative_operation( &self, operation: BinaryOp, @@ -238,10 +262,13 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { } else { debug_assert!(a_type.dyncast_array().is_some()); debug_assert!(b_type.dyncast_array().is_some()); + if a_type != b_type { + b = self.gcc_int_cast(b, a_type); + } let sign = if signed { "" } else { "u" }; let func_name = format!("__{}{}ti3", sign, operation_name); let param_a = self.context.new_parameter(self.location, a_type, "a"); - let param_b = self.context.new_parameter(self.location, b_type, "b"); + let param_b = self.context.new_parameter(self.location, a_type, "b"); let func = self.context.new_function( self.location, FunctionType::Extern, @@ -255,15 +282,13 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { } pub fn gcc_sdiv(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { - // FIXME(antoyo): check if the types are signed? // 128-bit, signed: __divti3 - // FIXME(antoyo): convert the arguments to signed? - self.multiplicative_operation(BinaryOp::Divide, "div", true, a, b) + self.division_operation(BinaryOp::Divide, "div", true, a, b) } pub fn gcc_udiv(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { // 128-bit, unsigned: __udivti3 - self.multiplicative_operation(BinaryOp::Divide, "div", false, a, b) + self.division_operation(BinaryOp::Divide, "div", false, a, b) } pub fn gcc_checked_binop( @@ -462,9 +487,18 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { lhs_high = self.context.new_cast(self.location, lhs_high, unsigned_type); rhs_high = self.context.new_cast(self.location, rhs_high, unsigned_type); } - // FIXME(antoyo): we probably need to handle signed comparison for unsigned - // integers. - _ => (), + IntPredicate::IntSGT + | IntPredicate::IntSGE + | IntPredicate::IntSLT + | IntPredicate::IntSLE => { + let signed_type = native_int_type.to_signed(self.cx); + lhs_high = self.context.new_cast(self.location, lhs_high, signed_type); + rhs_high = self.context.new_cast(self.location, rhs_high, signed_type); + } + IntPredicate::IntEQ | IntPredicate::IntNE => { + lhs_high = self.context.new_cast(self.location, lhs_high, unsigned_type); + rhs_high = self.context.new_cast(self.location, rhs_high, unsigned_type); + } } let condition = self.context.new_comparison( @@ -623,6 +657,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { } a ^ b } else { + if a_type != b_type { + b = self.gcc_int_cast(b, a_type); + } self.concat_low_high_rvalues( a_type, self.low(a) ^ self.low(b), @@ -832,6 +869,9 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { !a_native && !b_native, "both types should either be native or non-native for or operation" ); + if a_type != b_type { + b = self.gcc_int_cast(b, a_type); + } let native_int_type = a_type.dyncast_array().expect("get element type"); self.concat_low_high_rvalues( a_type, diff --git a/src/intrinsic/llvm.rs b/src/intrinsic/llvm.rs index 41efe3e8209bf..f3134edb72cb9 100644 --- a/src/intrinsic/llvm.rs +++ b/src/intrinsic/llvm.rs @@ -5,6 +5,7 @@ use rustc_codegen_ssa::traits::BuilderMethods; use crate::builder::Builder; use crate::context::{CodegenCx, new_array_type}; +use crate::type_::{StructAttribute, apply_struct_attributes}; fn encode_key_128_type<'a, 'gcc, 'tcx>( builder: &Builder<'a, 'gcc, 'tcx>, @@ -22,8 +23,7 @@ fn encode_key_128_type<'a, 'gcc, 'tcx>( "EncodeKey128Output", &[field1, field2, field3, field4, field5, field6, field7], ); - #[cfg(feature = "master")] - encode_type.as_type().set_packed(); + apply_struct_attributes(encode_type.as_type(), &[StructAttribute::Packed]); (encode_type.as_type(), field1, field2) } @@ -44,8 +44,7 @@ fn encode_key_256_type<'a, 'gcc, 'tcx>( "EncodeKey256Output", &[field1, field2, field3, field4, field5, field6, field7, field8], ); - #[cfg(feature = "master")] - encode_type.as_type().set_packed(); + apply_struct_attributes(encode_type.as_type(), &[StructAttribute::Packed]); (encode_type.as_type(), field1, field2) } @@ -57,8 +56,7 @@ fn aes_output_type<'a, 'gcc, 'tcx>( let field2 = builder.context.new_field(None, m128i, "field2"); let aes_output_type = builder.context.new_struct_type(None, "AesOutput", &[field1, field2]); let typ = aes_output_type.as_type(); - #[cfg(feature = "master")] - typ.set_packed(); + apply_struct_attributes(typ, &[StructAttribute::Packed]); (typ, field1, field2) } @@ -80,8 +78,7 @@ fn wide_aes_output_type<'a, 'gcc, 'tcx>( "WideAesOutput", &[field1, field2, field3, field4, field5, field6, field7, field8, field9], ); - #[cfg(feature = "master")] - aes_output_type.as_type().set_packed(); + apply_struct_attributes(aes_output_type.as_type(), &[StructAttribute::Packed]); (aes_output_type.as_type(), field1, field2) } diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs index 41fa7b3f9f162..0a24519a8091e 100644 --- a/src/intrinsic/mod.rs +++ b/src/intrinsic/mod.rs @@ -24,6 +24,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; +use rustc_session::config::OptLevel; use rustc_span::{Span, Symbol, bug, span_bug, sym}; use rustc_target::callconv::{ArgAbi, PassMode}; @@ -88,7 +89,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>( sym::round_ties_even_f64 => "rint", sym::roundf32 => "roundf", sym::roundf64 => "round", - sym::abort => "abort", _ => return None, }; Some(cx.context.get_builtin_function(gcc_name)) @@ -664,16 +664,26 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } fn abort(&mut self) { - let func = self.context.get_builtin_function("abort"); - let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; - self.call(self.type_void(), None, None, func, ReturnSlot::Direct, &[], None, None); + let func = self.context.get_builtin_function("__builtin_trap"); + self.block.add_eval(self.location, self.context.new_call(self.location, func, &[])); } fn assume(&mut self, value: Self::Value) { - // FIXME(antoyo): switch to assume when it exists. - // Or use something like this: - // #define __assume(cond) do { if (!(cond)) __builtin_unreachable(); } while (0) - self.expect(value, true); + // libgccjit currently has no direct equivalent of LLVM's `llvm.assume`, + // so use the idiom `if (!cond) __builtin_unreachable()`. + // FIXME: this should use IFN_ASSUME when we have internal functions in + // libgccjit. + if self.sess().opts.optimize == OptLevel::No { + return; + } + let then_block = self.append_sibling_block("assume_holds"); + let unreachable_block = self.append_sibling_block("assume_violated"); + self.block.end_with_conditional(self.location, value, then_block, unreachable_block); + + self.switch_to_block(unreachable_block); + self.unreachable(); + + self.switch_to_block(then_block); } fn expect(&mut self, cond: Self::Value, _expected: bool) -> Self::Value { diff --git a/src/intrinsic/simd.rs b/src/intrinsic/simd.rs index 1416f4eec9c4a..bb32a85f193ea 100644 --- a/src/intrinsic/simd.rs +++ b/src/intrinsic/simd.rs @@ -11,13 +11,13 @@ use rustc_codegen_ssa::diagnostics::ExpectedPointerMutability; use rustc_codegen_ssa::diagnostics::InvalidMonomorphization; use rustc_codegen_ssa::mir::operand::OperandRef; use rustc_codegen_ssa::mir::place::PlaceRef; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, LayoutTypeCodegenMethods}; #[cfg(feature = "master")] use rustc_hir as hir; use rustc_middle::mir::BinOp; -use rustc_middle::ty::layout::HasTyCtxt; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, Ty}; -use rustc_span::{ErrorGuaranteed, Span, Symbol, sym}; +use rustc_span::{ErrorGuaranteed, Span, Symbol, span_bug, sym}; use crate::builder::Builder; #[cfg(not(feature = "master"))] @@ -655,6 +655,39 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( return Ok(bx.context.new_rvalue_from_vector(bx.location, llret_ty, &values)); } + if name == sym::simd_arith_offset { + // This also checks that the first operand is a ptr type. + let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { + span_bug!(span, "must be called with a vector of pointer types as first argument") + }); + let layout = bx.layout_of(pointee); + // The second argument must be a ptr-sized integer. + // (We don't care about the signedness, this is wrapping anyway.) + let (_, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx()); + if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) { + span_bug!( + span, + "must be called with a vector of pointer-sized integers as second argument" + ); + } + + let pointee_type = bx.backend_type(layout); + let pointers = args[0].immediate(); + let offsets = args[1].immediate(); + let elem_type = llret_ty.dyncast_vector().expect("vector return type").get_element_type(); + let values: Vec<_> = (0..in_len) + .map(|i| { + let index = bx.context.new_rvalue_from_long(bx.usize_type, i as _); + let pointer = bx.extract_element(pointers, index); + let offset = bx.extract_element(offsets, index); + let pointer = bx.gep(pointee_type, pointer, &[offset]); + // GCC has no pointer vectors, so the lanes are `usize`. + bx.ptrtoint(pointer, elem_type) + }) + .collect(); + return Ok(bx.context.new_rvalue_from_vector(bx.location, llret_ty, &values)); + } + #[cfg(feature = "master")] if name == sym::simd_cast || name == sym::simd_as { require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); @@ -675,20 +708,28 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( return Ok(args[0].immediate()); } + #[derive(Copy, Clone)] + enum Sign { + Unsigned, + Signed, + } + use Sign::*; + enum Style { Float, - Int, + Int(Sign), Unsupported, } let in_style = match *in_elem.kind() { - ty::Int(_) | ty::Uint(_) => Style::Int, + ty::Int(_) => Style::Int(Signed), + ty::Uint(_) => Style::Int(Unsigned), ty::Float(_) => Style::Float, _ => Style::Unsupported, }; - let out_style = match *out_elem.kind() { - ty::Int(_) | ty::Uint(_) => Style::Int, + ty::Int(_) => Style::Int(Signed), + ty::Uint(_) => Style::Int(Unsigned), ty::Float(_) => Style::Float, _ => Style::Unsupported, }; @@ -707,6 +748,19 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( } ); } + (Style::Float, Style::Int(sign)) if name == sym::simd_as => { + let vector = args[0].immediate(); + let elem_type = + llret_ty.dyncast_vector().expect("vector return type").get_element_type(); + let values: Vec<_> = (0..in_len) + .map(|i| { + let index = bx.context.new_rvalue_from_long(bx.usize_type, i as _); + let value = bx.extract_element(vector, index); + bx.cast_float_to_int(matches!(sign, Sign::Signed), value, elem_type) + }) + .collect(); + return Ok(bx.context.new_rvalue_from_vector(bx.location, llret_ty, &values)); + } _ => return Ok(bx.context.convert_vector(None, args[0].immediate(), llret_ty)), } } @@ -1310,32 +1364,28 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( (true, false) => { // FIXME(antoyo): dyncast_vector should not require a call to unqualified. let arg_type = lhs.get_type().unqualified(); - // FIXME(antoyo): this uses the same algorithm from saturating add, but add the - // negative of the right operand. Find a proper subtraction algorithm. - let rhs = bx.context.new_unary_op(None, UnaryOp::Minus, arg_type, rhs); - // FIXME(antoyo): convert lhs and rhs to unsigned. - let sum = lhs + rhs; + let difference = lhs - rhs; let vector_type = arg_type.dyncast_vector().expect("vector type"); let unit = vector_type.get_num_units(); let a = bx.context.new_rvalue_from_int(elem_ty, ((elem_width as i32) << 3) - 1); let width = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![a; unit]); + // The subtraction overflows when the operands have different signs and the result + // has a different sign than the left operand. let xor1 = lhs ^ rhs; - let xor2 = lhs ^ sum; - let and = - bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, xor1) & xor2; - let mask = and >> width; + let xor2 = lhs ^ difference; + let mask = (xor1 & xor2) >> width; let one = bx.context.new_rvalue_one(elem_ty); let ones = bx.context.new_rvalue_from_vector(None, lhs.get_type(), &vec![one; unit]); let shift1 = ones << width; - let shift2 = sum >> width; + let shift2 = difference >> width; let mask_min = shift1 ^ shift2; - let and1 = - bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, mask) & sum; + let and1 = bx.context.new_unary_op(None, UnaryOp::BitwiseNegate, arg_type, mask) + & difference; let and2 = mask & mask_min; and1 + and2 diff --git a/src/lib.rs b/src/lib.rs index d7a3ef3b4a6c5..0784823c27b9e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -94,7 +94,7 @@ use rustc_middle::ty::TyCtxt; use rustc_session::config::{OptLevel, OutputFilenames}; use rustc_session::{CodegenBackendInit, EarlySession, IncrCompSession, Session}; use rustc_span::{Symbol, sym}; -use rustc_target::spec::{Arch, RelocModel}; +use rustc_target::spec::{RelocModel, TargetTuple}; use tempfile::TempDir; use crate::back::lto::ModuleBuffer; @@ -176,15 +176,21 @@ impl CodegenBackend for GccCodegenBackend { } fn init(&mut self, sess: &EarlySession) -> CodegenBackendInit { - fn file_path(sysroot_path: &Path, sess: &EarlySession) -> PathBuf { - let rustlib_path = - rustc_target::relative_target_rustlib_path(sysroot_path, &sess.host.llvm_target); - sysroot_path - .join(rustlib_path) - .join("codegen-backends") - .join("lib") - .join(sess.target.llvm_target.as_ref()) - .join("libgccjit.so") + fn file_paths(sysroot_path: &Path, sess: &EarlySession) -> Vec { + let rustlib_path = rustc_target::relative_target_rustlib_path( + sysroot_path, + rustc_session::config::host_tuple(), + ); + let lib_path = sysroot_path.join(rustlib_path).join("codegen-backends").join("lib"); + let rust_target_path = + lib_path.join(sess.opts.target_triple.tuple()).join("libgccjit.so"); + let mut paths = vec![rust_target_path]; + if matches!(sess.opts.target_triple, TargetTuple::TargetJson { .. }) { + let llvm_target_path = + lib_path.join(sess.target.llvm_target.as_ref()).join("libgccjit.so"); + paths.push(llvm_target_path); + } + paths } let global_backend_features = gcc_util::global_gcc_features(sess); @@ -192,21 +198,24 @@ impl CodegenBackend for GccCodegenBackend { // We use all_paths() instead of only path() in case the path specified by --sysroot is // invalid. // This is the case for instance in Rust for Linux where they specify --sysroot=/dev/null. - for path in sess.opts.sysroot.all_paths() { - let libgccjit_target_lib_file = file_path(path, sess); - if let Ok(true) = fs::exists(&libgccjit_target_lib_file) { - load_libgccjit_if_needed(&libgccjit_target_lib_file); - break; + 'sysroot: for path in sess.opts.sysroot.all_paths() { + for libgccjit_target_lib_file in file_paths(path, sess) { + if let Ok(true) = fs::exists(&libgccjit_target_lib_file) { + load_libgccjit_if_needed(&libgccjit_target_lib_file); + break 'sysroot; + } } } if !gccjit::is_loaded() { let mut paths = vec![]; for path in sess.opts.sysroot.all_paths() { - let libgccjit_target_lib_file = file_path(path, sess); - paths.push(libgccjit_target_lib_file); + for libgccjit_target_lib_file in file_paths(path, sess) { + paths.push(libgccjit_target_lib_file); + } } + paths.dedup(); panic!("Could not load libgccjit.so. Attempted paths: {:#?}", paths); } @@ -259,7 +268,7 @@ impl CodegenBackend for GccCodegenBackend { } fn target_cpu(&self, sess: &Session) -> String { - target_cpu(sess).to_owned() + target_cpu(sess).into_owned() } fn codegen_crate(&self, tcx: TyCtxt<'_>) -> Box { @@ -283,13 +292,6 @@ impl CodegenBackend for GccCodegenBackend { fn target_config(&self, sess: &EarlySession) -> TargetConfig { target_config(sess, &self.config().target_info) } -} - -fn new_context<'gcc, 'tcx>(tcx: TyCtxt<'tcx>) -> Context<'gcc> { - let context = Context::default(); - if matches!(tcx.sess.target.arch, Arch::X86 | Arch::X86_64) { - context.add_command_line_option("-masm=intel"); - } #[cfg(feature = "master")] { context.set_special_chars_allowed_in_func_names("$.*"); diff --git a/src/mono_item.rs b/src/mono_item.rs index fc92cc3d5c7a9..014a7ff90536d 100644 --- a/src/mono_item.rs +++ b/src/mono_item.rs @@ -1,15 +1,16 @@ #[cfg(feature = "master")] -use gccjit::{FnAttribute, VarAttribute}; +use gccjit::{FnAttribute, GlobalKind, ToRValue, Type, VarAttribute}; use rustc_codegen_ssa::traits::PreDefineCodegenMethods; use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_span::bug; +use crate::consts::const_alloc_type; use crate::context::CodegenCx; use crate::type_of::LayoutGccExt; use crate::{attributes, base}; @@ -19,23 +20,58 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { fn predefine_static( &mut self, def_id: DefId, - _linkage: Linkage, + linkage: Linkage, visibility: Visibility, symbol_name: &str, ) { let attrs = self.tcx.codegen_fn_attrs(def_id); let instance = Instance::mono(self.tcx, def_id); - let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; - // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out - // the gcc type from the actual evaluated initializer. - let ty = - if nested { self.tcx.types.unit } else { instance.ty(self.tcx, self.typing_env()) }; - let gcc_type = self.layout_of(ty).gcc_type(self); + // Declare the global with the type its initializer will have, so that `codegen_static` + // never has to retype it afterwards. The initializer is lowered as a packed struct of byte + // runs and relocations, which almost never matches the layout type. + let gcc_type = match self.tcx.eval_static_initializer(def_id) { + Ok(alloc) => const_alloc_type(self, alloc), + // The initializer failed to evaluate; `codegen_static` bails out on it too, so this + // type is never used to hold one. + Err(_) => { + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy one. + let ty = if nested { + self.tcx.types.unit + } else { + instance.ty(self.tcx, self.typing_env()) + }; + self.layout_of(ty).gcc_type(self) + } + }; let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); - let global = self.define_global(symbol_name, gcc_type, is_tls, attrs.link_section); + let global_kind = base::global_linkage_to_gcc(linkage); + let global = + self.define_global(global_name, gcc_type, global_kind, is_tls, attrs.link_section); #[cfg(feature = "master")] - global.add_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility))); + { + // Visibility is meaningless on an internal global: GCC ignores the attribute and + // warns about it. + if !matches!(global_kind, GlobalKind::Internal) { + // If we're compiling the compiler-builtins crate, e.g., the equivalent of + // compiler-rt, then we want to implicitly compile everything with hidden + // visibility as we're going to link this object all over the place but + // don't want the symbols to get exported. + let visibility = if self.tcx.is_compiler_builtins(LOCAL_CRATE) { + gccjit::Visibility::Hidden + } else { + base::visibility_to_gcc(visibility) + }; + global.add_attribute(VarAttribute::Visibility(visibility)); + } + if let Some(attribute) = base::global_linkage_attribute(linkage) { + global.add_attribute(attribute); + } + } + + #[cfg(feature = "master")] + self.add_static_aliases(gcc_type, global_name, attrs, &attrs.foreign_item_symbol_aliases); // FIXME(antoyo): set linkage. self.instances.borrow_mut().insert(instance, global); @@ -50,6 +86,104 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { ) { assert!(!instance.args.has_infer()); + let attrs = self.tcx.codegen_instance_attrs(instance.def); + + let decl = + self.predefine_without_aliases(instance, &attrs, linkage, visibility, symbol_name); + + #[cfg(feature = "master")] + self.add_function_aliases(instance, decl, &attrs, &attrs.foreign_item_symbol_aliases); + + self.functions.borrow_mut().insert(symbol_name.to_string(), decl); + self.function_instances.borrow_mut().insert(instance, decl); + } +} + +impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { + #[cfg(feature = "master")] + fn add_static_aliases( + &self, + gcc_type: Type<'gcc>, + aliased: &str, + attrs: &CodegenFnAttrs, + aliases: &[(DefId, Linkage, Visibility)], + ) { + let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); + + for &(alias, linkage, visibility) in aliases { + let instance = Instance::mono(self.tcx, alias); + let symbol_name = self.tcx.symbol_name(instance); + + let alias = self.declare_global( + symbol_name.name, + gcc_type, + GlobalKind::Imported, + is_tls, + attrs.link_section, + ); + alias.add_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility))); + alias.add_attribute(VarAttribute::Alias(aliased)); + if linkage == Linkage::WeakAny { + alias.add_attribute(VarAttribute::Weak); + } + + // Add the alias name to the set of cached items, so there is no duplicate + // instance added to it during the normal `external static` codegen + let prev_entry = self.instances.borrow_mut().insert(instance, alias); + + // If there already was a previous entry, then `add_static_aliases` was called multiple times for the same `alias` + // which would result in incorrect codegen + assert!(prev_entry.is_none(), "An instance was already present for {instance:?}"); + } + } + + #[cfg(feature = "master")] + fn add_function_aliases( + &self, + aliased_instance: Instance<'tcx>, + aliased: Function<'gcc>, + attrs: &CodegenFnAttrs, + aliases: &[(DefId, Linkage, Visibility)], + ) { + for &(alias, linkage, visibility) in aliases { + let symbol_name = self.tcx.symbol_name(Instance::mono(self.tcx, alias)); + + // predefine another copy of the original instance + // with a new symbol name + let alias_fn_decl = self.predefine_without_aliases( + aliased_instance, + attrs, + linkage, + visibility, + symbol_name.name, + ); + + let block = alias_fn_decl.new_block("start"); + let nb_params = alias_fn_decl.get_param_count(); + let mut args = Vec::with_capacity(nb_params); + for idx in 0..nb_params { + args.push(alias_fn_decl.get_param(idx as _).to_rvalue()); + } + + let void_type = self.context.new_type::<()>(); + let call = self.context.new_call(None, aliased, &args); + if alias_fn_decl.get_return_type() == void_type { + block.add_eval(None, call); + block.end_with_void_return(None); + } else { + block.end_with_return(None, call); + } + } + } + + fn predefine_without_aliases( + &self, + instance: Instance<'tcx>, + _attrs: &CodegenFnAttrs, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ) -> Function<'gcc> { let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); self.linkage.set(base::linkage_to_gcc(linkage)); let decl = self.declare_fn(symbol_name, fn_abi); @@ -57,6 +191,11 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { attributes::from_fn_attrs(self, decl, instance); + #[cfg(feature = "master")] + if base::linkage_needs_weak_attribute(linkage) { + fn_decl.add_attribute(FnAttribute::Weak); + } + // If we're compiling the compiler-builtins crate, e.g., the equivalent of // compiler-rt, then we want to implicitly compile everything with hidden // visibility as we're going to link this object all over the place but diff --git a/src/type_.rs b/src/type_.rs index 27b0d2079e63e..45828ce3e92b8 100644 --- a/src/type_.rs +++ b/src/type_.rs @@ -1,5 +1,6 @@ #[cfg(feature = "master")] use std::convert::TryInto; +use std::mem::discriminant; #[cfg(feature = "master")] use gccjit::CType; @@ -102,9 +103,10 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { self.bool_type } - pub fn type_struct(&self, fields: &[Type<'gcc>], packed: bool) -> Type<'gcc> { - let types = fields.to_vec(); - if let Some(typ) = self.struct_types.borrow().get(fields) { + pub fn type_struct(&self, fields: &[Type<'gcc>], attributes: &[StructAttribute]) -> Type<'gcc> { + let key = + StructTypeKey { fields: fields.to_vec(), attributes: canonical_attributes(attributes) }; + if let Some(typ) = self.struct_types.borrow().get(&key) { return *typ; } let fields: Vec<_> = fields @@ -115,15 +117,91 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { }) .collect(); let typ = self.context.new_struct_type(None, "struct", &fields).as_type(); - if packed { - #[cfg(feature = "master")] - typ.set_packed(); - } - self.struct_types.borrow_mut().insert(types, typ); + apply_struct_attributes(typ, &key.attributes); + self.struct_types.borrow_mut().insert(key, typ); typ } } +/// An attribute that can be set on a GCC struct type. +/// +/// This mirrors the subset of `gccjit::TypeAttribute` that cg_gcc needs, rather than using it +/// directly, because it must exist without the `master` feature and because it is what +/// `StructTypeKey` is keyed on. Adding a variant here is therefore all it takes to make a new +/// attribute part of the cache key: there is no second place to remember to update. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum StructAttribute { + /// Alignment, in bytes. + Aligned(u32), + /// Lay the fields out without inserting padding between them. + Packed, +} + +/// Identifies an anonymous struct type in `CodegenCx::struct_types`. +/// +/// Two Rust types with the same field list can still need distinct GCC types — `struct { a: u64, +/// b: u64 }` with and without `repr(align(16))` produces the same fields — so every attribute has +/// to be part of the key. Holding them as one [`StructAttribute`] list rather than as separate +/// fields is what keeps that true when a new attribute is added. +#[derive(Clone, Eq, Hash, PartialEq)] +pub struct StructTypeKey<'gcc> { + pub fields: Vec>, + pub attributes: Vec, +} + +/// The attributes a GCC struct needs in order to match the Rust layout it is built from. +pub fn struct_attributes(packed: bool, align: Option) -> Vec { + let mut attributes = Vec::new(); + if packed { + attributes.push(StructAttribute::Packed); + } + if let Some(align) = align + && align.bytes() > 1 + && align.bytes() <= MAX_STRUCT_ALIGNMENT + { + attributes.push(StructAttribute::Aligned(align.bytes() as u32)); + } + attributes +} + +/// The largest alignment GCC accepts on a type, in bytes. +const MAX_STRUCT_ALIGNMENT: u64 = 1 << 28; + +/// Put an attribute list into a canonical form so that it can be used as a cache key. +/// +/// Without this, `[Packed, Aligned(8)]` and `[Aligned(8), Packed]` would hash differently and mint +/// two GCC types for what is one Rust type. +fn canonical_attributes(attributes: &[StructAttribute]) -> Vec { + let mut attributes = attributes.to_vec(); + attributes.sort_unstable(); + attributes.dedup(); + debug_assert!( + attributes.windows(2).all(|pair| discriminant(&pair[0]) != discriminant(&pair[1])), + "contradictory struct attributes: {attributes:?}" + ); + attributes +} + +/// Set `attributes` on the struct type `typ`. +/// +/// This is the only place allowed to call `Type::add_attribute`; `clippy.toml` forbids it +/// everywhere else. An attribute set on a type that `CodegenCx::struct_types` handed out would +/// change every other use of that type, so attributes have to be decided when the type is created +/// and be part of its cache key. Going through [`StructAttribute`] is what enforces that. +#[cfg(feature = "master")] +#[allow(clippy::disallowed_methods)] +pub fn apply_struct_attributes(typ: Type<'_>, attributes: &[StructAttribute]) { + for attribute in attributes { + typ.add_attribute(match *attribute { + StructAttribute::Aligned(align) => TypeAttribute::Aligned(align), + StructAttribute::Packed => TypeAttribute::Packed, + }); + } +} + +#[cfg(not(feature = "master"))] +pub fn apply_struct_attributes(_typ: Type<'_>, _attributes: &[StructAttribute]) {} + impl<'gcc, 'tcx> BaseTypeCodegenMethods for CodegenCx<'gcc, 'tcx> { fn type_i8(&self) -> Type<'gcc> { self.i8_type @@ -325,17 +403,19 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { self.type_array(self.type_from_integer(unit), size / unit_size) } - pub fn set_struct_body(&self, typ: Struct<'gcc>, fields: &[Type<'gcc>], packed: bool) { + pub fn set_struct_body( + &self, + typ: Struct<'gcc>, + fields: &[Type<'gcc>], + attributes: &[StructAttribute], + ) { let fields: Vec<_> = fields .iter() .enumerate() .map(|(index, field)| self.context.new_field(None, *field, format!("field_{}", index))) .collect(); typ.set_fields(None, &fields); - if packed { - #[cfg(feature = "master")] - typ.as_type().set_packed(); - } + apply_struct_attributes(typ.as_type(), &canonical_attributes(attributes)); } pub fn type_named_struct(&self, name: &str) -> Struct<'gcc> { diff --git a/src/type_of.rs b/src/type_of.rs index 31654d0e6be81..833ba9efe4efb 100644 --- a/src/type_of.rs +++ b/src/type_of.rs @@ -17,7 +17,7 @@ use rustc_target::callconv::{CastTarget, FnAbi}; use crate::abi::{FnAbiGcc, FnAbiGccExt, GccType}; use crate::context::CodegenCx; -use crate::type_::struct_fields; +use crate::type_::{struct_attributes, struct_fields}; impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { fn type_from_unsigned_integer(&self, i: Integer) -> Type<'gcc> { @@ -81,7 +81,7 @@ fn uncached_gcc_type<'gcc, 'tcx>( layout.scalar_pair_element_gcc_type(cx, 0), layout.scalar_pair_element_gcc_type(cx, 1), ], - false, + &struct_attributes(false, Some(layout.align.abi)), ); } BackendRepr::Memory { .. } => {} @@ -129,11 +129,12 @@ fn uncached_gcc_type<'gcc, 'tcx>( FieldsShape::Primitive | FieldsShape::Union(_) => { let fill = cx.type_padding_filler(layout.size, layout.align.abi); let packed = false; + let attributes = struct_attributes(packed, Some(layout.align.abi)); match name { - None => cx.type_struct(&[fill], packed), + None => cx.type_struct(&[fill], &attributes), Some(ref name) => { let gcc_type = cx.type_named_struct(name); - cx.set_struct_body(gcc_type, &[fill], packed); + cx.set_struct_body(gcc_type, &[fill], &attributes); gcc_type.as_type() } } @@ -142,7 +143,7 @@ fn uncached_gcc_type<'gcc, 'tcx>( FieldsShape::Arbitrary { .. } => match name { None => { let (gcc_fields, packed) = struct_fields(cx, layout); - cx.type_struct(&gcc_fields, packed) + cx.type_struct(&gcc_fields, &struct_attributes(packed, Some(layout.align.abi))) } Some(ref name) => { let gcc_type = cx.type_named_struct(name); @@ -240,7 +241,11 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> { if let Some((deferred_ty, layout)) = defer { let (fields, packed) = struct_fields(cx, layout); - cx.set_struct_body(deferred_ty, &fields, packed); + cx.set_struct_body( + deferred_ty, + &fields, + &struct_attributes(packed, Some(layout.align.abi)), + ); } ty diff --git a/tests/asm/bulk_memory_alignment.rs b/tests/asm/bulk_memory_alignment.rs new file mode 100644 index 0000000000000..6115467447104 --- /dev/null +++ b/tests/asm/bulk_memory_alignment.rs @@ -0,0 +1,45 @@ +//@ assembly-output: emit-asm +//@ only-x86_64-unknown-linux-gnu +//@ compile-flags: -Copt-level=3 + +#![crate_type = "lib"] + +// The alignment reaches GCC's `memcpy`/`memset` expansion only through +// `__builtin_assume_aligned`; a pointer cast to an aligned type is stripped as a useless +// conversion. An over-aligned type therefore has to expand to aligned moves and a packed one +// to unaligned moves. The alignment is 64 so that the contrast holds whatever vector width +// the host picks. + +#[repr(align(64))] +pub struct Aligned([u8; 64]); + +#[repr(C, packed)] +pub struct Packed([u8; 64]); + +// CHECK-LABEL: "copy_aligned": +// CHECK: {{(v)?mov(dqa|aps)}} +#[no_mangle] +pub unsafe fn copy_aligned(destination: *mut Aligned, source: *const Aligned) { + core::ptr::copy_nonoverlapping(source, destination, 1); +} + +// CHECK-LABEL: "copy_packed": +// CHECK: {{(v)?mov(dqu|ups)}} +#[no_mangle] +pub unsafe fn copy_packed(destination: *mut Packed, source: *const Packed) { + core::ptr::copy_nonoverlapping(source, destination, 1); +} + +// CHECK-LABEL: "set_aligned": +// CHECK: {{(v)?mov(dqa|aps)}} +#[no_mangle] +pub unsafe fn set_aligned(destination: *mut Aligned) { + core::ptr::write_bytes(destination, 0, 1); +} + +// CHECK-LABEL: "set_packed": +// CHECK: {{(v)?mov(dqu|ups)}} +#[no_mangle] +pub unsafe fn set_packed(destination: *mut Packed) { + core::ptr::write_bytes(destination, 0, 1); +} diff --git a/tests/asm/volatile_bulk_memory.rs b/tests/asm/volatile_bulk_memory.rs new file mode 100644 index 0000000000000..6233b8ac74c0e --- /dev/null +++ b/tests/asm/volatile_bulk_memory.rs @@ -0,0 +1,37 @@ +//@ assembly-output: emit-asm +//@ only-x86_64-unknown-linux-gnu +//@ compile-flags: -Copt-level=3 + +#![feature(core_intrinsics)] +#![crate_type = "lib"] + +use std::intrinsics::{ + volatile_copy_memory, volatile_copy_nonoverlapping_memory, volatile_set_memory, +}; + +// The buffers below are never read back, so the writes only survive because they are volatile. +// The functions are ordered alphabetically because that is the order they are emitted in. + +// CHECK-LABEL: "volatile_copy": +// CHECK: mov +#[no_mangle] +pub unsafe fn volatile_copy(source: *const u8) { + let mut buffer = [1u8; 64]; + volatile_copy_memory(buffer.as_mut_ptr(), source, 64); +} + +// CHECK-LABEL: "volatile_copy_nonoverlapping": +// CHECK: mov +#[no_mangle] +pub unsafe fn volatile_copy_nonoverlapping(source: *const u8) { + let mut buffer = [1u8; 64]; + volatile_copy_nonoverlapping_memory(buffer.as_mut_ptr(), source, 64); +} + +// CHECK-LABEL: "volatile_set": +// CHECK: mov +#[no_mangle] +pub unsafe fn volatile_set() { + let mut buffer = [1u8; 64]; + volatile_set_memory(buffer.as_mut_ptr(), 0, 64); +} diff --git a/tests/c/import_linkage.c b/tests/c/import_linkage.c new file mode 100644 index 0000000000000..f2beb9603d08b --- /dev/null +++ b/tests/c/import_linkage.c @@ -0,0 +1,17 @@ +/* The symbols that `tests/run/import_linkage.rs` imports with an explicit `#[linkage]`. + * + * Such an import is a pointer whose value is the address of the symbol, so what the Rust side + * reads back is `&value_*`, not the pointer stored in it. The distinct values make a mix-up + * visible. */ + +#include + +int32_t external_value = 1; +int32_t available_externally_value = 2; +int32_t linkonce_value = 3; +int32_t linkonce_odr_value = 4; +int32_t weak_value = 5; +int32_t weak_odr_value = 6; +int32_t common_value = 7; +int32_t extern_weak_value = 8; +int32_t internal_value = 9; diff --git a/tests/c/overaligned_byval_abi.c b/tests/c/overaligned_byval_abi.c new file mode 100644 index 0000000000000..826a104f185ea --- /dev/null +++ b/tests/c/overaligned_byval_abi.c @@ -0,0 +1,54 @@ +/* Reference side of `tests/run/overaligned_byval_abi.rs`, compiled by the real GCC. + * + * `Aligned` is an over-aligned aggregate passed by value ("byval"): the ABI places it in a stack + * slot aligned to its own alignment, not packed right after the preceding argument. cg_gcc used + * to build the GCC struct type from the field list alone, which dropped Rust's `repr(align(64))`, + * so it placed the argument at an offset nobody else agreed on. + * + * The two functions here check both directions: `c_take_both` is a GCC-built callee for a cg_gcc + * caller, and `c_call_rust` is a GCC-built caller for a cg_gcc callee. + * + * The checks are on the *values* received rather than on the address of the argument: which + * alignment the ABI gives a stack slot is target-specific, but caller and callee agreeing on it + * is not. A disagreement makes the arguments arrive as garbage. */ + +#include + +struct Big { + int64_t a, b, c; +}; + +struct __attribute__((aligned(64))) Aligned { + int32_t x; +}; + +/* Defined on the Rust side. */ +extern int32_t rust_take_both(struct Big first, struct Aligned second, struct Big third, + struct Aligned fourth); + +/* Called from Rust: checks what a cg_gcc caller passed. */ +int32_t c_take_both(struct Big first, struct Aligned second, struct Big third, + struct Aligned fourth) +{ + if (first.a != 1 || first.b != 2 || first.c != 3) + return 1; + if (second.x != 42) + return 2; + if (third.a != 4 || third.b != 5 || third.c != 6) + return 3; + if (fourth.x != 43) + return 4; + return 0; +} + +/* Called from Rust: passes the arguments the way the ABI says, for a cg_gcc callee to read. */ +int32_t c_call_rust(void) +{ + struct Big first = {1, 2, 3}; + struct Big third = {4, 5, 6}; + struct Aligned second, fourth; + + second.x = 42; + fourth.x = 43; + return rust_take_both(first, second, third, fourth); +} diff --git a/tests/c/static_linkage.c b/tests/c/static_linkage.c new file mode 100644 index 0000000000000..787e61f9cf105 --- /dev/null +++ b/tests/c/static_linkage.c @@ -0,0 +1,37 @@ +/* Strong definitions of the statics that `tests/run/static_linkage.rs` also defines, but weakly. + * The linker has to keep these and drop the Rust ones; a backend that emits the Rust definitions + * as ordinary global symbols fails the link with a duplicate definition instead. + * + * `internal_static` is the opposite case: the Rust side keeps its own, and the two definitions + * coexist because the Rust one is local. */ + +#include + +int32_t weak_static = 1; +int32_t weak_odr_static = 2; +int32_t linkonce_static = 3; +int32_t linkonce_odr_static = 4; +int32_t common_static = 5; +int32_t internal_static = 200; + +/* `available_externally` promises the real definition lives elsewhere: a backend may read this one + * or emit an equivalent copy of the Rust initializer, so the two have to hold the same value. */ +int32_t available_externally_static = 7; + +/* Called from Rust, so that the reads also happen in a translation unit GCC compiled. */ +int32_t c_read_all(void) +{ + if (weak_static != 1) + return 11; + if (weak_odr_static != 2) + return 12; + if (linkonce_static != 3) + return 13; + if (linkonce_odr_static != 4) + return 14; + if (common_static != 5) + return 15; + if (internal_static != 200) + return 16; + return 0; +} diff --git a/tests/c/weak_function_linkage.c b/tests/c/weak_function_linkage.c new file mode 100644 index 0000000000000..25dcdedbcd950 --- /dev/null +++ b/tests/c/weak_function_linkage.c @@ -0,0 +1,54 @@ +/* Strong definitions of the functions that `tests/run/weak_function_linkage.rs` also defines, but + * weakly. The linker has to keep these and drop the Rust ones. + * + * A backend that emits the Rust definitions as ordinary global symbols does not merely pick the + * wrong one: the link fails outright with a duplicate definition. */ + +#include + +int32_t weak_function(void) +{ + return 1; +} + +int32_t weak_odr_function(void) +{ + return 2; +} + +int32_t linkonce_function(void) +{ + return 3; +} + +int32_t linkonce_odr_function(void) +{ + return 4; +} + +int32_t weak_inline_function(void) +{ + return 8; +} + +/* `available_externally` promises the real definition lives elsewhere: a backend may call this one + * or emit an equivalent copy of the Rust body, so the two have to return the same value. */ +int32_t available_externally_function(void) +{ + return 7; +} + +/* Called from Rust, so that the calls also go through a caller that GCC compiled: a cg_gcc caller + * could inline the weak body it can see instead of calling the symbol. */ +int32_t c_call_all(void) +{ + if (weak_function() != 1) + return 11; + if (weak_odr_function() != 2) + return 12; + if (linkonce_function() != 3) + return 13; + if (linkonce_odr_function() != 4) + return 14; + return 0; +} diff --git a/tests/compile/asm_noreturn_call.rs b/tests/compile/asm_noreturn_call.rs new file mode 100644 index 0000000000000..c9238697d8097 --- /dev/null +++ b/tests/compile/asm_noreturn_call.rs @@ -0,0 +1,15 @@ +// Compiler: + +// Regression test for https://github.com/rust-lang/rustc_codegen_gcc/issues/827 + +#![crate_type = "lib"] + +#[cfg(target_arch = "x86_64")] +pub type NoReturn = extern "sysv64" fn(&'static u8) -> !; + +#[cfg(target_arch = "x86_64")] +pub fn call_no_return(function: *const NoReturn) -> ! { + unsafe { + std::arch::asm!("call {}", in(reg) function, options(noreturn)); + } +} diff --git a/tests/compile/recursive_types.rs b/tests/compile/recursive_types.rs new file mode 100644 index 0000000000000..3c69074887f3e --- /dev/null +++ b/tests/compile/recursive_types.rs @@ -0,0 +1,22 @@ +// Compiler: + +// `Set` reaches itself by value through `*mut Root`, so a backend that emits `Root`'s +// fields with the still-incomplete `Set` type fails to compile this. + +#![crate_type = "lib"] + +#[repr(C)] +pub struct Set { + pub root: *mut Root, + pub first: usize, + pub second: usize, +} + +#[repr(C)] +pub struct Root { + pub default_set: Set, +} + +pub fn identity(set: Set) -> Set { + set +} diff --git a/tests/failing-ice-tests.txt b/tests/failing-ice-tests.txt index ff1b6f1489468..f9d6f99e7b42b 100644 --- a/tests/failing-ice-tests.txt +++ b/tests/failing-ice-tests.txt @@ -10,7 +10,6 @@ tests/ui/simd/intrinsic/generic-arithmetic-saturating-2.rs tests/ui/simd/intrinsic/generic-arithmetic-2.rs tests/ui/panics/default-backtrace-ice.rs tests/ui/mir/lint/storage-live.rs -tests/ui/layout/valid_range_oob.rs tests/ui/higher-ranked/trait-bounds/future.rs tests/ui/consts/const-eval/const-eval-query-stack.rs tests/ui/simd/masked-load-store.rs @@ -28,13 +27,21 @@ tests/ui/lto/thin-lto-global-allocator.rs tests/ui/lto/msvc-imp-present.rs tests/ui/lto/dylib-works.rs tests/ui/lto/all-crates.rs -tests/ui/issues/issue-47364.rs +tests/ui/codegen/no-segfault-with-multiple-codegen-units.rs tests/ui/functions-closures/parallel-codegen-closures.rs tests/ui/sepcomp/sepcomp-unwind.rs tests/ui/extern/issue-64655-extern-rust-must-allow-unwind.rs tests/ui/extern/issue-64655-allow-unwind-when-calling-panic-directly.rs -tests/ui/unwind-no-uwtable.rs +tests/ui/panics/unwind-force-no-unwind-tables.rs tests/ui/delegation/fn-header.rs tests/ui/simd/intrinsic/generic-arithmetic-pass.rs -tests/ui/simd/masked-load-store.rs -tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.rs +tests/ui/codegen/unknown-llvm-intrinsic.rs +tests/ui/codegen/incorrect-llvm-intrinsic-signature.rs +tests/ui/codegen/incorrect-arch-intrinsic.rs +tests/ui/codegen/custom-target-invalid-llvm-target.rs +tests/ui/asm/x86_64/naked_asm_escape.rs +tests/ui/lto/debuginfo-lto-alloc.rs +tests/ui/codegen/normalization-overflow/recursion-issue-118590.rs +tests/ui/codegen/normalization-overflow/recursion-issue-122823.rs +tests/ui/codegen/normalization-overflow/recursion-issue-131342.rs +tests/ui/codegen/normalization-overflow/recursion-issue-92004.rs diff --git a/tests/failing-lto-tests.txt b/tests/failing-lto-tests.txt index 4c62c35a512c1..7527005321991 100644 --- a/tests/failing-lto-tests.txt +++ b/tests/failing-lto-tests.txt @@ -1,6 +1,5 @@ tests/ui/lto/debuginfo-lto-alloc.rs -tests/ui/panic-runtime/lto-unwind.rs -tests/ui/uninhabited/uninhabited-transparent-return-abi.rs -tests/ui/coroutine/panic-drops-resume.rs -tests/ui/coroutine/panic-drops.rs -tests/ui/coroutine/panic-safe.rs +tests/ui/extern/issue-64655-allow-unwind-when-calling-panic-directly.rs +tests/ui/extern/issue-64655-extern-rust-must-allow-unwind.rs +tests/ui/lto/thin-lto-inlines2.rs +tests/ui/lto/lto-thin-rustc-loads-linker-plugin.rs diff --git a/tests/failing-run-make-tests.txt b/tests/failing-run-make-tests.txt index 528ee1df9f583..d5297e069f72f 100644 --- a/tests/failing-run-make-tests.txt +++ b/tests/failing-run-make-tests.txt @@ -11,4 +11,5 @@ tests/run-make/foreign-exceptions/ tests/run-make/glibc-staticlib-args/ tests/run-make/lto-smoke-c/ tests/run-make/return-non-c-like-enum/ -tests/run-make/short-ice +tests/run-make/short-ice/ +tests/run-make/embed-source-dwarf/ diff --git a/tests/failing-ui-tests.txt b/tests/failing-ui-tests.txt index e8a26a90890c1..7f96bfabedbea 100644 --- a/tests/failing-ui-tests.txt +++ b/tests/failing-ui-tests.txt @@ -1,113 +1,21 @@ tests/ui/asm/may_unwind.rs tests/ui/asm/x86_64/may_unwind.rs -tests/ui/drop/dynamic-drop-async.rs -tests/ui/cfg/cfg-panic-abort.rs tests/ui/intrinsics/panic-uninitialized-zeroed.rs -tests/ui/iterators/iter-sum-overflow-debug.rs -tests/ui/iterators/iter-sum-overflow-overflow-checks.rs -tests/ui/mir/mir_drop_order.rs -tests/ui/mir/mir_let_chains_drop_order.rs -tests/ui/mir/mir_match_guard_let_chains_drop_order.rs -tests/ui/panic-runtime/abort-link-to-unwinding-crates.rs -tests/ui/panic-runtime/abort.rs -tests/ui/panic-runtime/link-to-abort.rs -tests/ui/parser/unclosed-delimiter-in-dep.rs -tests/ui/consts/missing_span_in_backtrace.rs -tests/ui/drop/dynamic-drop.rs tests/ui/simd/issue-17170.rs tests/ui/simd/issue-39720.rs -tests/ui/drop/panic-during-drop-14875.rs -tests/ui/issues/issue-29948.rs tests/ui/process/println-with-broken-pipe.rs -tests/ui/lto/thin-lto-inlines2.rs -tests/ui/panic-runtime/lto-abort.rs -tests/ui/lto/lto-thin-rustc-loads-linker-plugin.rs -tests/ui/async-await/deep-futures-are-freeze.rs -tests/ui/coroutine/resume-after-return.rs -tests/ui/simd/masked-load-store.rs tests/ui/simd/repr_packed.rs -tests/ui/async-await/in-trait/dont-project-to-specializable-projection.rs -tests/ui/coroutine/unwind-abort-mix.rs -tests/ui/consts/issue-miri-1910.rs -tests/ui/consts/const_cmp_type_id.rs -tests/ui/consts/issue-94675.rs -tests/ui/traits/const-traits/const-drop-fail.rs -tests/ui/runtime/on-broken-pipe/child-processes.rs -tests/ui/sanitizer/cfi/assoc-ty-lifetime-issue-123053.rs -tests/ui/sanitizer/cfi/async-closures.rs -tests/ui/sanitizer/cfi/closures.rs -tests/ui/sanitizer/cfi/complex-receiver.rs -tests/ui/sanitizer/cfi/coroutine.rs -tests/ui/sanitizer/cfi/drop-in-place.rs -tests/ui/sanitizer/cfi/drop-no-principal.rs -tests/ui/sanitizer/cfi/fn-ptr.rs -tests/ui/sanitizer/cfi/self-ref.rs -tests/ui/sanitizer/cfi/supertraits.rs -tests/ui/sanitizer/cfi/virtual-auto.rs -tests/ui/sanitizer/cfi/sized-associated-ty.rs -tests/ui/sanitizer/cfi/can-reveal-opaques.rs -tests/ui/sanitizer/kcfi-mangling.rs -tests/ui/delegation/fn-header.rs -tests/ui/consts/const-eval/parse_ints.rs -tests/ui/simd/intrinsic/generic-as.rs -tests/ui/runtime/rt-explody-panic-payloads.rs -tests/ui/codegen/equal-pointers-unequal/as-cast/inline1.rs -tests/ui/codegen/equal-pointers-unequal/as-cast/inline2.rs -tests/ui/codegen/equal-pointers-unequal/as-cast/segfault.rs -tests/ui/codegen/equal-pointers-unequal/as-cast/zero.rs -tests/ui/codegen/equal-pointers-unequal/exposed-provenance/inline1.rs -tests/ui/codegen/equal-pointers-unequal/exposed-provenance/inline2.rs -tests/ui/codegen/equal-pointers-unequal/exposed-provenance/segfault.rs -tests/ui/codegen/equal-pointers-unequal/exposed-provenance/zero.rs -tests/ui/codegen/equal-pointers-unequal/strict-provenance/inline1.rs -tests/ui/codegen/equal-pointers-unequal/strict-provenance/inline2.rs -tests/ui/codegen/equal-pointers-unequal/strict-provenance/segfault.rs -tests/ui/codegen/equal-pointers-unequal/strict-provenance/zero.rs tests/ui/simd/simd-bitmask-notpow2.rs tests/ui/codegen/StackColoring-not-blowup-stack-issue-40883.rs tests/ui/numbers-arithmetic/u128-as-f32.rs tests/ui/process/nofile-limit.rs -tests/ui/linking/no-gc-encapsulation-symbols.rs -tests/ui/panics/unwind-force-no-unwind-tables.rs tests/ui/attributes/fn-align-dyn.rs tests/ui/linkage-attr/raw-dylib/elf/glibc-x86_64.rs -tests/ui/explicit-tail-calls/recursion-etc.rs -tests/ui/explicit-tail-calls/indexer.rs -tests/ui/explicit-tail-calls/drop-order.rs -tests/ui/c-variadic/valid.rs -tests/ui/c-variadic/inherent-method.rs -tests/ui/c-variadic/trait-method.rs -tests/ui/explicit-tail-calls/become-cast-return.rs -tests/ui/explicit-tail-calls/become-indirect-return.rs -tests/ui/panics/panic-abort-backtrace-without-debuginfo.rs -tests/ui/sanitizer/kcfi-c-variadic.rs -tests/ui/sanitizer/kcfi/fn-trait-objects.rs tests/ui/statics/const_generics.rs -tests/ui/test-attrs/test-panic-while-printing.rs tests/ui/thir-print/offset_of.rs -tests/ui/iterators/rangefrom-overflow-debug.rs -tests/ui/iterators/rangefrom-overflow-overflow-checks.rs -tests/ui/iterators/iter-filter-count-debug-check.rs -tests/ui/eii/linking/codegen_single_crate.rs -tests/ui/eii/linking/codegen_cross_crate.rs -tests/ui/eii/default/local_crate.rs -tests/ui/eii/duplicate/multiple_impls.rs -tests/ui/eii/default/call_default.rs -tests/ui/eii/linking/same-symbol.rs -tests/ui/eii/privacy1.rs -tests/ui/eii/default/call_impl.rs -tests/ui/c-variadic/copy.rs tests/ui/asm/x86_64/global_asm_escape.rs tests/ui/lto/all-crates.rs -tests/ui/consts/const-eval/c-variadic.rs -tests/ui/eii/default/call_default_panics.rs -tests/ui/explicit-tail-calls/indirect.rs -tests/ui/traits/inheritance/self-in-supertype.rs -tests/ui/fmt/fmt_debug/shallow.rs -tests/ui/c-variadic/roundtrip.rs -tests/ui/eii/eii_impl_with_contract.rs -tests/ui/eii/static/cross_crate_decl.rs -tests/ui/eii/static/cross_crate_def.rs -tests/ui/eii/static/same_address.rs -tests/ui/eii/static/simple.rs -tests/ui/explicit-tail-calls/default-trait-method.rs +tests/ui/explicit-tail-calls/tailcc-no-signature-restriction.rs +tests/ui/abi/rust-tail-cc.rs +tests/ui/abi/rust-preserve-none-cc.rs +tests/ui/extern/extern-types-field-offset.rs diff --git a/tests/lang_tests.rs b/tests/lang_tests.rs index 6afd54e1c3fe0..9c4708274280c 100644 --- a/tests/lang_tests.rs +++ b/tests/lang_tests.rs @@ -7,6 +7,68 @@ use std::process::Command; use lang_tester::LangTester; use tempfile::TempDir; +/// Directory holding the C files that the `tests/run` tests can link against. +/// +/// A `tests/c/.c` is compiled by the real GCC and linked into `tests/run/.rs`. +const C_TESTS_DIR: &str = "tests/c"; + +/// The m68k cross toolchain is not on the default `PATH` in CI. +// FIXME(antoyo): find a better way to add the PATH necessary locally. +const M68K_TOOLCHAIN_DIR: &str = "/opt/m68k-unknown-linux-gnu/bin"; + +fn target_path(test_target: &Option) -> Option { + test_target.as_ref().map(|_| { + let env_path = std::env::var("PATH").unwrap_or_default(); + format!("{}:{}", M68K_TOOLCHAIN_DIR, env_path) + }) +} + +/// Compile every C file in `tests/c` to an object file in `objects_dir`. +fn compile_c_files(objects_dir: &Path, test_target: &Option) { + let c_tests_dir = Path::new(C_TESTS_DIR); + if !c_tests_dir.is_dir() { + return; + } + std::fs::create_dir_all(objects_dir).expect("create the directory for the C object files"); + + let compiler = match test_target { + Some(target) => format!("{}-gcc", target), + None => "gcc".to_string(), + }; + + for entry in std::fs::read_dir(c_tests_dir).expect("read the C tests directory") { + let source = entry.expect("directory entry").path(); + if source.extension().and_then(|extension| extension.to_str()) != Some("c") { + continue; + } + let object = c_object_path(objects_dir, &source); + + let mut command = Command::new(&compiler); + command.arg("-c"); + // Optimize: an unoptimized C caller can happen to agree with a wrong callee. + command.arg("-O1"); + // GCC notes that the ABI of over-aligned arguments changed in GCC 4.6. That is the ABI + // being tested in overaligned_byval_abi, so the note is expected rather than a problem. + command.arg("-Wno-psabi"); + command.arg("-o"); + command.arg(&object); + command.arg(&source); + if let Some(env_path) = target_path(test_target) { + command.env("PATH", env_path); + } + + let status = command + .status() + .unwrap_or_else(|error| panic!("failed to run `{}`: {}", compiler, error)); + assert!(status.success(), "failed to compile `{}`", source.display()); + } +} + +/// The object file that a test source links against, if any: `tests/c/x.c` for `tests/run/x.rs`. +fn c_object_path(objects_dir: &Path, source: &Path) -> PathBuf { + objects_dir.join(source.file_stem().expect("file_stem")).with_extension("o") +} + fn compile_and_run_cmds( compiler_args: Vec, test_target: &Option, @@ -18,10 +80,7 @@ fn compile_and_run_cmds( // Test command 2: run `tempdir/x`. if test_target.is_some() { - let mut env_path = std::env::var("PATH").unwrap_or_default(); - // FIXME(antoyo): find a better way to add the PATH necessary locally. - env_path = format!("/opt/m68k-unknown-linux-gnu/bin:{}", env_path); - compiler.env("PATH", env_path); + compiler.env("PATH", target_path(test_target).expect("target PATH")); let mut commands = vec![("Compiler", compiler)]; if test_mode.should_run() { @@ -82,8 +141,10 @@ impl TestMode { } } +#[allow(clippy::too_many_arguments)] fn build_test_runner( tempdir: PathBuf, + c_objects_dir: PathBuf, current_dir: String, build_mode: BuildMode, test_kind: &str, @@ -159,6 +220,13 @@ fn build_test_runner( path.to_str().expect("to_str").into(), ]; + // Link against `tests/c/.c`, when the test has one. + let c_object = c_object_path(&c_objects_dir, path); + if c_object.exists() { + compiler_args.push("-C".into()); + compiler_args.push(format!("link-arg={}", c_object.display())); + } + if let Some(ref target) = test_target { compiler_args.extend_from_slice(&["--target".into(), target.into()]); @@ -193,9 +261,10 @@ fn build_test_runner( .run(); } -fn compile_tests(tempdir: PathBuf, current_dir: String) { +fn compile_tests(tempdir: PathBuf, c_objects_dir: PathBuf, current_dir: String) { build_test_runner( tempdir, + c_objects_dir, current_dir, BuildMode::Debug, "lang compile", @@ -205,24 +274,32 @@ fn compile_tests(tempdir: PathBuf, current_dir: String) { ); } -fn run_tests(tempdir: PathBuf, current_dir: String) { +fn run_tests(tempdir: PathBuf, c_objects_dir: PathBuf, current_dir: String) { build_test_runner( tempdir.clone(), + c_objects_dir.clone(), current_dir.clone(), BuildMode::Debug, "[DEBUG] lang run", "tests/run", TestMode::CompileAndRun, - &[], + &[ + // FIXME: remove this when the unwind issue is fixed in GCC m68k upstream. + "catch_unwind.rs", + ], ); build_test_runner( tempdir, + c_objects_dir, current_dir.to_string(), BuildMode::Release, "[RELEASE] lang run", "tests/run", TestMode::CompileAndRun, - &[], + &[ + // FIXME: remove this when the unwind issue is fixed in GCC m68k upstream. + "catch_unwind.rs", + ], ); } @@ -232,6 +309,11 @@ fn main() { let current_dir = current_dir.to_str().expect("current dir").to_string(); let tempdir_path: PathBuf = tempdir.as_ref().into(); - compile_tests(tempdir_path.clone(), current_dir.clone()); - run_tests(tempdir_path, current_dir); + let c_objects_dir = tempdir_path.join("c-objects"); + // FIXME(antoyo): find a way to send this via a cli argument. + let test_target = std::env::var("CG_GCC_TEST_TARGET").ok(); + compile_c_files(&c_objects_dir, &test_target); + + compile_tests(tempdir_path.clone(), c_objects_dir.clone(), current_dir.clone()); + run_tests(tempdir_path, c_objects_dir, current_dir); } diff --git a/tests/run/catch_unwind.rs b/tests/run/catch_unwind.rs new file mode 100644 index 0000000000000..919213db5a4a5 --- /dev/null +++ b/tests/run/catch_unwind.rs @@ -0,0 +1,25 @@ +// Compiler: +// +// Run-time: +// status: 0 +// stdout: Caught + +#![feature(fn_traits, unboxed_closures)] + +struct Wrapper(A); + +impl R> FnOnce<()> for Wrapper { + type Output = R; + + #[inline] + extern "rust-call" fn call_once(self, _args: ()) -> R { + (self.0)() + } +} + +fn main() { + std::panic::set_hook(Box::new(|_| {})); + let result = std::panic::catch_unwind(Wrapper(|| panic!())); + assert!(result.is_err()); + println!("Caught"); +} diff --git a/tests/run/custom_abort.rs b/tests/run/custom_abort.rs new file mode 100644 index 0000000000000..eafa4321a4324 --- /dev/null +++ b/tests/run/custom_abort.rs @@ -0,0 +1,27 @@ +// Compiler: +// +// Run-time: +// status: 42 + +// Check that a program can define its own `abort`. + +#![feature(no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +#[no_mangle] +extern "C" fn abort() { + unsafe { + libc::exit(42); + } +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + abort(); + 0 +} diff --git a/tests/run/import_linkage.rs b/tests/run/import_linkage.rs new file mode 100644 index 0000000000000..bf5cb9e532799 --- /dev/null +++ b/tests/run/import_linkage.rs @@ -0,0 +1,84 @@ +// Compiler: +// +// Run-time: +// status: 0 + +// Checks the `#[linkage]` flavours an `extern` static can be imported with, against the symbols +// `tests/c/import_linkage.c` defines. +// +// The value of such an import is the address of the symbol rather than its contents, which is why +// the types are pointers: an `extern_weak` import of a symbol nobody defines reads as null instead +// of failing the link. + +#![feature(linkage, no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +extern "C" { + #[linkage = "external"] + static external_value: *const i32; + #[linkage = "available_externally"] + static available_externally_value: *const i32; + #[linkage = "linkonce"] + static linkonce_value: *const i32; + #[linkage = "linkonce_odr"] + static linkonce_odr_value: *const i32; + #[linkage = "weak"] + static weak_value: *const i32; + #[linkage = "weak_odr"] + static weak_odr_value: *const i32; + #[linkage = "common"] + static common_value: *const i32; + #[linkage = "extern_weak"] + static extern_weak_value: *const i32; + // An import is an undefined reference whatever the flavour says. Upstream bug: rustc lowers + // this one to an internal declaration, which LLVM's verifier rejects ("Global is external, but + // doesn't have external or weak linkage!") and which crashes cg_llvm at -O3. + #[linkage = "internal"] + static internal_value: *const i32; + + // Nothing defines this one, so it stays null instead of breaking the link. + #[linkage = "extern_weak"] + static undefined_value: *const i32; +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + unsafe { + if *external_value != 1 { + return 1; + } + if *available_externally_value != 2 { + return 2; + } + if *linkonce_value != 3 { + return 3; + } + if *linkonce_odr_value != 4 { + return 4; + } + if *weak_value != 5 { + return 5; + } + if *weak_odr_value != 6 { + return 6; + } + if *common_value != 7 { + return 7; + } + if *extern_weak_value != 8 { + return 8; + } + if *internal_value != 9 { + return 9; + } + if undefined_value as usize != 0 { + return 10; + } + } + 0 +} diff --git a/tests/run/nonzero_div_ceil.rs b/tests/run/nonzero_div_ceil.rs new file mode 100644 index 0000000000000..e6bd3c2ec9845 --- /dev/null +++ b/tests/run/nonzero_div_ceil.rs @@ -0,0 +1,17 @@ +// Compiler: +// +// Run-time: +// status: 0 + +use std::hint::black_box; +use std::num::NonZero; + +fn main() { + for (dividend, divisor, expected) in + [(10u8, 3u8, 4u8), (1, 254, 1), (1, 255, 1), (2, 254, 1), (2, 255, 1), (200, 100, 2)] + { + let dividend = NonZero::new(black_box(dividend)).unwrap(); + let divisor = NonZero::new(black_box(divisor)).unwrap(); + assert_eq!(dividend.div_ceil(divisor).get(), expected); + } +} diff --git a/tests/run/overaligned_byval_abi.rs b/tests/run/overaligned_byval_abi.rs new file mode 100644 index 0000000000000..78ee35f05ca58 --- /dev/null +++ b/tests/run/overaligned_byval_abi.rs @@ -0,0 +1,89 @@ +// Compiler: +// +// Run-time: +// status: 0 + +// Checks that cg_gcc passes an over-aligned by-value ("byval") argument where the platform ABI +// says it goes, by calling in both directions with `tests/c/overaligned_byval_abi.c`, which is +// compiled by the real GCC. +// +// `tests/run/overaligned_byval_arg.rs` covers the Rust-visible half of the same bug. It cannot +// cover this one: with cg_gcc on both sides of a call, caller and callee place the argument at +// the same wrong offset and agree with each other. +// +// Two over-aligned arguments are used rather than one so that the failure is deterministic. A +// backend that drops `align(64)` packs the arguments at offsets 0, 24, 88 and 112 of the argument +// area; 112 - 24 = 88 is not a multiple of 64, so the two of them cannot both land on a 64-byte +// boundary however the argument area itself is aligned. With a single over-aligned argument the +// frame often happens to be 64-aligned and the bug hides. +// +// Only the values received are checked, never the address an argument landed at: which alignment +// a target gives a by-value stack slot differs between targets, but the two sides of a call +// agreeing on it does not. `overaligned_byval_arg.rs` is where the alignment itself is asserted. + +#![feature(no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +#[repr(C)] +struct Big { + a: i64, + b: i64, + c: i64, +} + +#[repr(C, align(64))] +struct Aligned { + x: i32, +} + +extern "C" { + fn c_take_both(first: Big, second: Aligned, third: Big, fourth: Aligned) -> i32; + fn c_call_rust() -> i32; +} + +// The callee for the GCC-built caller in `c_call_rust`. +// +// `#[no_mangle]` is not only about the symbol name: it makes the symbol externally visible, which +// pins the calling convention. Without it the function has internal linkage and GCC is free to +// clone it with a changed convention at `-O3` (the symbol comes out as `...constprop.0.isra.0`), +// so the arguments never travel through the stack slots and the release build passes spuriously. +#[no_mangle] +extern "C" fn rust_take_both(first: Big, second: Aligned, third: Big, fourth: Aligned) -> i32 { + if first.a as i32 != 1 || first.b as i32 != 2 || first.c as i32 != 3 { + return 5; + } + if second.x != 42 { + return 6; + } + if third.a as i32 != 4 || third.b as i32 != 5 || third.c as i32 != 6 { + return 7; + } + if fourth.x != 43 { + return 8; + } + 0 +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + // cg_gcc as the caller, GCC as the callee. + let result = unsafe { + c_take_both( + Big { a: 1, b: 2, c: 3 }, + Aligned { x: 42 }, + Big { a: 4, b: 5, c: 6 }, + Aligned { x: 43 }, + ) + }; + if result != 0 { + return result; + } + + // GCC as the caller, cg_gcc as the callee. + unsafe { c_call_rust() } +} diff --git a/tests/run/overaligned_byval_arg.rs b/tests/run/overaligned_byval_arg.rs new file mode 100644 index 0000000000000..e20ec11732962 --- /dev/null +++ b/tests/run/overaligned_byval_arg.rs @@ -0,0 +1,41 @@ +// Compiler: +// +// Run-time: +// status: 0 + +#![feature(no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +#[repr(C)] +struct Big { + a: i64, + b: i64, + c: i64, +} + +#[repr(C, align(64))] +struct Aligned { + x: i32, +} + +#[inline(never)] +#[no_mangle] +extern "C" fn check(_b1: Big, a1: Aligned, _b2: Big, a2: Aligned) -> i32 { + if (&a1 as *const Aligned as usize) % 64 != 0 { + return 1; + } + if (&a2 as *const Aligned as usize) % 64 != 0 { + return 2; + } + 0 +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + check(Big { a: 1, b: 2, c: 3 }, Aligned { x: 42 }, Big { a: 4, b: 5, c: 6 }, Aligned { x: 43 }) +} diff --git a/tests/run/ptr_to_int_div.rs b/tests/run/ptr_to_int_div.rs new file mode 100644 index 0000000000000..afc563d6c977b --- /dev/null +++ b/tests/run/ptr_to_int_div.rs @@ -0,0 +1,19 @@ +// Compiler: +// +// Run-time: +// status: 0 + +use std::hint::black_box; +use std::mem::transmute; + +fn main() { + let pointer = black_box(usize::MAX) as *const (); + + let unsigned = unsafe { transmute::<*const (), usize>(pointer) }; + assert_eq!(unsigned / black_box(2), usize::MAX / 2); + assert_eq!(unsigned % black_box(2), usize::MAX % 2); + + let signed = unsafe { transmute::<*const (), isize>(pointer) }; + assert_eq!(signed / black_box(2), -1isize / 2); + assert_eq!(signed % black_box(2), -1isize % 2); +} diff --git a/tests/run/simd.rs b/tests/run/simd.rs new file mode 100644 index 0000000000000..e0a23fdccf8ce --- /dev/null +++ b/tests/run/simd.rs @@ -0,0 +1,81 @@ +// Compiler: +// +// Run-time: +// status: 0 + +#![feature(portable_simd)] + +use std::hint::black_box; +use std::simd::prelude::*; + +fn test_saturating_add() { + let values = i32x4::from_array([i32::MIN, -2, 3, i32::MAX]); + let ones = i32x4::splat(1); + assert_eq!( + black_box(values).saturating_add(black_box(ones)).to_array(), + [i32::MIN + 1, -1, 4, i32::MAX] + ); + + let values = u32x4::from_array([0, 2, 3, u32::MAX]); + let ones = u32x4::splat(1); + assert_eq!(black_box(values).saturating_add(black_box(ones)).to_array(), [1, 3, 4, u32::MAX]); +} + +fn test_saturating_sub() { + let values = i32x4::from_array([i32::MIN, -2, 3, i32::MAX]); + let zero = i32x4::splat(0); + assert_eq!( + black_box(zero).saturating_sub(black_box(values)).to_array(), + [i32::MAX, 2, -3, i32::MIN + 1] + ); + assert_eq!(black_box(values).saturating_neg().to_array(), [i32::MAX, 2, -3, i32::MIN + 1]); + assert_eq!(black_box(values).saturating_abs().to_array(), [i32::MAX, 2, 3, i32::MAX]); + + let values = i32x4::from_array([i32::MIN, -2, 3, i32::MAX]); + let ones = i32x4::splat(1); + assert_eq!( + black_box(values).saturating_sub(black_box(ones)).to_array(), + [i32::MIN, -3, 2, i32::MAX - 1] + ); + + let values = u32x4::from_array([0, 2, 3, u32::MAX]); + let ones = u32x4::splat(1); + assert_eq!( + black_box(values).saturating_sub(black_box(ones)).to_array(), + [0, 1, 2, u32::MAX - 1] + ); +} + +fn test_float_cast() { + let floats = f32x4::from_array([1.9, -4.5, f32::INFINITY, f32::NAN]); + assert_eq!(black_box(floats).cast::().to_array(), [1, -4, i32::MAX, 0]); + + let floats = f32x4::from_array([f32::NEG_INFINITY, 1e20, -1e20, -0.0]); + assert_eq!(black_box(floats).cast::().to_array(), [i32::MIN, i32::MAX, i32::MIN, 0]); + + let floats = f32x4::from_array([-1.0, 3.7, f32::NAN, 1e20]); + assert_eq!(black_box(floats).cast::().to_array(), [0, 3, 0, u32::MAX]); + + let floats = f64x4::from_array([-1.5, 2.5, f64::NAN, f64::INFINITY]); + assert_eq!(black_box(floats).cast::().to_array(), [-1, 2, 0, i64::MAX]); +} + +fn test_arith_offset() { + let values = [10i32, 11, 12, 13, 14, 15, 16, 17]; + let indices = usizex4::from_array([7, 5, 3, 1]); + assert_eq!( + i32x4::gather_or_default(black_box(&values), black_box(indices)).to_array(), + [17, 15, 13, 11] + ); + + let mut destination = [0i32; 8]; + i32x4::from_array([1, 2, 3, 4]).scatter(black_box(&mut destination), black_box(indices)); + assert_eq!(destination, [0, 4, 0, 3, 0, 2, 0, 1]); +} + +fn main() { + test_saturating_add(); + test_saturating_sub(); + test_float_cast(); + test_arith_offset(); +} diff --git a/tests/run/static_alloc_shapes.rs b/tests/run/static_alloc_shapes.rs new file mode 100644 index 0000000000000..39a4d07bdf638 --- /dev/null +++ b/tests/run/static_alloc_shapes.rs @@ -0,0 +1,50 @@ +// Compiler: +// +// Run-time: +// status: 0 +// stdout: 8 +// 12 +// 5 +// 7 +// 7 +// 9 + +#![feature(no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +// One byte run of each length class that maps to a distinct array element type. +static mut BYTES8: [u8; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; +static mut BYTES12: [u8; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; +static mut BYTES5: [u8; 5] = [1, 2, 3, 4, 5]; + +static mut VALUE: isize = 7; +static mut OTHER: isize = 9; + +// An allocation that is exactly one relocation, so it ends on a pointer with no trailing bytes. +static mut PTR: &isize = unsafe { &VALUE }; + +struct TwoRefs { + first: &'static isize, + second: &'static isize, +} + +// Two adjacent relocations, with no byte run between them. +static mut TWO_REFS: TwoRefs = TwoRefs { first: unsafe { &VALUE }, second: unsafe { &OTHER } }; + +#[no_mangle] +extern "C" fn main(_argc: isize, _argv: *const *const u8) -> i32 { + unsafe { + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES8[7] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES12[11] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES5[4] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *PTR); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *TWO_REFS.first); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *TWO_REFS.second); + } + 0 +} diff --git a/tests/run/static_linkage.rs b/tests/run/static_linkage.rs new file mode 100644 index 0000000000000..7b911c064d797 --- /dev/null +++ b/tests/run/static_linkage.rs @@ -0,0 +1,79 @@ +// Compiler: +// +// Run-time: +// status: 0 + +// Checks that `#[linkage]` on a static that this crate defines reaches the symbol, against +// `tests/c/static_linkage.c`, which defines the overridable ones strongly. +// +// If `predefine_static` were to ignore its `linkage` argument outright, every static would come out as +// an ordinary global symbol: the overridable ones would clash with the C definitions at link time, and +// `internal` would export a symbol it should have kept private. + +#![feature(linkage, no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +#[linkage = "weak"] +#[no_mangle] +pub static weak_static: i32 = 0; + +#[linkage = "weak_odr"] +#[no_mangle] +pub static weak_odr_static: i32 = 0; + +#[linkage = "linkonce"] +#[no_mangle] +pub static linkonce_static: i32 = 0; + +#[linkage = "linkonce_odr"] +#[no_mangle] +pub static linkonce_odr_static: i32 = 0; + +// `common` is only valid on a mutable global: LLVM rejects a constant one. +#[linkage = "common"] +#[no_mangle] +pub static mut common_static: i32 = 0; + +// Private to this crate, so the C definition of the same name is a different object. +#[linkage = "internal"] +#[no_mangle] +pub static internal_static: i32 = 100; + +// Not overridden by the C side: the definition here is the one that survives. +#[linkage = "weak"] +#[no_mangle] +pub static only_weak_static: i32 = 6; + +// The real definition is the one in the C file; a backend may read it or emit an equivalent copy of +// this initializer, so both spell the same value. +#[linkage = "available_externally"] +#[no_mangle] +pub static available_externally_static: i32 = 7; + +extern "C" { + fn c_read_all() -> i32; +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + let result = unsafe { c_read_all() }; + if result != 0 { + return result; + } + + if internal_static != 100 { + return 1; + } + if only_weak_static != 6 { + return 2; + } + if available_externally_static != 7 { + return 3; + } + 0 +} diff --git a/tests/run/weak_function_linkage.rs b/tests/run/weak_function_linkage.rs new file mode 100644 index 0000000000000..677f01353401a --- /dev/null +++ b/tests/run/weak_function_linkage.rs @@ -0,0 +1,106 @@ +// Compiler: +// +// Run-time: +// status: 0 + +// Checks that the `#[linkage]` flavours another object file is allowed to override are emitted as +// weak symbols, by linking against `tests/c/weak_function_linkage.c`, which defines the same +// symbols strongly. + +#![feature(linkage, no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +#[linkage = "weak"] +#[no_mangle] +extern "C" fn weak_function() -> i32 { + 0 +} + +// `_odr` promises every definition of the symbol is equivalent, which lets a backend call this body +// instead of the one in the C file. They spell the same value for that reason. +#[linkage = "weak_odr"] +#[no_mangle] +extern "C" fn weak_odr_function() -> i32 { + 2 +} + +#[linkage = "linkonce"] +#[no_mangle] +extern "C" fn linkonce_function() -> i32 { + 0 +} + +#[linkage = "linkonce_odr"] +#[no_mangle] +extern "C" fn linkonce_odr_function() -> i32 { + 4 +} + +// `#[linkage = "common"]` is absent on purpose: a common symbol is `SHN_COMMON`, which the object +// format only allows for objects, so no backend can give a function that linkage. + +// Not overridden by the C side: the definition here is the one that runs. +#[linkage = "weak"] +#[no_mangle] +extern "C" fn only_weak_function() -> i32 { + 6 +} + +// The real definition is the one in the C file; a backend may call it or emit an equivalent copy of +// this body, so both spell the same value. +#[linkage = "available_externally"] +#[no_mangle] +extern "C" fn available_externally_function() -> i32 { + 7 +} + +// GCC drops `weak` from a function that is also `inline`: a backend that keeps the hint emits this +// as an ordinary global symbol and clashes with the C definition. rustc lints the hint as ignored +// on a function with an explicit `#[linkage]`, hence the `allow`. +#[linkage = "weak"] +#[inline] +#[no_mangle] +#[allow(unused_attributes)] +extern "C" fn weak_inline_function() -> i32 { + 0 +} + +extern "C" { + fn c_call_all() -> i32; +} + +#[no_mangle] +extern "C" fn main(_argc: i32, _argv: *const *const u8) -> i32 { + let result = unsafe { c_call_all() }; + if result != 0 { + return result; + } + + if weak_function() != 1 { + return 1; + } + if weak_odr_function() != 2 { + return 2; + } + if linkonce_function() != 3 { + return 3; + } + if linkonce_odr_function() != 4 { + return 4; + } + if only_weak_function() != 6 { + return 6; + } + if available_externally_function() != 7 { + return 7; + } + if weak_inline_function() != 8 { + return 8; + } + 0 +} diff --git a/tools/cspell_dicts/rust.txt b/tools/cspell_dicts/rust.txt deleted file mode 100644 index 379cbd77eef01..0000000000000 --- a/tools/cspell_dicts/rust.txt +++ /dev/null @@ -1,2 +0,0 @@ -lateout -repr diff --git a/tools/cspell_dicts/rustc_codegen_gcc.txt b/tools/cspell_dicts/rustc_codegen_gcc.txt deleted file mode 100644 index 4fb018b3ecd87..0000000000000 --- a/tools/cspell_dicts/rustc_codegen_gcc.txt +++ /dev/null @@ -1,78 +0,0 @@ -aapcs -addo -archs -ashl -ashr -cgcx -clzll -cmse -codegened -csky -ctfe -ctlz -ctpop -cttz -ctzll -flto -fmaximumf -fmuladd -fmuladdf -fminimumf -fmul -fptosi -fptosui -fptoui -fwrapv -gimple -hrtb -immediates -interner -liblto -llbb -llcx -llextra -llfn -lgcc -llmod -llresult -llret -ltrans -llty -llval -llvals -loong -lshr -masm -maximumf -maxnumf -mavx -mcmodel -minimumf -minnumf -miri -monomorphization -monomorphizations -monomorphized -monomorphizing -movnt -mulo -nvptx -pointee -powitf -reassoc -riscv -rlib -roundevenf -rustc -sitofp -sizet -spir -subo -sysv -tbaa -uitofp -unord -uninlined -utrunc -xabort -zext From 0787958c776c4f0e9e0bc252580aadffde5c0a43 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 18 Sep 2026 20:56:48 +0200 Subject: [PATCH 35/55] Fix source code divergence --- .github/workflows/ci.yml | 13 +- .github/workflows/stdarch.yml | 20 +- .gitignore | 2 +- CONTRIBUTING.md | 2 +- Cargo.lock | 92 +--- Readme.md | 8 +- build_system/Cargo.lock | 2 +- build_system/asm-tester/Cargo.lock | 507 ++++++++++++++++++ build_system/asm-tester/Cargo.toml | 13 + build_system/asm-tester/src/main.rs | 66 +++ build_system/src/build.rs | 2 +- build_system/src/clean.rs | 3 +- build_system/src/clippy.rs | 62 +++ build_system/src/config.rs | 13 +- build_system/src/fmt.rs | 5 +- build_system/src/main.rs | 112 ++-- build_system/src/rust_tools.rs | 2 +- build_system/src/test.rs | 141 ++++- build_system/src/todo.rs | 72 +++ build_system/src/utils.rs | 58 +- doc/subtree.md | 4 +- example/mini_core_hello_world.rs | 2 +- src/abi.rs | 41 +- src/asm.rs | 19 +- src/attributes.rs | 26 + src/back/lto.rs | 37 +- src/back/write.rs | 5 +- src/base.rs | 118 +--- src/builder.rs | 119 ++-- src/callee.rs | 2 +- src/consts.rs | 45 +- src/declare.rs | 32 +- src/diagnostics.rs | 4 - src/gcc_util.rs | 2 +- src/int.rs | 18 +- src/intrinsic/archs.rs | 86 ++- src/intrinsic/llvm.rs | 111 +++- src/intrinsic/mod.rs | 44 +- src/intrinsic/old_archs.rs | 4 + src/lib.rs | 22 +- src/mono_item.rs | 23 +- src/type_.rs | 4 +- tests/asm/asm/comments.rs | 12 + .../x86_64-naked-fn-no-cet-prolog.rs | 24 + tests/asm/panic-no-unwind-no-uwtable.rs | 8 + tests/asm/used.rs | 14 + tests/asm/x86_64-sse_crc.rs | 12 + .../compile/x86_interrupt_first_arg_byval.rs | 16 + tests/cpuid.def | 27 + tests/lang_tests.rs | 18 +- tests/run/asm.rs | 42 ++ tests/run/int.rs | 25 + tests/run/mir_preserve_ub_empty_switch.rs | 35 ++ tools/generate_intrinsics.py | 36 +- 54 files changed, 1739 insertions(+), 493 deletions(-) create mode 100644 build_system/asm-tester/Cargo.lock create mode 100644 build_system/asm-tester/Cargo.toml create mode 100644 build_system/asm-tester/src/main.rs create mode 100644 build_system/src/clippy.rs create mode 100644 build_system/src/todo.rs create mode 100644 tests/asm/asm/comments.rs create mode 100644 tests/asm/naked-functions/x86_64-naked-fn-no-cet-prolog.rs create mode 100644 tests/asm/panic-no-unwind-no-uwtable.rs create mode 100644 tests/asm/used.rs create mode 100644 tests/asm/x86_64-sse_crc.rs create mode 100644 tests/compile/x86_interrupt_first_arg_byval.rs create mode 100644 tests/cpuid.def create mode 100644 tests/run/mir_preserve_ub_empty_switch.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8888fdc3ee093..2f5cc409e363d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - { gcc: "gcc-15.deb" } - { gcc: "gcc-15-without-int128.deb" } commands: [ - "--std-tests", + "--std-tests --alloc-tests", # FIXME: re-enable asm tests when GCC can emit in the right syntax. # "--asm-tests", "--test-libcore", @@ -91,16 +91,17 @@ jobs: - name: Check formatting run: ./y.sh fmt --check - - name: clippy - run: | - cargo clippy --all-targets -- -D warnings - cargo clippy --all-targets --no-default-features -- -D warnings - cargo clippy --manifest-path build_system/Cargo.toml --all-targets -- -D warnings + - name: Check todo + run: ./y.sh check-todo + + - name: Check lints + run: ./y.sh clippy - name: Build run: | ./y.sh build --sysroot ./y.sh test --cargo-tests + CARGO_TEST_FLAGS="-Zmir-preserve-ub" ./y.sh test --cargo-tests -- mir_preserve_ub_empty_switch - name: Run y.sh cargo build run: | diff --git a/.github/workflows/stdarch.yml b/.github/workflows/stdarch.yml index 66f30b147b4c0..17d6449c85e08 100644 --- a/.github/workflows/stdarch.yml +++ b/.github/workflows/stdarch.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: cargo_runner: [ - "sde -future -rtm_mode full --", + "sde -cpuid-in /home/runner/work/rustc_codegen_gcc/rustc_codegen_gcc/tests/cpuid.def -rtm_mode full --", "", ] @@ -42,8 +42,14 @@ jobs: - name: Install more recent binutils run: | echo "deb http://archive.ubuntu.com/ubuntu plucky main universe" | sudo tee /etc/apt/sources.list.d/plucky-copies.list - sudo apt-get update + sudo apt-get update -o Acquire::Retries=3 sudo apt-get install binutils + installed="$(dpkg-query --showformat='${Version}' --show binutils)" + echo "Installed binutils: $installed" + if dpkg --compare-versions "$installed" lt "2.44"; then + echo "::error::binutils upgrade failed (got $installed, need >= 2.44); the apt fetch probably failed" + exit 1 + fi - name: Install Intel Software Development Emulator if: ${{ matrix.cargo_runner }} @@ -51,10 +57,9 @@ jobs: mkdir intel-sde cd intel-sde version=10.8.0-2026-03-15 - url_path=915934 dir=sde-external-$version-lin file=$dir.tar.xz - wget https://downloadmirror.intel.com/$url_path/$file + wget http://ci-mirrors.rust-lang.org/$file tar xvf $file sudo mkdir /usr/share/intel-sde sudo cp -r $dir/* /usr/share/intel-sde @@ -90,14 +95,15 @@ jobs: - name: Run stdarch tests if: ${{ !matrix.cargo_runner }} run: | - CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml + # FIXME: remove --skip test_tile_ and --skip --skip test__tile when it's implemented. + ./y.sh test --release --stdarch-tests -- --skip test_tile_ --skip test__tile - name: Run stdarch tests if: ${{ matrix.cargo_runner }} run: | # FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro. - # FIXME: remove --skip test_tile_ when it's implemented. - STDARCH_TEST_SKIP_FUNCTION="xsave,xsaveopt,xsave64,xsaveopt64" STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a --skip test_tile_ + # FIXME: remove --skip test_tile_ and --skip --skip test__tile when it's implemented. + STDARCH_TEST_SKIP_FUNCTION="xsave,xsaveopt,xsave64,xsaveopt64" STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a --skip test_tile_ --skip test__tile # Summary job for the merge queue. # ALL THE PREVIOUS JOBS NEED TO BE ADDED TO THE `needs` SECTION OF THIS JOB! diff --git a/.gitignore b/.gitignore index 2a8fdcda0b483..13bd0d0ffde9b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ perf.data.old *.events *.string* gimple* -*asm +*_asm res test-backend projects diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f81ecca445a8..c5c2a783b1ee7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,7 +112,7 @@ Full list of debugging options can be found in the [README](Readme.md#env-vars). ### Code Style Guidelines - Follow Rust standard coding conventions -- Ensure your code passes `rustfmt` and `clippy` +- Ensure your code passes `rustfmt` and `clippy` (you can run them with `y.sh fmt` and `y.sh clippy`) - Add comments explaining complex logic, especially in GCC interface code ## Additional Resources diff --git a/Cargo.lock b/Cargo.lock index c7e979a1112e3..c174628d0d188 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,9 +31,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", "windows-sys", @@ -117,15 +117,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.168" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "memchr" @@ -194,9 +194,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.7" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.20.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom", @@ -311,78 +311,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.59.0" +name = "windows-link" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-targets" -version = "0.52.6" +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-link", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/Readme.md b/Readme.md index 6b1f90b918855..9a7c624c9bc22 100644 --- a/Readme.md +++ b/Readme.md @@ -136,19 +136,21 @@ $ ./y.sh cargo build --manifest-path tests/hello-world/Cargo.toml ### Cargo ```bash -$ CHANNEL="release" $CG_GCCJIT_DIR/y.sh cargo run +$ CHANNEL=release $CG_GCCJIT_DIR/y.sh cargo run ``` -If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./y.sh test`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely. +If you compiled `cg_gcc` in debug mode (aka you didn't pass `--release` to `./y.sh build`) you should use `CHANNEL=debug` instead or omit `CHANNEL=release` completely. ### Rustc If you want to run `rustc` directly, you can do so with: ```bash -$ ./y.sh rustc my_crate.rs +$ CHANNEL=release ./y.sh rustc my_crate.rs ``` +If you compiled `cg_gcc` in debug mode (aka you didn't pass `--release` to `./y.sh build`) you should use `CHANNEL=debug` instead or omit `CHANNEL=release` completely. + You can do the same manually (although we don't recommend it): ```bash diff --git a/build_system/Cargo.lock b/build_system/Cargo.lock index e727561a2bfba..5e761149eb3bc 100644 --- a/build_system/Cargo.lock +++ b/build_system/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "boml" diff --git a/build_system/asm-tester/Cargo.lock b/build_system/asm-tester/Cargo.lock new file mode 100644 index 0000000000000..9ad96acfda407 --- /dev/null +++ b/build_system/asm-tester/Cargo.lock @@ -0,0 +1,507 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "asm-tester" +version = "0.1.0" +dependencies = [ + "compiletest_rs", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "compiletest_rs" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f150fe9105fcd2a57cad53f0c079a24de65195903ef670990f5909f695eac04c" +dependencies = [ + "diff", + "filetime", + "getopts", + "lazy_static", + "libc", + "log", + "miow", + "regex", + "rustfix", + "serde", + "serde_derive", + "serde_json", + "tester", + "windows-sys 0.59.0", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libredox" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +dependencies = [ + "libc", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miow" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "536bfad37a309d62069485248eeaba1e8d9853aaf951caaeaed0585a95346f08" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustfix" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82fa69b198d894d84e23afde8e9ab2af4400b2cba20d6bf2b428a8b01c222c5a" +dependencies = [ + "serde", + "serde_json", + "thiserror", + "tracing", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + +[[package]] +name = "tester" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e8bf7e0eb2dd7b4228cc1b6821fc5114cd6841ae59f652a85488c016091e5f" +dependencies = [ + "cfg-if", + "getopts", + "libc", + "num_cpus", + "term", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/build_system/asm-tester/Cargo.toml b/build_system/asm-tester/Cargo.toml new file mode 100644 index 0000000000000..eeefe61bdc75b --- /dev/null +++ b/build_system/asm-tester/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "asm-tester" +version = "0.1.0" +edition = "2024" + +[dependencies] +compiletest_rs = "0.11.2" + +[[bin]] +name = "asm-tester" +path = "src/main.rs" + +[workspace] diff --git a/build_system/asm-tester/src/main.rs b/build_system/asm-tester/src/main.rs new file mode 100644 index 0000000000000..00ee4ac936520 --- /dev/null +++ b/build_system/asm-tester/src/main.rs @@ -0,0 +1,66 @@ +use std::path::PathBuf; + +#[derive(Default)] +struct Config { + llvm_filecheck: Option, + filters: Vec, + rustc_flags: Vec, +} + +impl Config { + fn new() -> Result { + // We skip the program's name. + let mut args = std::env::args().skip(1); + let mut config = Self::default(); + + while let Some(arg) = args.next() { + match arg.as_str() { + "--llvm-filecheck" => { + config.llvm_filecheck = args.next().map(PathBuf::from); + } + "--filter" => { + if let Some(arg) = args.next() { + config.filters.push(arg); + } + } + "--" => { + config.rustc_flags.extend(&mut args); + // Nothing else to be read but the `break` makes it more clear. + break; + } + arg => return Err(format!("Unknown argument {arg:?}")), + } + } + if config.llvm_filecheck.is_none() { + Err("Missing `--llvm-filecheck` option".to_owned()) + } else if config.rustc_flags.is_empty() { + Err("Missing rustc flags (passed after `--`)".to_owned()) + } else { + Ok(config) + } + } +} + +fn main() { + let Config { llvm_filecheck, filters, rustc_flags } = match Config::new() { + Ok(c) => c, + Err(error) => { + eprintln!("{error}"); + std::process::exit(1); + } + }; + + let mut test_config = compiletest_rs::Config::default(); + + test_config.mode = compiletest_rs::common::Mode::Assembly; + test_config.src_base = PathBuf::from("tests/asm"); + test_config.llvm_filecheck = llvm_filecheck; + test_config.filters = filters; + test_config.strict_headers = true; + test_config.build_base = PathBuf::from("build/tests/asm"); + test_config.target_rustcflags = Some(rustc_flags.join(" ")); + test_config.link_deps(); + test_config.clean_rmeta(); + + compiletest_rs::run_tests(&test_config) +} diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 2f2900af5c88a..bcd386bfebdae 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -266,7 +266,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> { } run_command_with_output_and_env(&command, None, Some(&env))?; - args.config_info.setup(&mut env, false)?; + args.config_info.setup(&mut env, false, true)?; // We voluntarily ignore the error. let _ = fs::remove_dir_all("target/out"); diff --git a/build_system/src/clean.rs b/build_system/src/clean.rs index 43f01fdf35ecb..ec2092ee92ef5 100644 --- a/build_system/src/clean.rs +++ b/build_system/src/clean.rs @@ -74,7 +74,8 @@ fn clean_ui_tests() -> Result<(), String> { let path = Path::new(crate::BUILD_DIR) .join("rust/build/x86_64-unknown-linux-gnu/test/") .join(directory); - run_command(&[&"find", &path, &"-name", &"stamp", &"-delete"], None)?; + // The directory might not exist, so ignore the error. + let _ = run_command(&[&"find", &path, &"-name", &"stamp", &"-delete"], None); } Ok(()) } diff --git a/build_system/src/clippy.rs b/build_system/src/clippy.rs new file mode 100644 index 0000000000000..813d4b9141e1c --- /dev/null +++ b/build_system/src/clippy.rs @@ -0,0 +1,62 @@ +use std::path::Path; + +use crate::utils::{run_command_with_output, run_tool_and_install_it_if_not_present}; + +fn show_usage() { + println!( + r#" +`clippy` command help: + + --help : Show this help"# + ); +} + +pub fn run() -> Result<(), String> { + // We skip binary name and the `info` command. + let args = std::env::args().skip(2); + #[allow(clippy::never_loop)] + for arg in args { + match arg.as_str() { + "--help" => { + show_usage(); + return Ok(()); + } + _ => return Err(format!("Unknown option {arg}")), + } + } + + run_tool_and_install_it_if_not_present(&[ + &"cargo", + &"clippy", + &"--all-targets", + &"--", + &"-D", + &"warnings", + ])?; + run_command_with_output( + &[ + &"cargo", + &"clippy", + &"--all-targets", + &"--no-default-features", + &"--", + &"-D", + &"warnings", + ], + Some(Path::new(".")), + )?; + run_command_with_output( + &[ + &"cargo", + &"clippy", + &"--all-targets", + &"--manifest-path", + &"build_system/Cargo.toml", + &"--", + &"-D", + &"warnings", + ], + Some(Path::new(".")), + )?; + Ok(()) +} diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 8eb6d8f019e1c..fd78f691d1657 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -314,6 +314,7 @@ impl ConfigInfo { &mut self, env: &mut HashMap, use_system_gcc: bool, + generate_out_dir: bool, ) -> Result<(), String> { env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); @@ -444,12 +445,12 @@ impl ConfigInfo { self.rustc_command = vec![rustc]; self.rustc_command.extend_from_slice(&rustflags); - self.rustc_command.extend_from_slice(&[ - "-L".to_string(), - format!("crate={}", self.cargo_target_dir), - "--out-dir".to_string(), - self.cargo_target_dir.clone(), - ]); + self.rustc_command + .extend_from_slice(&["-L".to_string(), format!("crate={}", self.cargo_target_dir)]); + if generate_out_dir { + self.rustc_command + .extend_from_slice(&["--out-dir".to_string(), self.cargo_target_dir.clone()]); + } if !env.contains_key("RUSTC_LOG") { env.insert("RUSTC_LOG".to_string(), "warn".to_string()); diff --git a/build_system/src/fmt.rs b/build_system/src/fmt.rs index 91535f217e351..dc1ca1d3e82ae 100644 --- a/build_system/src/fmt.rs +++ b/build_system/src/fmt.rs @@ -1,7 +1,7 @@ use std::ffi::OsStr; use std::path::Path; -use crate::utils::{run_command_with_output, walk_dir}; +use crate::utils::{run_command_with_output, run_tool_and_install_it_if_not_present, walk_dir}; fn show_usage() { println!( @@ -31,8 +31,9 @@ pub fn run() -> Result<(), String> { let cmd: &[&dyn AsRef] = if check { &[&"cargo", &"fmt", &"--check"] } else { &[&"cargo", &"fmt"] }; - run_command_with_output(cmd, Some(Path::new(".")))?; + run_tool_and_install_it_if_not_present(cmd)?; run_command_with_output(cmd, Some(Path::new("build_system")))?; + run_command_with_output(cmd, Some(Path::new("build_system/asm-tester")))?; run_rustfmt_recursively("tests/run", check) } diff --git a/build_system/src/main.rs b/build_system/src/main.rs index d0b9811ac8ceb..37b1f306817fd 100644 --- a/build_system/src/main.rs +++ b/build_system/src/main.rs @@ -3,6 +3,7 @@ use std::{env, process}; mod abi_test; mod build; mod clean; +mod clippy; mod clone_gcc; mod config; mod fmt; @@ -12,6 +13,7 @@ mod prepare; mod rust_tools; mod rustc_info; mod test; +mod todo; mod utils; const BUILD_DIR: &str = "build"; @@ -24,43 +26,67 @@ macro_rules! arg_error { }}; } -fn usage() { - println!( - "\ +macro_rules! commands_decl { + ($($variant:ident: $doc_name:literal => $doc:literal ,)+) => { + enum Command { + $($variant),+ + } + + impl<'a> From> for Command { + fn from(arg: Option<&'a str>) -> Self { + match arg { + $(Some($doc_name) => Self::$variant,)+ + Some("--help") => { + usage(); + process::exit(0); + } + Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag), + Some(command) => arg_error!("Unknown command {}", command), + None => { + usage(); + process::exit(0); + } + } + } + } + + fn usage() { + println!("\ rustc_codegen_gcc build system Usage: build_system [command] [options] Options: - --help : Displays this help message. + --help : Displays this help message. + +Commands:", + ); + let mut commands = vec![$(($doc_name, $doc),)+]; + let longest = commands.iter().map(|(name, _)| name.len()).max().unwrap(); -Commands: - cargo : Executes a cargo command. - rustc : Compiles the program using the GCC compiler. - clean : Cleans the build directory, removing all compiled files and artifacts. - prepare : Prepares the environment for building, including fetching dependencies and setting up configurations. - build : Compiles the project. - test : Runs tests for the project. - info : Displays information about the build environment and project configuration. - clone-gcc : Clones the GCC compiler from a specified source. - fmt : Runs rustfmt - fuzz : Fuzzes `cg_gcc` using rustlantis - abi-test : Runs the abi-cafe test suite on the codegen, checking for ABI compatibility with LLVM" - ); + commands.sort_unstable_by(|a, b| a.0.cmp(b.0)); + for (name, doc) in commands { + let spacing = std::iter::repeat(' ').take(longest - name.len() + 1).collect::(); + eprintln!(" {name}{spacing}: {doc}."); + } + } + } } -pub enum Command { - Cargo, - Clean, - CloneGcc, - Prepare, - Build, - Rustc, - Test, - Info, - Fmt, - Fuzz, - AbiTest, +commands_decl! { + Cargo: "cargo" => "Executes a cargo command", + Clean: "clean" => "Cleans the build directory, removing all compiled files and artifacts", + Clippy: "clippy" => "Runs clippy", + CloneGcc: "clone-gcc" => "Clones the GCC compiler from a specified source", + Prepare: "prepare" => "Prepares the environment for building, including fetching dependencies and setting up configurations", + Build: "build" => "Compiles the project", + Rustc: "rustc" => "Compiles the program using the GCC compiler", + Test: "test" => "Runs tests for the project", + Info: "info" => "Displays information about the build environment and project configuration", + Fmt: "fmt" => "Runs rustfmt", + Fuzz: "fuzz" => "Fuzzes `cg_gcc` using `rustlantis`", + AbiTest: "abi-test" => "Runs the abi-cafe test suite on the codegen, checking for ABI compatibility with LLVM", + CheckTodo: "check-todo" => "Checks todo in the project", } fn main() { @@ -70,31 +96,7 @@ fn main() { } } - let command = match env::args().nth(1).as_deref() { - Some("cargo") => Command::Cargo, - Some("rustc") => Command::Rustc, - Some("clean") => Command::Clean, - Some("prepare") => Command::Prepare, - Some("build") => Command::Build, - Some("test") => Command::Test, - Some("info") => Command::Info, - Some("clone-gcc") => Command::CloneGcc, - Some("abi-test") => Command::AbiTest, - Some("fmt") => Command::Fmt, - Some("fuzz") => Command::Fuzz, - Some("--help") => { - usage(); - process::exit(0); - } - Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag), - Some(command) => arg_error!("Unknown command {}", command), - None => { - usage(); - process::exit(0); - } - }; - - if let Err(e) = match command { + if let Err(e) = match Command::from(env::args().nth(1).as_deref()) { Command::Cargo => rust_tools::run_cargo(), Command::Rustc => rust_tools::run_rustc(), Command::Clean => clean::run(), @@ -106,6 +108,8 @@ fn main() { Command::Fmt => fmt::run(), Command::Fuzz => fuzz::run(), Command::AbiTest => abi_test::run(), + Command::Clippy => clippy::run(), + Command::CheckTodo => todo::run(), } { eprintln!("Command failed to run: {e}"); // CI needs to tell a build system error apart from the test failures some suites expect. diff --git a/build_system/src/rust_tools.rs b/build_system/src/rust_tools.rs index b1faa27acc4a2..1b50f11c3d324 100644 --- a/build_system/src/rust_tools.rs +++ b/build_system/src/rust_tools.rs @@ -72,7 +72,7 @@ impl RustcTools { let mut env: HashMap = std::env::vars().collect(); let mut config = ConfigInfo::default(); - config.setup(&mut env, false)?; + config.setup(&mut env, false, false)?; let toolchain = get_toolchain()?; let toolchain_version = rustc_toolchain_version_info(&toolchain)?; diff --git a/build_system/src/test.rs b/build_system/src/test.rs index a4bdb80c94f28..a55c2dcf83ddc 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -11,8 +11,8 @@ use crate::build; use crate::config::{Channel, ConfigInfo}; use crate::utils::{ create_dir, get_sysroot_dir, get_toolchain, git_clone, git_clone_root_dir, remove_file, - run_command, run_command_with_env, run_command_with_output_and_env, rustc_version_info, - split_args, walk_dir, + run_command, run_command_with_env, run_command_with_output_and_env, + run_command_with_output_and_env_no_err, rustc_version_info, split_args, walk_dir, }; /// Exit code of `y.sh test` when the tests ran and reported failures, as opposed to the build @@ -39,6 +39,7 @@ fn get_runners() -> Runners { ("Run failing ui pattern tests", test_failing_ui_pattern_tests), ); runners.insert("--test-failing-rustc", ("Run failing rustc tests", test_failing_rustc)); + runners.insert("--run-ui-tests", ("Run specified rustc UI tests", run_ui_tests)); runners.insert("--projects", ("Run the tests of popular crates", test_projects)); runners.insert("--test-libcore", ("Run libcore tests", test_libcore)); runners.insert("--test-release-libcore", ("Run libcore tests", test_release_libcore)); @@ -56,8 +57,10 @@ fn get_runners() -> Runners { ); runners.insert("--extended-regex-tests", ("Run extended regex tests", extended_regex_tests)); runners.insert("--mini-tests", ("Run mini tests", mini_tests)); + runners.insert("--gcc-asm-tests", ("Run cg_gcc asm tests", test_asm)); runners.insert("--cargo-tests", ("Run cargo tests", cargo_tests)); runners.insert("--no-builtins-tests", ("Test #![no_builtins] attribute", no_builtins_tests)); + runners.insert("--stdarch-tests", ("Run stdarch tests", test_stdarch as Runner)); runners } @@ -519,6 +522,26 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { Ok(()) } +fn get_llvm_filecheck(env: &Env) -> Result { + match run_command_with_env( + &[ + &"bash", + &"-c", + &"which FileCheck-10 || \ + which FileCheck-11 || \ + which FileCheck-12 || \ + which FileCheck-13 || \ + which FileCheck-14 || \ + which FileCheck", + ], + None, + Some(env), + ) { + Ok(cmd) => Ok(String::from_utf8_lossy(&cmd.stdout).trim().to_string()), + Err(_) => Err("Failed to retrieve LLVM FileCheck, ignoring...".to_owned()), + } +} + fn setup_rustc(env: &mut Env, args: &TestArg) -> Result { let toolchain = format!( "+{channel}-{host}", @@ -562,23 +585,10 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result { let rustc = rustc.trim().to_owned(); if rustc.is_empty() { Err("`rustc` path is empty".to_string()) } else { Ok(rustc) } })?; - let llvm_filecheck = match run_command_with_env( - &[ - &"bash", - &"-c", - &"which FileCheck-10 || \ - which FileCheck-11 || \ - which FileCheck-12 || \ - which FileCheck-13 || \ - which FileCheck-14 || \ - which FileCheck", - ], - rust_dir, - Some(env), - ) { - Ok(cmd) => String::from_utf8_lossy(&cmd.stdout).to_string(), - Err(_) => { - eprintln!("Failed to retrieve LLVM FileCheck, ignoring..."); + let llvm_filecheck = match get_llvm_filecheck(env) { + Ok(l) => l, + Err(error) => { + eprintln!("{error}"); // FIXME: the test tests/run-make/no-builtins-attribute will fail if we cannot find // FileCheck. String::new() @@ -648,7 +658,7 @@ fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"0", &"--set", &"build.compiletest-allow-stage0=true", - &"tests/assembly-llvm/asm", + &"tests/assembly-gcc/asm", &"--compiletest-rustc-args", &rustc_args, ], @@ -921,6 +931,39 @@ fn test_libcore_doctests(env: &Env, args: &TestArg) -> Result<(), String> { Ok(()) } +fn test_stdarch(env: &Env, args: &TestArg) -> Result<(), String> { + println!("[TEST] stdarch"); + let manifest_path = get_sysroot_dir().join("sysroot_src/library/stdarch/Cargo.toml"); + let mut env = env.clone(); + + // `config.setup` already baked `CG_RUSTFLAGS` into `RUSTFLAGS`, so append the lint-allow to + // `RUSTFLAGS` directly (which `run_cargo_command` also propagates to `RUSTDOCFLAGS`). + let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + env.insert( + "RUSTFLAGS".to_string(), + format!("{rustflags} -Ainternal_features").trim().to_owned(), + ); + env.insert("TARGET".to_string(), args.config_info.target_triple.clone()); + + let mut command: Vec<&dyn AsRef> = + vec![&"test", &"--manifest-path", &manifest_path, &"--"]; + for test_name in &args.test_args { + command.push(test_name); + } + run_cargo_command(&command, None, &env, args)?; + Ok(()) +} + +fn test_alloc(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] alloc"); + let path = get_sysroot_dir().join("sysroot_src/library/alloctests"); + let _ = remove_dir_all(path.join("target")); + // FIXME(antoyo): run in release mode when we fix the failures. + run_cargo_command(&[&"test"], Some(&path), env, args)?; + Ok(()) +} + fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { if !args.is_using_gcc_master_branch() { println!("Not using GCC master branch. Skipping `extended_rand_tests`."); @@ -1065,7 +1108,6 @@ fn contains_ui_error_patterns(file_path: &Path, keep_lto_tests: bool) -> Result< "//@ known-bug", "-Cllvm-args", "//~", - "thread", ] .iter() .any(|check| line.contains(check)) @@ -1548,6 +1590,60 @@ fn remove_files_callback(file_path: &str) -> impl Fn(&Path) -> Result Result<(), String> { + fn is_path_time_more_recent(ref_time: std::time::SystemTime, path: &str) -> bool { + std::fs::metadata(path) + .and_then(|metadata| metadata.modified()) + .is_ok_and(|time| ref_time < time) + } + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] cg_gcc assembly"); + let llvm_filecheck = get_llvm_filecheck(env)?; + + let target_dir = std::env::current_dir().unwrap().join("build_system/asm-tester/target"); + + // All this code is because `cargo` keeps recompiling this file, and we can't figure out why. + let binary_file_path = "build_system/asm-tester/target/debug/asm-tester"; + let mut need_recompilation = true; + if let Ok(metadata) = std::fs::metadata(binary_file_path) + && let Ok(ref_time) = metadata.modified() + && !is_path_time_more_recent(ref_time, "build_system/asm-tester/Cargo.toml") + && !is_path_time_more_recent(ref_time, "build_system/asm-tester/Cargo.lock") + && !is_path_time_more_recent(ref_time, "build_system/asm-tester/src/main.rs") + { + need_recompilation = false; + } + + if need_recompilation { + let build_asm_args: Vec<&dyn AsRef> = vec![ + &"cargo", + &"build", + &"--manifest-path", + &"build_system/asm-tester/Cargo.toml", + &"--target-dir", + &target_dir, + &"--", + ]; + run_command_with_output_and_env_no_err(&build_asm_args, Some(Path::new(".")), Some(env))?; + } + + let mut test_asm_args: Vec<&dyn AsRef> = vec![ + &"build_system/asm-tester/target/debug/asm-tester", + &"--llvm-filecheck", + &llvm_filecheck, + ]; + for test_arg in &args.test_args { + test_asm_args.push(&"--filter"); + test_asm_args.push(test_arg); + } + test_asm_args.push(&"--"); + for arg in args.config_info.rustc_command_vec().into_iter().skip(1) { + test_asm_args.push(arg); + } + run_command_with_output_and_env_no_err(&test_asm_args, Some(Path::new(".")), Some(env)) +} + fn run_all(env: &Env, args: &TestArg) -> Result<(), String> { clean(env, args)?; mini_tests(env, args)?; @@ -1559,6 +1655,7 @@ fn run_all(env: &Env, args: &TestArg) -> Result<(), String> { cargo_tests(env, args)?; no_builtins_tests(env, args)?; test_rustc(env, args)?; + test_asm(env, args)?; Ok(()) } @@ -1580,7 +1677,7 @@ pub fn run() -> Result<(), String> { return Ok(()); } - args.config_info.setup(&mut env, args.use_system_gcc)?; + args.config_info.setup(&mut env, args.use_system_gcc, true)?; if args.runners.is_empty() { run_all(&env, &args)?; diff --git a/build_system/src/todo.rs b/build_system/src/todo.rs new file mode 100644 index 0000000000000..5b89410844788 --- /dev/null +++ b/build_system/src/todo.rs @@ -0,0 +1,72 @@ +use std::ffi::OsStr; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const EXTENSIONS: &[&str] = + &["rs", "py", "js", "sh", "c", "cpp", "h", "md", "css", "ftl", "toml", "yml", "yaml"]; + +fn has_supported_extension(path: &Path) -> bool { + path.extension().is_some_and(|ext| EXTENSIONS.iter().any(|e| ext == OsStr::new(e))) +} + +fn list_tracked_files() -> Result, String> { + let output = Command::new("git") + .args(["ls-files", "-z"]) + .output() + .map_err(|e| format!("Failed to run `git ls-files`: {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("`git ls-files` failed: {stderr}")); + } + + let mut files = Vec::new(); + for entry in output.stdout.split(|b| *b == 0) { + if entry.is_empty() { + continue; + } + let path = std::str::from_utf8(entry).unwrap(); + files.push(PathBuf::from(path)); + } + + Ok(files) +} + +pub(crate) fn run() -> Result<(), String> { + let files = list_tracked_files()?; + let mut error_count = 0; + // Avoid embedding the task marker in source so greps only find real occurrences. + let todo_marker = "todo".to_ascii_uppercase(); + + for file in files { + if !has_supported_extension(&file) { + continue; + } + + let file_handle = + File::open(&file).map_err(|e| format!("Failed to open {}: {e}", file.display()))?; + let reader = BufReader::new(file_handle); + + for (i, line) in reader.lines().enumerate() { + let line = line.map_err(|e| format!("Failed to read {}: {e}", file.display()))?; + let trimmed = line.trim(); + if trimmed.contains(&todo_marker) { + eprintln!( + "{}:{}: {} is used for tasks that should be done before merging a PR; if you want to leave a message in the codebase use FIXME", + file.display(), + i + 1, + todo_marker + ); + error_count += 1; + } + } + } + + if error_count == 0 { + return Ok(()); + } + + Err(format!("found {} {}(s)", error_count, todo_marker)) +} diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 112322f8688c1..4c67156a85fb2 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -2,10 +2,11 @@ use std::collections::HashMap; use std::ffi::OsStr; use std::fmt::Debug; use std::fs; +use std::io::{BufReader, Read}; #[cfg(unix)] use std::os::unix::process::ExitStatusExt; use std::path::{Path, PathBuf}; -use std::process::{Command, ExitStatus, Output}; +use std::process::{Command, ExitStatus, Output, Stdio}; fn exec_command( input: &[&dyn AsRef], @@ -47,7 +48,7 @@ pub(crate) fn get_command_inner( command } -fn check_exit_status( +pub(crate) fn check_exit_status( input: &[&dyn AsRef], cwd: Option<&Path>, exit_status: ExitStatus, @@ -115,6 +116,30 @@ pub fn run_command_with_output( check_exit_status(input, cwd, exit_status, None, true) } +pub fn run_command_with_output_and_get_it( + input: &[&dyn AsRef], + cwd: Option<&Path>, +) -> Result<(ExitStatus, String), String> { + let mut child = get_command_inner(input, cwd, None) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| command_error(input, &cwd, e))?; + + let stderr = child.stderr.take().expect("Failed to capture stderr"); + let mut captured = String::new(); + BufReader::new(stderr).read_to_string(&mut captured).expect("failed to read stderr"); + + let status = child.wait().map_err(|e| command_error(input, &cwd, e))?; + #[cfg(unix)] + { + if let Some(signal) = status.signal() { + // In case the signal didn't kill the current process. + return Err(command_error(input, &cwd, format!("Process received signal {signal}"))); + } + } + Ok((status, captured)) +} + pub fn run_command_with_output_and_env( input: &[&dyn AsRef], cwd: Option<&Path>, @@ -124,7 +149,6 @@ pub fn run_command_with_output_and_env( check_exit_status(input, cwd, exit_status, None, true) } -#[cfg(not(unix))] pub fn run_command_with_output_and_env_no_err( input: &[&dyn AsRef], cwd: Option<&Path>, @@ -419,6 +443,34 @@ pub fn get_sysroot_dir() -> PathBuf { Path::new(crate::BUILD_DIR).join("build_sysroot") } +pub fn run_tool_and_install_it_if_not_present(cmd: &[&dyn AsRef]) -> Result<(), String> { + let (exit_status, stderr) = run_command_with_output_and_get_it(cmd, Some(Path::new(".")))?; + if exit_status.success() { + return Ok(()); + } + let mut iter = stderr.split('\n'); + if let Some(line) = iter.next() + && line.contains("is not installed for the toolchain") + && let Some(line) = iter.next() + && line.contains("run `rustup component add") + && let Some(cmd) = line.split('`').nth(1) + && let Some(tool_name) = cmd.rsplit(' ').next() + { + println!("`{tool_name}` is not installed for this toolchain, installing it..."); + // A weird round-about way to get a `&&str` so I can get a `&dyn AsRef` but + // as long as it works... + let cmd = cmd.split(' ').collect::>(); + let cmd = cmd.iter().map(|s: &&str| s as &dyn AsRef).collect::>(); + run_command_with_output(cmd.as_slice(), Some(Path::new(".")))?; + } else { + // If the component is installed, then it's something else. In this case we fail like we + // should have and let the user handles the error. + return check_exit_status(cmd, Some(Path::new(".")), exit_status, None, true); + } + // We retry the command... + run_command_with_output(cmd, Some(Path::new("."))) +} + #[cfg(test)] mod tests { use super::*; diff --git a/doc/subtree.md b/doc/subtree.md index a81b6c9c74bdd..fcac399e46542 100644 --- a/doc/subtree.md +++ b/doc/subtree.md @@ -1,7 +1,7 @@ # git subtree sync `rustc_codegen_gcc` is a subtree of the rust compiler. As such, it needs to be -sync from time to time to ensure changes that happened on their side are also +synced from time to time to ensure changes that happened on their side are also included on our side. ### How to install a forked git-subtree @@ -41,6 +41,8 @@ cd ../rust git pull origin master git checkout -b subtree-update_cg_gcc_YYYY-MM-DD PATH="$HOME/bin:$PATH" ~/bin/git-subtree pull --prefix=compiler/rustc_codegen_gcc/ https://github.com/rust-lang/rustc_codegen_gcc.git master +# Don't forget to update the `gcc` submodule to the same version as the +# one in `rustc_codegen_gcc/libgccjit.version`. git push # Immediately merge the merge commit into cg_gcc to prevent merge conflicts when syncing from rust-lang/rust later. diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index 6e155f89ee5cc..ab841d51a7f53 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -6,7 +6,7 @@ )] #![no_core] #![allow(dead_code, internal_features, non_camel_case_types)] -#![rustfmt_skip] +#![cfg_attr(rustfmt, rustfmt_skip)] extern crate mini_core; diff --git a/src/abi.rs b/src/abi.rs index 09e009700a80c..63eaf52ce9f01 100644 --- a/src/abi.rs +++ b/src/abi.rs @@ -146,12 +146,23 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NonNull) { non_null_args.push(arg_index as i32 + 1); } + // There are a few others `ArgAttribute` variants" + // + // * ArgAttribute::ReadOnly: `access(read_only())`, but it's only used for emitting + // warning, not for optimization. + // * ArgAttribute::NoUndef: No equivalent in GCC + // * ArgAttribute::Writable: `access(read_write())` or `access(write_only())`, but it's + // only used for emitting warning, not for optimization. + // * ArgAttribute::NoFree: No equivalent in GCC ty }; #[cfg(not(feature = "master"))] let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes, _arg_index: usize| ty; - for arg in self.args.iter() { + for (source_arg_index, arg) in self.args.iter().enumerate() { + #[cfg(not(feature = "master"))] + let _ = source_arg_index; + let arg_ty = match arg.mode { PassMode::Ignore => continue, PassMode::Pair(a, b) => { @@ -179,9 +190,31 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { apply_attrs(ty, &cast.attrs, argument_tys.len()) } PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => { - // This is a "byval" argument, so we don't apply the `restrict` attribute on it. - on_stack_param_indices.insert(argument_tys.len()); - arg.layout.gcc_type(cx) + let x86_interrupt_first_arg = { + #[cfg(feature = "master")] + { + source_arg_index == 0 + && matches!(self.conv, CanonAbi::Interrupt(InterruptKind::X86)) + } + #[cfg(not(feature = "master"))] + { + false + } + }; + + if x86_interrupt_first_arg { + // Rust lowers the first `x86-interrupt` argument as a byval stack slot. + // LLVM represents that as a pointer parameter with `byval`; GCC's + // interrupt attribute likewise requires a pointer-shaped first parameter. + // Do not add this parameter to `on_stack_param_indices`: that set is only + // needed when GCC represents a byval argument as a value parameter, while + // this parameter is already pointer-shaped. + cx.type_ptr_to(arg.layout.gcc_type(cx)) + } else { + // This is a "byval" argument, so we don't apply the `restrict` attribute on it. + on_stack_param_indices.insert(argument_tys.len()); + arg.layout.gcc_type(cx) + } } PassMode::Direct(attrs) => { apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len()) diff --git a/src/asm.rs b/src/asm.rs index e84cf54cb1147..75a48c217c49b 100644 --- a/src/asm.rs +++ b/src/asm.rs @@ -297,7 +297,9 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { out_place, }); - if !readwrite { + if readwrite { + self.llbb().add_assignment(None, tmp_var, in_value.immediate()); + } else { let out_gcc_idx = outputs.len() - 1; let constraint = Cow::Owned(out_gcc_idx.to_string()); @@ -363,7 +365,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { let ty = value.layout.gcc_type(self.cx); let reg_var = self.current_func().new_local(None, ty, "input_register"); reg_var.set_register_name(reg_name); - self.llbb().add_assignment(None, reg_var, value.immediate()); + // FIXME: We should remove this when switching to "untyped" pointers + let value = value.immediate(); + let value = if value.get_type() != ty { + self.context.new_cast(None, value, ty) + } else { + value + }; + self.llbb().add_assignment(None, reg_var, value); inputs.push(AsmInOperand { constraint: "r".into(), @@ -602,6 +611,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { self.llbb().add_eval(None, self.context.new_call(None, builtin_unreachable, &[])); } + if !options.contains(InlineAsmOptions::NORETURN) + && let Some(dest) = dest + { + self.switch_to_block(dest); + } + // Write results to outputs. // // We need to do this because: diff --git a/src/attributes.rs b/src/attributes.rs index 87c989031678d..41db5e83bdcc9 100644 --- a/src/attributes.rs +++ b/src/attributes.rs @@ -2,6 +2,8 @@ use gccjit::FnAttribute; use gccjit::Function; #[cfg(feature = "master")] +use rustc_abi::{CanonAbi, InterruptKind}; +#[cfg(feature = "master")] use rustc_hir::attrs::InlineAttr; use rustc_hir::attrs::InstructionSetAttr; #[cfg(feature = "master")] @@ -9,6 +11,7 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; #[cfg(feature = "master")] use rustc_middle::mir::TerminatorKind; use rustc_middle::ty; +use rustc_target::callconv::FnAbi; #[cfg(feature = "master")] use rustc_target::spec::Arch; @@ -84,12 +87,23 @@ fn inline_attr<'gcc, 'tcx>( } } +#[cfg(feature = "master")] +fn is_x86_interrupt<'tcx>(fn_abi: Option<&FnAbi<'tcx, ty::Ty<'tcx>>>) -> bool { + matches!( + fn_abi, + Some(fn_abi) if matches!(fn_abi.conv, CanonAbi::Interrupt(InterruptKind::X86)) + ) +} + /// Composite function which sets GCC attributes for function depending on its AST (`#[attribute]`) /// attributes. pub fn from_fn_attrs<'gcc, 'tcx>( cx: &CodegenCx<'gcc, 'tcx>, #[cfg_attr(not(feature = "master"), expect(unused_variables))] func: Function<'gcc>, instance: ty::Instance<'tcx>, + #[cfg_attr(not(feature = "master"), expect(unused_variables))] fn_abi: Option< + &FnAbi<'tcx, ty::Ty<'tcx>>, + >, ) { let codegen_fn_attrs = cx.tcx.codegen_instance_attrs(instance.def); @@ -132,6 +146,11 @@ pub fn from_fn_attrs<'gcc, 'tcx>( } } + #[cfg(feature = "master")] + let x86_interrupt = is_x86_interrupt(fn_abi); + #[cfg(not(feature = "master"))] + let x86_interrupt = false; + let mut function_features = codegen_fn_attrs .target_features .iter() @@ -147,6 +166,13 @@ pub fn from_fn_attrs<'gcc, 'tcx>( // Check if GCC requires the same. let mut global_features = cx.tcx.sess.global_backend_features.iter().map(|s| s.as_str()); function_features.extend(&mut global_features); + if x86_interrupt { + // GCC does not preserve SSE, MMX, or x87 state in interrupt handlers and rejects + // them whenever those instruction sets are enabled, even if the handler does not + // emit such instructions. Restrict the function to general registers so the + // interrupt attribute works with the default x86_64 target features. + function_features.push("general-regs-only"); + } let target_features = function_features .iter() .filter_map(|feature| { diff --git a/src/back/lto.rs b/src/back/lto.rs index b0de1ead56ed1..baf1fda02e258 100644 --- a/src/back/lto.rs +++ b/src/back/lto.rs @@ -20,6 +20,7 @@ use std::ffi::CString; use std::fs::{self, File}; use std::path::{Path, PathBuf}; +use std::sync::Arc; use gccjit::OutputKind; use object::read::archive::ArchiveFile; @@ -29,9 +30,9 @@ use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, SharedEmitter} use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::memmap::Mmap; -use rustc_data_structures::profiling::SelfProfilerRef; use rustc_errors::{DiagCtxt, DiagCtxtHandle}; use rustc_log::tracing::info; +use rustc_session::Session; use tempfile::{TempDir, tempdir}; use crate::back::write::{codegen, save_temp_bitcode}; @@ -103,8 +104,8 @@ fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> { /// Performs fat LTO by merging all modules into a single one and returning it /// for further optimization. pub(crate) fn run_fat( + sess: &Session, cgcx: &CodegenContext, - prof: &SelfProfilerRef, shared_emitter: &SharedEmitter, each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, @@ -115,8 +116,8 @@ pub(crate) fn run_fat( /*let symbols_below_threshold = lto_data.symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>();*/ fat_lto( + sess, cgcx, - prof, dcx, modules, lto_data.upstream_modules, @@ -126,15 +127,15 @@ pub(crate) fn run_fat( } fn fat_lto( + sess: &Session, cgcx: &CodegenContext, - prof: &SelfProfilerRef, dcx: DiagCtxtHandle<'_>, modules: Vec>, mut serialized_modules: Vec<(SerializedModule, CString)>, tmp_path: TempDir, //symbols_below_threshold: &[String], ) -> CompiledModule { - let _timer = prof.generic_activity("GCC_fat_lto_build_monolithic_module"); + let _timer = sess.prof.generic_activity("GCC_fat_lto_build_monolithic_module"); info!("going for a fat lto"); // Sort out all our lists of incoming modules into two lists. @@ -184,17 +185,16 @@ fn fat_lto( // module and create a linker with it. let mut module: ModuleCodegen = match costliest_module { Some((_cost, i)) => in_memory.remove(i), - None => { - unimplemented!("Incremental"); - /*assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); - let (buffer, name) = serialized_modules.remove(0); - info!("no in-memory regular modules to choose from, parsing {:?}", name); - ModuleCodegen { - module_llvm: GccContext::parse(cgcx, &name, buffer.data(), dcx)?, - name: name.into_string().unwrap(), - kind: ModuleKind::Regular, - }*/ - } + None => ModuleCodegen::new_regular( + "lto_module".to_string(), + GccContext { + context: Arc::new(SyncContext::new(new_context(sess))), + relocation_model: sess.relocation_model(), + lto_supported: true, + lto_mode: LtoMode::None, + temp_dir: None, + }, + ), }; { info!("using {:?} as a base module", module.name); @@ -221,7 +221,8 @@ fn fat_lto( // We add the object files and save in should_combine_object_files that we should combine // them into a single object file when compiling later. for (bc_decoded, name) in serialized_modules { - let _timer = prof + let _timer = sess + .prof .generic_activity_with_arg_recorder("GCC_fat_lto_link_module", |recorder| { recorder.record_arg(format!("{:?}", name)) }); @@ -259,7 +260,7 @@ fn fat_lto( // of now. module.module_llvm.temp_dir = Some(tmp_path); - codegen(cgcx, prof, dcx, module, &cgcx.module_config) + codegen(cgcx, &sess.prof, dcx, module, &cgcx.module_config) } pub struct ModuleBuffer(PathBuf); diff --git a/src/back/write.rs b/src/back/write.rs index cf5514412f745..1f4fd8a314ad2 100644 --- a/src/back/write.rs +++ b/src/back/write.rs @@ -11,8 +11,8 @@ use rustc_log::tracing::debug; use rustc_session::config::OutputType; use rustc_target::spec::SplitDebuginfo; -use crate::base::add_pic_option; use crate::diagnostics::CopyBitcode; +use crate::gcc_util::add_pic_option; use crate::{GccContext, LtoMode}; pub(crate) fn codegen( @@ -60,9 +60,6 @@ pub(crate) fn codegen( let _timer = prof .generic_activity_with_arg("GCC_module_codegen_embed_bitcode", &*module.name); if lto_supported { - // FIXME(antoyo): maybe we should call embed_bitcode to have the proper iOS fixes? - //embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data); - context.add_command_line_option("-flto=auto"); context.add_command_line_option("-flto-partition=one"); context.add_command_line_option("-ffat-lto-objects"); diff --git a/src/base.rs b/src/base.rs index 7cb734d523273..436a9e5227352 100644 --- a/src/base.rs +++ b/src/base.rs @@ -1,5 +1,3 @@ -use std::collections::HashSet; -use std::env; use std::sync::Arc; use std::time::Instant; @@ -19,7 +17,6 @@ use rustc_session::config::DebugInfo; use rustc_span::Symbol; #[cfg(feature = "master")] use rustc_target::spec::SymbolVisibility; -use rustc_target::spec::{Arch, RelocModel}; use crate::builder::Builder; use crate::context::CodegenCx; @@ -144,41 +141,7 @@ pub fn compile_codegen_unit( ) -> ModuleCodegen { let cgu = tcx.codegen_unit(cgu_name); // Instantiate monomorphizations without filling out definitions yet... - let context = new_context(tcx); - - if tcx.sess.panic_strategy().unwinds() { - context.add_command_line_option("-fexceptions"); - context.add_driver_option("-fexceptions"); - } - - let disabled_features: HashSet<_> = tcx - .sess - .opts - .cg - .target_feature - .split(',') - .filter(|feature| feature.starts_with('-')) - .map(|string| &string[1..]) - .collect(); - - if !disabled_features.contains("avx") && tcx.sess.target.arch == Arch::X86_64 { - // NOTE: we always enable AVX because the equivalent of llvm.x86.sse2.cmp.pd in GCC for - // SSE2 is multiple builtins, so we use the AVX __builtin_ia32_cmppd instead. - // FIXME(antoyo): use the proper builtins for llvm.x86.sse2.cmp.pd and similar. - context.add_command_line_option("-mavx"); - } - - for arg in &tcx.sess.opts.cg.llvm_args { - context.add_command_line_option(arg); - } - // NOTE: This is needed to compile the file src/intrinsic/archs.rs during a bootstrap of rustc. - context.add_command_line_option("-fno-var-tracking-assignments"); - // NOTE: an optimization (https://github.com/rust-lang/rustc_codegen_gcc/issues/53). - context.add_command_line_option("-fno-semantic-interposition"); - // NOTE: Rust relies on LLVM not doing TBAA (https://github.com/rust-lang/unsafe-code-guidelines/issues/292). - context.add_command_line_option("-fno-strict-aliasing"); - // NOTE: Rust relies on LLVM doing wrapping on overflow. - context.add_command_line_option("-fwrapv"); + let context = new_context(tcx.sess); // NOTE: We need to honor the `#![no_builtins]` attribute to prevent GCC from // replacing code patterns (like loops) with calls to builtins (like memset). @@ -191,64 +154,6 @@ pub fn compile_codegen_unit( context.add_command_line_option("-fno-tree-loop-distribute-patterns"); } - if let Some(model) = tcx.sess.code_model() { - use rustc_target::spec::CodeModel; - - context.add_command_line_option(match model { - CodeModel::Tiny => "-mcmodel=tiny", - CodeModel::Small => "-mcmodel=small", - CodeModel::Kernel => "-mcmodel=kernel", - CodeModel::Medium => "-mcmodel=medium", - CodeModel::Large => "-mcmodel=large", - }); - } - - add_pic_option(&context, tcx.sess.relocation_model()); - - let target_cpu = gcc_util::target_cpu(tcx.sess); - if target_cpu != "generic" { - context.add_command_line_option(format!("-march={}", target_cpu)); - } - - if tcx - .sess - .opts - .unstable_opts - .function_sections - .unwrap_or(tcx.sess.target.function_sections) - { - context.add_command_line_option("-ffunction-sections"); - context.add_command_line_option("-fdata-sections"); - } - - if env::var("CG_GCCJIT_DUMP_RTL").as_deref() == Ok("1") { - context.add_command_line_option("-fdump-rtl-vregs"); - } - if env::var("CG_GCCJIT_DUMP_RTL_ALL").as_deref() == Ok("1") { - context.add_command_line_option("-fdump-rtl-all"); - } - if env::var("CG_GCCJIT_DUMP_TREE_ALL").as_deref() == Ok("1") { - context.add_command_line_option("-fdump-tree-all-eh"); - } - if env::var("CG_GCCJIT_DUMP_IPA_ALL").as_deref() == Ok("1") { - context.add_command_line_option("-fdump-ipa-all-eh"); - } - if env::var("CG_GCCJIT_DUMP_CODE").as_deref() == Ok("1") { - context.set_dump_code_on_compile(true); - } - if env::var("CG_GCCJIT_DUMP_GIMPLE").as_deref() == Ok("1") { - context.set_dump_initial_gimple(true); - } - if env::var("CG_GCCJIT_DUMP_EVERYTHING").as_deref() == Ok("1") { - context.set_dump_everything(true); - } - if env::var("CG_GCCJIT_KEEP_INTERMEDIATES").as_deref() == Ok("1") { - context.set_keep_intermediates(true); - } - if env::var("CG_GCCJIT_VERBOSE").as_deref() == Ok("1") { - context.add_driver_option("-v"); - } - // NOTE: The codegen generates unreachable blocks. context.set_allow_unreachable_blocks(true); @@ -317,24 +222,3 @@ pub fn compile_codegen_unit( (module, cost) } - -pub fn add_pic_option<'gcc>(context: &Context<'gcc>, relocation_model: RelocModel) { - match relocation_model { - rustc_target::spec::RelocModel::Static => { - context.add_command_line_option("-fno-pie"); - context.add_driver_option("-fno-pie"); - } - rustc_target::spec::RelocModel::Pic => { - context.add_command_line_option("-fPIC"); - // NOTE: we use both add_command_line_option and add_driver_option because the usage in - // this module (compile_codegen_unit) requires add_command_line_option while the usage - // in the back::write module (codegen) requires add_driver_option. - context.add_driver_option("-fPIC"); - } - rustc_target::spec::RelocModel::Pie => { - context.add_command_line_option("-fPIE"); - context.add_driver_option("-fPIE"); - } - model => eprintln!("Unsupported relocation model: {:?}", model), - } -} diff --git a/src/builder.rs b/src/builder.rs index 6de2827e63dee..a1eab8f448990 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -4,8 +4,8 @@ use std::convert::TryFrom; use std::ops::Deref; use gccjit::{ - BinaryOp, Block, ComparisonOp, Context, Function, LValue, Location, RValue, ToRValue, Type, - UnaryOp, + BinaryOp, Block, CType, ComparisonOp, Context, Function, LValue, Location, RValue, ToRValue, + Type, UnaryOp, }; use rustc_abi as abi; use rustc_abi::{Align, HasDataLayout, Size, TargetDataLayout, WrappingRange}; @@ -118,7 +118,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { ); let previous_var = func.new_local(self.location, previous_value.get_type(), "previous_value"); - let return_value = func.new_local(self.location, previous_value.get_type(), "return_value"); + let return_value = self.new_temp(func, self.location, previous_value.get_type()); self.llbb().add_assignment(self.location, previous_var, previous_value); self.llbb().add_assignment(self.location, return_value, previous_var.to_rvalue()); @@ -389,29 +389,27 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { func: Function<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>, + must_tail: bool, ) -> RValue<'gcc> { let args = self.check_call("call", func, args); + let call = self.cx.context.new_call(self.location, func, &args); + if must_tail { + // Return the bare tail call, don't assign or `add_eval` it yet. + return call; + } + // gccjit requires to use the result of functions, even when it's not used. // That's why we assign the result to a local or call add_eval(). let return_type = func.get_return_type(); let void_type = self.context.new_type::<()>(); let current_func = self.block.get_function(); if return_type != void_type { - let result = current_func.new_local( - self.location, - return_type, - format!("returnValue{}", self.next_value_counter()), - ); - self.block.add_assignment( - self.location, - result, - self.cx.context.new_call(self.location, func, &args), - ); + let result = self.new_temp(current_func, self.location, return_type); + self.block.add_assignment(self.location, result, call); result.to_rvalue() } else { - self.block - .add_eval(self.location, self.cx.context.new_call(self.location, func, &args)); + self.block.add_eval(self.location, call); // Return dummy value when not having return value. self.context.new_rvalue_zero(self.isize_type) } @@ -424,6 +422,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { mut func_ptr: RValue<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>, + must_tail: bool, ) -> RValue<'gcc> { let func_ptr_type = { let func_ptr_type = func_ptr.get_type(); @@ -448,6 +447,12 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { let args_adjusted = args.len() != previous_arg_count; let args = self.check_ptr_call("call", func_ptr, &args, &on_stack_param_indices); + if must_tail { + // Return the bare tail call, don't assign or `add_eval` it yet. + let call = self.cx.context.new_call_through_ptr(self.location, func_ptr, &args); + return call; + } + // gccjit requires to use the result of functions, even when it's not used. // That's why we assign the result to a local or call add_eval(). let return_type = gcc_func.get_return_type(); @@ -464,11 +469,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { args_adjusted, orig_args, ); - let result = current_func.new_local( - self.location, - return_value.get_type(), - format!("ptrReturnValue{}", self.next_value_counter()), - ); + let result = self.new_temp(current_func, self.location, return_value.get_type()); self.block.add_assignment(self.location, result, return_value); result.to_rvalue() } else { @@ -490,8 +491,16 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.location, self.cx.context.new_call_through_ptr(self.location, func_ptr, &args), ); - // Return dummy value when not having return value. - self.context.new_rvalue_zero(self.isize_type) + // Return dummy value when not having return value, unless the intrinsic adapter + // needs to synthesize a non-void LLVM-level result from out-parameters. + llvm::adjust_intrinsic_return_value( + self, + self.context.new_rvalue_zero(self.isize_type), + &func_name, + &args, + args_adjusted, + orig_args, + ) } } @@ -506,11 +515,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { let return_type = self.context.new_type::(); let current_func = self.block.get_function(); // FIXME(antoyo): return the new_call() directly? Since the overflow function has no side-effects. - let result = current_func.new_local( - self.location, - return_type, - format!("overflowReturnValue{}", self.next_value_counter()), - ); + let result = self.new_temp(current_func, self.location, return_type); self.block.add_assignment( self.location, result, @@ -642,6 +647,18 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { default_block: Block<'gcc>, cases: impl ExactSizeIterator)>, ) { + // A switch with no cases is equivalent to an unconditional jump to the + // default block. Such a `SwitchInt` (one with only an `otherwise` target) + // is normally simplified into a `goto`, but `-Z mir-preserve-ub` keeps it, + // so it can reach here with e.g. the `bool` discriminant produced by a + // range-pattern comparison. `gcc_jit_block_end_with_switch` rejects a + // discriminant that is not of integer type, so emit a plain jump instead + // of a (pointless) switch. + if cases.len() == 0 { + self.block.end_with_jump(self.location, default_block); + return; + } + let mut gcc_cases = vec![]; let typ = self.val_ty(value); // FIXME(FractalFir): This is a workaround for a libgccjit limitation. @@ -1070,11 +1087,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { // the current basic block. Otherwise, it could be used in another basic block, causing a // dereference after a drop, for instance. let deref = ptr.dereference(self.location).to_rvalue(); - let loaded_value = function.new_local( - self.location, - aligned_type, - format!("loadedValue{}", self.next_value_counter()), - ); + let loaded_value = self.new_temp(function, self.location, aligned_type); block.add_assignment(self.location, loaded_value, deref); loaded_value.to_rvalue() } @@ -1193,7 +1206,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let next_bb = self.append_sibling_block("repeat_loop_next"); let ptr_type = start.get_type(); - let current = self.llbb().get_function().new_local(self.location, ptr_type, "loop_var"); + let current = self.new_temp(self.llbb().get_function(), self.location, ptr_type); let current_val = current.to_rvalue(); self.assign(current, start); @@ -1567,7 +1580,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { mut else_val: RValue<'gcc>, ) -> RValue<'gcc> { let func = self.current_func(); - let variable = func.new_local(self.location, then_val.get_type(), "selectVar"); + let variable = self.new_temp(func, self.location, then_val.get_type()); let then_block = func.new_block("then"); let else_block = func.new_block("else"); let after_block = func.new_block("after"); @@ -1589,8 +1602,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { variable.to_rvalue() } - fn va_arg(&mut self, _list: RValue<'gcc>, _ty: Type<'gcc>) -> RValue<'gcc> { - unimplemented!(); + fn va_arg(&mut self, list: RValue<'gcc>, ty: Type<'gcc>) -> RValue<'gcc> { + let va_list_type = self.context.new_c_type(CType::VaList); + let list = self.context.new_cast(self.location, list, va_list_type.make_pointer()); + self.context.new_va_arg(self.location, list, ty) } #[cfg(feature = "master")] @@ -1705,11 +1720,9 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { #[cfg(not(feature = "master"))] fn cleanup_landing_pad(&mut self, _pers_fn: Function<'gcc>) -> (RValue<'gcc>, RValue<'gcc>) { let value1 = self - .current_func() - .new_local(self.location, self.u8_type.make_pointer(), "landing_pad0") + .new_temp(self.current_func(), self.location, self.u8_type.make_pointer()) .to_rvalue(); - let value2 = - self.current_func().new_local(self.location, self.i32_type, "landing_pad1").to_rvalue(); + let value2 = self.new_temp(self.current_func(), self.location, self.i32_type).to_rvalue(); (value1, value2) } @@ -1772,7 +1785,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { // NOTE: since success contains the call to the intrinsic, it must be added to the basic block before // expected so that we store expected after the call. - let success_var = self.current_func().new_local(self.location, self.bool_type, "success"); + let success_var = self.new_temp(self.current_func(), self.location, self.bool_type); self.llbb().add_assignment(self.location, success_var, success); (expected.to_rvalue(), success_var.to_rvalue()) @@ -1867,7 +1880,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn tail_call( &mut self, - _llty: Self::Type, + llty: Self::Type, _fn_attrs: Option<&CodegenFnAttrs>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Value, @@ -2475,11 +2488,31 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.bitcast_if_needed(res, result_type) } + /// Create a temporary variable. + /// + /// GCC will use more stack space with a local variable than with a temporary variable in debug mode, + /// so in order to avoid having the stack probe test fail in CI, we avoid creating local variables for temporaries. + pub fn new_temp( + &self, + function: Function<'gcc>, + location: Option>, + typ: Type<'gcc>, + ) -> LValue<'gcc> { + #[cfg(feature = "master")] + { + function.new_temp(location, typ) + } + #[cfg(not(feature = "master"))] + { + function.new_local(location, typ, format!("temp{}", self.next_value_counter())) + } + } + // GCC doesn't like deeply nested expressions. // By assigning intermediate expressions to a variable, this allow us to avoid deeply nested // expressions and GCC will use much less RAM. fn assign_to_var(&self, value: RValue<'gcc>) -> RValue<'gcc> { - let var = self.current_func().new_local(self.location, value.get_type(), "opResult"); + let var = self.new_temp(self.current_func(), self.location, value.get_type()); self.llbb().add_assignment(self.location, var, value); var.to_rvalue() } diff --git a/src/callee.rs b/src/callee.rs index 00f095ed54371..d3f412180da55 100644 --- a/src/callee.rs +++ b/src/callee.rs @@ -70,7 +70,7 @@ pub fn get_fn<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, instance: Instance<'tcx>) cx.linkage.set(FunctionType::Extern); let func = cx.declare_fn(sym, fn_abi); - attributes::from_fn_attrs(cx, func, instance); + attributes::from_fn_attrs(cx, func, instance, Some(fn_abi)); #[cfg(feature = "master")] { diff --git a/src/consts.rs b/src/consts.rs index e6acbc6625d44..8576dfe079163 100644 --- a/src/consts.rs +++ b/src/consts.rs @@ -171,29 +171,52 @@ impl<'gcc, 'tcx> StaticCodegenMethods for CodegenCx<'gcc, 'tcx> { } // Wasm statics with custom link sections get special treatment as they - // go into custom sections of the wasm executable. - if self.tcx.sess.target.is_like_wasm { + // go into custom sections of the wasm executable. The exception to this + // is the `.init_array` section which are treated specially by the wasm linker. + if self.tcx.sess.target.is_like_wasm + && attrs + .link_section + .map(|link_section| !link_section.as_str().starts_with(".init_array")) + .unwrap_or(true) + { if let Some(_section) = attrs.link_section { unimplemented!(); } - } else { - // FIXME(antoyo): set link section. + } else if let Some(_section) = attrs.link_section { + #[cfg(feature = "master")] + global.add_attribute(VarAttribute::Section(_section.as_str())); } - if attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER) - || attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) - { - self.add_used_global(global.to_rvalue()); + if attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER) { + // To copy the conditions from the LLVM backend... + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)); + self.add_used_global(global); + } + if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { + // To copy the conditions from the LLVM backend... + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER)); + self.add_retained_global(global); } } } impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { - /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. - pub fn add_used_global(&mut self, _global: RValue<'gcc>) { - // FIXME(antoyo) + /// Need to have the `SHF_GNU_RETAIN` flag, so needs to use the `retain` attribute instead of + /// `used`. This is used by `#[used(linker)]`. + pub fn add_retained_global(&mut self, global: LValue<'gcc>) { + // We need to add the `used` C attribute in any case. + self.add_used_global(global); + #[cfg(feature = "master")] + global.add_attribute(VarAttribute::Retain); + } + + /// This is used by `#[used(compiler)]` and `#[used]`. + pub fn add_used_global(&mut self, _global: LValue<'gcc>) { + #[cfg(feature = "master")] + _global.add_attribute(VarAttribute::Used); } + // No need to have the `SHF_GNU_RETAIN` flag, so `used` attribute is ok. #[cfg_attr(not(feature = "master"), expect(unused_variables))] pub fn add_used_function(&self, function: Function<'gcc>) { #[cfg(feature = "master")] diff --git a/src/declare.rs b/src/declare.rs index a9503574a03ed..32bb7c3aa349e 100644 --- a/src/declare.rs +++ b/src/declare.rs @@ -1,12 +1,12 @@ #[cfg(feature = "master")] -use gccjit::{FnAttribute, ToRValue}; +use gccjit::{FnAttribute, ToRValue, VarAttribute}; use gccjit::{Function, FunctionType, GlobalKind, LValue, RValue, Type}; use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; use rustc_middle::ty::Ty; use rustc_span::Symbol; use rustc_target::callconv::FnAbi; -use crate::abi::{FnAbiGcc, FnAbiGccExt}; +use crate::abi::FnAbiGccExt; use crate::context::CodegenCx; impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { @@ -25,6 +25,9 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { global.set_tls_model(self.tls_model); } if let Some(link_section) = link_section { + #[cfg(feature = "master")] + global.add_attribute(VarAttribute::Section(link_section.as_str())); + #[cfg(not(feature = "master"))] global.set_link_section(link_section.as_str()); } global @@ -74,6 +77,9 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { global.set_tls_model(self.tls_model); } if let Some(link_section) = link_section { + #[cfg(feature = "master")] + global.add_attribute(VarAttribute::Section(link_section.as_str())); + #[cfg(not(feature = "master"))] global.set_link_section(link_section.as_str()); } let global_address = global.get_address(None); @@ -111,22 +117,22 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { } pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> { - let FnAbiGcc { - return_type, - arguments_type, - is_c_variadic, - on_stack_param_indices, - #[cfg(feature = "master")] - fn_attributes, - } = fn_abi.gcc_type(self); + let fn_abi_gcc = fn_abi.gcc_type(self); #[cfg(feature = "master")] let conv = fn_abi.gcc_cconv(self); #[cfg(not(feature = "master"))] let conv = None; - let func = declare_raw_fn(self, name, conv, return_type, &arguments_type, is_c_variadic); - self.on_stack_function_params.borrow_mut().insert(func, on_stack_param_indices); + let func = declare_raw_fn( + self, + name, + conv, + fn_abi_gcc.return_type, + &fn_abi_gcc.arguments_type, + fn_abi_gcc.is_c_variadic, + ); + self.on_stack_function_params.borrow_mut().insert(func, fn_abi_gcc.on_stack_param_indices); #[cfg(feature = "master")] - for fn_attr in fn_attributes { + for fn_attr in fn_abi_gcc.fn_attributes { func.add_attribute(fn_attr); } func diff --git a/src/diagnostics.rs b/src/diagnostics.rs index de633d3bdde79..67723ebd2f30b 100644 --- a/src/diagnostics.rs +++ b/src/diagnostics.rs @@ -20,10 +20,6 @@ pub(crate) struct LtoBitcodeFromRlib { pub gcc_err: String, } -#[derive(Diagnostic)] -#[diag("explicit tail calls with the 'become' keyword are not implemented in the GCC backend")] -pub(crate) struct ExplicitTailCallsUnsupported; - #[derive(Diagnostic)] #[diag("asm contains a NUL byte")] pub(crate) struct NulBytesInAsm { diff --git a/src/gcc_util.rs b/src/gcc_util.rs index 4b0fef32cac03..0628171e488b3 100644 --- a/src/gcc_util.rs +++ b/src/gcc_util.rs @@ -4,7 +4,7 @@ use std::env; use gccjit::Context; #[cfg(feature = "master")] -use gccjit::Context; +use gccjit::Version; use rustc_codegen_ssa::target_features; use rustc_data_structures::smallvec::{SmallVec, smallvec}; use rustc_session::config::NATIVE_CPU; diff --git a/src/int.rs b/src/int.rs index 021f05c666d46..4e4b911666143 100644 --- a/src/int.rs +++ b/src/int.rs @@ -457,7 +457,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { if self.is_non_native_int_type(a_type) || self.is_non_native_int_type(b_type) { // This algorithm is based on compiler-rt's __cmpti2: // https://github.com/llvm-mirror/compiler-rt/blob/f0745e8476f069296a7c71accedd061dce4cdf79/lib/builtins/cmpti2.c#L21 - let result = self.current_func().new_local(self.location, self.int_type, "icmp_result"); + let result = self.new_temp(self.current_func(), self.location, self.int_type); let block1 = self.current_func().new_block("block1"); let block2 = self.current_func().new_block("block2"); let block3 = self.current_func().new_block("block3"); @@ -636,9 +636,17 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { rhs = self.context.new_cast(self.location, rhs, unsigned_type); } } - // FIXME(antoyo): we probably need to handle signed comparison for unsigned - // integers. - _ => (), + IntPredicate::IntSGT + | IntPredicate::IntSGE + | IntPredicate::IntSLT + | IntPredicate::IntSLE => { + if !a_type.is_vector() { + let signed_type = a_type.to_signed(self.cx); + lhs = self.context.new_cast(self.location, lhs, signed_type); + rhs = self.context.new_cast(self.location, rhs, signed_type); + } + } + IntPredicate::IntEQ | IntPredicate::IntNE => (), } self.context.new_comparison(self.location, op.to_gcc_comparison(), lhs, rhs) } @@ -902,7 +910,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { self.bitwise_operation(BinaryOp::BitwiseOr, a, b, loc) } - // FIXME(antoyo): can we use https://github.com/rust-lang/compiler-builtins/blob/master/src/int/mod.rs#L379 instead? + // FIXME(antoyo): can we use https://github.com/rust-lang/compiler-builtins/blob/1a99c2aa295bb2d507fa0e67a3b5eef64fba92a0/libm/src/math/support/int_traits.rs#L485 instead? pub fn gcc_int_cast(&self, value: RValue<'gcc>, dest_typ: Type<'gcc>) -> RValue<'gcc> { let value_type = value.get_type(); if self.is_native_int_type_or_bool(dest_typ) && self.is_native_int_type_or_bool(value_type) diff --git a/src/intrinsic/archs.rs b/src/intrinsic/archs.rs index 3c1698df6dec2..1856c2468616d 100644 --- a/src/intrinsic/archs.rs +++ b/src/intrinsic/archs.rs @@ -24,6 +24,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "gcsss" => "__builtin_arm_gcsss", "isb" => "__builtin_arm_isb", "prefetch" => "__builtin_arm_prefetch", + "prefetch.ir" => "__builtin_arm_prefetch_ir", "range.prefetch" => "__builtin_arm_range_prefetch", "sme.in.streaming.mode" => "__builtin_arm_in_streaming_mode", "sve.aesd" => "__builtin_sve_svaesd_u8", @@ -53,6 +54,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "alignbyte" => "__builtin_amdgcn_alignbyte", "ashr.pk.i8.i32" => "__builtin_amdgcn_ashr_pk_i8_i32", "ashr.pk.u8.i32" => "__builtin_amdgcn_ashr_pk_u8_i32", + "asyncmark" => "__builtin_amdgcn_asyncmark", "buffer.wbinvl1" => "__builtin_amdgcn_buffer_wbinvl1", "buffer.wbinvl1.sc" => "__builtin_amdgcn_buffer_wbinvl1_sc", "buffer.wbinvl1.vol" => "__builtin_amdgcn_buffer_wbinvl1_vol", @@ -270,6 +272,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "fdot2c.f32.bf16" => "__builtin_amdgcn_fdot2c_f32_bf16", "flat.prefetch" => "__builtin_amdgcn_flat_prefetch", "fmul.legacy" => "__builtin_amdgcn_fmul_legacy", + "global.load.async.lds" => "__builtin_amdgcn_global_load_async_lds", "global.load.async.to.lds.b128" => { "__builtin_amdgcn_global_load_async_to_lds_b128" } @@ -361,11 +364,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "perm.pk16.b4.u4" => "__builtin_amdgcn_perm_pk16_b4_u4", "perm.pk16.b6.u4" => "__builtin_amdgcn_perm_pk16_b6_u4", "perm.pk16.b8.u4" => "__builtin_amdgcn_perm_pk16_b8_u4", - "permlane.bcast" => "__builtin_amdgcn_permlane_bcast", - "permlane.down" => "__builtin_amdgcn_permlane_down", "permlane.idx.gen" => "__builtin_amdgcn_permlane_idx_gen", - "permlane.up" => "__builtin_amdgcn_permlane_up", - "permlane.xor" => "__builtin_amdgcn_permlane_xor", "permlane16.var" => "__builtin_amdgcn_permlane16_var", "permlanex16.var" => "__builtin_amdgcn_permlanex16_var", "pk.add.max.i16" => "__builtin_amdgcn_pk_add_max_i16", @@ -375,6 +374,9 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "prng.b32" => "__builtin_amdgcn_prng_b32", "qsad.pk.u16.u8" => "__builtin_amdgcn_qsad_pk_u16_u8", "queue.ptr" => "__builtin_amdgcn_queue_ptr", + "raw.ptr.buffer.load.async.lds" => { + "__builtin_amdgcn_raw_ptr_buffer_load_async_lds" + } "raw.ptr.buffer.load.lds" => "__builtin_amdgcn_raw_ptr_buffer_load_lds", "rcp.legacy" => "__builtin_amdgcn_rcp_legacy", "rsq.legacy" => "__builtin_amdgcn_rsq_legacy", @@ -386,6 +388,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "s.barrier.signal.isfirst" => "__builtin_amdgcn_s_barrier_signal_isfirst", "s.barrier.signal.var" => "__builtin_amdgcn_s_barrier_signal_var", "s.barrier.wait" => "__builtin_amdgcn_s_barrier_wait", + "s.bitreplicate" => "__builtin_amdgcn_s_bitreplicate", "s.buffer.prefetch.data" => "__builtin_amdgcn_s_buffer_prefetch_data", "s.cluster.barrier" => "__builtin_amdgcn_s_cluster_barrier", "s.dcache.inv" => "__builtin_amdgcn_s_dcache_inv", @@ -412,6 +415,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "s.ttracedata" => "__builtin_amdgcn_s_ttracedata", "s.ttracedata.imm" => "__builtin_amdgcn_s_ttracedata_imm", "s.wait.asynccnt" => "__builtin_amdgcn_s_wait_asynccnt", + "s.wait.event" => "__builtin_amdgcn_s_wait_event", "s.wait.event.export.ready" => "__builtin_amdgcn_s_wait_event_export_ready", "s.wait.tensorcnt" => "__builtin_amdgcn_s_wait_tensorcnt", "s.waitcnt" => "__builtin_amdgcn_s_waitcnt", @@ -462,16 +466,18 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "smfmac.i32.16x16x64.i8" => "__builtin_amdgcn_smfmac_i32_16x16x64_i8", "smfmac.i32.32x32x32.i8" => "__builtin_amdgcn_smfmac_i32_32x32x32_i8", "smfmac.i32.32x32x64.i8" => "__builtin_amdgcn_smfmac_i32_32x32x64_i8", + "struct.ptr.buffer.load.async.lds" => { + "__builtin_amdgcn_struct_ptr_buffer_load_async_lds" + } "struct.ptr.buffer.load.lds" => "__builtin_amdgcn_struct_ptr_buffer_load_lds", "sudot4" => "__builtin_amdgcn_sudot4", "sudot8" => "__builtin_amdgcn_sudot8", "tensor.load.to.lds" => "__builtin_amdgcn_tensor_load_to_lds", - "tensor.load.to.lds.d2" => "__builtin_amdgcn_tensor_load_to_lds_d2", "tensor.store.from.lds" => "__builtin_amdgcn_tensor_store_from_lds", - "tensor.store.from.lds.d2" => "__builtin_amdgcn_tensor_store_from_lds_d2", "udot2" => "__builtin_amdgcn_udot2", "udot4" => "__builtin_amdgcn_udot4", "udot8" => "__builtin_amdgcn_udot8", + "wait.asyncmark" => "__builtin_amdgcn_wait_asyncmark", "wave.barrier" => "__builtin_amdgcn_wave_barrier", "wavefrontsize" => "__builtin_amdgcn_wavefrontsize", "workgroup.id.x" => "__builtin_amdgcn_workgroup_id_x", @@ -4844,7 +4850,11 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "add.rn.f" => "__nvvm_add_rn_f", "add.rn.ftz.f" => "__nvvm_add_rn_ftz_f", "add.rn.ftz.sat.f" => "__nvvm_add_rn_ftz_sat_f", + "add.rn.ftz.sat.f16" => "__nvvm_add_rn_ftz_sat_f16", + "add.rn.ftz.sat.v2f16" => "__nvvm_add_rn_ftz_sat_v2f16", "add.rn.sat.f" => "__nvvm_add_rn_sat_f", + "add.rn.sat.f16" => "__nvvm_add_rn_sat_f16", + "add.rn.sat.v2f16" => "__nvvm_add_rn_sat_v2f16", "add.rp.d" => "__nvvm_add_rp_d", "add.rp.f" => "__nvvm_add_rp_f", "add.rp.ftz.f" => "__nvvm_add_rp_ftz_f", @@ -5063,18 +5073,10 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "fma.rn.bf16x2" => "__nvvm_fma_rn_bf16x2", "fma.rn.d" => "__nvvm_fma_rn_d", "fma.rn.f" => "__nvvm_fma_rn_f", - "fma.rn.ftz.bf16" => "__nvvm_fma_rn_ftz_bf16", - "fma.rn.ftz.bf16x2" => "__nvvm_fma_rn_ftz_bf16x2", "fma.rn.ftz.f" => "__nvvm_fma_rn_ftz_f", - "fma.rn.ftz.relu.bf16" => "__nvvm_fma_rn_ftz_relu_bf16", - "fma.rn.ftz.relu.bf16x2" => "__nvvm_fma_rn_ftz_relu_bf16x2", - "fma.rn.ftz.sat.bf16" => "__nvvm_fma_rn_ftz_sat_bf16", - "fma.rn.ftz.sat.bf16x2" => "__nvvm_fma_rn_ftz_sat_bf16x2", "fma.rn.ftz.sat.f" => "__nvvm_fma_rn_ftz_sat_f", "fma.rn.relu.bf16" => "__nvvm_fma_rn_relu_bf16", "fma.rn.relu.bf16x2" => "__nvvm_fma_rn_relu_bf16x2", - "fma.rn.sat.bf16" => "__nvvm_fma_rn_sat_bf16", - "fma.rn.sat.bf16x2" => "__nvvm_fma_rn_sat_bf16x2", "fma.rn.sat.f" => "__nvvm_fma_rn_sat_f", "fma.rp.d" => "__nvvm_fma_rp_d", "fma.rp.f" => "__nvvm_fma_rp_f", @@ -5195,6 +5197,10 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "mul.rn.d" => "__nvvm_mul_rn_d", "mul.rn.f" => "__nvvm_mul_rn_f", "mul.rn.ftz.f" => "__nvvm_mul_rn_ftz_f", + "mul.rn.ftz.sat.f16" => "__nvvm_mul_rn_ftz_sat_f16", + "mul.rn.ftz.sat.v2f16" => "__nvvm_mul_rn_ftz_sat_v2f16", + "mul.rn.sat.f16" => "__nvvm_mul_rn_sat_f16", + "mul.rn.sat.v2f16" => "__nvvm_mul_rn_sat_v2f16", "mul.rp.d" => "__nvvm_mul_rp_d", "mul.rp.f" => "__nvvm_mul_rp_f", "mul.rp.ftz.f" => "__nvvm_mul_rp_ftz_f", @@ -5827,8 +5833,10 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "altivec.vmuleuh" => "__builtin_altivec_vmuleuh", "altivec.vmuleuw" => "__builtin_altivec_vmuleuw", "altivec.vmulhsd" => "__builtin_altivec_vmulhsd", + "altivec.vmulhsh" => "__builtin_altivec_vmulhsh", "altivec.vmulhsw" => "__builtin_altivec_vmulhsw", "altivec.vmulhud" => "__builtin_altivec_vmulhud", + "altivec.vmulhuh" => "__builtin_altivec_vmulhuh", "altivec.vmulhuw" => "__builtin_altivec_vmulhuw", "altivec.vmulosb" => "__builtin_altivec_vmulosb", "altivec.vmulosd" => "__builtin_altivec_vmulosd", @@ -5912,22 +5920,45 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "altivec.vsum4shs" => "__builtin_altivec_vsum4shs", "altivec.vsum4ubs" => "__builtin_altivec_vsum4ubs", "altivec.vsumsws" => "__builtin_altivec_vsumsws", + "altivec.vucmprhb" => "__builtin_altivec_vucmprhb", + "altivec.vucmprhh" => "__builtin_altivec_vucmprhh", + "altivec.vucmprhn" => "__builtin_altivec_vucmprhn", + "altivec.vucmprlb" => "__builtin_altivec_vucmprlb", + "altivec.vucmprlh" => "__builtin_altivec_vucmprlh", + "altivec.vucmprln" => "__builtin_altivec_vucmprln", "altivec.vupkhpx" => "__builtin_altivec_vupkhpx", "altivec.vupkhsb" => "__builtin_altivec_vupkhsb", "altivec.vupkhsh" => "__builtin_altivec_vupkhsh", + "altivec.vupkhsntob" => "__builtin_altivec_vupkhsntob", "altivec.vupkhsw" => "__builtin_altivec_vupkhsw", + "altivec.vupkint4tobf16" => "__builtin_altivec_vupkint4tobf16", + "altivec.vupkint4tofp32" => "__builtin_altivec_vupkint4tofp32", + "altivec.vupkint8tobf16" => "__builtin_altivec_vupkint8tobf16", + "altivec.vupkint8tofp32" => "__builtin_altivec_vupkint8tofp32", "altivec.vupklpx" => "__builtin_altivec_vupklpx", "altivec.vupklsb" => "__builtin_altivec_vupklsb", "altivec.vupklsh" => "__builtin_altivec_vupklsh", + "altivec.vupklsntob" => "__builtin_altivec_vupklsntob", "altivec.vupklsw" => "__builtin_altivec_vupklsw", "amo.ldat" => "__builtin_amo_ldat", + "amo.ldat.cond" => "__builtin_amo_ldat_cond", + "amo.ldat.csne" => "__builtin_amo_ldat_csne", "amo.lwat" => "__builtin_amo_lwat", + "amo.lwat.cond" => "__builtin_amo_lwat_cond", + "amo.lwat.csne" => "__builtin_amo_lwat_csne", + "amo.stdat" => "__builtin_amo_stdat", + "amo.stwat" => "__builtin_amo_stwat", "bcdadd" => "__builtin_ppc_bcdadd", "bcdadd.p" => "__builtin_ppc_bcdadd_p", "bcdcopysign" => "__builtin_ppc_bcdcopysign", "bcdsetsign" => "__builtin_ppc_bcdsetsign", + "bcdshift" => "__builtin_ppc_bcdshift", + "bcdshiftround" => "__builtin_ppc_bcdshiftround", "bcdsub" => "__builtin_ppc_bcdsub", "bcdsub.p" => "__builtin_ppc_bcdsub_p", + "bcdtruncate" => "__builtin_ppc_bcdtruncate", + "bcdunsignedshift" => "__builtin_ppc_bcdunsignedshift", + "bcdunsignedtruncate" => "__builtin_ppc_bcdunsignedtruncate", "bpermd" => "__builtin_bpermd", "cbcdtd" => "__builtin_cbcdtd", "cbcdtdd" => "__builtin_ppc_cbcdtd", @@ -6126,6 +6157,27 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "vsx.xxinsertw" => "__builtin_vsx_xxinsertw", "vsx.xxleqv" => "__builtin_vsx_xxleqv", "vsx.xxpermx" => "__builtin_vsx_xxpermx", + "xsaddaddsuqm" => "__builtin_xsaddaddsuqm", + "xsaddadduqm" => "__builtin_xsaddadduqm", + "xsaddsubsuqm" => "__builtin_xsaddsubsuqm", + "xsaddsubuqm" => "__builtin_xsaddsubuqm", + "xsmerge2t1uqm" => "__builtin_xsmerge2t1uqm", + "xsmerge2t2uqm" => "__builtin_xsmerge2t2uqm", + "xsmerge2t3uqm" => "__builtin_xsmerge2t3uqm", + "xsmerge3t1uqm" => "__builtin_xsmerge3t1uqm", + "xsrebase2t1uqm" => "__builtin_xsrebase2t1uqm", + "xsrebase2t2uqm" => "__builtin_xsrebase2t2uqm", + "xsrebase2t3uqm" => "__builtin_xsrebase2t3uqm", + "xsrebase2t4uqm" => "__builtin_xsrebase2t4uqm", + "xsrebase3t1uqm" => "__builtin_xsrebase3t1uqm", + "xsrebase3t2uqm" => "__builtin_xsrebase3t2uqm", + "xsrebase3t3uqm" => "__builtin_xsrebase3t3uqm", + "xxmulmul" => "__builtin_xxmulmul", + "xxmulmulhiadd" => "__builtin_xxmulmulhiadd", + "xxmulmulloadd" => "__builtin_xxmulmulloadd", + "xxssumudm" => "__builtin_xxssumudm", + "xxssumudmc" => "__builtin_xxssumudmc", + "xxssumudmcext" => "__builtin_xxssumudmcext", "zoned2packed" => "__builtin_ppc_zoned2packed", _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"), } @@ -6388,13 +6440,13 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { // spv "group.memory.barrier.with.group.sync" => "__builtin_spirv_group_barrier", "num.subgroups" => "__builtin_spirv_num_subgroups", + "subgroup.ballot" => "__builtin_spirv_subgroup_ballot", "subgroup.id" => "__builtin_spirv_subgroup_id", "subgroup.local.invocation.id" => { "__builtin_spirv_subgroup_local_invocation_id" } "subgroup.max.size" => "__builtin_spirv_subgroup_max_size", "subgroup.size" => "__builtin_spirv_subgroup_size", - "wave.ballot" => "__builtin_spirv_subgroup_ballot", _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"), } } @@ -8661,10 +8713,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "bmi.bextr.64" => "__builtin_ia32_bextr_u64", "bmi.bzhi.32" => "__builtin_ia32_bzhi_si", "bmi.bzhi.64" => "__builtin_ia32_bzhi_di", - "bmi.pdep.32" => "__builtin_ia32_pdep_si", - "bmi.pdep.64" => "__builtin_ia32_pdep_di", - "bmi.pext.32" => "__builtin_ia32_pext_si", - "bmi.pext.64" => "__builtin_ia32_pext_di", "cldemote" => "__builtin_ia32_cldemote", "clflushopt" => "__builtin_ia32_clflushopt", "clrssbsy" => "__builtin_ia32_clrssbsy", diff --git a/src/intrinsic/llvm.rs b/src/intrinsic/llvm.rs index f3134edb72cb9..ef381715c1ea2 100644 --- a/src/intrinsic/llvm.rs +++ b/src/intrinsic/llvm.rs @@ -475,6 +475,26 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>( new_args.push(variable.get_address(None)); args = new_args.into(); } + "__builtin_ia32_2intersectd128" + | "__builtin_ia32_2intersectq128" + | "__builtin_ia32_2intersectd256" + | "__builtin_ia32_2intersectq256" + | "__builtin_ia32_2intersectd512" + | "__builtin_ia32_2intersectq512" => { + let old_args = args.to_vec(); + let mut new_args = vec![]; + let arg1_type = gcc_func.get_param_type(0); + let first_mask = + builder.current_func().new_local(None, arg1_type, "return_2intersect_arg1"); + let arg2_type = gcc_func.get_param_type(1); + let second_mask = + builder.current_func().new_local(None, arg2_type, "return_2intersect_arg2"); + new_args.push(first_mask.get_address(None)); + new_args.push(second_mask.get_address(None)); + new_args.push(old_args[0]); + new_args.push(old_args[1]); + args = new_args.into(); + } "__builtin_ia32_vpermt2varqi512_mask" | "__builtin_ia32_vpermt2varqi256_mask" | "__builtin_ia32_vpermt2varqi128_mask" @@ -486,6 +506,23 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>( let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1); args = vec![new_args[1], new_args[0], new_args[2], minus_one].into(); } + "__builtin_ia32_fpclassph128_mask" + | "__builtin_ia32_fpclassph256_mask" + | "__builtin_ia32_fpclassph512_mask" + | "__builtin_ia32_fpclasspd128_mask" + | "__builtin_ia32_fpclassps128_mask" + | "__builtin_ia32_fpclasspd256_mask" + | "__builtin_ia32_fpclassps256_mask" + | "__builtin_ia32_fpclasspd512_mask" + | "__builtin_ia32_fpclassps512_mask" + | "__builtin_ia32_vpshufbitqmb128_mask" + | "__builtin_ia32_vpshufbitqmb256_mask" + | "__builtin_ia32_vpshufbitqmb512_mask" => { + let new_args = args.to_vec(); + let arg3_type = gcc_func.get_param_type(2); + let minus_one = builder.context.new_rvalue_from_int(arg3_type, -1); + args = vec![new_args[0], new_args[1], minus_one].into(); + } "__builtin_ia32_xrstor" | "__builtin_ia32_xrstor64" | "__builtin_ia32_xsavec" @@ -837,7 +874,7 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>( "__builtin_ia32_rdrand64_step" => { let random_number = args[0].dereference(None).to_rvalue(); let success_variable = - builder.current_func().new_local(None, return_value.get_type(), "success"); + builder.new_temp(builder.current_func(), None, return_value.get_type()); builder.llbb().add_assignment(None, success_variable, return_value); let field1 = builder.context.new_field(None, random_number.get_type(), "random_number"); @@ -851,6 +888,25 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>( &[random_number, success_variable.to_rvalue()], ); } + "__builtin_ia32_2intersectd128" + | "__builtin_ia32_2intersectq128" + | "__builtin_ia32_2intersectd256" + | "__builtin_ia32_2intersectq256" + | "__builtin_ia32_2intersectd512" + | "__builtin_ia32_2intersectq512" => { + let first_mask = args[0].dereference(None).to_rvalue(); + let second_mask = args[1].dereference(None).to_rvalue(); + let field1 = builder.context.new_field(None, first_mask.get_type(), "first_mask"); + let field2 = builder.context.new_field(None, second_mask.get_type(), "second_mask"); + let struct_type = + builder.context.new_struct_type(None, "vp2intersect_result", &[field1, field2]); + return_value = builder.context.new_struct_constructor( + None, + struct_type.as_type(), + None, + &[first_mask, second_mask], + ); + } "fma" => { let f16_type = builder.context.new_c_type(CType::Float16); return_value = builder.context.new_cast(None, return_value, f16_type); @@ -1179,6 +1235,9 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function "llvm.x86.avx512.mask.vpshufbitqmb.512" => "__builtin_ia32_vpshufbitqmb512_mask", "llvm.x86.avx512.mask.vpshufbitqmb.256" => "__builtin_ia32_vpshufbitqmb256_mask", "llvm.x86.avx512.mask.vpshufbitqmb.128" => "__builtin_ia32_vpshufbitqmb128_mask", + "llvm.x86.avx512.vpshufbitqmb.512" => "__builtin_ia32_vpshufbitqmb512_mask", + "llvm.x86.avx512.vpshufbitqmb.256" => "__builtin_ia32_vpshufbitqmb256_mask", + "llvm.x86.avx512.vpshufbitqmb.128" => "__builtin_ia32_vpshufbitqmb128_mask", "llvm.x86.avx512.mask.ucmp.w.512" => "__builtin_ia32_ucmpw512_mask", "llvm.x86.avx512.mask.ucmp.w.256" => "__builtin_ia32_ucmpw256_mask", "llvm.x86.avx512.mask.ucmp.w.128" => "__builtin_ia32_ucmpw128_mask", @@ -1336,11 +1395,20 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function "llvm.x86.avx512bf16.cvtne2ps2bf16.128" => "__builtin_ia32_cvtne2ps2bf16_v8bf", "llvm.x86.avx512bf16.cvtne2ps2bf16.256" => "__builtin_ia32_cvtne2ps2bf16_v16bf", "llvm.x86.avx512bf16.cvtne2ps2bf16.512" => "__builtin_ia32_cvtne2ps2bf16_v32bf", + "llvm.x86.vcvtneps2bf16128" => "__builtin_ia32_cvtneps2bf16_v4sf", + "llvm.x86.vcvtneps2bf16256" => "__builtin_ia32_cvtneps2bf16_v8sf", + "llvm.x86.avx512bf16.mask.cvtneps2bf16.128" => "__builtin_ia32_cvtneps2bf16_v4sf_mask", "llvm.x86.avx512bf16.cvtneps2bf16.256" => "__builtin_ia32_cvtneps2bf16_v8sf", "llvm.x86.avx512bf16.cvtneps2bf16.512" => "__builtin_ia32_cvtneps2bf16_v16sf", "llvm.x86.avx512bf16.dpbf16ps.128" => "__builtin_ia32_dpbf16ps_v4sf", "llvm.x86.avx512bf16.dpbf16ps.256" => "__builtin_ia32_dpbf16ps_v8sf", "llvm.x86.avx512bf16.dpbf16ps.512" => "__builtin_ia32_dpbf16ps_v16sf", + "llvm.x86.avx512.vp2intersect.d.128" => "__builtin_ia32_2intersectd128", + "llvm.x86.avx512.vp2intersect.q.128" => "__builtin_ia32_2intersectq128", + "llvm.x86.avx512.vp2intersect.d.256" => "__builtin_ia32_2intersectd256", + "llvm.x86.avx512.vp2intersect.q.256" => "__builtin_ia32_2intersectq256", + "llvm.x86.avx512.vp2intersect.d.512" => "__builtin_ia32_2intersectd512", + "llvm.x86.avx512.vp2intersect.q.512" => "__builtin_ia32_2intersectq512", "llvm.x86.pclmulqdq.512" => "__builtin_ia32_vpclmulqdq_v8di", "llvm.x86.pclmulqdq.256" => "__builtin_ia32_vpclmulqdq_v4di", "llvm.x86.avx512.pmulhu.w.512" => "__builtin_ia32_pmulhuw512_mask", @@ -1574,38 +1642,79 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function "llvm.x86.avx512.uitofp.round.v4f64.v4i64" => "__builtin_ia32_cvtuqq2pd256_mask", "llvm.x86.avx512.uitofp.round.v8f32.v8i64" => "__builtin_ia32_cvtuqq2ps512_mask", "llvm.x86.avx512.uitofp.round.v4f32.v4i64" => "__builtin_ia32_cvtuqq2ps256_mask", + "llvm.x86.avx512fp16.fpclass.ph.128" => "__builtin_ia32_fpclassph128_mask", + "llvm.x86.avx512fp16.mask.cmp.ph.128" => "__builtin_ia32_cmpph128_mask", + "llvm.x86.avx512fp16.fpclass.ph.256" => "__builtin_ia32_fpclassph256_mask", + "llvm.x86.avx512fp16.fpclass.ph.512" => "__builtin_ia32_fpclassph512_mask", + "llvm.x86.avx512fp16.mask.cmp.ph.256" => "__builtin_ia32_cmpph256_mask", + "llvm.x86.avx512fp16.mask.cmp.ph.512" => "__builtin_ia32_cmpph512_mask_round", + "llvm.x86.avx512.fpclass.pd.128" => "__builtin_ia32_fpclasspd128_mask", + "llvm.x86.avx512.fpclass.ps.128" => "__builtin_ia32_fpclassps128_mask", + "llvm.x86.avx512.fpclass.pd.256" => "__builtin_ia32_fpclasspd256_mask", + "llvm.x86.avx512.fpclass.ps.256" => "__builtin_ia32_fpclassps256_mask", + "llvm.x86.avx512.fpclass.pd.512" => "__builtin_ia32_fpclasspd512_mask", + "llvm.x86.avx512.fpclass.ps.512" => "__builtin_ia32_fpclassps512_mask", // FIXME: support the tile builtins: "llvm.x86.ldtilecfg" => "__builtin_trap", "llvm.x86.sttilecfg" => "__builtin_trap", "llvm.x86.tileloadd64" => "__builtin_trap", + "llvm.x86.tileloadd64.internal" => "__builtin_trap", "llvm.x86.tilerelease" => "__builtin_trap", "llvm.x86.tilestored64" => "__builtin_trap", + "llvm.x86.tilestored64.internal" => "__builtin_trap", "llvm.x86.tileloaddrs64" => "__builtin_trap", + "llvm.x86.tileloaddrs64.internal" => "__builtin_trap", "llvm.x86.tileloaddt164" => "__builtin_trap", + "llvm.x86.tileloaddt164.internal" => "__builtin_trap", "llvm.x86.tileloaddrst164" => "__builtin_trap", + "llvm.x86.tileloaddrst164.internal" => "__builtin_trap", "llvm.x86.tilezero" => "__builtin_trap", + "llvm.x86.tilezero.internal" => "__builtin_trap", "llvm.x86.tilemovrow" => "__builtin_trap", + "llvm.x86.tilemovrow.internal" => "__builtin_trap", "llvm.x86.tilemovrowi" => "__builtin_trap", "llvm.x86.tdpbhf8ps" => "__builtin_trap", + "llvm.x86.tdpbhf8ps.internal" => "__builtin_trap", "llvm.x86.tdphbf8ps" => "__builtin_trap", + "llvm.x86.tdphbf8ps.internal" => "__builtin_trap", "llvm.x86.tdpbf8ps" => "__builtin_trap", + "llvm.x86.tdpbf8ps.internal" => "__builtin_trap", "llvm.x86.tdphf8ps" => "__builtin_trap", + "llvm.x86.tdphf8ps.internal" => "__builtin_trap", "llvm.x86.tdpbf16ps" => "__builtin_trap", + "llvm.x86.tdpbf16ps.internal" => "__builtin_trap", "llvm.x86.tdpbssd" => "__builtin_trap", + "llvm.x86.tdpbssd.internal" => "__builtin_trap", "llvm.x86.tdpbsud" => "__builtin_trap", + "llvm.x86.tdpbsud.internal" => "__builtin_trap", "llvm.x86.tdpbusd" => "__builtin_trap", + "llvm.x86.tdpbusd.internal" => "__builtin_trap", "llvm.x86.tdpbuud" => "__builtin_trap", + "llvm.x86.tdpbuud.internal" => "__builtin_trap", "llvm.x86.tdpfp16ps" => "__builtin_trap", + "llvm.x86.tdpfp16ps.internal" => "__builtin_trap", "llvm.x86.tmmultf32ps" => "__builtin_trap", + "llvm.x86.tmmultf32ps.internal" => "__builtin_trap", "llvm.x86.tcvtrowps2phh" => "__builtin_trap", + "llvm.x86.tcvtrowps2phh.internal" => "__builtin_trap", "llvm.x86.tcvtrowps2phl" => "__builtin_trap", + "llvm.x86.tcvtrowps2phl.internal" => "__builtin_trap", "llvm.x86.tcvtrowd2ps" => "__builtin_trap", + "llvm.x86.tcvtrowd2ps.internal" => "__builtin_trap", "llvm.x86.tcvtrowd2psi" => "__builtin_trap", "llvm.x86.tcvtrowps2phhi" => "__builtin_trap", "llvm.x86.tcvtrowps2phli" => "__builtin_trap", + "llvm.x86.tcvtrowps2bf16h" => "__builtin_trap", + "llvm.x86.tcvtrowps2bf16h.internal" => "__builtin_trap", + "llvm.x86.tcvtrowps2bf16hi" => "__builtin_trap", + "llvm.x86.tcvtrowps2bf16l" => "__builtin_trap", + "llvm.x86.tcvtrowps2bf16l.internal" => "__builtin_trap", + "llvm.x86.tcvtrowps2bf16li" => "__builtin_trap", "llvm.x86.tcmmimfp16ps" => "__builtin_trap", + "llvm.x86.tcmmimfp16ps.internal" => "__builtin_trap", "llvm.x86.tcmmrlfp16ps" => "__builtin_trap", + "llvm.x86.tcmmrlfp16ps.internal" => "__builtin_trap", // NOTE: this file is generated by https://github.com/GuillaumeGomez/llvmint/blob/master/generate_list.py _ => map_arch_intrinsic(name), diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs index 0a24519a8091e..4d2590ac81e41 100644 --- a/src/intrinsic/mod.rs +++ b/src/intrinsic/mod.rs @@ -4,7 +4,7 @@ mod simd; #[cfg(feature = "master")] use std::iter; -use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, Type, UnaryOp}; +use gccjit::{CType, ComparisonOp, Function, FunctionType, RValue, ToRValue, Type, UnaryOp}; use rustc_abi::{Align, BackendRepr, HasDataLayout, WrappingRange}; use rustc_codegen_ssa::base::wants_msvc_seh; use rustc_codegen_ssa::common::IntPredicate; @@ -81,7 +81,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>( sym::floorf64 => "floor", sym::ceilf32 => "ceilf", sym::ceilf64 => "ceil", - sym::powf128 => return float_intrinsic(cx, cx.type_f128(), "powf128"), sym::truncf32 => "truncf", sym::truncf64 => "trunc", // We match the LLVM backend and lower this to `rint`. @@ -180,14 +179,11 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc let simple = get_simple_intrinsic(self, name); let value = match name { - _ if simple.is_some() => { - let func = simple.expect("simple intrinsic function"); - self.cx.context.new_call( - self.location, - func, - &args.iter().map(|arg| arg.immediate()).collect::>(), - ) - } + _ if let Some(func) = simple => self.cx.context.new_call( + self.location, + func, + &args.iter().map(|arg| arg.immediate()).collect::>(), + ), // FIXME(antoyo): We can probably remove these and use the fallback intrinsic implementation. sym::minimumf32 | sym::minimumf64 | sym::maximumf32 | sym::maximumf64 => { let (ty, func_name) = match name { @@ -322,7 +318,9 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc unimplemented!(); } sym::va_arg => { - unimplemented!(); + let va_list = args[0].immediate(); + let gcc_type = self.immediate_backend_type(result.layout); + self.va_arg(va_list, gcc_type) } sym::volatile_load | sym::unaligned_volatile_load => { @@ -611,7 +609,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc self.on_stack_function_params.borrow_mut().insert(func, FxHashSet::default()); - crate::attributes::from_fn_attrs(self, func, instance); + crate::attributes::from_fn_attrs(self, func, instance, None); func }; @@ -701,8 +699,18 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc self.context.new_rvalue_from_int(self.int_type, 0) } - fn va_start(&mut self, _va_list: RValue<'gcc>) { - unimplemented!(); + fn va_start(&mut self, va_list: RValue<'gcc>) { + let func = self.context.get_builtin_function("__builtin_va_start"); + + let va_list_type = self.context.new_c_type(CType::VaList); + let va_list = self.context.new_cast(self.location, va_list, va_list_type.make_pointer()); + + // Pre-C23 requires that the last "normal" argument was passed to va_start. + // Just pass 0, this appears to be handled correctly. + let last_normal_arg = self.context.new_rvalue_from_int(self.int_type, 0); + + let call = self.context.new_call(self.location, func, &[va_list, last_normal_arg]); + self.block.add_eval(self.location, call); } fn retag_reg(&mut self, _ptr: Self::Value, _info: &RetagInfo) -> Self::Value { @@ -960,7 +968,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { let else_block = func.new_block("else"); let after_block = func.new_block("after"); - let result = func.new_local(None, self.u32_type, "zeros"); + let result = self.new_temp(func, None, self.u32_type); let zero = self.cx.gcc_zero(arg.get_type()); let cond = self.gcc_icmp(IntPredicate::IntEQ, arg, zero); self.llbb().end_with_conditional(None, cond, then_block, else_block); @@ -1041,7 +1049,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { // else call it on the 64 high bits and add 64. In the else case, 64 high bits can't be 0 // because arg is not 0. - let result = self.current_func().new_local(None, result_type, "count_zeroes_results"); + let result = self.new_temp(self.current_func(), None, result_type); let cz_then_block = self.current_func().new_block("cz_then"); let cz_else_block = self.current_func().new_block("cz_else"); @@ -1156,8 +1164,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { let loop_tail = func.new_block("tail"); let counter_type = self.int_type; - let counter = self.current_func().new_local(None, counter_type, "popcount_counter"); - let val = self.current_func().new_local(None, value_type, "popcount_value"); + let counter = self.new_temp(self.current_func(), None, counter_type); + let val = self.new_temp(self.current_func(), None, value_type); let zero = self.gcc_zero(counter_type); self.llbb().add_assignment(self.location, counter, zero); self.llbb().add_assignment(self.location, val, value); diff --git a/src/intrinsic/old_archs.rs b/src/intrinsic/old_archs.rs index 8d3e3487b5cb4..1aac52c28d220 100644 --- a/src/intrinsic/old_archs.rs +++ b/src/intrinsic/old_archs.rs @@ -1240,6 +1240,10 @@ pub(crate) fn old_archs(arch: &str, name: &str) -> ArchCheckResult { "avx512.vbroadcast.sd.pd.512" => "__builtin_ia32_vbroadcastsd_pd512", "avx512.vbroadcast.ss.512" => "__builtin_ia32_vbroadcastss512", "avx512.vbroadcast.ss.ps.512" => "__builtin_ia32_vbroadcastss_ps512", + "bmi.pdep.32" => "__builtin_ia32_pdep_si", + "bmi.pdep.64" => "__builtin_ia32_pdep_di", + "bmi.pext.32" => "__builtin_ia32_pext_si", + "bmi.pext.64" => "__builtin_ia32_pext_di", "fma.mask.vfmadd.pd.512" => "__builtin_ia32_vfmaddpd512_mask", "fma.mask.vfmadd.ps.512" => "__builtin_ia32_vfmaddps512_mask", "fma.mask.vfmaddsub.pd.512" => "__builtin_ia32_vfmaddsubpd512_mask", diff --git a/src/lib.rs b/src/lib.rs index 0784823c27b9e..695bca1d69ec1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -74,9 +74,9 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; -use gccjit::{CType, Context, OptimizationLevel}; #[cfg(feature = "master")] -use gccjit::{TargetInfo, Version}; +use gccjit::TargetInfo; +use gccjit::{CType, Context, OptimizationLevel}; use rustc_ast::expand::allocator::AllocatorMethod; use rustc_codegen_ssa::back::lto::ThinModule; use rustc_codegen_ssa::back::write::{ @@ -292,20 +292,6 @@ impl CodegenBackend for GccCodegenBackend { fn target_config(&self, sess: &EarlySession) -> TargetConfig { target_config(sess, &self.config().target_info) } - #[cfg(feature = "master")] - { - context.set_special_chars_allowed_in_func_names("$.*"); - let version = Version::get(); - let version = format!("{}.{}.{}", version.major, version.minor, version.patch); - context.set_output_ident(&format!( - "rustc version {} with libgccjit {}", - rustc_interface::util::rustc_version_str().unwrap_or("unknown version"), - version, - )); - } - // FIXME(antoyo): check if this should only be added when using -Cforce-unwind-tables=n. - context.add_command_line_option("-fno-asynchronous-unwind-tables"); - context } impl ExtraBackendMethods for GccCodegenBackend { @@ -318,7 +304,7 @@ impl ExtraBackendMethods for GccCodegenBackend { methods: &[AllocatorMethod], ) -> Self::Module { let mut mods = GccContext { - context: Arc::new(SyncContext::new(new_context(tcx))), + context: Arc::new(SyncContext::new(gcc_util::new_context(tcx.sess))), relocation_model: tcx.sess.relocation_model(), lto_mode: LtoMode::None, lto_supported: self.config().lto_supported, @@ -411,7 +397,7 @@ impl WriteBackendMethods for GccCodegenBackend { each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, ) -> CompiledModule { - back::lto::run_fat(cgcx, &sess.prof, shared_emitter, each_linked_rlib_for_lto, modules) + back::lto::run_fat(sess, cgcx, shared_emitter, each_linked_rlib_for_lto, modules) } fn run_thin_lto( diff --git a/src/mono_item.rs b/src/mono_item.rs index 014a7ff90536d..d8170fbb085a7 100644 --- a/src/mono_item.rs +++ b/src/mono_item.rs @@ -1,3 +1,4 @@ +use gccjit::Function; #[cfg(feature = "master")] use gccjit::{FnAttribute, GlobalKind, ToRValue, Type, VarAttribute}; use rustc_codegen_ssa::traits::PreDefineCodegenMethods; @@ -22,7 +23,7 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { def_id: DefId, linkage: Linkage, visibility: Visibility, - symbol_name: &str, + global_name: &str, ) { let attrs = self.tcx.codegen_fn_attrs(def_id); let instance = Instance::mono(self.tcx, def_id); @@ -73,7 +74,6 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { #[cfg(feature = "master")] self.add_static_aliases(gcc_type, global_name, attrs, &attrs.foreign_item_symbol_aliases); - // FIXME(antoyo): set linkage. self.instances.borrow_mut().insert(instance, global); } @@ -186,10 +186,9 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { ) -> Function<'gcc> { let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); self.linkage.set(base::linkage_to_gcc(linkage)); - let decl = self.declare_fn(symbol_name, fn_abi); - //let attrs = self.tcx.codegen_instance_attrs(instance.def); + let fn_decl = self.declare_fn(symbol_name, fn_abi); - attributes::from_fn_attrs(self, decl, instance); + attributes::from_fn_attrs(self, fn_decl, instance, Some(fn_abi)); #[cfg(feature = "master")] if base::linkage_needs_weak_attribute(linkage) { @@ -202,17 +201,21 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { // don't want the symbols to get exported. if linkage != Linkage::Internal && self.tcx.is_compiler_builtins(LOCAL_CRATE) { #[cfg(feature = "master")] - decl.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden)); + fn_decl.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden)); } else if visibility != Visibility::Default { #[cfg(feature = "master")] - decl.add_attribute(FnAttribute::Visibility(base::visibility_to_gcc(visibility))); + fn_decl.add_attribute(FnAttribute::Visibility(base::visibility_to_gcc(visibility))); + } + + #[cfg(feature = "master")] + if let Some(section) = _attrs.link_section { + fn_decl.add_attribute(FnAttribute::Section(section.as_str())); } - // FIXME(antoyo): call set_link_section() to allow initializing argc/argv. // FIXME(antoyo): set unique comdat. // FIXME(antoyo): use inline attribute from there in linkage.set() above. + // FIXME: Should we handle dso? - self.functions.borrow_mut().insert(symbol_name.to_string(), decl); - self.function_instances.borrow_mut().insert(instance, decl); + fn_decl } } diff --git a/src/type_.rs b/src/type_.rs index 45828ce3e92b8..55b37ee89b63f 100644 --- a/src/type_.rs +++ b/src/type_.rs @@ -3,7 +3,7 @@ use std::convert::TryInto; use std::mem::discriminant; #[cfg(feature = "master")] -use gccjit::CType; +use gccjit::{CType, TypeAttribute}; use gccjit::{RValue, Struct, Type}; use rustc_abi::{AddressSpace, Align, Integer, Size}; use rustc_codegen_ssa::common::TypeKind; @@ -232,7 +232,7 @@ impl<'gcc, 'tcx> BaseTypeCodegenMethods for CodegenCx<'gcc, 'tcx> { if self.supports_f16_type { return self.context.new_c_type(CType::Float16); } - bug!("unsupported float width 16") + self.u16_type } fn type_f32(&self) -> Type<'gcc> { diff --git a/tests/asm/asm/comments.rs b/tests/asm/asm/comments.rs new file mode 100644 index 0000000000000..603bb014930c4 --- /dev/null +++ b/tests/asm/asm/comments.rs @@ -0,0 +1,12 @@ +//@ assembly-output: emit-asm +//@ only-x86_64 +// Check that comments in assembly get passed + +#![crate_type = "lib"] + +// CHECK-LABEL: "test_comments": +#[no_mangle] +pub fn test_comments() { + // CHECK: example comment + unsafe { core::arch::asm!("nop // example comment") }; +} diff --git a/tests/asm/naked-functions/x86_64-naked-fn-no-cet-prolog.rs b/tests/asm/naked-functions/x86_64-naked-fn-no-cet-prolog.rs new file mode 100644 index 0000000000000..81ee9b13b4eca --- /dev/null +++ b/tests/asm/naked-functions/x86_64-naked-fn-no-cet-prolog.rs @@ -0,0 +1,24 @@ +//@ compile-flags: -C no-prepopulate-passes -Zcf-protection=full +//@ assembly-output: emit-asm +//@ needs-asm-support +//@ only-x86_64 + +#![crate_type = "lib"] + +use std::arch::naked_asm; + +// The problem at hand: Rust has adopted a fairly strict meaning for "naked functions", +// meaning "no prologue whatsoever, no, really, not one instruction." +// Unfortunately, x86's control-flow enforcement, specifically indirect branch protection, +// works by using an instruction for each possible landing site, +// and LLVM implements this via making sure of that. +#[no_mangle] +#[unsafe(naked)] +pub extern "sysv64" fn will_halt() -> ! { + // CHECK-NOT: endbr{{32|64}} + // CHECK: hlt + naked_asm!("hlt") +} + +// what about aarch64? +// "branch-protection"=false diff --git a/tests/asm/panic-no-unwind-no-uwtable.rs b/tests/asm/panic-no-unwind-no-uwtable.rs new file mode 100644 index 0000000000000..b51b173e9616e --- /dev/null +++ b/tests/asm/panic-no-unwind-no-uwtable.rs @@ -0,0 +1,8 @@ +//@ assembly-output: emit-asm +//@ only-x86_64-unknown-linux-gnu +//@ compile-flags: -C panic=unwind -C force-unwind-tables=n -Copt-level=3 + +#![crate_type = "lib"] + +// CHECK-NOT: .cfi_startproc +pub fn foo() {} diff --git a/tests/asm/used.rs b/tests/asm/used.rs new file mode 100644 index 0000000000000..deb0c69dc48fa --- /dev/null +++ b/tests/asm/used.rs @@ -0,0 +1,14 @@ +//@ assembly-output: emit-asm +//@ only-x86_64-unknown-linux-gnu + +#![feature(used_with_arg)] +#![crate_type = "lib"] + +// CHECK: .section .rodata.X,"a" +#[used(compiler)] +#[no_mangle] +pub static X: u32 = 12; +// CHECK: .section .rodata.Y,"aR" +#[used(linker)] +#[no_mangle] +pub static Y: u32 = 12; diff --git a/tests/asm/x86_64-sse_crc.rs b/tests/asm/x86_64-sse_crc.rs new file mode 100644 index 0000000000000..bde58955a2146 --- /dev/null +++ b/tests/asm/x86_64-sse_crc.rs @@ -0,0 +1,12 @@ +//@ only-x86_64 +//@ assembly-output: emit-asm +//@ compile-flags: --crate-type staticlib -Ctarget-feature=+sse4.2 + +// CHECK-LABEL: banana +// CHECK: crc32 +#[no_mangle] +pub unsafe fn banana(v: u8) -> u32 { + use std::arch::x86_64::*; + let out = !0u32; + _mm_crc32_u8(out, v) +} diff --git a/tests/compile/x86_interrupt_first_arg_byval.rs b/tests/compile/x86_interrupt_first_arg_byval.rs new file mode 100644 index 0000000000000..4b6bbd48f7ad5 --- /dev/null +++ b/tests/compile/x86_interrupt_first_arg_byval.rs @@ -0,0 +1,16 @@ +// Compiler: + +// Test that `x86-interrupt` functions whose first argument is passed by value +// emit pointer-shaped GCC parameters and compile with interrupt-safe target features. + +#![feature(abi_x86_interrupt)] +#![crate_type = "lib"] + +#[repr(C)] +pub struct Frame { + ip: u64, +} + +pub extern "x86-interrupt" fn scalar(_a: i64) {} + +pub extern "x86-interrupt" fn aggregate(_frame: Frame) {} diff --git a/tests/cpuid.def b/tests/cpuid.def new file mode 100644 index 0000000000000..05fe8e94a8282 --- /dev/null +++ b/tests/cpuid.def @@ -0,0 +1,27 @@ +# Input => Output +# EAX ECX => EAX EBX ECX EDX +00000000 ******** => 00000024 756e6547 6c65746e 49656e69 #Processor ID and Manufacturer +00000001 ******** => 00400f10 00100800 7ffaf3ff bfebfbff +00000007 00000000 => 00000002 f3bfbfbf bac05ffe 03d54130 #Extended Features +00000007 00000001 => 98ee00bf 00000002 00000020 1d29cd3e +0000000d 00000000 => 000e02e7 00002b00 00002b00 00000000 #xcr0 +0000000d 00000001 => 0000001f 00000240 00000100 00000000 #Supervisor State +0000000d 00000002 => 00000100 00000240 00000000 00000000 +0000000d 00000005 => 00000040 00000440 00000000 00000000 #zmasks +0000000d 00000006 => 00000200 00000480 00000000 00000000 #zmmh +0000000d 00000007 => 00000400 00000680 00000000 00000000 #zmm +0000000d 00000011 => 00000040 00000ac0 00000002 00000000 #tileconfig +0000000d 00000012 => 00002000 00000b00 00000006 00000000 #tiles +0000000d 00000013 => 00000080 000003c0 00000000 00000000 #APX +00000019 ******** => 00000000 00000005 00000000 00000000 #Key Locker +0000001d 00000000 => 00000001 00000000 00000000 00000000 #AMX Tile +0000001d 00000001 => 04002000 00080040 00000010 00000000 #AMX Palette1 +0000001e 00000000 => 00000001 00004010 00000000 00000000 #AMX Tmul +0000001e 00000001 => 000001ff 00000000 00000000 00000000 +00000024 00000000 => 00000001 00070002 00000000 00000000 #AVX10 +00000024 00000001 => 00000000 00000000 00000004 00000000 +80000000 ******** => 80000004 00000000 00000000 00000000 +80000001 ******** => 00000000 00000000 00000121 2c100000 +80000002 ******** => 00000000 00000000 00000000 00000000 +80000003 ******** => 00000000 00000000 00000000 00000000 +80000004 ******** => 00000000 00000000 00000000 00000000 diff --git a/tests/lang_tests.rs b/tests/lang_tests.rs index 9c4708274280c..7ec0ab877b025 100644 --- a/tests/lang_tests.rs +++ b/tests/lang_tests.rs @@ -240,6 +240,16 @@ fn build_test_runner( } } + // Extra flags passed at run time (as opposed to the compile-time + // `TEST_FLAGS`). This lets a single test opt into flags like + // `-Zmir-preserve-ub` via an `ignore-if` directive that checks + // whether `CARGO_TEST_FLAGS` is set. + if let Ok(flags) = std::env::var("CARGO_TEST_FLAGS") { + for flag in flags.split_whitespace() { + compiler_args.push(flag.into()); + } + } + if build_mode.is_debug() { compiler_args .extend_from_slice(&["-C".to_string(), "llvm-args=sanitize-undefined".into()]); @@ -270,7 +280,13 @@ fn compile_tests(tempdir: PathBuf, c_objects_dir: PathBuf, current_dir: String) "lang compile", "tests/compile", TestMode::Compile, - &["simd-ffi.rs", "asm_nul_byte.rs", "global_asm_nul_byte.rs", "naked_asm_nul_byte.rs"], + &[ + "simd-ffi.rs", + "asm_nul_byte.rs", + "global_asm_nul_byte.rs", + "naked_asm_nul_byte.rs", + "x86_interrupt_first_arg_byval.rs", + ], ); } diff --git a/tests/run/asm.rs b/tests/run/asm.rs index 01775c92ffc8a..42141c671b596 100644 --- a/tests/run/asm.rs +++ b/tests/run/asm.rs @@ -3,6 +3,8 @@ // Run-time: // status: 0 +#![feature(asm_goto_with_outputs)] + #[cfg(target_arch = "x86_64")] use std::arch::{asm, global_asm}; @@ -32,6 +34,20 @@ pub unsafe fn mem_cpy(dst: *mut u8, src: *const u8, len: usize) { ); } +#[cfg(target_arch = "x86_64")] +#[unsafe(no_mangle)] +pub fn asm_goto_test(mut a: i16) -> i16 { + unsafe { + std::arch::asm!( + "jmp {op}", + inout("eax") a, + op = label { a = 7; }, + options(nostack,nomem) + ); + a + } +} + #[cfg(target_arch = "x86_64")] fn asm() { unsafe { @@ -190,6 +206,14 @@ fn asm() { } assert_eq!((x, y), (8, 8)); + // Regression test for + // typed pointer inputs to explicit registers need a cast. + let mut x = 123_i32; + unsafe { + asm!("", in("rdi") &mut x, options(nostack, preserves_flags)); + } + assert_eq!(x, 123); + // sysv64 is the default calling convention on unix systems. The rdi register is // used to pass arguments in the sysv64 calling convention, so this register will be clobbered #[cfg(unix)] @@ -227,6 +251,24 @@ fn asm() { out("r15b") _, ); } + + // Make sure the input value from inout is assigned to the input value + unsafe { + // Use a very distinctive value unlikely to live in any register. + let input: u64 = 0x1234567890ABCDEF; + let mut output: u64; + + asm!( + "push {1}", + "pop {0}", + out(reg) output, + inout(reg) input => _, + ); + + assert_eq!(output, 0x1234567890ABCDEF); + } + + asm_goto_test(0); } #[cfg(not(target_arch = "x86_64"))] diff --git a/tests/run/int.rs b/tests/run/int.rs index 78675acb5447b..ef825b4d80185 100644 --- a/tests/run/int.rs +++ b/tests/run/int.rs @@ -319,4 +319,29 @@ fn main() { const VAL5: T = 73236519889708027473620326106273939584_i128; check_ops128!(); } + + { + #[allow(dead_code)] + #[repr(u8)] + enum Inner { + L0 = 0, + H255 = 255, + } + #[allow(dead_code)] + enum O { + A(Inner), + B, + C, + } + + #[inline(never)] + fn which(o: &O) -> &'static str { + match o { + O::A(_) => "a", + O::B => "b", + O::C => "c", + } + } + assert_eq!(which(black_box(&O::A(Inner::H255))), "a"); + } } diff --git a/tests/run/mir_preserve_ub_empty_switch.rs b/tests/run/mir_preserve_ub_empty_switch.rs new file mode 100644 index 0000000000000..26056360b9212 --- /dev/null +++ b/tests/run/mir_preserve_ub_empty_switch.rs @@ -0,0 +1,35 @@ +// ignore-if: test -z "$CARGO_TEST_FLAGS" +// Compiler: +// +// Run-time: +// status: 0 + +// Regression test for https://github.com/rust-lang/rustc_codegen_gcc/issues/881 +// +// This needs `-Zmir-preserve-ub`, so it is skipped unless that flag is passed +// through `CARGO_TEST_FLAGS` (see the `ignore-if` directive above). Run it with: +// CARGO_TEST_FLAGS="-Zmir-preserve-ub" ./y.sh test --cargo-tests -- mir_preserve_ub_empty_switch + +#![feature(no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use intrinsics::black_box; +use mini_core::*; + +#[no_mangle] +extern "C" fn main(argc: i32, _argv: *const *const u8) -> i32 { + // With `-Zmir-preserve-ub`, the range pattern below is lowered to a pair of + // comparisons and the second one becomes a `SwitchInt` with no cases (only + // an `otherwise` target) whose discriminant is the `bool` comparison + // result. `gcc_jit_block_end_with_switch` rejects a non-integer + // discriminant, so the backend must emit a plain jump for it instead. + let value = black_box(argc); + match value { + 0..=9 => (), + _ => (), + } + 0 +} diff --git a/tools/generate_intrinsics.py b/tools/generate_intrinsics.py index 5390323407779..06425f682a88b 100644 --- a/tools/generate_intrinsics.py +++ b/tools/generate_intrinsics.py @@ -84,6 +84,10 @@ def update_intrinsics(llvm_path): # This speeds up the comparison, and makes our code considerably smaller. # Since all intrinsic names start with "llvm.", we skip that prefix. print("Updating content of `{}`...".format(output_file)) + indent4 = " " + indent8 = indent4 + indent4 + indent12 = indent8 + indent4 + indent16 = indent12 + indent4 with open(output_file, "w", encoding="utf8") as out: out.write("""// File generated by `rustc_codegen_gcc/tools/generate_intrinsics.py` // DO NOT EDIT IT! @@ -95,33 +99,35 @@ def update_intrinsics(llvm_path): if let ArchCheckResult::Ok(res) = old_arch_res { return res; } -match arch {""") + match arch { +""") for arch in archs: if len(intrinsics[arch]) == 0: continue attribute = "#[expect(non_snake_case)]" if arch[0].isupper() else "" - out.write("\"{}\" => {{ {} fn {}(name: &str,full_name:&str) -> &'static str {{ match name {{".format(arch, attribute, arch)) + out.write(f"""{indent4}"{arch}" => {{ +{indent8}{attribute} fn {arch}(name: &str,full_name:&str) -> &'static str {{ +{indent12}match name {{""") intrinsics[arch].sort(key=lambda x: (x[0], x[1])) - out.write(' // {}\n'.format(arch)) + out.write(f'{indent16}// {arch}\n') for entry in intrinsics[arch]: llvm_name = entry[0].removeprefix("llvm."); llvm_name = llvm_name.removeprefix(arch); llvm_name = llvm_name.removeprefix("."); if "_round_mask" in entry[1]: - out.write(' // [INVALID CONVERSION]: "{}" => "{}",\n'.format(llvm_name, entry[1])) + out.write(f'{indent16}// [INVALID CONVERSION]: "{llvm_name}" => "{entry[1]}",\n') else: - out.write(' "{}" => "{}",\n'.format(llvm_name, entry[1])) - out.write(' _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"),\n') - out.write("}} }} {}(name,full_name) }}\n,".format(arch)) - out.write(""" _ => { - match old_arch_res { - ArchCheckResult::UnknownIntrinsic => unimplemented!("***** unsupported LLVM intrinsic {full_name}"), - ArchCheckResult::UnknownArch => unimplemented!("***** unsupported LLVM architecture {arch}, intrinsic: {full_name}"), - ArchCheckResult::Ok(_) => unreachable!(), - } - }""") + out.write(f'{indent16}"{llvm_name}" => "{entry[1]}",\n') + out.write(f'{indent16}_ => unimplemented!("***** unsupported LLVM intrinsic {{full_name}}"),\n') + out.write(f"{indent16}}}\n{indent12}}}\n{indent8}{arch}(name,full_name)\n{indent8}}}\n,") + out.write(f"""{indent4}_ => {{ +{indent8}match old_arch_res {{ +{indent8}ArchCheckResult::UnknownIntrinsic => unimplemented!("***** unsupported LLVM intrinsic {{full_name}}"), +{indent8}ArchCheckResult::UnknownArch => unimplemented!("***** unsupported LLVM architecture {{arch}}, intrinsic: {{full_name}}"), +{indent8}ArchCheckResult::Ok(_) => unreachable!(), +{indent4}}} +}}""") out.write("}\n}") - subprocess.call(["rustfmt", output_file]) print("Done!") From 8d490ac4f16cf4437d6aa4a05e8b9a83f9217f94 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 18 Sep 2026 22:52:45 +0200 Subject: [PATCH 36/55] Remove forgotten file --- ...1-Add-stdarch-Cargo.toml-for-testing.patch | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 patches/0001-Add-stdarch-Cargo.toml-for-testing.patch diff --git a/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch b/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch deleted file mode 100644 index 3a8c37a8b8d9a..0000000000000 --- a/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 190e26c9274b3c93a9ee3516b395590e6bd9213b Mon Sep 17 00:00:00 2001 -From: None -Date: Sun, 3 Aug 2025 19:54:56 -0400 -Subject: [PATCH] Patch 0001-Add-stdarch-Cargo.toml-for-testing.patch - ---- - library/stdarch/Cargo.toml | 20 ++++++++++++++++++++ - 1 file changed, 20 insertions(+) - create mode 100644 library/stdarch/Cargo.toml - -diff --git a/library/stdarch/Cargo.toml b/library/stdarch/Cargo.toml -new file mode 100644 -index 0000000..bd6725c ---- /dev/null -+++ b/library/stdarch/Cargo.toml -@@ -0,0 +1,20 @@ -+[workspace] -+resolver = "1" -+members = [ -+ "crates/*", -+ #"examples/" -+] -+exclude = [ -+ "crates/wasm-assert-instr-tests", -+ "rust_programs", -+] -+ -+[profile.release] -+debug = true -+opt-level = 3 -+incremental = true -+ -+[profile.bench] -+debug = 1 -+opt-level = 3 -+incremental = true --- -2.50.1 - From fcddd10cfcaea293978a33db7e078aec493deb86 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sun, 20 Sep 2026 15:03:11 +0200 Subject: [PATCH 37/55] s390x: use funnel shift --- library/stdarch/crates/core_arch/src/s390x/vector.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/s390x/vector.rs b/library/stdarch/crates/core_arch/src/s390x/vector.rs index 6703691f080b2..6899f482e270d 100644 --- a/library/stdarch/crates/core_arch/src/s390x/vector.rs +++ b/library/stdarch/crates/core_arch/src/s390x/vector.rs @@ -122,8 +122,6 @@ unsafe extern "llvm-intrinsic" { #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char; #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char; - #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16; - #[link_name = "llvm.s390.verimb"] fn verimb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char, d: i32) -> vector_signed_char; #[link_name = "llvm.s390.verimh"] fn verimh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short, d: i32) -> vector_signed_short; #[link_name = "llvm.s390.verimf"] fn verimf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int, d: i32) -> vector_signed_int; @@ -3664,12 +3662,7 @@ mod sealed { #[target_feature(enable = "vector-enhancements-2")] unsafe fn vec_srdb(self, b: Self) -> Self { static_assert_uimm_bits!(C, 3); - transmute(vsrd(transmute(self), transmute(b), C)) - // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129955#issuecomment-3207488190 - // LLVM currently rewrites `fshr` to `fshl`, and the logic in the s390x - // backend cannot deal with that yet. - // #[link_name = "llvm.fshr.i128"] fn fshr_i128(a: u128, b: u128, c: u128) -> u128; - // transmute(fshr_i128(transmute(self), transmute(b), const { C as u128 })) + transmute(u128::funnel_shr(transmute(self), transmute(b), C)) } } )* From 3da3866635c74f7d666422697daba0b759878129 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Sun, 20 Sep 2026 15:09:09 +0200 Subject: [PATCH 38/55] s390x: use u128 arithmetic --- .../stdarch/crates/core_arch/src/s390x/vector.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/library/stdarch/crates/core_arch/src/s390x/vector.rs b/library/stdarch/crates/core_arch/src/s390x/vector.rs index 6899f482e270d..8fd9fd83a64cd 100644 --- a/library/stdarch/crates/core_arch/src/s390x/vector.rs +++ b/library/stdarch/crates/core_arch/src/s390x/vector.rs @@ -138,9 +138,6 @@ unsafe extern "llvm-intrinsic" { #[link_name = "llvm.s390.vsumqf"] fn vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128; #[link_name = "llvm.s390.vsumqg"] fn vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128; - #[link_name = "llvm.s390.vaccq"] fn vaccq(a: u128, b: u128) -> u128; - #[link_name = "llvm.s390.vacccq"] fn vacccq(a: u128, b: u128, c: u128) -> u128; - #[link_name = "llvm.s390.vscbiq"] fn vscbiq(a: u128, b: u128) -> u128; #[link_name = "llvm.s390.vsbiq"] fn vsbiq(a: u128, b: u128, c: u128) -> u128; #[link_name = "llvm.s390.vsbcbiq"] fn vsbcbiq(a: u128, b: u128, c: u128) -> u128; @@ -4842,9 +4839,7 @@ pub unsafe fn vec_addc_u128( ) -> vector_unsigned_char { let a: u128 = transmute(a); let b: u128 = transmute(b); - // FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557 - // transmute(a.overflowing_add(b).1 as u128) - transmute(vaccq(a, b)) + transmute(a.overflowing_add(b).1 as u128) } /// Vector Add With Carry unsigned 128-bits @@ -4879,10 +4874,8 @@ pub unsafe fn vec_addec_u128( let a: u128 = transmute(a); let b: u128 = transmute(b); let c: u128 = transmute(c); - // FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557 - // let (_d, carry) = a.carrying_add(b, c & 1 != 0); - // transmute(carry as u128) - transmute(vacccq(a, b, c)) + let (_d, carry) = a.carrying_add(b, c & 1 != 0); + transmute(carry as u128) } /// Vector Subtract with Carryout From acc7501018ff1a3fa1c341d39b79bc524535f5a2 Mon Sep 17 00:00:00 2001 From: khyperia <953151+khyperia@users.noreply.github.com> Date: Mon, 21 Sep 2026 07:05:12 +0200 Subject: [PATCH 39/55] gca: fix unreachable --- .../src/builder/expr/as_constant.rs | 53 ++++++++++++------- .../gca/const-reference-to-constructor.rs | 10 ++++ 2 files changed, 43 insertions(+), 20 deletions(-) create mode 100644 tests/ui/const-generics/gca/const-reference-to-constructor.rs diff --git a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs index eb9f56d887684..43936770ff762 100644 --- a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs +++ b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs @@ -74,28 +74,41 @@ pub(crate) fn as_constant_inner<'tcx>( ExprKind::NamedConst { def_id, args, ref user_ty } => { let user_ty = user_ty.as_ref().and_then(push_cuta); - // Under generic_const_args, `def_id` might be a regular const declared in a trait, but - // is `impl`d as a directly represented const. We do not know whether it is here, so we - // must use type system normalization for all consts under generic_const_args. - // FIXME(generic_const_args): there's a lot to consider here! `Const::Ty` uses valtrees - // and `Const::Unevaluated` does not, we should revisit this before stabilization. - let def_kind = tcx.def_kind(def_id); - if tcx.features().generic_const_args() - || matches!(def_kind, DefKind::Const | DefKind::AssocConst) - && tcx.is_direct_const(def_id) - { - let kind = match def_kind { - DefKind::AssocConst => { - if let DefKind::Impl { of_trait: false } = tcx.def_kind(tcx.parent(def_id)) - { - ty::AliasConstKind::InherentImpl { def_id } - } else { - ty::AliasConstKind::Projection { def_id } - } + let get_kind = |def_id, def_kind| match def_kind { + DefKind::AssocConst => { + if let DefKind::Impl { of_trait: false } = tcx.def_kind(tcx.parent(def_id)) { + ty::AliasConstKind::InherentImpl { def_id } + } else { + ty::AliasConstKind::Projection { def_id } } - DefKind::Const => ty::AliasConstKind::Free { def_id }, - _ => unreachable!(), + } + DefKind::Const => ty::AliasConstKind::Free { def_id }, + kind => bug!("unexpected DefKind in THIR ExprKind::NamedConst: {kind:?}"), + }; + + let could_be_direct_const = |def_id| { + let def_kind = tcx.def_kind(def_id); + let (DefKind::Const | DefKind::AssocConst) = def_kind else { + return None; }; + if tcx.is_direct_const(def_id) { + return Some(get_kind(def_id, def_kind)); + } + // Under generic_const_args, `def_id` might be a regular const declared in a trait, + // but is `impl`d as a directly represented const. We do not know whether it is + // here, so we must use type system normalization for all const projections. + // FIXME(generic_const_args): there's a lot to consider here! `Const::Ty` uses + // valtrees and `Const::Unevaluated` does not, we should revisit this before + // stabilization. + if tcx.features().generic_const_args() + && let kind @ ty::AliasConstKind::Projection { .. } = get_kind(def_id, def_kind) + { + return Some(kind); + } + None + }; + + if let Some(kind) = could_be_direct_const(def_id) { let alias = ty::AliasConst::new(tcx, kind, args); let ct = ty::Const::new_alias(tcx, ty::IsRigid::No, alias); let const_ = Const::Ty(ty, ct); diff --git a/tests/ui/const-generics/gca/const-reference-to-constructor.rs b/tests/ui/const-generics/gca/const-reference-to-constructor.rs new file mode 100644 index 0000000000000..bafaf9f532545 --- /dev/null +++ b/tests/ui/const-generics/gca/const-reference-to-constructor.rs @@ -0,0 +1,10 @@ +//@ check-pass +//@ compile-flags: -Znext-solver +//! https://github.com/rust-lang/rust/issues/162923 +#![feature(min_generic_const_args)] +#![feature(generic_const_args)] +enum T::B as u8 }> { + A = 2, + B, +} +fn main() {} From e587f8b32f0dabd62fe4b882ca55d2ae38d9ae77 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Mon, 21 Sep 2026 08:57:52 +0100 Subject: [PATCH 40/55] Wire up f16b in backends --- src/base.rs | 2 ++ src/context.rs | 3 +++ src/lib.rs | 2 ++ src/type_.rs | 8 ++++++++ 4 files changed, 15 insertions(+) diff --git a/src/base.rs b/src/base.rs index 101af0bb0bff1..498ea4b45a8f8 100644 --- a/src/base.rs +++ b/src/base.rs @@ -214,6 +214,7 @@ pub fn compile_codegen_unit( // -fsyntax-only), forbid the compilation when get_target_info() is called on a // context. let f16_type_supported = target_info.supports_target_dependent_type(CType::Float16); + let f16b_type_supported = target_info.supports_target_dependent_type(CType::BFloat16); let f32_type_supported = target_info.supports_target_dependent_type(CType::Float32); let f64_type_supported = target_info.supports_target_dependent_type(CType::Float64); let f128_type_supported = target_info.supports_target_dependent_type(CType::Float128); @@ -225,6 +226,7 @@ pub fn compile_codegen_unit( tcx, u128_type_supported, f16_type_supported, + f16b_type_supported, f32_type_supported, f64_type_supported, f128_type_supported, diff --git a/src/context.rs b/src/context.rs index 38e0e5f329f76..a41255be9e1ea 100644 --- a/src/context.rs +++ b/src/context.rs @@ -72,6 +72,7 @@ pub struct CodegenCx<'gcc, 'tcx> { pub supports_128bit_integers: bool, pub supports_f16_type: bool, + pub supports_f16b_type: bool, pub supports_f32_type: bool, pub supports_f64_type: bool, pub supports_f128_type: bool, @@ -140,6 +141,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { tcx: TyCtxt<'tcx>, supports_128bit_integers: bool, supports_f16_type: bool, + supports_f16b_type: bool, supports_f32_type: bool, supports_f64_type: bool, supports_f128_type: bool, @@ -276,6 +278,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { supports_128bit_integers, supports_f16_type, + supports_f16b_type, supports_f32_type, supports_f64_type, supports_f128_type, diff --git a/src/lib.rs b/src/lib.rs index d7a3ef3b4a6c5..79802a670a3f0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -503,6 +503,7 @@ fn target_config(sess: &EarlySession, target_info: &SharedTargetInfo) -> TargetC ); let has_reliable_f16 = target_info.supports_target_dependent_type(CType::Float16); + let has_reliable_f16b = target_info.supports_target_dependent_type(CType::BFloat16); let has_reliable_f128 = target_info.supports_target_dependent_type(CType::Float128); TargetConfig { @@ -510,6 +511,7 @@ fn target_config(sess: &EarlySession, target_info: &SharedTargetInfo) -> TargetC // There are no known bugs with GCC support for f16 or f128 has_reliable_f16, has_reliable_f16_math: has_reliable_f16, + has_reliable_f16b, has_reliable_f128, has_reliable_f128_math: has_reliable_f128, } diff --git a/src/type_.rs b/src/type_.rs index 27b0d2079e63e..1d8582fe337ad 100644 --- a/src/type_.rs +++ b/src/type_.rs @@ -157,6 +157,14 @@ impl<'gcc, 'tcx> BaseTypeCodegenMethods for CodegenCx<'gcc, 'tcx> { bug!("unsupported float width 16") } + fn type_f16b(&self) -> Type<'gcc> { + #[cfg(feature = "master")] + if self.supports_f16b_type { + return self.context.new_c_type(CType::BFloat16); + } + bug!("unsupported type bfloat16") + } + fn type_f32(&self) -> Type<'gcc> { #[cfg(feature = "master")] if self.supports_f32_type { From fc49efd517cabd80f27969c5313ce58a9f2905a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 21 Sep 2026 11:06:46 +0200 Subject: [PATCH 41/55] Prepare for merging from rust-lang/rust This updates the rust-version file to 220b36c420c49c59923f54cd4a76634fac98a067. --- library/stdarch/rust-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/stdarch/rust-version b/library/stdarch/rust-version index 18fea436747c7..368cecc0b82f9 100644 --- a/library/stdarch/rust-version +++ b/library/stdarch/rust-version @@ -1 +1 @@ -32d94cc9be3f6e6c3fa1deaea9e0ab93c4980dba +220b36c420c49c59923f54cd4a76634fac98a067 From c263e3de4d240f871206d00075188918eb6e1bbb Mon Sep 17 00:00:00 2001 From: Amirhossein Akhlaghpour Date: Thu, 10 Sep 2026 16:02:27 +0330 Subject: [PATCH 42/55] avoid accessing uninferred closure upvars in diagnostics Signed-off-by: Amirhossein Akhlaghpour --- .../src/error_reporting/traits/suggestions.rs | 11 ++++- ...-capture-uninferred-upvars-issue-162440.rs | 12 +++++ ...ture-uninferred-upvars-issue-162440.stderr | 41 ++++++++++++++++ ...-capture-uninferred-upvars-with-capture.rs | 16 ++++++ ...ture-uninferred-upvars-with-capture.stderr | 49 +++++++++++++++++++ 5 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 tests/ui/traits/next-solver/closure-capture-uninferred-upvars-issue-162440.rs create mode 100644 tests/ui/traits/next-solver/closure-capture-uninferred-upvars-issue-162440.stderr create mode 100644 tests/ui/traits/next-solver/closure-capture-uninferred-upvars-with-capture.rs create mode 100644 tests/ui/traits/next-solver/closure-capture-uninferred-upvars-with-capture.stderr diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs index 4f5d93a2fe286..6ede57df7d0f5 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs @@ -3787,9 +3787,18 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if typeck_results.hir_owner.to_def_id() != typeck_root { return false; } + + // Error reporting can run before closure capture analysis has inferred the + // tuple of upvar types. avoid accessing upvar types until they are available. + let upvar_tys = match upvar_args.tupled_upvars_ty().kind() { + ty::Tuple(args) => args, + ty::Error(_) => ty::List::empty(), + ty::Infer(_) => return false, + ty => unreachable!("unexpected upvar types tuple: {ty:?}"), + }; + let captures: Vec<_> = typeck_results.closure_min_captures_flattened(closure_def_id).collect(); - let upvar_tys = upvar_args.upvar_tys(); if captures.len() != upvar_tys.len() { return false; } diff --git a/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-issue-162440.rs b/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-issue-162440.rs new file mode 100644 index 0000000000000..9b6562ff578bf --- /dev/null +++ b/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-issue-162440.rs @@ -0,0 +1,12 @@ +// Regression test for #162440. Diagnostics can run before closure capture +// analysis has inferred the tuple of upvar types. The capture-specific note +// must fall back instead of trying to access uninferred upvar types. + +//@ compile-flags: -Znext-solver=globally + +fn main() { + Some([0]).map(|s| s[..]); + //~^ ERROR the size for values of type `[{integer}]` cannot be known at compilation time + //~| ERROR the size for values of type `[{integer}]` cannot be known at compilation time + //~| ERROR the size for values of type `[{integer}]` cannot be known at compilation time +} diff --git a/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-issue-162440.stderr b/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-issue-162440.stderr new file mode 100644 index 0000000000000..4aed7e179b8ef --- /dev/null +++ b/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-issue-162440.stderr @@ -0,0 +1,41 @@ +error[E0277]: the size for values of type `[{integer}]` cannot be known at compilation time + --> $DIR/closure-capture-uninferred-upvars-issue-162440.rs:8:15 + | +LL | Some([0]).map(|s| s[..]); + | ^^^ doesn't have a size known at compile-time + | + = help: the trait `Sized` is not implemented for `[{integer}]` +note: required by an implicit `Sized` bound in `Option::::map` + --> $SRC_DIR/core/src/option.rs:LL:COL + +error[E0277]: the size for values of type `[{integer}]` cannot be known at compilation time + --> $DIR/closure-capture-uninferred-upvars-issue-162440.rs:8:23 + | +LL | Some([0]).map(|s| s[..]); + | ^^^^^ doesn't have a size known at compile-time + | + = help: the trait `Sized` is not implemented for `[{integer}]` + = note: the return type of a function must have a statically known size + +error[E0277]: the size for values of type `[{integer}]` cannot be known at compilation time + --> $DIR/closure-capture-uninferred-upvars-issue-162440.rs:8:19 + | +LL | Some([0]).map(|s| s[..]); + | --- ---^^^^^^ + | | | + | | doesn't have a size known at compile-time + | | within this `{closure@$DIR/closure-capture-uninferred-upvars-issue-162440.rs:8:19: 8:22}` + | required by a bound introduced by this call + | + = help: within `{closure@$DIR/closure-capture-uninferred-upvars-issue-162440.rs:8:19: 8:22}`, the trait `Sized` is not implemented for `[{integer}]` +note: required because it's used within this closure + --> $DIR/closure-capture-uninferred-upvars-issue-162440.rs:8:19 + | +LL | Some([0]).map(|s| s[..]); + | ^^^ +note: required by a bound in `Option::::map` + --> $SRC_DIR/core/src/option.rs:LL:COL + +error: aborting due to 3 previous errors + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-with-capture.rs b/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-with-capture.rs new file mode 100644 index 0000000000000..ff4a9b67a3fee --- /dev/null +++ b/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-with-capture.rs @@ -0,0 +1,16 @@ +// Diagnostics may run before closure capture analysis has inferred the upvar +// tuple, even for a closure which actually captures a value. + +//@ compile-flags: -Znext-solver=globally + +fn main() { + let x = String::new(); + + Some([0]).map(|s| { + //~^ ERROR the size for values of type `[{integer}]` cannot be known at compilation time + //~| ERROR the size for values of type `[{integer}]` cannot be known at compilation time + //~| ERROR the size for values of type `[{integer}]` cannot be known at compilation time + let _ = &x; + s[..] + }); +} diff --git a/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-with-capture.stderr b/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-with-capture.stderr new file mode 100644 index 0000000000000..b19dca701f2fb --- /dev/null +++ b/tests/ui/traits/next-solver/closure-capture-uninferred-upvars-with-capture.stderr @@ -0,0 +1,49 @@ +error[E0277]: the size for values of type `[{integer}]` cannot be known at compilation time + --> $DIR/closure-capture-uninferred-upvars-with-capture.rs:9:15 + | +LL | Some([0]).map(|s| { + | ^^^ doesn't have a size known at compile-time + | + = help: the trait `Sized` is not implemented for `[{integer}]` +note: required by an implicit `Sized` bound in `Option::::map` + --> $SRC_DIR/core/src/option.rs:LL:COL + +error[E0277]: the size for values of type `[{integer}]` cannot be known at compilation time + --> $DIR/closure-capture-uninferred-upvars-with-capture.rs:9:23 + | +LL | Some([0]).map(|s| { + | _______________________^ +... | +LL | | s[..] +LL | | }); + | |_____^ doesn't have a size known at compile-time + | + = help: the trait `Sized` is not implemented for `[{integer}]` + = note: the return type of a function must have a statically known size + +error[E0277]: the size for values of type `[{integer}]` cannot be known at compilation time + --> $DIR/closure-capture-uninferred-upvars-with-capture.rs:9:19 + | +LL | Some([0]).map(|s| { + | --- ^-- + | | | + | _______________|___within this `{closure@$DIR/closure-capture-uninferred-upvars-with-capture.rs:9:19: 9:22}` + | | | + | | required by a bound introduced by this call +... | +LL | | s[..] +LL | | }); + | |_____^ doesn't have a size known at compile-time + | + = help: within `{closure@$DIR/closure-capture-uninferred-upvars-with-capture.rs:9:19: 9:22}`, the trait `Sized` is not implemented for `[{integer}]` +note: required because it's used within this closure + --> $DIR/closure-capture-uninferred-upvars-with-capture.rs:9:19 + | +LL | Some([0]).map(|s| { + | ^^^ +note: required by a bound in `Option::::map` + --> $SRC_DIR/core/src/option.rs:LL:COL + +error: aborting due to 3 previous errors + +For more information about this error, try `rustc --explain E0277`. From 46d8bca83e4cf09bc298e900d21ae9c46c062c62 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Thu, 17 Sep 2026 09:36:40 +0100 Subject: [PATCH 43/55] `use ConstExt` for methods that do not yet exist in `rustc_type_ir` --- src/intrinsic/simd.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/intrinsic/simd.rs b/src/intrinsic/simd.rs index bb32a85f193ea..54013b6be6173 100644 --- a/src/intrinsic/simd.rs +++ b/src/intrinsic/simd.rs @@ -15,6 +15,7 @@ use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, LayoutTy #[cfg(feature = "master")] use rustc_hir as hir; use rustc_middle::mir::BinOp; +use rustc_middle::ty::consts::ConstExt; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, Ty}; use rustc_span::{ErrorGuaranteed, Span, Symbol, span_bug, sym}; From db0b931b23cf226407979eece7c69c2aa2dbd8aa Mon Sep 17 00:00:00 2001 From: Antoni Boucher Date: Tue, 22 Sep 2026 09:21:06 -0400 Subject: [PATCH 44/55] Update to nightly-2026-09-22 --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 0c81f7c7c7398..f3106ee20c331 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2026-09-18" +channel = "nightly-2026-09-22" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 7a8706e98c13918cad687d8e8d1423280b6d68fd Mon Sep 17 00:00:00 2001 From: Antoni Boucher Date: Tue, 22 Sep 2026 10:22:49 -0400 Subject: [PATCH 45/55] Skip test_mm_srav_epi64 as it's broken upstream --- .github/workflows/stdarch.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stdarch.yml b/.github/workflows/stdarch.yml index 17d6449c85e08..34499c6e7b522 100644 --- a/.github/workflows/stdarch.yml +++ b/.github/workflows/stdarch.yml @@ -96,14 +96,16 @@ jobs: if: ${{ !matrix.cargo_runner }} run: | # FIXME: remove --skip test_tile_ and --skip --skip test__tile when it's implemented. - ./y.sh test --release --stdarch-tests -- --skip test_tile_ --skip test__tile + # FIXME: remove --skip test_mm_srav_epi64 when it's fixed upstream. + ./y.sh test --release --stdarch-tests -- --skip test_tile_ --skip test__tile --skip test_mm_srav_epi64 - name: Run stdarch tests if: ${{ matrix.cargo_runner }} run: | # FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro. # FIXME: remove --skip test_tile_ and --skip --skip test__tile when it's implemented. - STDARCH_TEST_SKIP_FUNCTION="xsave,xsaveopt,xsave64,xsaveopt64" STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a --skip test_tile_ --skip test__tile + # FIXME: remove --skip test_mm_srav_epi64 when it's fixed upstream. + STDARCH_TEST_SKIP_FUNCTION="xsave,xsaveopt,xsave64,xsaveopt64" STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a --skip test_tile_ --skip test__tile --skip test_mm_srav_epi64 # Summary job for the merge queue. # ALL THE PREVIOUS JOBS NEED TO BE ADDED TO THE `needs` SECTION OF THIS JOB! From dc249dca94155267c02a52a6dbacc92f1ffd8331 Mon Sep 17 00:00:00 2001 From: Flakebi Date: Tue, 1 Sep 2026 09:40:37 +0200 Subject: [PATCH 46/55] Add address_space and byref to abi PassMode::Indirect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both will be used by the amdgpu target to implement the `gpu-kernel` ABI. `address_space` specifies the address space of an indirect argument. `AmdgpuKernelArg` translates to LLVM’s byref, which is similar to on_stack/byval, however, there is no extra copy made, the pointer may not point to the stack but can point to some other address space, and the passed argument should not be modified. byval and byref are mutually exclusive, so change on_stack to an enum with the new states, Pointer (none), OnStack and AmdgpuKernelArg. --- src/abi.rs | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/abi.rs b/src/abi.rs index 63eaf52ce9f01..5b88cebb4f174 100644 --- a/src/abi.rs +++ b/src/abi.rs @@ -11,7 +11,7 @@ use rustc_middle::ty::layout::LayoutOf; #[cfg(feature = "master")] use rustc_session::{Session, config}; use rustc_span::bug; -use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode}; #[cfg(feature = "master")] use rustc_target::spec::Arch; @@ -189,7 +189,12 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { let ty = cast.gcc_type(cx); apply_attrs(ty, &cast.attrs, argument_tys.len()) } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => { + PassMode::Indirect { + attrs: _, + meta_attrs: None, + address_space: _, + mode: IndirectMode::OnStack, + } => { let x86_interrupt_first_arg = { #[cfg(feature = "master")] { @@ -216,14 +221,32 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { arg.layout.gcc_type(cx) } } + PassMode::Indirect { + attrs: _, + meta_attrs: None, + address_space: _, + mode: IndirectMode::AmdgpuKernelArg, + } => { + unimplemented!("unsupported amdgpu kernel argument") + } PassMode::Direct(attrs) => { apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len()) } - PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { + PassMode::Indirect { + attrs, + meta_attrs: None, + address_space: _, + mode: IndirectMode::Pointer, + } => { apply_attrs(cx.type_ptr_to(arg.layout.gcc_type(cx)), &attrs, argument_tys.len()) } - PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { - assert!(!on_stack); + PassMode::Indirect { + attrs, + meta_attrs: Some(meta_attrs), + address_space: _, + mode, + } => { + assert!(mode == IndirectMode::Pointer); // Construct the type of a (wide) pointer to `ty`, and pass its two fields. // Any two ABI-compatible unsized types have the same metadata type and // moreover the same metadata value leads to the same dynamic size and From d9e4b39c549474cc12239d6ee797acd4d7c1f2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 23 Sep 2026 13:10:03 +0200 Subject: [PATCH 47/55] Rename rust-toolchain to rust-toolchain.toml For better compatibility with josh-sync. The old filename is deprecated anyway. --- rust-toolchain => rust-toolchain.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename rust-toolchain => rust-toolchain.toml (100%) diff --git a/rust-toolchain b/rust-toolchain.toml similarity index 100% rename from rust-toolchain rename to rust-toolchain.toml From 047bdcbe109cbed3a78ea01ff1a5637ef6c8c3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 23 Sep 2026 13:13:00 +0200 Subject: [PATCH 48/55] Update apt packages before installing them --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f5cc409e363d..f37123c124c74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,9 @@ jobs: - name: Install packages # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests. - run: sudo apt-get install ninja-build ripgrep llvm-14-tools llvm + run: | + sudo apt-get update + sudo apt-get install ninja-build ripgrep llvm-14-tools llvm - name: Install the libraries needed to build librsvg if: ${{ contains(matrix.commands, '--projects') }} From 3ca6e6cf2033c561330e813e571edd490a72396e Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 23 Sep 2026 14:47:31 +0200 Subject: [PATCH 49/55] fix `test_mm_srav_epi64` shift direction --- library/stdarch/crates/core_arch/src/x86_64/avx512f.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs b/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs index 832384b11e68b..9c261784553bd 100644 --- a/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86_64/avx512f.rs @@ -9583,7 +9583,7 @@ mod tests { let a = _mm_set_epi64x(-1, -2); let b = _mm_set_epi64x(64, 65); let r = _mm_srav_epi64(a, b); - let e = _mm_set_epi64x((-1i64).unbounded_shl(64), (-2i64).unbounded_shl(65)); + let e = _mm_set_epi64x((-1i64).unbounded_shr(64), (-2i64).unbounded_shr(65)); assert_eq_m128i(r, e); let e = _mm_set_epi64x(-1, -1); assert_eq_m128i(r, e); From d67606b1a0ff66ce94cf799308b6678e7456cd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 23 Sep 2026 15:38:23 +0200 Subject: [PATCH 50/55] Update code to use the new file --- Readme.md | 2 +- build_system/src/abi_test.rs | 2 +- build_system/src/utils.rs | 6 +++--- doc/tips.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Readme.md b/Readme.md index 9a7c624c9bc22..14c0e3ad0f21c 100644 --- a/Readme.md +++ b/Readme.md @@ -154,7 +154,7 @@ If you compiled `cg_gcc` in debug mode (aka you didn't pass `--release` to `./y. You can do the same manually (although we don't recommend it): ```bash -$ LIBRARY_PATH="[gcc-path value]" LD_LIBRARY_PATH="[gcc-path value]" rustc +$(cat $CG_GCCJIT_DIR/rust-toolchain | grep 'channel' | cut -d '=' -f 2 | sed 's/"//g' | sed 's/ //g') -Cpanic=abort -Zcodegen-backend=$CG_GCCJIT_DIR/target/release/librustc_codegen_gcc.so --sysroot $CG_GCCJIT_DIR/build_sysroot/sysroot my_crate.rs +$ LIBRARY_PATH="[gcc-path value]" LD_LIBRARY_PATH="[gcc-path value]" rustc +$(cat $CG_GCCJIT_DIR/rust-toolchain.toml | grep 'channel' | cut -d '=' -f 2 | sed 's/"//g' | sed 's/ //g') -Cpanic=abort -Zcodegen-backend=$CG_GCCJIT_DIR/target/release/librustc_codegen_gcc.so --sysroot $CG_GCCJIT_DIR/build_sysroot/sysroot my_crate.rs ``` ## Environment variables diff --git a/build_system/src/abi_test.rs b/build_system/src/abi_test.rs index a85886d87f365..9fc06dc8dbc04 100644 --- a/build_system/src/abi_test.rs +++ b/build_system/src/abi_test.rs @@ -34,7 +34,7 @@ pub fn run() -> Result<(), String> { .map_err(|err| format!("Git clone failed with message: {err:?}!"))?; // Configure abi-cafe to use the exact same rustc version we use - this is crucial. // Otherwise, the concept of ABI compatibility becomes meanignless. - std::fs::copy("rust-toolchain", "clones/abi-cafe/rust-toolchain") + std::fs::copy("rust-toolchain.toml", "clones/abi-cafe/rust-toolchain.toml") .expect("Could not copy toolchain configs!"); // Get the backend path. // We will use the *debug* build of the backend - it has more checks enabled. diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 4c67156a85fb2..dda44e4d16d41 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -242,9 +242,9 @@ fn rustc_version_info_inner( } pub fn get_toolchain() -> Result { - let content = match fs::read_to_string("rust-toolchain") { + let content = match fs::read_to_string("rust-toolchain.toml") { Ok(content) => content, - Err(_) => return Err("No `rust-toolchain` file found".to_string()), + Err(_) => return Err("No `rust-toolchain.toml` file found".to_string()), }; match content .split('\n') @@ -259,7 +259,7 @@ pub fn get_toolchain() -> Result { .next() { Some(toolchain) => Ok(toolchain.to_string()), - None => Err("Couldn't find `channel` in `rust-toolchain` file".to_string()), + None => Err("Couldn't find `channel` in `rust-toolchain.toml` file".to_string()), } } diff --git a/doc/tips.md b/doc/tips.md index dc40ee4d39952..28964f66403bf 100644 --- a/doc/tips.md +++ b/doc/tips.md @@ -41,7 +41,7 @@ COLLECT_NO_DEMANGLE=1 ### How to use a custom-build rustc * Build the stage2 compiler (`rustup toolchain link debug-current build/x86_64-unknown-linux-gnu/stage2`). - * Clean and rebuild the codegen with `debug-current` in the file `rust-toolchain`. + * Clean and rebuild the codegen with `debug-current` in the file `rust-toolchain.toml`. ### How to use a custom sysroot source path From 603ca3fbf97a90478114056d068965e6c3a45ec4 Mon Sep 17 00:00:00 2001 From: Yukang Date: Thu, 24 Sep 2026 09:11:43 +0800 Subject: [PATCH 51/55] Add regression tests for await macro suggestions --- .../await-keyword/await-macro-rustfix.rs | 22 ++++++++ .../await-keyword/await-macro-rustfix.stderr | 50 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 tests/ui/async-await/await-keyword/await-macro-rustfix.rs create mode 100644 tests/ui/async-await/await-keyword/await-macro-rustfix.stderr diff --git a/tests/ui/async-await/await-keyword/await-macro-rustfix.rs b/tests/ui/async-await/await-keyword/await-macro-rustfix.rs new file mode 100644 index 0000000000000..e9fc64bf6b92e --- /dev/null +++ b/tests/ui/async-await/await-keyword/await-macro-rustfix.rs @@ -0,0 +1,22 @@ +//@ edition: 2024 + +// Replacing `await!(...)` must preserve grouping and any following `?`. + +use std::future::ready; + +async fn check() -> Result<(), ()> { + let future = ready(1); + let _: i32 = await!(future); + //~^ ERROR incorrect use of `await` + let _: i32 = await!(ready(1)); + //~^ ERROR incorrect use of `await` + let _: i32 = await!(ready(Ok::(1)))?; + //~^ ERROR incorrect use of `await` + let _: i32 = await!(/* keep */ &mut ready(1) /* keep */); + //~^ ERROR incorrect use of `await` + Ok(()) +} + +fn main() { + let _ = check(); +} diff --git a/tests/ui/async-await/await-keyword/await-macro-rustfix.stderr b/tests/ui/async-await/await-keyword/await-macro-rustfix.stderr new file mode 100644 index 0000000000000..02a60fff3371f --- /dev/null +++ b/tests/ui/async-await/await-keyword/await-macro-rustfix.stderr @@ -0,0 +1,50 @@ +error: incorrect use of `await` + --> $DIR/await-macro-rustfix.rs:9:18 + | +LL | let _: i32 = await!(future); + | ^^^^^^^^^^^^^^ + | +help: `await` is a postfix operation + | +LL - let _: i32 = await!(future); +LL + let _: i32 = future.await); + | + +error: incorrect use of `await` + --> $DIR/await-macro-rustfix.rs:11:18 + | +LL | let _: i32 = await!(ready(1)); + | ^^^^^^^^^^^^^^^^ + | +help: `await` is a postfix operation + | +LL - let _: i32 = await!(ready(1)); +LL + let _: i32 = ready(1).await); + | + +error: incorrect use of `await` + --> $DIR/await-macro-rustfix.rs:13:18 + | +LL | let _: i32 = await!(ready(Ok::(1)))?; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: `await` is a postfix operation + | +LL - let _: i32 = await!(ready(Ok::(1)))?; +LL + let _: i32 = ready(Ok::(1)).await)?; + | + +error: incorrect use of `await` + --> $DIR/await-macro-rustfix.rs:15:18 + | +LL | let _: i32 = await!(/* keep */ &mut ready(1) /* keep */); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: `await` is a postfix operation + | +LL - let _: i32 = await!(/* keep */ &mut ready(1) /* keep */); +LL + let _: i32 = &mut ready(1).await /* keep */); + | + +error: aborting due to 4 previous errors + From 3ea2995de898fd22bd562c286cc4d7b26176cc30 Mon Sep 17 00:00:00 2001 From: Yukang Date: Thu, 24 Sep 2026 09:22:16 +0800 Subject: [PATCH 52/55] Fix incorrect use of await in parser suggestion --- .../rustc_parse/src/parser/diagnostics.rs | 29 ++++++++++++------- .../await-keyword/await-macro-rustfix.fixed | 23 +++++++++++++++ .../await-keyword/await-macro-rustfix.rs | 1 + .../await-keyword/await-macro-rustfix.stderr | 16 +++++----- .../incorrect-syntax-suggestions.stderr | 8 ++--- 5 files changed, 55 insertions(+), 22 deletions(-) create mode 100644 tests/ui/async-await/await-keyword/await-macro-rustfix.fixed diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 928518c88fec6..0630a65e56cb4 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -3,7 +3,7 @@ use std::ops::{Deref, DerefMut}; use ast::token::IdentKind; use rustc_ast::token::{self, Lit, LitKind, Token, TokenKind}; -use rustc_ast::util::parser::AssocOp; +use rustc_ast::util::parser::{AssocOp, ExprPrecedence}; use rustc_ast::{ self as ast, AngleBracketedArg, AngleBracketedArgs, AnonConst, AttrVec, BinOpKind, BindingMode, Block, BlockCheckMode, Expr, ExprKind, GenericArg, GenericArgs, Generics, Item, ItemKind, @@ -1719,26 +1719,35 @@ impl<'a> Parser<'a> { &mut self, await_sp: Span, ) -> PResult<'a, Box> { - let (hi, expr, is_question) = if self.token == token::Bang { + let (hi, expr_span, is_question) = if self.token == token::Bang { // Handle `await!()`. self.recover_await_macro()? } else { self.recover_await_prefix(await_sp)? }; - let (sp, guar) = self.error_on_incorrect_await(await_sp, hi, &expr, is_question); + let (sp, guar) = self.error_on_incorrect_await(await_sp, hi, expr_span, is_question); let expr = self.mk_expr_err(await_sp.to(sp), guar); self.maybe_recover_from_bad_qpath(expr) } - fn recover_await_macro(&mut self) -> PResult<'a, (Span, Box, bool)> { + fn recover_await_macro(&mut self) -> PResult<'a, (Span, Span, bool)> { self.expect(exp!(Bang))?; self.expect(exp!(OpenParen))?; + let open = self.prev_token.span; let expr = self.parse_expr()?; self.expect(exp!(CloseParen))?; - Ok((self.prev_token.span, expr, false)) + let close = self.prev_token.span; + // Keep parentheses when needed for `.await`, e.g. `(&mut future).await`. + // Use the delimiters to preserve any comments around the operand. + let expr_span = if expr.precedence() < ExprPrecedence::Unambiguous { + open.to(close) + } else { + open.shrink_to_hi().to(close.shrink_to_lo()) + }; + Ok((close, expr_span, false)) } - fn recover_await_prefix(&mut self, await_sp: Span) -> PResult<'a, (Span, Box, bool)> { + fn recover_await_prefix(&mut self, await_sp: Span) -> PResult<'a, (Span, Span, bool)> { let is_question = self.eat(exp!(Question)); // Handle `await? `. let expr = if self.token == token::OpenBrace { // Handle `await { }`. @@ -1752,22 +1761,22 @@ impl<'a> Parser<'a> { err.span_label(await_sp, format!("while parsing this incorrect await expression")); err })?; - Ok((expr.span, expr, is_question)) + Ok((expr.span, expr.span, is_question)) } fn error_on_incorrect_await( &self, lo: Span, hi: Span, - expr: &Expr, + expr_span: Span, is_question: bool, ) -> (Span, ErrorGuaranteed) { let span = lo.to(hi); let guar = self.dcx().emit_err(IncorrectAwait { span, suggestion: AwaitSuggestion { - removal: lo.until(expr.span), - dot_await: expr.span.shrink_to_hi(), + removal: lo.until(expr_span), + dot_await: expr_span.shrink_to_hi().to(hi.shrink_to_hi()), question_mark: if is_question { "?" } else { "" }, }, }); diff --git a/tests/ui/async-await/await-keyword/await-macro-rustfix.fixed b/tests/ui/async-await/await-keyword/await-macro-rustfix.fixed new file mode 100644 index 0000000000000..2946c05ad51b5 --- /dev/null +++ b/tests/ui/async-await/await-keyword/await-macro-rustfix.fixed @@ -0,0 +1,23 @@ +//@ run-rustfix +//@ edition: 2024 + +// Replacing `await!(...)` must preserve grouping and any following `?`. + +use std::future::ready; + +async fn check() -> Result<(), ()> { + let future = ready(1); + let _: i32 = future.await; + //~^ ERROR incorrect use of `await` + let _: i32 = ready(1).await; + //~^ ERROR incorrect use of `await` + let _: i32 = ready(Ok::(1)).await?; + //~^ ERROR incorrect use of `await` + let _: i32 = (/* keep */ &mut ready(1) /* keep */).await; + //~^ ERROR incorrect use of `await` + Ok(()) +} + +fn main() { + let _ = check(); +} diff --git a/tests/ui/async-await/await-keyword/await-macro-rustfix.rs b/tests/ui/async-await/await-keyword/await-macro-rustfix.rs index e9fc64bf6b92e..7272a2084843d 100644 --- a/tests/ui/async-await/await-keyword/await-macro-rustfix.rs +++ b/tests/ui/async-await/await-keyword/await-macro-rustfix.rs @@ -1,3 +1,4 @@ +//@ run-rustfix //@ edition: 2024 // Replacing `await!(...)` must preserve grouping and any following `?`. diff --git a/tests/ui/async-await/await-keyword/await-macro-rustfix.stderr b/tests/ui/async-await/await-keyword/await-macro-rustfix.stderr index 02a60fff3371f..42a1bdfdafcaa 100644 --- a/tests/ui/async-await/await-keyword/await-macro-rustfix.stderr +++ b/tests/ui/async-await/await-keyword/await-macro-rustfix.stderr @@ -1,5 +1,5 @@ error: incorrect use of `await` - --> $DIR/await-macro-rustfix.rs:9:18 + --> $DIR/await-macro-rustfix.rs:10:18 | LL | let _: i32 = await!(future); | ^^^^^^^^^^^^^^ @@ -7,11 +7,11 @@ LL | let _: i32 = await!(future); help: `await` is a postfix operation | LL - let _: i32 = await!(future); -LL + let _: i32 = future.await); +LL + let _: i32 = future.await; | error: incorrect use of `await` - --> $DIR/await-macro-rustfix.rs:11:18 + --> $DIR/await-macro-rustfix.rs:12:18 | LL | let _: i32 = await!(ready(1)); | ^^^^^^^^^^^^^^^^ @@ -19,11 +19,11 @@ LL | let _: i32 = await!(ready(1)); help: `await` is a postfix operation | LL - let _: i32 = await!(ready(1)); -LL + let _: i32 = ready(1).await); +LL + let _: i32 = ready(1).await; | error: incorrect use of `await` - --> $DIR/await-macro-rustfix.rs:13:18 + --> $DIR/await-macro-rustfix.rs:14:18 | LL | let _: i32 = await!(ready(Ok::(1)))?; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -31,11 +31,11 @@ LL | let _: i32 = await!(ready(Ok::(1)))?; help: `await` is a postfix operation | LL - let _: i32 = await!(ready(Ok::(1)))?; -LL + let _: i32 = ready(Ok::(1)).await)?; +LL + let _: i32 = ready(Ok::(1)).await?; | error: incorrect use of `await` - --> $DIR/await-macro-rustfix.rs:15:18 + --> $DIR/await-macro-rustfix.rs:16:18 | LL | let _: i32 = await!(/* keep */ &mut ready(1) /* keep */); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -43,7 +43,7 @@ LL | let _: i32 = await!(/* keep */ &mut ready(1) /* keep */); help: `await` is a postfix operation | LL - let _: i32 = await!(/* keep */ &mut ready(1) /* keep */); -LL + let _: i32 = &mut ready(1).await /* keep */); +LL + let _: i32 = (/* keep */ &mut ready(1) /* keep */).await; | error: aborting due to 4 previous errors diff --git a/tests/ui/async-await/await-keyword/incorrect-syntax-suggestions.stderr b/tests/ui/async-await/await-keyword/incorrect-syntax-suggestions.stderr index 0ccde7d8709f1..a6ec0b3f8542d 100644 --- a/tests/ui/async-await/await-keyword/incorrect-syntax-suggestions.stderr +++ b/tests/ui/async-await/await-keyword/incorrect-syntax-suggestions.stderr @@ -187,7 +187,7 @@ LL | let _ = await!(bar()); help: `await` is a postfix operation | LL - let _ = await!(bar()); -LL + let _ = bar().await); +LL + let _ = bar().await; | error: incorrect use of `await` @@ -199,7 +199,7 @@ LL | let _ = await!(bar())?; help: `await` is a postfix operation | LL - let _ = await!(bar())?; -LL + let _ = bar().await)?; +LL + let _ = bar().await?; | error: incorrect use of `await` @@ -211,7 +211,7 @@ LL | let _ = await!(bar())?; help: `await` is a postfix operation | LL - let _ = await!(bar())?; -LL + let _ = bar().await)?; +LL + let _ = bar().await?; | error: incorrect use of `await` @@ -223,7 +223,7 @@ LL | let _ = await!(bar())?; help: `await` is a postfix operation | LL - let _ = await!(bar())?; -LL + let _ = bar().await)?; +LL + let _ = bar().await?; | error: expected expression, found `=>` From f8f1dc284d48ec282abd94e9d43d4c3de149117c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=9D=E5=80=89=E6=B0=B4=E5=B8=8C?= Date: Wed, 23 Sep 2026 23:59:51 +0800 Subject: [PATCH 53/55] Enable EII tests for cg_gcc --- tests/ui/eii/default/call_default.rs | 1 - tests/ui/eii/default/call_default_panics.rs | 1 - tests/ui/eii/default/call_impl.rs | 1 - tests/ui/eii/default/local_crate.rs | 1 - tests/ui/eii/default/local_crate_explicit.rs | 1 - tests/ui/eii/default/local_crate_explicit.stderr | 2 +- tests/ui/eii/duplicate/both_decl_and_impl.rs | 1 - tests/ui/eii/duplicate/both_decl_and_impl.stderr | 8 ++++---- tests/ui/eii/duplicate/duplicate1.rs | 1 - tests/ui/eii/duplicate/duplicate2.rs | 1 - tests/ui/eii/duplicate/duplicate3.rs | 1 - tests/ui/eii/duplicate/dylib_default_duplicate.rs | 1 - tests/ui/eii/duplicate/dylib_default_duplicate.stderr | 2 +- tests/ui/eii/duplicate/multiple_impls.rs | 1 - tests/ui/eii/duplicate/multiple_impls.stderr | 8 ++++---- tests/ui/eii/eii_impl_with_contract.rs | 1 - tests/ui/eii/linking/codegen_cross_crate.rs | 1 - tests/ui/eii/linking/codegen_single_crate.rs | 1 - tests/ui/eii/linking/same-symbol.rs | 1 - tests/ui/eii/linking/track_caller_cross_crate.rs | 1 - tests/ui/eii/privacy1.rs | 1 - tests/ui/eii/shadow_builtin.rs | 1 - tests/ui/eii/shadow_builtin.stderr | 6 +++--- tests/ui/eii/static/argument_required.rs | 1 - tests/ui/eii/static/argument_required.stderr | 2 +- tests/ui/eii/static/cross_crate_decl.rs | 1 - tests/ui/eii/static/cross_crate_def.rs | 1 - tests/ui/eii/static/default.rs | 1 - tests/ui/eii/static/default_apple.rs | 1 - tests/ui/eii/static/default_apple.stderr | 2 +- tests/ui/eii/static/default_cross_crate.rs | 1 - tests/ui/eii/static/default_cross_crate_explicit.rs | 1 - tests/ui/eii/static/default_explicit.rs | 1 - tests/ui/eii/static/duplicate.rs | 1 - tests/ui/eii/static/duplicate.stderr | 2 +- tests/ui/eii/static/mismatch_fn_static.rs | 1 - tests/ui/eii/static/mismatch_fn_static.stderr | 2 +- tests/ui/eii/static/mismatch_mut.rs | 1 - tests/ui/eii/static/mismatch_mut.stderr | 4 ++-- tests/ui/eii/static/mismatch_mut2.rs | 1 - tests/ui/eii/static/mismatch_mut2.stderr | 2 +- tests/ui/eii/static/mismatch_safety.rs | 1 - tests/ui/eii/static/mismatch_safety.stderr | 2 +- tests/ui/eii/static/mismatch_safety2.rs | 1 - tests/ui/eii/static/mismatch_safety2.stderr | 2 +- tests/ui/eii/static/mismatch_static_fn.rs | 1 - tests/ui/eii/static/mismatch_static_fn.stderr | 2 +- tests/ui/eii/static/mut.rs | 1 - tests/ui/eii/static/mut.stderr | 2 +- tests/ui/eii/static/same_address.rs | 1 - tests/ui/eii/static/simple.rs | 1 - tests/ui/eii/static/subtype.rs | 1 - tests/ui/eii/static/subtype_wrong.rs | 1 - tests/ui/eii/static/subtype_wrong.stderr | 2 +- tests/ui/eii/static/wrong_ty.rs | 1 - tests/ui/eii/static/wrong_ty.stderr | 4 ++-- tests/ui/eii/track_caller.rs | 1 - 57 files changed, 27 insertions(+), 67 deletions(-) diff --git a/tests/ui/eii/default/call_default.rs b/tests/ui/eii/default/call_default.rs index 7769ae00bb0f5..db5b7da45d470 100644 --- a/tests/ui/eii/default/call_default.rs +++ b/tests/ui/eii/default/call_default.rs @@ -3,7 +3,6 @@ //@ aux-build: decl_with_default.rs //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests EIIs with default implementations. diff --git a/tests/ui/eii/default/call_default_panics.rs b/tests/ui/eii/default/call_default_panics.rs index d027748b0ffb3..526e71856046b 100644 --- a/tests/ui/eii/default/call_default_panics.rs +++ b/tests/ui/eii/default/call_default_panics.rs @@ -5,7 +5,6 @@ //@ run-pass //@ needs-unwind //@ exec-env:RUST_BACKTRACE=1 -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // A small test to make sure that unwinding works properly. diff --git a/tests/ui/eii/default/call_impl.rs b/tests/ui/eii/default/call_impl.rs index 4b64033a940f5..7f7d5d0163cfb 100644 --- a/tests/ui/eii/default/call_impl.rs +++ b/tests/ui/eii/default/call_impl.rs @@ -4,7 +4,6 @@ //@ aux-build: impl1.rs //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests EIIs with default implementations. diff --git a/tests/ui/eii/default/local_crate.rs b/tests/ui/eii/default/local_crate.rs index d6e992c409453..bf97c8bd2fd68 100644 --- a/tests/ui/eii/default/local_crate.rs +++ b/tests/ui/eii/default/local_crate.rs @@ -1,6 +1,5 @@ //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests EIIs with default implementations. diff --git a/tests/ui/eii/default/local_crate_explicit.rs b/tests/ui/eii/default/local_crate_explicit.rs index 7c29fa0edd6ae..c2e52f8bd174a 100644 --- a/tests/ui/eii/default/local_crate_explicit.rs +++ b/tests/ui/eii/default/local_crate_explicit.rs @@ -1,6 +1,5 @@ //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests EIIs with default implementations. diff --git a/tests/ui/eii/default/local_crate_explicit.stderr b/tests/ui/eii/default/local_crate_explicit.stderr index d80acf14c516d..51b4d9b545b3d 100644 --- a/tests/ui/eii/default/local_crate_explicit.stderr +++ b/tests/ui/eii/default/local_crate_explicit.stderr @@ -1,5 +1,5 @@ warning: function `decl1` is never used - --> $DIR/local_crate_explicit.rs:11:8 + --> $DIR/local_crate_explicit.rs:10:8 | LL | pub fn decl1(x: u64) { | ^^^^^ diff --git a/tests/ui/eii/duplicate/both_decl_and_impl.rs b/tests/ui/eii/duplicate/both_decl_and_impl.rs index a2fc571d3f497..9cf122e40aac3 100644 --- a/tests/ui/eii/duplicate/both_decl_and_impl.rs +++ b/tests/ui/eii/duplicate/both_decl_and_impl.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests that one item can't both define and impl an EII at the same time diff --git a/tests/ui/eii/duplicate/both_decl_and_impl.stderr b/tests/ui/eii/duplicate/both_decl_and_impl.stderr index 1cec485a90cff..d109eaee95a5e 100644 --- a/tests/ui/eii/duplicate/both_decl_and_impl.stderr +++ b/tests/ui/eii/duplicate/both_decl_and_impl.stderr @@ -1,23 +1,23 @@ error: a single item cannot both declare and implement EIIs - --> $DIR/both_decl_and_impl.rs:11:1 + --> $DIR/both_decl_and_impl.rs:10:1 | LL | #[eii] | ^^^^^^ error: only a small subset of attributes are supported on externally implementable items - --> $DIR/both_decl_and_impl.rs:21:1 + --> $DIR/both_decl_and_impl.rs:20:1 | LL | fn d(x: u64) {} | ^^^^^^^^^^^^ | note: this attribute is not supported - --> $DIR/both_decl_and_impl.rs:20:1 + --> $DIR/both_decl_and_impl.rs:19:1 | LL | #[c] | ^^^^ error: `#[c]` function required, but not found - --> $DIR/both_decl_and_impl.rs:16:4 + --> $DIR/both_decl_and_impl.rs:15:4 | LL | fn c(x: u64); | ^ expected because `#[c]` was declared here in crate `both_decl_and_impl` diff --git a/tests/ui/eii/duplicate/duplicate1.rs b/tests/ui/eii/duplicate/duplicate1.rs index 54803f31c85f4..9dc9038bec059 100644 --- a/tests/ui/eii/duplicate/duplicate1.rs +++ b/tests/ui/eii/duplicate/duplicate1.rs @@ -2,7 +2,6 @@ //@[dylib] needs-crate-type: dylib //@ aux-build: impl1.rs //@ aux-build: impl2.rs -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // tests that EIIs error properly, even if the conflicting implementations live in another crate. diff --git a/tests/ui/eii/duplicate/duplicate2.rs b/tests/ui/eii/duplicate/duplicate2.rs index 8b4d7c4913c91..95121e1e6d9b8 100644 --- a/tests/ui/eii/duplicate/duplicate2.rs +++ b/tests/ui/eii/duplicate/duplicate2.rs @@ -1,7 +1,6 @@ //@ aux-build: impl1.rs //@ aux-build: impl2.rs //@ aux-build: impl3.rs -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests the error message when there are multiple implementations of an EII in many crates. diff --git a/tests/ui/eii/duplicate/duplicate3.rs b/tests/ui/eii/duplicate/duplicate3.rs index 96d6543130ccb..d50b87865109b 100644 --- a/tests/ui/eii/duplicate/duplicate3.rs +++ b/tests/ui/eii/duplicate/duplicate3.rs @@ -2,7 +2,6 @@ //@ aux-build: impl2.rs //@ aux-build: impl3.rs //@ aux-build: impl4.rs -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests the error message when there are multiple implementations of an EII in many crates. diff --git a/tests/ui/eii/duplicate/dylib_default_duplicate.rs b/tests/ui/eii/duplicate/dylib_default_duplicate.rs index 5480bb13b4a49..a72a137eb9709 100644 --- a/tests/ui/eii/duplicate/dylib_default_duplicate.rs +++ b/tests/ui/eii/duplicate/dylib_default_duplicate.rs @@ -1,7 +1,6 @@ //@ aux-build: dylib_default.rs //@ needs-crate-type: dylib //@ compile-flags: --emit link -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Regression test for https://github.com/rust-lang/rust/issues/156320. diff --git a/tests/ui/eii/duplicate/dylib_default_duplicate.stderr b/tests/ui/eii/duplicate/dylib_default_duplicate.stderr index 04c6a13710e28..91c90edb02449 100644 --- a/tests/ui/eii/duplicate/dylib_default_duplicate.stderr +++ b/tests/ui/eii/duplicate/dylib_default_duplicate.stderr @@ -1,5 +1,5 @@ error: multiple implementations of `#[eii1]` - --> $DIR/dylib_default_duplicate.rs:15:1 + --> $DIR/dylib_default_duplicate.rs:14:1 | LL | fn other(x: u64) { | ^^^^^^^^^^^^^^^^ first implemented here in crate `dylib_default_duplicate` diff --git a/tests/ui/eii/duplicate/multiple_impls.rs b/tests/ui/eii/duplicate/multiple_impls.rs index 3e541cb16b131..8f1feaa530adb 100644 --- a/tests/ui/eii/duplicate/multiple_impls.rs +++ b/tests/ui/eii/duplicate/multiple_impls.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests that one item can't implement two EIIs diff --git a/tests/ui/eii/duplicate/multiple_impls.stderr b/tests/ui/eii/duplicate/multiple_impls.stderr index efeec635c859a..b2e8871d2087a 100644 --- a/tests/ui/eii/duplicate/multiple_impls.stderr +++ b/tests/ui/eii/duplicate/multiple_impls.stderr @@ -1,17 +1,17 @@ error: a single item cannot implement multiple EIIs - --> $DIR/multiple_impls.rs:15:1 + --> $DIR/multiple_impls.rs:14:1 | LL | #[b] | ^^^^ error: a single item cannot implement multiple EIIs - --> $DIR/multiple_impls.rs:29:1 + --> $DIR/multiple_impls.rs:28:1 | LL | #[d] | ^^^^ error: `#[a]` function required, but not found - --> $DIR/multiple_impls.rs:8:4 + --> $DIR/multiple_impls.rs:7:4 | LL | fn a(x: u64); | ^ expected because `#[a]` was declared here in crate `multiple_impls` @@ -19,7 +19,7 @@ LL | fn a(x: u64); = help: expected at least one implementation in crate `multiple_impls` or any of its dependencies error: `#[c]` static required, but not found - --> $DIR/multiple_impls.rs:21:7 + --> $DIR/multiple_impls.rs:20:7 | LL | #[eii(c)] | ^ expected because `#[c]` was declared here in crate `multiple_impls` diff --git a/tests/ui/eii/eii_impl_with_contract.rs b/tests/ui/eii/eii_impl_with_contract.rs index 1789ec92ea60d..a33d7a0c23751 100644 --- a/tests/ui/eii/eii_impl_with_contract.rs +++ b/tests/ui/eii/eii_impl_with_contract.rs @@ -1,5 +1,4 @@ //@ run-pass -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu diff --git a/tests/ui/eii/linking/codegen_cross_crate.rs b/tests/ui/eii/linking/codegen_cross_crate.rs index ab0d4d4b9eb25..f11c8a04661b6 100644 --- a/tests/ui/eii/linking/codegen_cross_crate.rs +++ b/tests/ui/eii/linking/codegen_cross_crate.rs @@ -2,7 +2,6 @@ //@ check-run-results //@ aux-build: codegen_cross_crate_other_crate.rs //@ compile-flags: -O -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether calling EIIs works with the declaration in another crate. diff --git a/tests/ui/eii/linking/codegen_single_crate.rs b/tests/ui/eii/linking/codegen_single_crate.rs index 4faa30a79040f..81b56cd4261ba 100644 --- a/tests/ui/eii/linking/codegen_single_crate.rs +++ b/tests/ui/eii/linking/codegen_single_crate.rs @@ -1,6 +1,5 @@ //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether calling EIIs works with the declaration in the same crate. diff --git a/tests/ui/eii/linking/same-symbol.rs b/tests/ui/eii/linking/same-symbol.rs index 02518f6bced03..57af6357f3de4 100644 --- a/tests/ui/eii/linking/same-symbol.rs +++ b/tests/ui/eii/linking/same-symbol.rs @@ -1,6 +1,5 @@ //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu #![feature(extern_item_impls)] diff --git a/tests/ui/eii/linking/track_caller_cross_crate.rs b/tests/ui/eii/linking/track_caller_cross_crate.rs index e8ba5dc2b864b..b8d0d53336c39 100644 --- a/tests/ui/eii/linking/track_caller_cross_crate.rs +++ b/tests/ui/eii/linking/track_caller_cross_crate.rs @@ -2,7 +2,6 @@ //@ check-run-results //@ aux-build: track_caller_cross_other_crate.rs //@ compile-flags: -O -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests that `#[track_caller]` on an EII declaration in one crate is derived diff --git a/tests/ui/eii/privacy1.rs b/tests/ui/eii/privacy1.rs index 95544ae867d89..e342441546513 100644 --- a/tests/ui/eii/privacy1.rs +++ b/tests/ui/eii/privacy1.rs @@ -1,7 +1,6 @@ //@ run-pass //@ check-run-results //@ aux-build: other_crate_privacy1.rs -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether re-exports work. diff --git a/tests/ui/eii/shadow_builtin.rs b/tests/ui/eii/shadow_builtin.rs index cd4c14514dc46..f9447b85273af 100644 --- a/tests/ui/eii/shadow_builtin.rs +++ b/tests/ui/eii/shadow_builtin.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether calling EIIs works with the declaration in the same crate. diff --git a/tests/ui/eii/shadow_builtin.stderr b/tests/ui/eii/shadow_builtin.stderr index 33f60b2d2f13f..bdb580785d15c 100644 --- a/tests/ui/eii/shadow_builtin.stderr +++ b/tests/ui/eii/shadow_builtin.stderr @@ -1,5 +1,5 @@ error[E0659]: `inline` is ambiguous - --> $DIR/shadow_builtin.rs:11:3 + --> $DIR/shadow_builtin.rs:10:3 | LL | #[inline] | ^^^^^^ ambiguous name @@ -7,14 +7,14 @@ LL | #[inline] = note: ambiguous because of a name conflict with a builtin attribute = note: `inline` could refer to a built-in attribute note: `inline` could also refer to the attribute macro defined here - --> $DIR/shadow_builtin.rs:7:1 + --> $DIR/shadow_builtin.rs:6:1 | LL | #[eii(inline)] | ^^^^^^^^^^^^^^ = help: use `crate::inline` to refer to this attribute macro unambiguously error: `#[inline]` function required, but not found - --> $DIR/shadow_builtin.rs:7:7 + --> $DIR/shadow_builtin.rs:6:7 | LL | #[eii(inline)] | ^^^^^^ expected because `#[inline]` was declared here in crate `shadow_builtin` diff --git a/tests/ui/eii/static/argument_required.rs b/tests/ui/eii/static/argument_required.rs index 9b00dcf194387..e8f66e083e983 100644 --- a/tests/ui/eii/static/argument_required.rs +++ b/tests/ui/eii/static/argument_required.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/argument_required.stderr b/tests/ui/eii/static/argument_required.stderr index 9e5ee398f77d8..9f722d2065e1e 100644 --- a/tests/ui/eii/static/argument_required.stderr +++ b/tests/ui/eii/static/argument_required.stderr @@ -1,5 +1,5 @@ error: `#[eii]` requires the name as an explicit argument when used on a static - --> $DIR/argument_required.rs:7:1 + --> $DIR/argument_required.rs:6:1 | LL | #[eii] | ^^^^^^ diff --git a/tests/ui/eii/static/cross_crate_decl.rs b/tests/ui/eii/static/cross_crate_decl.rs index 75333c48e40ef..ba038102099f5 100644 --- a/tests/ui/eii/static/cross_crate_decl.rs +++ b/tests/ui/eii/static/cross_crate_decl.rs @@ -2,7 +2,6 @@ //@ check-run-results //@ aux-build: cross_crate_decl.rs //@ compile-flags: -O -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether calling EIIs works with the declaration in another crate. diff --git a/tests/ui/eii/static/cross_crate_def.rs b/tests/ui/eii/static/cross_crate_def.rs index 19dbdaadeef05..0461c1dfe7398 100644 --- a/tests/ui/eii/static/cross_crate_def.rs +++ b/tests/ui/eii/static/cross_crate_def.rs @@ -4,7 +4,6 @@ //@ check-run-results //@ aux-build: cross_crate_def.rs //@ compile-flags: -O -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether calling EIIs works with the declaration and definition in another crate. diff --git a/tests/ui/eii/static/default.rs b/tests/ui/eii/static/default.rs index beb777cc32e82..c4e2208337914 100644 --- a/tests/ui/eii/static/default.rs +++ b/tests/ui/eii/static/default.rs @@ -1,6 +1,5 @@ //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // FIXME(#157649): static EII defaults currently fail to link on Apple targets. diff --git a/tests/ui/eii/static/default_apple.rs b/tests/ui/eii/static/default_apple.rs index f638ee480f4cb..38446a2d17150 100644 --- a/tests/ui/eii/static/default_apple.rs +++ b/tests/ui/eii/static/default_apple.rs @@ -1,5 +1,4 @@ //@ only-apple -//@ ignore-backends: gcc #![feature(extern_item_impls)] #![crate_type = "lib"] diff --git a/tests/ui/eii/static/default_apple.stderr b/tests/ui/eii/static/default_apple.stderr index d4ced5948970d..3fbcde6dbb9a5 100644 --- a/tests/ui/eii/static/default_apple.stderr +++ b/tests/ui/eii/static/default_apple.stderr @@ -1,5 +1,5 @@ error: `#[eii]` cannot be used on statics with a value on Apple targets - --> $DIR/default_apple.rs:7:25 + --> $DIR/default_apple.rs:6:25 | LL | pub static DECL1: u64 = 5; | ^ diff --git a/tests/ui/eii/static/default_cross_crate.rs b/tests/ui/eii/static/default_cross_crate.rs index 9d6df257ffbef..a53c73876d977 100644 --- a/tests/ui/eii/static/default_cross_crate.rs +++ b/tests/ui/eii/static/default_cross_crate.rs @@ -3,7 +3,6 @@ //@ aux-build: decl_with_default.rs //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // FIXME(#157649): static EII defaults currently fail to link on Apple targets. diff --git a/tests/ui/eii/static/default_cross_crate_explicit.rs b/tests/ui/eii/static/default_cross_crate_explicit.rs index 6ad015b9b601b..4835e2ed323ab 100644 --- a/tests/ui/eii/static/default_cross_crate_explicit.rs +++ b/tests/ui/eii/static/default_cross_crate_explicit.rs @@ -4,7 +4,6 @@ //@ aux-build: impl_default_override.rs //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // FIXME(#157649): static EII defaults currently fail to link on Apple targets. diff --git a/tests/ui/eii/static/default_explicit.rs b/tests/ui/eii/static/default_explicit.rs index 8237b18106031..bc3f44e7e01b5 100644 --- a/tests/ui/eii/static/default_explicit.rs +++ b/tests/ui/eii/static/default_explicit.rs @@ -1,6 +1,5 @@ //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // FIXME(#157649): static EII defaults currently fail to link on Apple targets. diff --git a/tests/ui/eii/static/duplicate.rs b/tests/ui/eii/static/duplicate.rs index a8db02f33051d..54cab82f617e7 100644 --- a/tests/ui/eii/static/duplicate.rs +++ b/tests/ui/eii/static/duplicate.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/duplicate.stderr b/tests/ui/eii/static/duplicate.stderr index 270664c8c74c6..e967e9c4e65c8 100644 --- a/tests/ui/eii/static/duplicate.stderr +++ b/tests/ui/eii/static/duplicate.stderr @@ -1,5 +1,5 @@ error: multiple implementations of `#[hello]` - --> $DIR/duplicate.rs:11:1 + --> $DIR/duplicate.rs:10:1 | LL | static HELLO_IMPL1: u64 = 5; | ^^^^^^^^^^^^^^^^^^^^^^^ first implemented here in crate `duplicate` diff --git a/tests/ui/eii/static/mismatch_fn_static.rs b/tests/ui/eii/static/mismatch_fn_static.rs index 8d8fa5b4d06c4..49003388526f8 100644 --- a/tests/ui/eii/static/mismatch_fn_static.rs +++ b/tests/ui/eii/static/mismatch_fn_static.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/mismatch_fn_static.stderr b/tests/ui/eii/static/mismatch_fn_static.stderr index e8fa5f85b1f13..e8be6ccb7ef57 100644 --- a/tests/ui/eii/static/mismatch_fn_static.stderr +++ b/tests/ui/eii/static/mismatch_fn_static.stderr @@ -1,5 +1,5 @@ error: `#[hello]` must be used on a function - --> $DIR/mismatch_fn_static.rs:10:1 + --> $DIR/mismatch_fn_static.rs:9:1 | LL | #[hello] | ^^^^^^^^ diff --git a/tests/ui/eii/static/mismatch_mut.rs b/tests/ui/eii/static/mismatch_mut.rs index 61c806fb976ca..b0e35b2bf84dc 100644 --- a/tests/ui/eii/static/mismatch_mut.rs +++ b/tests/ui/eii/static/mismatch_mut.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/mismatch_mut.stderr b/tests/ui/eii/static/mismatch_mut.stderr index 537ac0de3c3a2..0812652a6a1bc 100644 --- a/tests/ui/eii/static/mismatch_mut.stderr +++ b/tests/ui/eii/static/mismatch_mut.stderr @@ -1,11 +1,11 @@ error: `#[eii]` cannot be used on mutable statics - --> $DIR/mismatch_mut.rs:7:1 + --> $DIR/mismatch_mut.rs:6:1 | LL | #[eii(hello)] | ^^^^^^^^^^^^^ error: mutability does not match with the definition of`#[hello]` - --> $DIR/mismatch_mut.rs:11:1 + --> $DIR/mismatch_mut.rs:10:1 | LL | #[hello] | ^^^^^^^^ diff --git a/tests/ui/eii/static/mismatch_mut2.rs b/tests/ui/eii/static/mismatch_mut2.rs index ff21af5f0d714..f083e713d1769 100644 --- a/tests/ui/eii/static/mismatch_mut2.rs +++ b/tests/ui/eii/static/mismatch_mut2.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/mismatch_mut2.stderr b/tests/ui/eii/static/mismatch_mut2.stderr index 6ac3df57697db..b4b953d61a23f 100644 --- a/tests/ui/eii/static/mismatch_mut2.stderr +++ b/tests/ui/eii/static/mismatch_mut2.stderr @@ -1,5 +1,5 @@ error: mutability does not match with the definition of`#[hello]` - --> $DIR/mismatch_mut2.rs:10:1 + --> $DIR/mismatch_mut2.rs:9:1 | LL | #[hello] | ^^^^^^^^ diff --git a/tests/ui/eii/static/mismatch_safety.rs b/tests/ui/eii/static/mismatch_safety.rs index b8d503adc2d21..667c6fff14189 100644 --- a/tests/ui/eii/static/mismatch_safety.rs +++ b/tests/ui/eii/static/mismatch_safety.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/mismatch_safety.stderr b/tests/ui/eii/static/mismatch_safety.stderr index d4fa85778ae11..19a3b0dd86639 100644 --- a/tests/ui/eii/static/mismatch_safety.stderr +++ b/tests/ui/eii/static/mismatch_safety.stderr @@ -1,5 +1,5 @@ error: safety does not match with the definition of`#[hello]` - --> $DIR/mismatch_safety.rs:10:1 + --> $DIR/mismatch_safety.rs:9:1 | LL | #[hello] | ^^^^^^^^ diff --git a/tests/ui/eii/static/mismatch_safety2.rs b/tests/ui/eii/static/mismatch_safety2.rs index 412e2a694c1fc..5240184f6ada5 100644 --- a/tests/ui/eii/static/mismatch_safety2.rs +++ b/tests/ui/eii/static/mismatch_safety2.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/mismatch_safety2.stderr b/tests/ui/eii/static/mismatch_safety2.stderr index 6957a6202b614..c49f55f2b911a 100644 --- a/tests/ui/eii/static/mismatch_safety2.stderr +++ b/tests/ui/eii/static/mismatch_safety2.stderr @@ -1,5 +1,5 @@ error: static items cannot be declared with `unsafe` safety qualifier outside of `extern` block - --> $DIR/mismatch_safety2.rs:11:1 + --> $DIR/mismatch_safety2.rs:10:1 | LL | unsafe static HELLO_IMPL: u64 = 5; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/ui/eii/static/mismatch_static_fn.rs b/tests/ui/eii/static/mismatch_static_fn.rs index 06e3d95b7cdcb..b4fd139959d68 100644 --- a/tests/ui/eii/static/mismatch_static_fn.rs +++ b/tests/ui/eii/static/mismatch_static_fn.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/mismatch_static_fn.stderr b/tests/ui/eii/static/mismatch_static_fn.stderr index 639e3cfa3beb3..56ea50c00fdd0 100644 --- a/tests/ui/eii/static/mismatch_static_fn.stderr +++ b/tests/ui/eii/static/mismatch_static_fn.stderr @@ -1,5 +1,5 @@ error: `#[hello]` must be used on a static - --> $DIR/mismatch_static_fn.rs:10:1 + --> $DIR/mismatch_static_fn.rs:9:1 | LL | #[hello] | ^^^^^^^^ diff --git a/tests/ui/eii/static/mut.rs b/tests/ui/eii/static/mut.rs index 4c9d84061fb25..c351a6df8fbd2 100644 --- a/tests/ui/eii/static/mut.rs +++ b/tests/ui/eii/static/mut.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/mut.stderr b/tests/ui/eii/static/mut.stderr index cd3a0ca23c7f4..299737d1f07b6 100644 --- a/tests/ui/eii/static/mut.stderr +++ b/tests/ui/eii/static/mut.stderr @@ -1,5 +1,5 @@ error: `#[eii]` cannot be used on mutable statics - --> $DIR/mut.rs:7:1 + --> $DIR/mut.rs:6:1 | LL | #[eii(hello)] | ^^^^^^^^^^^^^ diff --git a/tests/ui/eii/static/same_address.rs b/tests/ui/eii/static/same_address.rs index de316b9f5f9ff..a49ffd9a552b7 100644 --- a/tests/ui/eii/static/same_address.rs +++ b/tests/ui/eii/static/same_address.rs @@ -1,6 +1,5 @@ //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs and their declarations share the same address diff --git a/tests/ui/eii/static/simple.rs b/tests/ui/eii/static/simple.rs index a592609e9153a..84d49bcef638a 100644 --- a/tests/ui/eii/static/simple.rs +++ b/tests/ui/eii/static/simple.rs @@ -1,6 +1,5 @@ //@ run-pass //@ check-run-results -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests whether EIIs work on statics diff --git a/tests/ui/eii/static/subtype.rs b/tests/ui/eii/static/subtype.rs index 4553d7b8c59c0..a417838e2f7b0 100644 --- a/tests/ui/eii/static/subtype.rs +++ b/tests/ui/eii/static/subtype.rs @@ -1,5 +1,4 @@ //@ check-pass -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests that mismatching types of the declaration and definition are rejected diff --git a/tests/ui/eii/static/subtype_wrong.rs b/tests/ui/eii/static/subtype_wrong.rs index ac975592a0f08..9d67d7b458cc7 100644 --- a/tests/ui/eii/static/subtype_wrong.rs +++ b/tests/ui/eii/static/subtype_wrong.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests that mismatching types of the declaration and definition are rejected diff --git a/tests/ui/eii/static/subtype_wrong.stderr b/tests/ui/eii/static/subtype_wrong.stderr index a20074947c15d..77a1353aecc32 100644 --- a/tests/ui/eii/static/subtype_wrong.stderr +++ b/tests/ui/eii/static/subtype_wrong.stderr @@ -1,5 +1,5 @@ error[E0308]: mismatched types - --> $DIR/subtype_wrong.rs:13:1 + --> $DIR/subtype_wrong.rs:12:1 | LL | static HELLO_IMPL: for<'a> fn(&'a u8) -> &'a u8 = |_| todo!(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ one type is more general than the other diff --git a/tests/ui/eii/static/wrong_ty.rs b/tests/ui/eii/static/wrong_ty.rs index 40b7859b06b01..4f320603c512f 100644 --- a/tests/ui/eii/static/wrong_ty.rs +++ b/tests/ui/eii/static/wrong_ty.rs @@ -1,4 +1,3 @@ -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests that mismatching types of the declaration and definition are rejected diff --git a/tests/ui/eii/static/wrong_ty.stderr b/tests/ui/eii/static/wrong_ty.stderr index 5095513527747..b7569ea82b77f 100644 --- a/tests/ui/eii/static/wrong_ty.stderr +++ b/tests/ui/eii/static/wrong_ty.stderr @@ -1,11 +1,11 @@ error[E0806]: static `HELLO_IMPL` has a type that is incompatible with the declaration of `#[hello]` - --> $DIR/wrong_ty.rs:13:1 + --> $DIR/wrong_ty.rs:12:1 | LL | static HELLO_IMPL: bool = true; | ^^^^^^^^^^^^^^^^^^^^^^^ | note: expected this because of this attribute - --> $DIR/wrong_ty.rs:12:1 + --> $DIR/wrong_ty.rs:11:1 | LL | #[hello] | ^^^^^^^^ diff --git a/tests/ui/eii/track_caller.rs b/tests/ui/eii/track_caller.rs index 5e298fba9675a..52807428f20d0 100644 --- a/tests/ui/eii/track_caller.rs +++ b/tests/ui/eii/track_caller.rs @@ -1,5 +1,4 @@ //@ run-pass -//@ ignore-backends: gcc // FIXME(#125418): linking on Windows GNU targets is not yet supported. //@ ignore-windows-gnu // Tests that `#[track_caller]` on an EII declaration is threaded through both From 67afe3876d125429c626a3d92199480c992df385 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 23 Sep 2026 22:47:46 +0200 Subject: [PATCH 54/55] Update rustc version to `2026-09-24` --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f3106ee20c331..a7774784e3afa 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2026-09-22" +channel = "nightly-2026-09-24" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 5ed0180b646917d5bd4c414ab6ffa5d3e9d317f0 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:03:01 +0000 Subject: [PATCH 55/55] Build a new incr comp session dir from scratch every time Rather than copying the old incr comp dir and then modifying it. This saves a copy/hardlink for files that are modified. And it removes the need for accurate work product tracking to avoid accumulating cruft, which is non-trivial. We don't accurately track the pre-LTO bitcode files for ThinLTO for example. --- .../rustc_codegen_cranelift/src/driver/aot.rs | 1 - compiler/rustc_codegen_llvm/src/back/lto.rs | 36 +- compiler/rustc_codegen_ssa/src/back/write.rs | 50 ++- compiler/rustc_codegen_ssa/src/base.rs | 4 + compiler/rustc_codegen_ssa/src/lib.rs | 2 - compiler/rustc_incremental/src/diagnostics.rs | 22 -- compiler/rustc_incremental/src/lib.rs | 3 +- compiler/rustc_incremental/src/persist/fs.rs | 338 +++++++----------- .../rustc_incremental/src/persist/fs/tests.rs | 27 +- .../rustc_incremental/src/persist/load.rs | 43 +-- compiler/rustc_incremental/src/persist/mod.rs | 2 +- .../rustc_incremental/src/persist/save.rs | 19 +- .../src/persist/work_product.rs | 24 +- compiler/rustc_interface/src/queries.rs | 1 - compiler/rustc_metadata/src/rmeta/encoder.rs | 5 +- compiler/rustc_session/src/session.rs | 9 +- .../run-make/incremental-session-gc/rmake.rs | 3 - 17 files changed, 216 insertions(+), 373 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index cd82df386f5ce..73d4adf8e3cf2 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -140,7 +140,6 @@ fn emit_module( bytecode: None, assembly: None, llvm_ir: None, - links_from_incr_cache: Vec::new(), }) } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index cb12c103834af..d52e34d30fb7d 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -463,23 +463,29 @@ fn thin_lto( info!("thin LTO data created"); - let (key_map_path, prev_key_map, curr_key_map) = if let Some(ref incr_comp_session_dir) = - cgcx.incr_comp_session_dir + let new_key_map_path = cgcx + .new_incr_comp_session_dir + .as_ref() + .map(|dir| dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME)); + + let prev_key_map = if let Some(ref old_incr_comp_session_dir) = + cgcx.old_incr_comp_session_dir { - let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME); + let old_path = old_incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME); + // If the previous file was deleted, or we get an IO error // reading the file, then we'll just use `None` as the // prev_key_map, which will force the code to be recompiled. - let prev = - if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None }; - let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names); - (Some(path), prev, curr) + if old_path.exists() { ThinLTOKeysMap::load_from_file(&old_path).ok() } else { None } + } else { + assert!(green_modules.is_empty()); + None + }; + let curr_key_map = if cgcx.new_incr_comp_session_dir.is_some() { + ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names) } else { - // If we don't compile incrementally, we don't need to load the - // import data from LLVM. assert!(green_modules.is_empty()); - let curr = ThinLTOKeysMap::default(); - (None, None, curr) + ThinLTOKeysMap::default() }; info!("thin LTO cache key map loaded"); info!("prev_key_map: {:#?}", prev_key_map); @@ -500,7 +506,8 @@ fn thin_lto( if let (Some(prev_key_map), true) = (prev_key_map.as_ref(), green_modules.contains_key(module_name)) { - assert!(cgcx.incr_comp_session_dir.is_some()); + assert!(cgcx.old_incr_comp_session_dir.is_some()); + assert!(cgcx.new_incr_comp_session_dir.is_some()); // If a module exists in both the current and the previous session, // and has the same LTO cache key in both sessions, then we can re-use it @@ -508,7 +515,6 @@ fn thin_lto( let work_product = green_modules[module_name].clone(); copy_jobs.push(work_product); info!(" - {}: re-used", module_name); - assert!(cgcx.incr_comp_session_dir.is_some()); continue; } } @@ -518,8 +524,8 @@ fn thin_lto( } // Save the current ThinLTO import information for the next compilation - // session, overwriting the previous serialized data (if any). - if let Some(path) = key_map_path + // session. + if let Some(path) = new_key_map_path && let Err(err) = curr_key_map.save_to_file(&path) { write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err }); diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 0f609464593dc..7633ae31af38f 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -15,7 +15,9 @@ use rustc_errors::{ Level, MultiSpan, Style, Sublevel, Suggestions, catch_fatal_errors, }; use rustc_fs_util::link_or_copy; -use rustc_incremental::{copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir_sess}; +use rustc_incremental::{ + copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir_sess, in_old_incr_comp_dir_sess, +}; use rustc_macros::{Decodable, Encodable}; use rustc_metadata::fs::copy_to_stdout; use rustc_middle::dep_graph::{WorkProduct, WorkProductMap}; @@ -351,9 +353,12 @@ pub struct CodegenContext { /// Directory into which should the LLVM optimization remarks be written. /// If `None`, they will be written to stderr. pub remark_dir: Option, + /// The previous incremental compilation session directory, or None if we + /// are not compiling incrementally or there is no previous session. + pub old_incr_comp_session_dir: Option, /// The incremental compilation session directory, or None if we are not /// compiling incrementally - pub incr_comp_session_dir: Option, + pub new_incr_comp_session_dir: Option, /// `Some(limit)` if the codegen should be run in parallel. /// /// Depends on [`WriteBackendMethods::supports_parallel()`] and `--jobs-backend`. @@ -497,7 +502,6 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( incr_comp_session.unwrap(), &module.name, files.as_slice(), - &module.links_from_incr_cache, ); work_products.insert(id, product); } @@ -839,7 +843,7 @@ fn execute_optimize_work_item( // save our module to disk first. let bitcode = if cgcx.module_config.emit_pre_lto_bc { let filename = pre_lto_bitcode_filename(&module.name); - cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) + cgcx.new_incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) } else { None }; @@ -886,11 +890,9 @@ fn execute_copy_from_cache_work_item( let dcx = DiagCtxt::new(Box::new(shared_emitter)); let dcx = dcx.handle(); - let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap(); - - let mut links_from_incr_cache = Vec::new(); + let incr_comp_session_dir = cgcx.old_incr_comp_session_dir.as_ref().unwrap(); - let mut load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| { + let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| { let source_file_in_incr_comp_dir = incr_comp_session_dir.join(saved_path); debug!( "copying preexisting module `{}` from {:?} to {}", @@ -899,10 +901,7 @@ fn execute_copy_from_cache_work_item( output_path.display() ); match link_or_copy(&source_file_in_incr_comp_dir, &output_path) { - Ok(_) => { - links_from_incr_cache.push(source_file_in_incr_comp_dir); - Some(output_path) - } + Ok(_) => Some(output_path), Err(error) => { dcx.emit_err(diagnostics::CopyPathBuf { source_file: source_file_in_incr_comp_dir, @@ -925,7 +924,7 @@ fn execute_copy_from_cache_work_item( load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file) }); - let mut load_from_incr_cache = |perform, output_type: OutputType| { + let load_from_incr_cache = |perform, output_type: OutputType| { if perform { let saved_file = module.source.saved_files.get(output_type.extension())?; let output_path = cgcx.output_filenames.temp_path_for_cgu(output_type, &module.name); @@ -953,7 +952,6 @@ fn execute_copy_from_cache_work_item( } CompiledModule { - links_from_incr_cache, kind: ModuleKind::Regular, name: module.name, object, @@ -1294,10 +1292,15 @@ fn start_executing_work( time_trace: sess.opts.unstable_opts.llvm_time_trace, remark: sess.opts.cg.remark.clone(), remark_dir, - incr_comp_session_dir: tcx + old_incr_comp_session_dir: tcx + .incr_comp_session + .as_ref() + .and_then(|incr_comp_session| incr_comp_session.old_session_directory.as_deref()) + .map(ToOwned::to_owned), + new_incr_comp_session_dir: tcx .incr_comp_session .as_ref() - .map(|incr_comp_session| (&*incr_comp_session.session_directory).to_owned()), + .map(|incr_comp_session| (&*incr_comp_session.new_session_directory).to_owned()), output_filenames: Arc::clone(tcx.output_filenames(())), module_config: regular_config, opt_level, @@ -2262,7 +2265,22 @@ pub(crate) fn submit_pre_lto_module_to_llvm( module: CachedModuleCodegen, ) { let filename = pre_lto_bitcode_filename(&module.name); + let old_bitcode_path = + in_old_incr_comp_dir_sess(tcx.incr_comp_session.unwrap(), &filename).unwrap(); let bitcode_path = in_incr_comp_dir_sess(tcx.incr_comp_session.unwrap(), &filename); + + match link_or_copy(&old_bitcode_path, &bitcode_path) { + Ok(_) => {} + Err(error) => { + tcx.sess.dcx().emit_err(diagnostics::CopyPathBuf { + source_file: old_bitcode_path, + output_path: bitcode_path, + error, + }); + return; + } + } + // Schedule the module to be loaded drop( coordinator diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index de921a9cb0d1e..2c15a55c84334 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -877,6 +877,10 @@ pub fn codegen_crate< source: cgu.previous_work_product(tcx), }, ); + // This will unwind if there are errors, which triggers our `AbortCodegenOnDrop` + // guard. Unfortunately, just skipping the `submit_pre_lto_module_to_llvm` makes + // compilation hang on post-monomorphization errors. + tcx.dcx().abort_if_errors(); } CguReuse::PostLto => { submit_post_lto_module_to_llvm( diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 1272b26ca0612..5ae8ba4f7fded 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -114,7 +114,6 @@ impl ModuleCodegen { bytecode, assembly, llvm_ir, - links_from_incr_cache: Vec::new(), } } } @@ -129,7 +128,6 @@ pub struct CompiledModule { pub bytecode: Option, pub assembly: Option, // --emit=asm pub llvm_ir: Option, // --emit=llvm-ir, llvm-bc is in bytecode - pub links_from_incr_cache: Vec, } impl CompiledModule { diff --git a/compiler/rustc_incremental/src/diagnostics.rs b/compiler/rustc_incremental/src/diagnostics.rs index 6e291b7ea3abb..b9ac4662dcbdc 100644 --- a/compiler/rustc_incremental/src/diagnostics.rs +++ b/compiler/rustc_incremental/src/diagnostics.rs @@ -169,21 +169,6 @@ pub(crate) struct DeleteLock<'a> { pub err: std::io::Error, } -#[derive(Diagnostic)] -#[diag( - "hard linking files in the incremental compilation cache failed. copying files instead. consider moving the cache directory to a file system which supports hard linking in session dir `{$path}`" -)] -pub(crate) struct HardLinkFailed<'a> { - pub path: &'a Path, -} - -#[derive(Diagnostic)] -#[diag("failed to delete partly initialized session dir `{$path}`: {$err}")] -pub(crate) struct DeletePartial<'a> { - pub path: &'a Path, - pub err: std::io::Error, -} - #[derive(Diagnostic)] #[diag("did not finalize incremental compilation session directory `{$path}`: {$err}")] #[help("the next build will not be able to reuse work from this compilation")] @@ -266,13 +251,6 @@ pub(crate) struct CopyWorkProductToCache<'a> { pub err: std::io::Error, } -#[derive(Diagnostic)] -#[diag("file-system error deleting outdated file `{$path}`: {$err}")] -pub(crate) struct DeleteWorkProduct<'a> { - pub path: &'a Path, - pub err: std::io::Error, -} - #[derive(Diagnostic)] #[diag( "corrupt incremental compilation artifact found at `{$path}`. This file will automatically be ignored and deleted. If you see this message repeatedly or can provoke it without manually manipulating the compiler's artifacts, please file an issue. The incremental compilation system relies on hardlinks and filesystem locks behaving correctly, and may not deal well with OS crashes, so whatever information you can provide about your filesystem or other state may be very relevant" diff --git a/compiler/rustc_incremental/src/lib.rs b/compiler/rustc_incremental/src/lib.rs index 83646cb086d8d..b5470d224ffbd 100644 --- a/compiler/rustc_incremental/src/lib.rs +++ b/compiler/rustc_incremental/src/lib.rs @@ -3,6 +3,7 @@ // tidy-alphabetical-start #![deny(missing_docs)] #![feature(file_buffered)] +#![feature(try_blocks)] // tidy-alphabetical-end mod assert_dep_graph; @@ -11,7 +12,7 @@ mod persist; pub use persist::{ copy_cgu_workproduct_to_incr_comp_cache_dir, finalize_session_directory, in_incr_comp_dir_sess, - load_query_result_cache, save_work_product_index, setup_dep_graph, + in_old_incr_comp_dir_sess, load_query_result_cache, save_work_product_index, setup_dep_graph, }; use rustc_middle::util::Providers; diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs index 374e0b0dd6d1a..3f896584eaef2 100644 --- a/compiler/rustc_incremental/src/persist/fs.rs +++ b/compiler/rustc_incremental/src/persist/fs.rs @@ -1,14 +1,14 @@ //! This module manages how the incremental compilation cache is represented in //! the file system. //! -//! Incremental compilation caches are managed according to a copy-on-write -//! strategy: Once a complete, consistent cache version is finalized, it is -//! never modified. Instead, when a subsequent compilation session is started, -//! the compiler will allocate a new version of the cache that starts out as -//! a copy of the previous version. Then only this new copy is modified and it -//! will not be visible to other processes until it is finalized. This ensures -//! that multiple compiler processes can be executed concurrently for the same -//! crate without interfering with each other or blocking each other. +//! Incremental compilation caches are managed according to a rebuild from +//! scratch strategy: Once a complete, consistent cache version is finalized, it +//! is never modified. Instead, when a subsequent compilation session is started, +//! the compiler will allocate a new version of the cache that starts out empty. +//! Then only this new directory is written to and it will not be visible to +//! other processes until it is finalized. This ensures that multiple compiler +//! processes can be executed concurrently for the same crate without +//! interfering with each other or blocking each other. //! //! More concretely this is implemented via the following protocol: //! @@ -22,12 +22,12 @@ //! considered finalized if the "-working" suffix in the directory name has //! been replaced by the SVH of the crate. //! 3. Once the compiler has found a valid, finalized session directory, it will -//! hard-link/copy its contents into the new "-working" directory. If all -//! goes well, it will have its own, private copy of the source directory and -//! subsequently not have to worry about synchronizing with other compiler -//! processes. +//! obtain a shared lock on the directory. If this succeeds, it will have +//! read-only access to the old session directory without having to worry +//! about synchronizing with other compiler processes. //! 4. Now the compiler can do its normal compilation process, which involves -//! reading and updating its private session directory. +//! writing to its private session directory. Possibly by hardlinking +//! existing files from the old session directory if they haven't changed. //! 5. When compilation finishes without errors, the private session directory //! will be in a state where it can be used as input for other compilation //! sessions. That is, it will contain a dependency graph and cache artifacts @@ -71,23 +71,15 @@ //! //! Another case that has to be considered is what happens if one process //! deletes a finalized session directory that another process is currently -//! trying to copy from. This case is also handled via the lock file. Before -//! a process starts copying a finalized session directory, it will acquire a -//! shared lock on the directory's lock file. Any garbage collecting process, -//! on the other hand, will acquire an exclusive lock on the lock file. -//! Thus, if a directory is being collected, any reader process will fail -//! acquiring the shared lock and will leave the directory alone. Conversely, -//! if a collecting process can't acquire the exclusive lock because the -//! directory is currently being read from, it will leave collecting that -//! directory to another process at a later point in time. -//! The exact same scheme is also used when reading the metadata hashes file -//! from an extern crate. When a crate is compiled, the hash values of its -//! metadata are stored in a file in its session directory. When the -//! compilation session of another crate imports the first crate's metadata, -//! it also has to read in the accompanying metadata hashes. It thus will access -//! the finalized session directory of all crates it links to and while doing -//! so, it will also place a read lock on that the respective session directory -//! so that it won't be deleted while the metadata hashes are loaded. +//! reading from. This case is also handled via the lock file. Before a process +//! starts reading from a finalized session directory, it will acquire a shared +//! lock on the directory's lock file. Any garbage collecting process, on the +//! other hand, will acquire an exclusive lock on the lock file. Thus, if a +//! directory is being collected, any reader process will fail acquiring the +//! shared lock and will leave the directory alone. Conversely, if a collecting +//! process can't acquire the exclusive lock because the directory is currently +//! being read from, it will leave collecting that directory to another process +//! at a later point in time. //! //! ## Preconditions //! @@ -110,11 +102,11 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use rand::{RngCore, rng}; use rustc_data_structures::base_n::{BaseNString, CASE_INSENSITIVE, ToBaseN}; -use rustc_data_structures::fx::{FxHashSet, FxIndexSet}; +use rustc_data_structures::fx::FxIndexSet; use rustc_data_structures::svh::Svh; use rustc_data_structures::unord::{UnordMap, UnordSet}; use rustc_data_structures::{base_n, flock}; -use rustc_fs_util::{LinkOrCopy, link_or_copy, try_canonicalize}; +use rustc_fs_util::try_canonicalize; use rustc_middle::dep_graph::WorkProduct; use rustc_session::config::OutputType; use rustc_session::{IncrCompSession, Session, StableCrateId}; @@ -138,6 +130,11 @@ const QUERY_CACHE_FILENAME: &str = "query-cache.bin"; // case-sensitive (as opposed to base64, for example). const INT_ENCODE_BASE: usize = base_n::CASE_INSENSITIVE; +/// Returns the path to a previous session's dependency graph. +pub(crate) fn old_dep_graph_path(incr_comp_session: &IncrCompSession) -> Option { + in_old_incr_comp_dir_sess(incr_comp_session, DEP_GRAPH_FILENAME) +} + /// Returns the path to a session's dependency graph. pub(crate) fn dep_graph_path(incr_comp_session: &IncrCompSession) -> PathBuf { in_incr_comp_dir_sess(incr_comp_session, DEP_GRAPH_FILENAME) @@ -151,10 +148,19 @@ pub(crate) fn staging_dep_graph_path(incr_comp_session: &IncrCompSession) -> Pat in_incr_comp_dir_sess(incr_comp_session, STAGING_DEP_GRAPH_FILENAME) } +pub(crate) fn old_work_products_path(incr_comp_session: &IncrCompSession) -> Option { + in_old_incr_comp_dir_sess(incr_comp_session, WORK_PRODUCTS_FILENAME) +} + pub(crate) fn work_products_path(incr_comp_session: &IncrCompSession) -> PathBuf { in_incr_comp_dir_sess(incr_comp_session, WORK_PRODUCTS_FILENAME) } +/// Returns the path to a previous session's query cache. +pub(crate) fn old_query_cache_path(incr_comp_session: &IncrCompSession) -> Option { + in_old_incr_comp_dir_sess(incr_comp_session, QUERY_CACHE_FILENAME) +} + /// Returns the path to a session's query cache. pub(crate) fn query_cache_path(incr_comp_session: &IncrCompSession) -> PathBuf { in_incr_comp_dir_sess(incr_comp_session, QUERY_CACHE_FILENAME) @@ -182,10 +188,19 @@ fn lock_file_path(session_dir: &Path) -> PathBuf { crate_dir.join(&directory_name[0..dash_indices[2]]).with_extension(&LOCK_FILE_EXT[1..]) } +/// Returns the path for a given filename within the incremental compilation directory +/// in the previous session. +pub fn in_old_incr_comp_dir_sess( + incr_comp_session: &IncrCompSession, + file_name: &str, +) -> Option { + incr_comp_session.old_session_directory.as_ref().map(|dir| dir.join(file_name)) +} + /// Returns the path for a given filename within the incremental compilation directory /// in the current session. pub fn in_incr_comp_dir_sess(incr_comp_session: &IncrCompSession, file_name: &str) -> PathBuf { - incr_comp_session.session_directory.join(file_name) + incr_comp_session.new_session_directory.join(file_name) } /// Allocates the private session directory. @@ -230,62 +245,30 @@ pub(crate) fn prepare_session_directory( } }; - let mut source_directories_already_tried = FxHashSet::default(); - - loop { - // Generate a session directory of the form: - // - // {incr-comp-dir}/{crate-name-and-disambiguator}/s-{timestamp}-{random}-working - let session_directory = generate_session_dir_path(&crate_dir); - debug!("session-dir: {}", session_directory.display()); - - // Lock the new session directory. If this fails, return an - // error without retrying - let (session_directory, lock_file_path) = - lock_and_create_directory(sess, &session_directory); - - // Find a suitable source directory to copy from. Ignore those that we - // have already tried before. - let source_directory = find_source_directory(&crate_dir, &source_directories_already_tried); - - let Some(source_directory) = source_directory else { - // There's nowhere to copy from, we're done - debug!( - "no source directory found. Continuing with empty session \ - directory." - ); - - return IncrCompSession { session_directory }; - }; - - debug!("attempting to copy data from source: {}", source_directory.display()); - - // Try copying over all files from the source directory - if let Ok(allows_links) = copy_files(sess, &session_directory, &source_directory) { - debug!("successfully copied data from: {}", source_directory.display()); + // Generate a session directory of the form: + // + // {incr-comp-dir}/{crate-name-and-disambiguator}/s-{timestamp}-{random}-working + let new_session_dir = generate_session_dir_path(&crate_dir); + debug!("session-dir: {}", new_session_dir.display()); - if !allows_links { - sess.dcx().emit_warn(diagnostics::HardLinkFailed { path: &session_directory }); - } - - return IncrCompSession { session_directory }; - } else { - debug!("copying failed - trying next directory"); + // Lock the new session directory. If this fails, return an + // error without retrying + let new_session_directory = lock_directory(sess, &new_session_dir, true /* new_session */) + .expect("should emit fatal error on lock fail"); - // Something went wrong while trying to copy/link files from the - // source directory. Try again with a different one. - source_directories_already_tried.insert(source_directory); + // Find a suitable source directory to copy from. Ignore those that we + // have already tried before. + let old_source_directory = find_source_directory(sess, &crate_dir); - // Try to remove the session directory we just allocated. We don't - // know if there's any garbage in it from the failed copy action. - if let Err(err) = std_fs::remove_dir_all(&*session_directory) { - sess.dcx().emit_warn(diagnostics::DeletePartial { path: &session_directory, err }); - } + let old_session_directory = if let Some(old_source_directory) = old_source_directory { + debug!("attempting to use: {}", old_source_directory.display()); + Some(old_source_directory) + } else { + debug!("no source directory found. Continuing with empty session directory."); + None + }; - delete_session_dir_lock_file(sess, &lock_file_path); - drop(session_directory); - } - } + IncrCompSession { old_session_directory, new_session_directory } } /// This function finalizes and thus 'publishes' the session directory by @@ -301,13 +284,13 @@ pub fn finalize_session_directory( if sess.opts.incremental.is_none() { return; } - let incr_comp_session = incr_comp_session.unwrap(); + let mut incr_comp_session = incr_comp_session.unwrap(); // The svh is always produced when incr. comp. is enabled. let svh = svh.unwrap(); let _timer = sess.timer("incr_comp_finalize_session_directory"); - let incr_comp_session_dir = &*incr_comp_session.session_directory; + let incr_comp_session_dir = &*incr_comp_session.new_session_directory; debug!("finalize_session_directory() - session directory: {}", incr_comp_session_dir.display()); @@ -367,77 +350,28 @@ pub fn finalize_session_directory( } } - let _ = garbage_collect_session_directories( - sess, - &incr_comp_session, - false, // keep_most_recent - ); -} + // Unlock the old session directory now that we will no longer read from it. + incr_comp_session.old_session_directory = None; -pub(crate) fn delete_all_session_dir_contents( - incr_comp_session: &IncrCompSession, -) -> io::Result<()> { - let sess_dir_iterator = incr_comp_session.session_directory.read_dir()?; - for entry in sess_dir_iterator { - let entry = entry?; - safe_remove_file(&entry.path())? - } - Ok(()) + let _ = garbage_collect_session_directories(sess, &incr_comp_session); } -fn copy_files(sess: &Session, target_dir: &Path, source_dir: &Path) -> Result { - // We acquire a shared lock on the lock file of the directory, so that - // nobody deletes it out from under us while we are reading from it. - let lock_file_path = lock_file_path(source_dir); - - // not exclusive - let Ok(_lock) = flock::Lock::try_lock( - &lock_file_path, - false, // don't create - false, - ) else { - // Could not acquire the lock, don't try to copy from here - return Err(()); - }; - - let Ok(source_dir_iterator) = source_dir.read_dir() else { - return Err(()); - }; - - let mut files_linked = 0; - let mut files_copied = 0; - - for entry in source_dir_iterator { - match entry { - Ok(entry) => { - let file_name = entry.file_name(); - - let target_file_path = target_dir.join(file_name); - let source_path = entry.path(); - - debug!("copying into session dir: {}", source_path.display()); - match link_or_copy(source_path, target_file_path) { - Ok(LinkOrCopy::Link) => files_linked += 1, - Ok(LinkOrCopy::Copy) => files_copied += 1, - Err(_) => return Err(()), - } +pub(crate) fn invalidate_old_session_dir(sess: &Session, incr_comp_session: &mut IncrCompSession) { + if let Some(old_incr_comp_session_dir) = incr_comp_session.old_session_directory.take() { + let res = try { + let sess_dir_iterator = old_incr_comp_session_dir.read_dir()?; + for entry in sess_dir_iterator { + let entry = entry?; + safe_remove_file(&entry.path())? } - Err(_) => return Err(()), + }; + if let Err(err) = res { + sess.dcx().emit_err(diagnostics::DeleteIncompatible { + path: (*old_incr_comp_session_dir).to_owned(), + err, + }); } } - - if sess.opts.unstable_opts.incremental_info { - eprintln!( - "[incremental] session directory: \ - {files_linked} files hard-linked" - ); - eprintln!( - "[incremental] session directory: \ - {files_copied} files copied" - ); - } - - Ok(files_linked > 0 || files_copied == 0) } /// Generates unique directory path of the form: @@ -470,33 +404,44 @@ fn create_dir(sess: &Session, path: &Path, dir_tag: &str) { } } -/// Allocate the lock-file, lock it and create the session directory. -fn lock_and_create_directory(sess: &Session, session_dir: &Path) -> (flock::LockedDir, PathBuf) { +/// Allocate the lock-file, lock it and create the session directory if requested. +fn lock_directory( + sess: &Session, + session_dir: &Path, + new_session: bool, +) -> Option { let lock_file_path = lock_file_path(session_dir); debug!("lock_directory() - lock_file: {}", lock_file_path.display()); match flock::LockedDir::try_lock( session_dir.to_owned(), &lock_file_path, - true, // create the lock file - true, + new_session, // create + new_session, // exclusive ) { - // the lock should be exclusive Ok(lock) => { // Now that we have the lock, we can actually create the session // directory - create_dir(sess, &session_dir, "session"); + if new_session { + create_dir(sess, &session_dir, "session"); + } - (lock, lock_file_path) + Some(lock) } Err(lock_err) => { let is_unsupported_lock = flock::Lock::error_unsupported(&lock_err); - sess.dcx().emit_fatal(diagnostics::CreateLock { + let diag = diagnostics::CreateLock { lock_err, session_dir, is_unsupported_lock, is_cargo: rustc_session::utils::was_invoked_from_cargo(), - }); + }; + if new_session { + sess.dcx().emit_fatal(diag); + } else { + sess.dcx().emit_warn(diag); + None + } } } } @@ -507,24 +452,18 @@ fn delete_session_dir_lock_file(sess: &Session, lock_file_path: &Path) { } } -/// Finds the most recent published session directory that is not in the -/// ignore-list. -fn find_source_directory( - crate_dir: &Path, - source_directories_already_tried: &FxHashSet, -) -> Option { +/// Finds the most recent published session directory. +fn find_source_directory(sess: &Session, crate_dir: &Path) -> Option { let iter = crate_dir .read_dir() .unwrap() // FIXME .filter_map(|e| e.ok().map(|e| e.path())); - find_source_directory_in_iter(iter, source_directories_already_tried) + find_source_directory_in_iter(iter) + .and_then(|session_dir| lock_directory(sess, &session_dir, false /* new_session */)) } -fn find_source_directory_in_iter( - iter: I, - source_directories_already_tried: &FxHashSet, -) -> Option +fn find_source_directory_in_iter(iter: I) -> Option where I: Iterator, { @@ -538,10 +477,7 @@ where continue; }; - if source_directories_already_tried.contains(&session_dir) - || !is_session_directory(&directory_name) - || !is_finalized(&directory_name) - { + if !is_session_directory(&directory_name) || !is_finalized(&directory_name) { debug!("find_source_directory_in_iter - ignoring"); continue; } @@ -619,11 +555,10 @@ fn is_old_enough_to_be_collected(timestamp: SystemTime) -> bool { pub(crate) fn garbage_collect_session_directories( sess: &Session, incr_comp_session: &IncrCompSession, - keep_most_recent: bool, ) -> io::Result<()> { debug!("garbage_collect_session_directories() - begin"); - let session_directory = &*incr_comp_session.session_directory; + let session_directory = &*incr_comp_session.new_session_directory; debug!( "garbage_collect_session_directories() - session directory: {}", @@ -760,10 +695,7 @@ pub(crate) fn garbage_collect_session_directories( ); // Note that we are holding on to the lock - return Some(( - (timestamp, crate_directory.join(directory_name)), - Some(lock), - )); + return Some((crate_directory.join(directory_name), lock)); } Err(_) => { debug!( @@ -818,25 +750,22 @@ pub(crate) fn garbage_collect_session_directories( } None }); - let deletion_candidates = deletion_candidates.into(); // Delete all but the most recent of the candidates - all_except_maybe_most_recent(deletion_candidates, keep_most_recent).into_items().all( - |(path, lock)| { - debug!("garbage_collect_session_directories() - deleting `{}`", path.display()); + deletion_candidates.all(|(path, lock)| { + debug!("garbage_collect_session_directories() - deleting `{}`", path.display()); - if let Err(err) = std_fs::remove_dir_all(&path) { - sess.dcx().emit_warn(diagnostics::FinalizedGcFailed { path: &path, err }); - } else { - delete_session_dir_lock_file(sess, &lock_file_path(&path)); - } + if let Err(err) = std_fs::remove_dir_all(&path) { + sess.dcx().emit_warn(diagnostics::FinalizedGcFailed { path: &path, err }); + } else { + delete_session_dir_lock_file(sess, &lock_file_path(&path)); + } - // Let's make it explicit that the file lock is released at this point, - // or rather, that we held on to it until here - drop(lock); - true - }, - ); + // Let's make it explicit that the file lock is released at this point, + // or rather, that we held on to it until here + drop(lock); + true + }); Ok(()) } @@ -851,21 +780,6 @@ fn delete_old(sess: &Session, path: &Path) { } } -fn all_except_maybe_most_recent( - deletion_candidates: UnordMap<(SystemTime, PathBuf), Option>, - keep_most_recent: bool, -) -> UnordMap> { - let most_recent = keep_most_recent - .then(|| deletion_candidates.items().map(|(&(timestamp, _), _)| timestamp).max()) - .flatten(); - - deletion_candidates - .into_items() - .filter(|&((timestamp, _), _)| Some(timestamp) != most_recent) - .map(|((_, path), lock)| (path, lock)) - .collect() -} - fn safe_remove_file(p: &Path) -> io::Result<()> { match std_fs::remove_file(p) { Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), diff --git a/compiler/rustc_incremental/src/persist/fs/tests.rs b/compiler/rustc_incremental/src/persist/fs/tests.rs index 75b573dd4944a..112afd8c7bd1d 100644 --- a/compiler/rustc_incremental/src/persist/fs/tests.rs +++ b/compiler/rustc_incremental/src/persist/fs/tests.rs @@ -1,25 +1,5 @@ use super::*; -#[test] -fn test_all_except_most_recent() { - let input: UnordMap<_, Option> = UnordMap::from_iter([ - ((UNIX_EPOCH + Duration::new(4, 0), PathBuf::from("4")), None), - ((UNIX_EPOCH + Duration::new(1, 0), PathBuf::from("1")), None), - ((UNIX_EPOCH + Duration::new(5, 0), PathBuf::from("5")), None), - ((UNIX_EPOCH + Duration::new(3, 0), PathBuf::from("3")), None), - ((UNIX_EPOCH + Duration::new(2, 0), PathBuf::from("2")), None), - ]); - assert_eq!( - all_except_maybe_most_recent(input, true) - .into_items() - .map(|(path, _)| path) - .into_sorted_stable_ord(), - vec![PathBuf::from("1"), PathBuf::from("2"), PathBuf::from("3"), PathBuf::from("4")] - ); - - assert!(all_except_maybe_most_recent(UnordMap::default(), true).is_empty()); -} - #[test] fn test_timestamp_serialization() { for i in 0..1_000u64 { @@ -31,8 +11,6 @@ fn test_timestamp_serialization() { #[test] fn test_find_source_directory_in_iter() { - let already_visited = FxHashSet::default(); - // Find newest assert_eq!( find_source_directory_in_iter( @@ -42,7 +20,6 @@ fn test_find_source_directory_in_iter() { PathBuf::from("crate-dir/s-1234-0000-svh") ] .into_iter(), - &already_visited ), Some(PathBuf::from("crate-dir/s-3234-0000-svh")) ); @@ -56,13 +33,12 @@ fn test_find_source_directory_in_iter() { PathBuf::from("crate-dir/s-1234-0000-svh") ] .into_iter(), - &already_visited ), Some(PathBuf::from("crate-dir/s-2234-0000-svh")) ); // Handle empty - assert_eq!(find_source_directory_in_iter([].into_iter(), &already_visited), None); + assert_eq!(find_source_directory_in_iter([].into_iter()), None); // Handle only working assert_eq!( @@ -73,7 +49,6 @@ fn test_find_source_directory_in_iter() { PathBuf::from("crate-dir/s-1234-0000-working") ] .into_iter(), - &already_visited ), None ); diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index fb83149e65b22..76e0bf92c6f0f 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -16,8 +16,8 @@ use rustc_span::Symbol; use tracing::{debug, warn}; use super::data::*; +use super::file_format; use super::fs::*; -use super::{file_format, work_product}; use crate::diagnostics; use crate::persist::file_format::{OpenFile, OpenFileError}; @@ -32,15 +32,6 @@ enum LoadResult { IoError { path: PathBuf, err: io::Error }, } -fn delete_dirty_work_product( - sess: &Session, - incr_comp_session: &IncrCompSession, - swp: SerializedWorkProduct, -) { - debug!("delete_dirty_work_product({:?})", swp); - work_product::delete_workproduct_files(sess, incr_comp_session, &swp.work_product); -} - fn load_dep_graph(sess: &Session, incr_comp_session: &IncrCompSession) -> LoadResult { assert!(sess.opts.incremental.is_some()); @@ -48,12 +39,16 @@ fn load_dep_graph(sess: &Session, incr_comp_session: &IncrCompSession) -> LoadRe // Calling `sess.incr_comp_session_dir()` will panic if `sess.opts.incremental.is_none()`. // Fortunately, we just checked that this isn't the case. - let path = dep_graph_path(incr_comp_session); + let Some(path) = old_dep_graph_path(incr_comp_session) else { + return LoadResult::DataOutOfDate; + }; let expected_hash = sess.opts.dep_tracking_hash(false); let mut prev_work_products = UnordMap::default(); - let work_products_path = work_products_path(incr_comp_session); + let Some(work_products_path) = old_work_products_path(incr_comp_session) else { + return LoadResult::DataOutOfDate; + }; if let Ok(OpenFile { mmap, start_pos }) = file_format::open_incremental_file(sess, &work_products_path) @@ -68,7 +63,7 @@ fn load_dep_graph(sess: &Session, incr_comp_session: &IncrCompSession) -> LoadRe for swp in work_products { let all_files_exist = swp.work_product.saved_files.items().all(|(_, path)| { - let exists = in_incr_comp_dir_sess(incr_comp_session, path).exists(); + let exists = in_old_incr_comp_dir_sess(incr_comp_session, path).unwrap().exists(); if !exists && sess.opts.unstable_opts.incremental_info { eprintln!("incremental: could not find file for work product: {path}",); } @@ -80,7 +75,7 @@ fn load_dep_graph(sess: &Session, incr_comp_session: &IncrCompSession) -> LoadRe prev_work_products.insert(swp.id, swp.work_product); } else { debug!("reconcile_work_products: some file for {:?} does not exist", swp); - delete_dirty_work_product(sess, incr_comp_session, swp); + return LoadResult::DataOutOfDate; } } } @@ -134,7 +129,9 @@ pub fn load_query_result_cache( let _prof_timer = sess.prof.generic_activity("incr_comp_load_query_result_cache"); - let path = query_cache_path(incr_comp_session); + let Some(path) = old_query_cache_path(incr_comp_session) else { + return Some(OnDiskCache::new_empty()); + }; match file_format::open_incremental_file(sess, &path) { Ok(OpenFile { mmap, start_pos }) => { let cache = OnDiskCache::new(sess, mmap, start_pos).unwrap_or_else(|()| { @@ -190,16 +187,12 @@ pub fn setup_dep_graph( } // `load_dep_graph` can only be called after `prepare_session_directory`. - let incr_comp_session = prepare_session_directory(sess, crate_name, stable_crate_id); + let mut incr_comp_session = prepare_session_directory(sess, crate_name, stable_crate_id); // Try to load the previous session's dep graph and work products. let load_result = load_dep_graph(sess, &incr_comp_session); sess.time("incr_comp_garbage_collect_session_directories", || { - if let Err(e) = garbage_collect_session_directories( - sess, - &incr_comp_session, - true, // keep_most_recent - ) { + if let Err(e) = garbage_collect_session_directories(sess, &incr_comp_session) { warn!( "Error while trying to garbage collect incremental compilation \ cache directory: {e}", @@ -213,15 +206,11 @@ pub fn setup_dep_graph( let (prev_graph, prev_work_products) = match load_result { LoadResult::IoError { path, err } => { sess.dcx().emit_warn(diagnostics::LoadDepGraph { path, err }); + invalidate_old_session_dir(sess, &mut incr_comp_session); Default::default() } LoadResult::DataOutOfDate => { - if let Err(err) = delete_all_session_dir_contents(&incr_comp_session) { - sess.dcx().emit_err(diagnostics::DeleteIncompatible { - path: dep_graph_path(&incr_comp_session), - err, - }); - } + invalidate_old_session_dir(sess, &mut incr_comp_session); Default::default() } LoadResult::Ok { prev_graph, prev_work_products } => (prev_graph, prev_work_products), diff --git a/compiler/rustc_incremental/src/persist/mod.rs b/compiler/rustc_incremental/src/persist/mod.rs index 7d486cc394b80..fb318357b26cb 100644 --- a/compiler/rustc_incremental/src/persist/mod.rs +++ b/compiler/rustc_incremental/src/persist/mod.rs @@ -10,7 +10,7 @@ mod load; mod save; mod work_product; -pub use fs::{finalize_session_directory, in_incr_comp_dir_sess}; +pub use fs::{finalize_session_directory, in_incr_comp_dir_sess, in_old_incr_comp_dir_sess}; pub use load::{load_query_result_cache, setup_dep_graph}; pub(crate) use save::save_dep_graph; pub use save::save_work_product_index; diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs index 12f674fe2a859..46f47d6c8623c 100644 --- a/compiler/rustc_incremental/src/persist/save.rs +++ b/compiler/rustc_incremental/src/persist/save.rs @@ -11,7 +11,7 @@ use tracing::debug; use super::data::*; use super::fs::*; -use super::{clean, file_format, work_product}; +use super::{clean, file_format}; use crate::assert_dep_graph::assert_dep_graph; use crate::diagnostics; @@ -112,23 +112,6 @@ pub fn save_work_product_index( e.finish() }); - // We also need to clean out old work-products, as not all of them are - // deleted during invalidation. Some object files don't change their - // content, they are just not needed anymore. - let previous_work_products = dep_graph.previous_work_products(); - for (id, wp) in previous_work_products.to_sorted_stable_ord() { - if !new_work_products.contains_key(id) { - work_product::delete_workproduct_files(sess, incr_comp_session.unwrap(), wp); - debug_assert!( - !wp.saved_files.items().all(|(_, path)| in_incr_comp_dir_sess( - incr_comp_session.unwrap(), - path - ) - .exists()) - ); - } - } - // Check that we did not delete one of the current work-products: debug_assert!({ new_work_products.items().all(|(_, wp)| { diff --git a/compiler/rustc_incremental/src/persist/work_product.rs b/compiler/rustc_incremental/src/persist/work_product.rs index 7bb66fee4d1a3..0aaa9aa8e96c2 100644 --- a/compiler/rustc_incremental/src/persist/work_product.rs +++ b/compiler/rustc_incremental/src/persist/work_product.rs @@ -1,9 +1,8 @@ -//! Functions for saving and removing intermediate [work products]. +//! Function for saving intermediate [work products]. //! //! [work products]: WorkProduct -use std::fs as std_fs; -use std::path::{Path, PathBuf}; +use std::path::Path; use rustc_data_structures::unord::UnordMap; use rustc_fs_util::link_or_copy; @@ -23,7 +22,6 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( incr_comp_session: &IncrCompSession, cgu_name: &str, files: &[(&'static str, &Path)], - known_links: &[PathBuf], ) -> (WorkProductId, WorkProduct) { debug!(?cgu_name, ?files); assert!(sess.opts.incremental.is_some()); @@ -32,10 +30,6 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( for (ext, path) in files { let file_name = format!("{cgu_name}.{ext}"); let path_in_incr_dir = in_incr_comp_dir_sess(incr_comp_session, &file_name); - if known_links.contains(&path_in_incr_dir) { - let _ = saved_files.insert(ext.to_string(), file_name); - continue; - } match link_or_copy(path, &path_in_incr_dir) { Ok(_) => { let _ = saved_files.insert(ext.to_string(), file_name); @@ -55,17 +49,3 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir( let work_product_id = WorkProductId::from_cgu_name(cgu_name); (work_product_id, work_product) } - -/// Removes files for a given work product. -pub(crate) fn delete_workproduct_files( - sess: &Session, - incr_comp_session: &IncrCompSession, - work_product: &WorkProduct, -) { - for (_, path) in work_product.saved_files.items().into_sorted_stable_ord() { - let path = in_incr_comp_dir_sess(incr_comp_session, path); - if let Err(err) = std_fs::remove_file(&path) { - sess.dcx().emit_warn(diagnostics::DeleteWorkProduct { path: &path, err }); - } - } -} diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs index 2f196c5e5d609..759297bc69592 100644 --- a/compiler/rustc_interface/src/queries.rs +++ b/compiler/rustc_interface/src/queries.rs @@ -101,7 +101,6 @@ impl Linker { incr_comp_session.as_ref().unwrap(), WorkProduct::METADATA_WORKPRODUCT_CGU_NAME, &[(OutputType::Metadata.extension(), path)], - &[], ); work_products.insert(id, product); } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 8481db9d3c523..f64b2913d40d9 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2502,14 +2502,15 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { // If the metadata dep-node is green, try to reuse the saved work product. if tcx.dep_graph.is_fully_enabled() + && let incr_comp_session = tcx.incr_comp_session.unwrap() + && let Some(old_incr_comp_session_dir) = &incr_comp_session.old_session_directory && let work_product_id = WorkProductId::from_cgu_name(WorkProduct::METADATA_WORKPRODUCT_CGU_NAME) && let Some(work_product) = tcx.dep_graph.previous_work_product(&work_product_id) && tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { let saved_path = &work_product.saved_files[OutputType::Metadata.extension()]; - let incr_comp_session_dir = &tcx.incr_comp_session.unwrap().session_directory; - let source_file_in_incr_dir = &incr_comp_session_dir.join(saved_path); + let source_file_in_incr_dir = &old_incr_comp_session_dir.join(saved_path); debug!("copying preexisting metadata from {source_file_in_incr_dir:?} to {path:?}"); match rustc_fs_util::link_or_copy(&source_file_in_incr_dir, path) { Ok(_) => {} diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index acfbb08b9b630..0ebf5dd1b99a1 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -1841,10 +1841,11 @@ fn validate_commandline_args_with_session_available(sess: &Session) { /// Holds data on the current incremental compilation session, if there is one. pub struct IncrCompSession { - /// The directory containing all cached data. Cached data from a previous - /// session can be read out of it and new data for the current session will - /// be written into it. - pub session_directory: flock::LockedDir, + /// The directory from which cached data of a previous session can be read. + pub old_session_directory: Option, + /// The directory to which cached data for the current session can be + /// written to. + pub new_session_directory: flock::LockedDir, } /// A wrapper around an [`DiagCtxt`] that is used for early error emissions. diff --git a/tests/run-make/incremental-session-gc/rmake.rs b/tests/run-make/incremental-session-gc/rmake.rs index 341a21690f215..405708abb7e3c 100644 --- a/tests/run-make/incremental-session-gc/rmake.rs +++ b/tests/run-make/incremental-session-gc/rmake.rs @@ -12,14 +12,12 @@ fn main() { compile(); let mut previous = session_dir(); - rfs::write(previous.join("sentinel"), "previous session"); for _ in 0..2 { compile(); let current = session_dir(); assert_ne!(previous, current); assert!(!previous.exists(), "superseded session was not collected: {previous:?}"); - assert_eq!(rfs::read_to_string(current.join("sentinel")), "previous session"); previous = current; } @@ -36,7 +34,6 @@ fn main() { let current = session_dir(); assert_ne!(previous, newer); assert!(!newer.exists(), "superseded session was not collected: {previous:?}"); - assert_eq!(rfs::read_to_string(current.join("sentinel")), "previous session"); } fn session_dir() -> PathBuf {