sm4: simd support for armv8, x86_64

sm4/Cargo.toml

@@ -14,6 +14,10 @@ categories = ["cryptography", "no-std"]
 
 [dependencies]
 cipher = "0.5"
+cfg-if = "1"
+
+[target.'cfg(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "x86"))'.dependencies]
+cpufeatures = "0.3"
 
 [dev-dependencies]
 cipher = { version = "0.5", features = ["dev"] }
@@ -24,3 +28,6 @@ zeroize = ["cipher/zeroize"]
 
 [package.metadata.docs.rs]
 all-features = true
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(sm4_force_soft)'] }

sm4/src/armv8/autodetect/linux.rs

@@ -0,0 +1,121 @@
+#![allow(unsafe_code)]
+
+use cipher::{
+    AlgorithmName, BlockCipherDecClosure, BlockCipherDecrypt, BlockCipherEncClosure,
+    BlockCipherEncrypt, BlockSizeUser, Key, KeyInit, KeySizeUser, consts::U16,
+};
+use core::{fmt, mem::ManuallyDrop};
+
+#[cfg(feature = "zeroize")]
+use cipher::zeroize::ZeroizeOnDrop;
+
+use crate::armv8::{neon::Sm4 as NeonSm4, sm4e::Sm4 as CryptoExtensionSm4};
+
+cpufeatures::new!(sm4_intrinsics, "sm4");
+
+union Sm4Cipher {
+    sm4: ManuallyDrop<CryptoExtensionSm4>,
+    neon: ManuallyDrop<NeonSm4>,
+}
+
+/// SM4 block cipher.
+pub struct Sm4 {
+    cipher: Sm4Cipher,
+    token: sm4_intrinsics::InitToken,
+}
+
+impl KeySizeUser for Sm4 {
+    type KeySize = U16;
+}
+
+impl KeyInit for Sm4 {
+    #[inline]
+    fn new(key: &Key<Self>) -> Self {
+        let (token, intrinsics_presence) = sm4_intrinsics::init_get();
+
+        let cipher = if intrinsics_presence {
+            Sm4Cipher {
+                sm4: ManuallyDrop::new(CryptoExtensionSm4::new(key)),
+            }
+        } else {
+            Sm4Cipher {
+                neon: ManuallyDrop::new(NeonSm4::new(key)),
+            }
+        };
+
+        Self { cipher, token }
+    }
+}
+
+impl Clone for Sm4 {
+    fn clone(&self) -> Self {
+        let cipher = if self.token.get() {
+            Sm4Cipher {
+                sm4: unsafe { self.cipher.sm4.clone() },
+            }
+        } else {
+            Sm4Cipher {
+                neon: unsafe { self.cipher.neon.clone() },
+            }
+        };
+
+        Self {
+            cipher,
+            token: self.token,
+        }
+    }
+}
+
+impl BlockSizeUser for Sm4 {
+    type BlockSize = U16;
+}
+
+impl BlockCipherEncrypt for Sm4 {
+    fn encrypt_with_backend(&self, f: impl BlockCipherEncClosure<BlockSize = U16>) {
+        unsafe {
+            if self.token.get() {
+                self.cipher.sm4.encrypt_with_backend(f);
+            } else {
+                self.cipher.neon.encrypt_with_backend(f);
+            }
+        }
+    }
+}
+
+impl BlockCipherDecrypt for Sm4 {
+    fn decrypt_with_backend(&self, f: impl BlockCipherDecClosure<BlockSize = U16>) {
+        unsafe {
+            if self.token.get() {
+                self.cipher.sm4.decrypt_with_backend(f);
+            } else {
+                self.cipher.neon.decrypt_with_backend(f);
+            }
+        }
+    }
+}
+
+impl fmt::Debug for Sm4 {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        f.write_str(concat!(stringify!(Sm4), " { .. }"))
+    }
+}
+
+impl AlgorithmName for Sm4 {
+    fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(stringify!(Sm4))
+    }
+}
+
+impl Drop for Sm4 {
+    fn drop(&mut self) {
+        if self.token.get() {
+            unsafe { ManuallyDrop::drop(&mut self.cipher.sm4) }
+        } else {
+            unsafe { ManuallyDrop::drop(&mut self.cipher.neon) }
+        }
+    }
+}
+
+#[cfg(feature = "zeroize")]
+#[cfg_attr(docsrs, doc(cfg(feature = "zeroize")))]
+impl ZeroizeOnDrop for Sm4 {}

sm4/src/armv8/autodetect/mod.rs

@@ -0,0 +1,11 @@
+use cfg_if::cfg_if;
+
+cfg_if! {
+    if #[cfg(any(target_os = "linux", target_os = "android"))] {
+        mod linux;
+        pub use self::linux::*;
+    } else {
+        mod others;
+        pub use self::others::*;
+    }
+}

sm4/src/armv8/autodetect/others.rs

@@ -0,0 +1,3 @@
+//! On targets other than Linux/Android the SM4 crypto extensions cannot be
+//! detected at runtime, so the NEON backend is used unconditionally.
+pub use crate::armv8::neon::Sm4;

sm4/src/armv8/intrinsics.rs

@@ -0,0 +1,31 @@
+//! ARMv8 extension intrinsics
+
+#![allow(unsafe_code, unsafe_op_in_unsafe_fn)]
+
+use core::arch::{aarch64::*, asm};
+
+#[inline]
+#[target_feature(enable = "sm4")]
+pub(super) unsafe fn vsm4eq_u32(mut a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
+    asm!(
+        "SM4E {d:v}.4S, {n:v}.4S",
+        d = inout(vreg) a,
+        n = in(vreg) b,
+        options(pure, nomem, nostack, preserves_flags)
+    );
+    a
+}
+
+#[inline]
+#[target_feature(enable = "sm4")]
+pub(super) unsafe fn vsm4ekeyq_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
+    let mut key: uint32x4_t;
+    asm!(
+        "SM4EKEY {d:v}.4S, {n:v}.4S, {m:v}.4S",
+        d = out(vreg) key,
+        n = in(vreg) a,
+        m = in(vreg) b,
+        options(pure, nomem, nostack, preserves_flags)
+    );
+    key
+}

sm4/src/armv8/mod.rs

@@ -0,0 +1,8 @@
+pub mod autodetect;
+// The SM4 crypto extension can only be detected at runtime on Linux/Android,
+// so the `sm4e` backend and its `intrinsics` are compiled only for those OSes.
+#[cfg(any(target_os = "linux", target_os = "android"))]
+mod intrinsics;
+mod neon;
+#[cfg(any(target_os = "linux", target_os = "android"))]
+mod sm4e;