poly1305: AVX2 detection

Automatically detects the availability of CLMUL based on CPUID, and
falls back to the "soft" implementation if unavailable.

Author: tarcieri

.github/workflows/poly1305.yml

@@ -39,6 +39,7 @@ jobs:
           target: ${{ matrix.target }}
           override: true
       - run: cargo build --target ${{ matrix.target }} --release
+      - run: cargo build --target ${{ matrix.target }} --release --features force-soft
 
   # Tests for the portable software backend
   soft:
@@ -70,6 +71,8 @@ jobs:
       - run: ${{ matrix.deps }}
       - run: cargo check --target ${{ matrix.target }} --all-features
       - run: cargo test --target ${{ matrix.target }} --release
+      - run: cargo test --target ${{ matrix.target }} --release --features force-soft
+      - run: cargo test --target ${{ matrix.target }} --release --features std
       - run: cargo test --target ${{ matrix.target }} --release --all-features
 
   # Tests for the AVX2 backend
@@ -104,6 +107,8 @@ jobs:
       - run: ${{ matrix.deps }}
       - run: cargo check --target ${{ matrix.target }} --all-features
       - run: cargo test --target ${{ matrix.target }} --release
+      - run: cargo test --target ${{ matrix.target }} --release --features force-soft
+      - run: cargo test --target ${{ matrix.target }} --release --features std
       - run: cargo test --target ${{ matrix.target }} --release --all-features
 
   # Cross-compiled tests
@@ -135,4 +140,6 @@ jobs:
           override: true
       - run: cargo install cross
       - run: cross test --target ${{ matrix.target }} --release
+      - run: cross test --target ${{ matrix.target }} --release --features force-soft
+      - run: cross test --target ${{ matrix.target }} --release --features std
       - run: cross test --target ${{ matrix.target }} --release --all-features

Cargo.lock

@@ -15,8 +15,12 @@ edition = "2018"
 universal-hash = { version = "0.4", default-features = false }
 zeroize = { version = "1", optional = true, default-features = false }
 
+[target.'cfg(any(target_arch = "x86_64", target_arch = "x86"))'.dependencies]
+cpuid-bool = "0.2"
+
 [dev-dependencies]
 hex-literal = "0.2"
 
 [features]
+force-soft = []
 std = ["universal-hash/std"]

poly1305/Cargo.toml

@@ -0,0 +1,85 @@
+//! Autodetection support for AVX2 CPU intrinsics on x86 CPUs, with fallback
+//! to the "soft" backend when it's unavailable.
+
+use crate::{backend, Block, Key, Tag};
+
+cpuid_bool::new!(avx2_cpuid, "avx2");
+
+pub struct State {
+    inner: Inner,
+    token: avx2_cpuid::InitToken,
+}
+
+union Inner {
+    avx2: backend::avx2::State,
+    soft: backend::soft::State,
+}
+
+impl State {
+    /// Initialize Poly1305 [`State`] with the given key
+    #[inline]
+    pub(crate) fn new(key: &Key) -> State {
+        let (token, avx2_present) = avx2_cpuid::init_get();
+
+        let inner = if avx2_present {
+            Inner {
+                avx2: backend::avx2::State::new(key),
+            }
+        } else {
+            Inner {
+                soft: backend::soft::State::new(key),
+            }
+        };
+
+        Self { inner, token }
+    }
+
+    /// Reset internal state
+    #[inline]
+    pub(crate) fn reset(&mut self) {
+        if self.token.get() {
+            unsafe { self.inner.avx2.reset() }
+        } else {
+            unsafe { self.inner.soft.reset() }
+        }
+    }
+
+    /// Compute a Poly1305 block
+    #[inline]
+    pub(crate) fn compute_block(&mut self, block: &Block, partial: bool) {
+        if self.token.get() {
+            unsafe { self.inner.avx2.compute_block(block, partial) }
+        } else {
+            unsafe { self.inner.soft.compute_block(block, partial) }
+        }
+    }
+
+    /// Finalize output producing a [`Tag`]
+    #[inline]
+    pub(crate) fn finalize(&mut self) -> Tag {
+        if self.token.get() {
+            unsafe { self.inner.avx2.finalize() }
+        } else {
+            unsafe { self.inner.soft.finalize() }
+        }
+    }
+}
+
+impl Clone for State {
+    fn clone(&self) -> Self {
+        let inner = if self.token.get() {
+            Inner {
+                avx2: unsafe { self.inner.avx2 },
+            }
+        } else {
+            Inner {
+                soft: unsafe { self.inner.soft },
+            }
+        };
+
+        Self {
+            inner,
+            token: self.token,
+        }
+    }
+}

poly1305/src/autodetect.rs

@@ -1,27 +1,9 @@
+//! Poly1305 backends
+
 #[cfg(all(
     any(target_arch = "x86", target_arch = "x86_64"),
-    target_feature = "avx2"
+    not(feature = "force-soft")
 ))]
 pub(crate) mod avx2;
 
-#[cfg(any(
-    not(all(
-        any(target_arch = "x86", target_arch = "x86_64"),
-        target_feature = "avx2"
-    )),
-    fuzzing,
-    test,
-))]
 pub(crate) mod soft;
-
-#[cfg(all(
-    any(target_arch = "x86", target_arch = "x86_64"),
-    target_feature = "avx2",
-))]
-pub(crate) use avx2::State;
-
-#[cfg(not(all(
-    any(target_arch = "x86", target_arch = "x86_64"),
-    target_feature = "avx2",
-)))]
-pub(crate) use soft::State;

poly1305/src/backend.rs

@@ -23,14 +23,14 @@ use crate::{Block, Key, Tag};
 mod helpers;
 use self::helpers::*;
 
-#[derive(Clone)]
+#[derive(Copy, Clone)]
 struct Initialized {
     p: Aligned4x130,
     m: SpacedMultiplier4x130,
     r4: PrecomputedMultiplier,
 }
 
-#[derive(Clone)]
+#[derive(Copy, Clone)]
 pub(crate) struct State {
     k: AdditionKey,
     r1: PrecomputedMultiplier,
@@ -42,7 +42,7 @@ pub(crate) struct State {
 }
 
 impl State {
-    /// Initialize Poly1305 state with the given key
+    /// Initialize Poly1305 [`State`] with the given key
     pub(crate) fn new(key: &Key) -> Self {
         // Prepare addition key and polynomial key.
         let (k, r1) = prepare_keys(key);
@@ -67,6 +67,7 @@ impl State {
         self.num_cached_blocks = 0;
     }
 
+    /// Compute a Poly1305 block
     pub(crate) fn compute_block(&mut self, block: &Block, partial: bool) {
         // We can cache a single partial block.
         if partial {
@@ -99,6 +100,7 @@ impl State {
         }
     }
 
+    /// Finalize output producing a [`Tag`]
     pub(crate) fn finalize(&mut self) -> Tag {
         assert!(self.num_cached_blocks < 4);
         let mut data = &self.cached_blocks[..];

poly1305/src/backend/avx2.rs

@@ -890,7 +890,7 @@ impl SpacedMultiplier4x130 {
 ///
 /// Unlike `Aligned2x130` which wraps two `Aligned130`s, this struct represents the four
 /// integers as 20 limbs spread across three 256-bit vectors.
-#[derive(Clone, Debug)]
+#[derive(Copy, Clone, Debug)]
 pub(super) struct Aligned4x130 {
     v0: __m256i,
     v1: __m256i,
@@ -1116,7 +1116,7 @@ impl Mul<PrecomputedMultiplier> for &Aligned4x130 {
             // x.v0 = [  x32,   x30,   x22,   x20,   x12,   x10,   x02,   x00]
             // y =    [5·r_4, 5·r_3, 5·r_2,   r_4,   r_3,   r_2,   r_1,   r_0]
             // z =    [5·r_1, 5·r_1, 5·r_1, 5·r_1, 5·r_1, 5·r_1, 5·r_1, 5·r_1]
-            let mut x = self.clone();
+            let mut x = *self;
             let y = other.a;
             let z = other.a_5;
 

poly1305/src/backend/avx2/helpers.rs

@@ -19,15 +19,15 @@ use zeroize::Zeroize;
 
 use crate::{Block, Key, Tag};
 
-#[derive(Clone, Default)]
+#[derive(Copy, Clone, Default)]
 pub(crate) struct State {
     r: [u32; 5],
     h: [u32; 5],
     pad: [u32; 4],
 }
 
 impl State {
-    /// Initialize Poly1305State with the given key
+    /// Initialize Poly1305 [`State`] with the given key
     pub(crate) fn new(key: &Key) -> State {
         let mut poly = State::default();
 
@@ -47,7 +47,6 @@ impl State {
     }
 
     /// Reset internal state
-    #[allow(dead_code)]
     pub(crate) fn reset(&mut self) {
         self.h = Default::default();
     }
@@ -144,6 +143,7 @@ impl State {
         self.h[4] = h4;
     }
 
+    /// Finalize output producing a [`Tag`]
     pub(crate) fn finalize(&mut self) -> Tag {
         // fully carry h
         let mut h0 = self.h[0];

poly1305/src/backend/soft.rs

@@ -70,15 +70,33 @@ use universal_hash::{
     NewUniversalHash, UniversalHash,
 };
 
+#[cfg(all(
+    any(target_arch = "x86", target_arch = "x86_64"),
+    not(feature = "force-soft")
+))]
+mod autodetect;
+
 mod backend;
 
 #[cfg(all(
     any(target_arch = "x86", target_arch = "x86_64"),
-    target_feature = "avx2",
+    not(feature = "force-soft"),
     any(fuzzing, test)
 ))]
 mod fuzz;
 
+#[cfg(all(
+    any(target_arch = "x86", target_arch = "x86_64"),
+    not(feature = "force-soft")
+))]
+use crate::autodetect::State;
+
+#[cfg(not(all(
+    any(target_arch = "x86", target_arch = "x86_64"),
+    not(feature = "force-soft")
+)))]
+use crate::backend::soft::State;
+
 /// Size of a Poly1305 key
 pub const KEY_SIZE: usize = 32;
 
@@ -102,7 +120,7 @@ pub type Tag = universal_hash::Output<Poly1305>;
 /// For this reason it doesn't impl the `crypto_mac::Mac` trait.
 #[derive(Clone)]
 pub struct Poly1305 {
-    state: backend::State,
+    state: State,
 }
 
 impl NewUniversalHash for Poly1305 {
@@ -111,7 +129,7 @@ impl NewUniversalHash for Poly1305 {
     /// Initialize Poly1305 with the given key
     fn new(key: &Key) -> Poly1305 {
         Poly1305 {
-            state: backend::State::new(key),
+            state: State::new(key),
         }
     }
 }
@@ -158,7 +176,7 @@ impl Poly1305 {
 
 #[cfg(all(
     any(target_arch = "x86", target_arch = "x86_64"),
-    target_feature = "avx2",
+    not(feature = "force-soft"),
     any(fuzzing, test)
 ))]
 pub use crate::fuzz::fuzz_avx2;