poly1305: AVX2 detection (#97)

* poly1305: AVX2 detection

Automatically detects the availability of CLMUL based on CPUID, and
falls back to the "soft" implementation if unavailable.

* poly1305: apply #[target_feature(enable="avx2")]

* Extract problematic lambdas into named functions

* poly1305: impl Zeroize for autodetect::State

Author: tarcieri

.github/workflows/poly1305.yml

@@ -39,6 +39,7 @@ jobs:
           target: ${{ matrix.target }}
           override: true
       - run: cargo build --target ${{ matrix.target }} --release
+      - run: cargo build --target ${{ matrix.target }} --release --features force-soft
 
   # Tests for the portable software backend
   soft:
@@ -70,6 +71,8 @@ jobs:
       - run: ${{ matrix.deps }}
       - run: cargo check --target ${{ matrix.target }} --all-features
       - run: cargo test --target ${{ matrix.target }} --release
+      - run: cargo test --target ${{ matrix.target }} --release --features force-soft
+      - run: cargo test --target ${{ matrix.target }} --release --features std
       - run: cargo test --target ${{ matrix.target }} --release --all-features
 
   # Tests for the AVX2 backend
@@ -104,6 +107,8 @@ jobs:
       - run: ${{ matrix.deps }}
       - run: cargo check --target ${{ matrix.target }} --all-features
       - run: cargo test --target ${{ matrix.target }} --release
+      - run: cargo test --target ${{ matrix.target }} --release --features force-soft
+      - run: cargo test --target ${{ matrix.target }} --release --features std
       - run: cargo test --target ${{ matrix.target }} --release --all-features
 
   # Cross-compiled tests
@@ -135,4 +140,6 @@ jobs:
           override: true
       - run: cargo install cross
       - run: cross test --target ${{ matrix.target }} --release
+      - run: cross test --target ${{ matrix.target }} --release --features force-soft
+      - run: cross test --target ${{ matrix.target }} --release --features std
       - run: cross test --target ${{ matrix.target }} --release --all-features

Cargo.lock

@@ -15,8 +15,12 @@ edition = "2018"
 universal-hash = { version = "0.4", default-features = false }
 zeroize = { version = "1", optional = true, default-features = false }
 
+[target.'cfg(any(target_arch = "x86_64", target_arch = "x86"))'.dependencies]
+cpuid-bool = "0.2"
+
 [dev-dependencies]
 hex-literal = "0.2"
 
 [features]
+force-soft = []
 std = ["universal-hash/std"]

poly1305/Cargo.toml

@@ -0,0 +1,97 @@
+//! Autodetection support for AVX2 CPU intrinsics on x86 CPUs, with fallback
+//! to the "soft" backend when it's unavailable.
+
+use crate::{backend, Block, Key, Tag};
+
+cpuid_bool::new!(avx2_cpuid, "avx2");
+
+pub struct State {
+    inner: Inner,
+    token: avx2_cpuid::InitToken,
+}
+
+union Inner {
+    avx2: backend::avx2::State,
+    soft: backend::soft::State,
+}
+
+impl State {
+    /// Initialize Poly1305 [`State`] with the given key
+    #[inline]
+    pub(crate) fn new(key: &Key) -> State {
+        let (token, avx2_present) = avx2_cpuid::init_get();
+
+        let inner = if avx2_present {
+            Inner {
+                avx2: backend::avx2::State::new(key),
+            }
+        } else {
+            Inner {
+                soft: backend::soft::State::new(key),
+            }
+        };
+
+        Self { inner, token }
+    }
+
+    /// Reset internal state
+    #[inline]
+    pub(crate) fn reset(&mut self) {
+        if self.token.get() {
+            unsafe { self.inner.avx2.reset() }
+        } else {
+            unsafe { self.inner.soft.reset() }
+        }
+    }
+
+    /// Compute a Poly1305 block
+    #[inline]
+    pub(crate) fn compute_block(&mut self, block: &Block, partial: bool) {
+        if self.token.get() {
+            unsafe { self.inner.avx2.compute_block(block, partial) }
+        } else {
+            unsafe { self.inner.soft.compute_block(block, partial) }
+        }
+    }
+
+    /// Finalize output producing a [`Tag`]
+    #[inline]
+    pub(crate) fn finalize(&mut self) -> Tag {
+        if self.token.get() {
+            unsafe { self.inner.avx2.finalize() }
+        } else {
+            unsafe { self.inner.soft.finalize() }
+        }
+    }
+}
+
+impl Clone for State {
+    fn clone(&self) -> Self {
+        let inner = if self.token.get() {
+            Inner {
+                avx2: unsafe { self.inner.avx2 },
+            }
+        } else {
+            Inner {
+                soft: unsafe { self.inner.soft },
+            }
+        };
+
+        Self {
+            inner,
+            token: self.token,
+        }
+    }
+}
+
+#[cfg(feature = "zeroize")]
+impl Drop for State {
+    fn drop(&mut self) {
+        use zeroize::Zeroize;
+        const SIZE: usize = core::mem::size_of::<State>();
+
+        let inner_array = unsafe { &mut *(self as *mut State as *mut [u8; SIZE]) };
+
+        inner_slice.zeroize();
+    }
+}

poly1305/src/autodetect.rs

@@ -1,27 +1,9 @@
+//! Poly1305 backends
+
 #[cfg(all(
     any(target_arch = "x86", target_arch = "x86_64"),
-    target_feature = "avx2"
+    not(feature = "force-soft")
 ))]
 pub(crate) mod avx2;
 
-#[cfg(any(
-    not(all(
-        any(target_arch = "x86", target_arch = "x86_64"),
-        target_feature = "avx2"
-    )),
-    fuzzing,
-    test,
-))]
 pub(crate) mod soft;
-
-#[cfg(all(
-    any(target_arch = "x86", target_arch = "x86_64"),
-    target_feature = "avx2",
-))]
-pub(crate) use avx2::State;
-
-#[cfg(not(all(
-    any(target_arch = "x86", target_arch = "x86_64"),
-    target_feature = "avx2",
-)))]
-pub(crate) use soft::State;

poly1305/src/backend.rs

@@ -23,14 +23,14 @@ use crate::{Block, Key, Tag};
 mod helpers;
 use self::helpers::*;
 
-#[derive(Clone)]
+#[derive(Copy, Clone)]
 struct Initialized {
     p: Aligned4x130,
     m: SpacedMultiplier4x130,
     r4: PrecomputedMultiplier,
 }
 
-#[derive(Clone)]
+#[derive(Copy, Clone)]
 pub(crate) struct State {
     k: AdditionKey,
     r1: PrecomputedMultiplier,
@@ -42,10 +42,10 @@ pub(crate) struct State {
 }
 
 impl State {
-    /// Initialize Poly1305 state with the given key
+    /// Initialize Poly1305 [`State`] with the given key
     pub(crate) fn new(key: &Key) -> Self {
         // Prepare addition key and polynomial key.
-        let (k, r1) = prepare_keys(key);
+        let (k, r1) = unsafe { prepare_keys(key) };
 
         // Precompute R^2.
         let r2 = (r1 * r1).reduce();
@@ -67,7 +67,9 @@ impl State {
         self.num_cached_blocks = 0;
     }
 
-    pub(crate) fn compute_block(&mut self, block: &Block, partial: bool) {
+    /// Compute a Poly1305 block
+    #[target_feature(enable = "avx2")]
+    pub(crate) unsafe fn compute_block(&mut self, block: &Block, partial: bool) {
         // We can cache a single partial block.
         if partial {
             assert!(self.partial_block.is_none());
@@ -99,7 +101,9 @@ impl State {
         }
     }
 
-    pub(crate) fn finalize(&mut self) -> Tag {
+    /// Finalize output producing a [`Tag`]
+    #[target_feature(enable = "avx2")]
+    pub(crate) unsafe fn finalize(&mut self) -> Tag {
         assert!(self.num_cached_blocks < 4);
         let mut data = &self.cached_blocks[..];