chore: update blake3 and use new hazmat api (#63)

* minimal change to adapt to the new guts api of upstream blake3

* Use new API everywhere

* Adapt to latest ~guts~ hazmat api

* Update hazmat dep

* Use published hazmat API

* Use div_ceil to get clippy to shut up.

TBH I think the code was fine before, but whatever...

* require serde for mixed ror now

* fix imports

* remove last mentions of guts

* remove test that compared guts hash_subtree with our own recursive_hash_subtree

The bug is gone now, and since this is now an official feature of blake3 we rely on their tests

Author: rklaehn

Cargo.lock

@@ -13,7 +13,7 @@ rust-version = "1.75"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-iroh-blake3 = "1.4.3"
+
 range-collections = { version = "0.4.5", features = ["new_unchecked"] }
 smallvec = "1"
 
@@ -24,13 +24,14 @@ iroh-io = { version = "0.6.0", default-features = false, optional = true }
 positioned-io = { version = "0.3.1", default-features = false }
 genawaiter = { version = "0.99.1", features = ["futures03"], optional = true }
 tokio = { version = "1", features = ["sync"], default-features = false, optional = true }
+blake3 = { version = "1.8" }
 serde = { version = "1", features = ["derive"], optional = true }
 
 [features]
 serde = ["dep:serde", "bytes/serde"]
 tokio_fsm = ["dep:futures-lite", "dep:iroh-io"]
 validate = ["dep:genawaiter"]
-experimental-mixed = ["dep:tokio"]
+experimental-mixed = ["dep:tokio", "dep:serde"]
 default = ["tokio_fsm", "validate", "serde"]
 
 [dev-dependencies]

Cargo.toml

@@ -58,11 +58,6 @@ fn hash_benches_large(c: &mut Criterion) {
             hasher.finalize()
         })
     });
-    c.bench_function("hash_subtree", |b| {
-        b.iter(|| {
-            blake3::guts::hash_subtree(0, &data, true);
-        })
-    });
 }
 
 criterion_group!(benches, offset_benches, iter_benches, hash_benches_large,);

benches/tree_bench.rs

@@ -14,7 +14,6 @@ use std::{
     result,
 };
 
-use blake3::guts::parent_cv;
 use bytes::Bytes;
 pub use iroh_io::{AsyncSliceReader, AsyncSliceWriter};
 use iroh_io::{AsyncStreamReader, AsyncStreamWriter};
@@ -30,6 +29,7 @@ use crate::{
         Leaf, Parent,
     },
     iter::{BaoChunk, ResponseIter},
+    parent_cv,
     rec::{encode_selected_rec, truncate_ranges, truncate_ranges_owned},
     BaoTree, BlockSize, ChunkRanges, ChunkRangesRef, TreeNode,
 };
@@ -757,8 +757,8 @@ mod validate {
 
     use super::Outboard;
     use crate::{
-        blake3, hash_subtree, io::LocalBoxFuture, rec::truncate_ranges, split, BaoTree, ChunkNum,
-        ChunkRangesRef, TreeNode,
+        blake3, hash_subtree, io::LocalBoxFuture, parent_cv, rec::truncate_ranges, split, BaoTree,
+        ChunkNum, ChunkRangesRef, TreeNode,
     };
 
     /// Given a data file and an outboard, compute all valid ranges.
@@ -867,7 +867,7 @@ mod validate {
                     // outboard is incomplete, we can't validate
                     return Ok(());
                 };
-                let actual = blake3::guts::parent_cv(&l_hash, &r_hash, is_root);
+                let actual = parent_cv(&l_hash, &r_hash, is_root);
                 if &actual != parent_hash {
                     // hash mismatch, we can't validate
                     return Ok(());
@@ -970,7 +970,7 @@ mod validate {
                     // outboard is incomplete, we can't validate
                     return Ok(());
                 };
-                let actual = blake3::guts::parent_cv(&l_hash, &r_hash, is_root);
+                let actual = parent_cv(&l_hash, &r_hash, is_root);
                 if &actual != parent_hash {
                     // hash mismatch, we can't validate
                     return Ok(());

src/io/fsm.rs

@@ -1,16 +1,15 @@
 //! Read from sync, send to tokio sender
 use std::{future::Future, result};
 
+use blake3;
 use bytes::Bytes;
-use iroh_blake3 as blake3;
-use iroh_blake3::guts::parent_cv;
 use serde::{Deserialize, Serialize};
 use smallvec::SmallVec;
 
 use super::{sync::Outboard, EncodeError, Leaf, Parent};
 use crate::{
-    hash_subtree, iter::BaoChunk, rec::truncate_ranges, split_inner, ChunkNum, ChunkRangesRef,
-    TreeNode,
+    hash_subtree, iter::BaoChunk, parent_cv, rec::truncate_ranges, split_inner, ChunkNum,
+    ChunkRangesRef, TreeNode,
 };
 
 /// A content item for the bao streaming protocol.
@@ -230,7 +229,7 @@ pub fn traverse_selected_rec(
     emit_data: bool,
     res: &mut Vec<EncodedItem>,
 ) -> blake3::Hash {
-    use blake3::guts::{ChunkState, CHUNK_LEN};
+    use blake3::CHUNK_LEN;
     if data.len() <= CHUNK_LEN {
         if emit_data && !query.is_empty() {
             res.push(
@@ -241,9 +240,7 @@ pub fn traverse_selected_rec(
                 .into(),
             );
         }
-        let mut hasher = ChunkState::new(start_chunk.0);
-        hasher.update(&data);
-        hasher.finalize(is_root)
+        hash_subtree(start_chunk.0, &data, is_root)
     } else {
         let chunks = data.len() / CHUNK_LEN + (data.len() % CHUNK_LEN != 0) as usize;
         let chunks = chunks.next_power_of_two();
@@ -256,7 +253,7 @@ pub fn traverse_selected_rec(
         // for full ranges where the level is below min_level, we want to emit
         // just the data.
         //
-        // todo: maybe call into blake3::guts::hash_subtree directly for this case? it would be faster.
+        // todo: maybe call into blake3::hazmat::hash_subtree directly for this case? it would be faster.
         let full = query.is_all();
         let emit_parent = !query.is_empty() && (!full || level >= min_level);
         let hash_offset = if emit_parent {

src/io/mixed.rs

@@ -7,7 +7,6 @@ use std::{
     result,
 };
 
-use blake3::guts::parent_cv;
 use bytes::BytesMut;
 pub use positioned_io::{ReadAt, Size, WriteAt};
 use smallvec::SmallVec;
@@ -22,6 +21,7 @@ use crate::{
         Leaf, Parent,
     },
     iter::{BaoChunk, ResponseIterRef},
+    parent_cv,
     rec::encode_selected_rec,
     BaoTree, BlockSize, ChunkRangesRef, TreeNode,
 };
@@ -663,8 +663,8 @@ mod validate {
 
     use super::Outboard;
     use crate::{
-        blake3, hash_subtree, io::LocalBoxFuture, rec::truncate_ranges, split, BaoTree, ChunkNum,
-        ChunkRangesRef, TreeNode,
+        blake3, hash_subtree, io::LocalBoxFuture, parent_cv, rec::truncate_ranges, split, BaoTree,
+        ChunkNum, ChunkRangesRef, TreeNode,
     };
 
     /// Given a data file and an outboard, compute all valid ranges.
@@ -777,7 +777,7 @@ mod validate {
                     // outboard is incomplete, we can't validate
                     return Ok(());
                 };
-                let actual = blake3::guts::parent_cv(&l_hash, &r_hash, is_root);
+                let actual = parent_cv(&l_hash, &r_hash, is_root);
                 if &actual != parent_hash {
                     // hash mismatch, we can't validate
                     return Ok(());
@@ -879,7 +879,7 @@ mod validate {
                     // outboard is incomplete, we can't validate
                     return Ok(());
                 };
-                let actual = blake3::guts::parent_cv(&l_hash, &r_hash, is_root);
+                let actual = parent_cv(&l_hash, &r_hash, is_root);
                 if &actual != parent_hash {
                     // hash mismatch, we can't validate
                     return Ok(());

src/io/sync.rs

@@ -214,7 +214,7 @@ mod tree;
 use iter::*;
 pub use tree::{BlockSize, ChunkNum};
 pub mod io;
-pub use iroh_blake3 as blake3;
+pub use blake3;
 
 #[cfg(all(test, feature = "tokio_fsm"))]
 mod tests;
@@ -232,34 +232,32 @@ pub type ByteRanges = range_collections::RangeSet2<u64>;
 /// [ChunkRanges] implements [`AsRef<ChunkRangesRef>`].
 pub type ChunkRangesRef = range_collections::RangeSetRef<ChunkNum>;
 
-/// Quickly hash a subtree
-///
-/// This is a wrapper that passes through to the blake3::guts implementation if the size is a power of 2, and
-/// falls back to a recursive implementation if it is not. There is a bug in the guts implementation that
-/// requires this workaround.
-pub fn hash_subtree(start_chunk: u64, data: &[u8], is_root: bool) -> blake3::Hash {
-    if data.len().is_power_of_two() {
-        blake3::guts::hash_subtree(start_chunk, data, is_root)
+fn hash_subtree(start_chunk: u64, data: &[u8], is_root: bool) -> blake3::Hash {
+    use blake3::hazmat::{ChainingValue, HasherExt};
+    if is_root {
+        debug_assert!(start_chunk == 0);
+        blake3::hash(data)
     } else {
-        recursive_hash_subtree(start_chunk, data, is_root)
+        let mut hasher = blake3::Hasher::new();
+        hasher.set_input_offset(start_chunk * 1024);
+        hasher.update(data);
+        let non_root_hash: ChainingValue = hasher.finalize_non_root();
+        blake3::Hash::from(non_root_hash)
     }
 }
 
-/// This is a recursive version of [`hash_subtree`], for testing.
-fn recursive_hash_subtree(start_chunk: u64, data: &[u8], is_root: bool) -> blake3::Hash {
-    use blake3::guts::{ChunkState, CHUNK_LEN};
-    if data.len() <= CHUNK_LEN {
-        let mut hasher = ChunkState::new(start_chunk);
-        hasher.update(data);
-        hasher.finalize(is_root)
+fn parent_cv(left_child: &blake3::Hash, right_child: &blake3::Hash, is_root: bool) -> blake3::Hash {
+    use blake3::hazmat::{merge_subtrees_non_root, merge_subtrees_root, ChainingValue, Mode};
+    let left_child: ChainingValue = *left_child.as_bytes();
+    let right_child: ChainingValue = *right_child.as_bytes();
+    if is_root {
+        merge_subtrees_root(&left_child, &right_child, Mode::Hash)
     } else {
-        let chunks = data.len() / CHUNK_LEN + (data.len() % CHUNK_LEN != 0) as usize;
-        let chunks = chunks.next_power_of_two();
-        let mid = chunks / 2;
-        let mid_bytes = mid * CHUNK_LEN;
-        let left = recursive_hash_subtree(start_chunk, &data[..mid_bytes], false);
-        let right = recursive_hash_subtree(start_chunk + mid as u64, &data[mid_bytes..], false);
-        blake3::guts::parent_cv(&left, &right, is_root)
+        blake3::Hash::from(merge_subtrees_non_root(
+            &left_child,
+            &right_child,
+            Mode::Hash,
+        ))
     }
 }
 

src/lib.rs

@@ -2,7 +2,7 @@
 //!
 //! Encocding is used to compute hashes, decoding is only used in tests as a
 //! reference implementation.
-use crate::{blake3, split_inner, ChunkNum, ChunkRangesRef};
+use crate::{blake3, hash_subtree, parent_cv, split_inner, ChunkNum, ChunkRangesRef};
 
 /// Given a set of chunk ranges, adapt them for a tree of the given size.
 ///
@@ -105,14 +105,12 @@ pub(crate) fn encode_selected_rec(
     emit_data: bool,
     res: &mut Vec<u8>,
 ) -> blake3::Hash {
-    use blake3::guts::{ChunkState, CHUNK_LEN};
+    use blake3::CHUNK_LEN;
     if data.len() <= CHUNK_LEN {
         if emit_data && !query.is_empty() {
             res.extend_from_slice(data);
         }
-        let mut hasher = ChunkState::new(start_chunk.0);
-        hasher.update(data);
-        hasher.finalize(is_root)
+        hash_subtree(start_chunk.0, data, is_root)
     } else {
         let chunks = data.len() / CHUNK_LEN + (data.len() % CHUNK_LEN != 0) as usize;
         let chunks = chunks.next_power_of_two();
@@ -125,7 +123,7 @@ pub(crate) fn encode_selected_rec(
         // for full ranges where the level is below min_level, we want to emit
         // just the data.
         //
-        // todo: maybe call into blake3::guts::hash_subtree directly for this case? it would be faster.
+        // todo: maybe call into blake3::hazmat::hash_subtree directly for this case? it would be faster.
         let full = query.is_all();
         let emit_parent = !query.is_empty() && (!full || level >= min_level);
         let hash_offset = if emit_parent {
@@ -159,7 +157,7 @@ pub(crate) fn encode_selected_rec(
             res[o..o + 32].copy_from_slice(left.as_bytes());
             res[o + 32..o + 64].copy_from_slice(right.as_bytes());
         }
-        blake3::guts::parent_cv(&left, &right, is_root)
+        parent_cv(&left, &right, is_root)
     }
 }
 
@@ -203,7 +201,7 @@ mod test_support {
         if ranges.is_empty() {
             return;
         }
-        use blake3::guts::CHUNK_LEN;
+        use blake3::CHUNK_LEN;
 
         if size <= CHUNK_LEN {
             emit(BaoChunk::Leaf {