Add rabin chunker

The program now supports content defined chunking via a rabin chunker
implementation.

Author: inverted-tree

src/chunker/chunker.rs

@@ -1,11 +1,17 @@
-use crate::chunker::dynamicChunker::DynamicChunker;
 use crate::chunker::fileChunker::FileChunker;
+use crate::chunker::rabinChunker::RabinChunker;
 use crate::chunker::staticChunker::StaticChunker;
-use std::slice::Chunks;
 
 use clap::ValueEnum;
 use memmap2::Mmap;
 
+#[derive(Debug, Clone, Copy)]
+pub enum ChunkingScheme {
+    FILE,
+    STATIC,
+    CONTENT,
+}
+
 #[derive(ValueEnum, Clone, Copy, Debug)]
 pub enum ChunkerType {
     FILE,
@@ -45,42 +51,46 @@ impl ChunkerType {
             ChunkerType::CDC64K => 1 << 16,
         }
     }
-}
-
-pub trait Chunker {
-    fn chunk<'a>(&self, mmap: &'a Mmap) -> Chunks<'a, u8>;
-}
-
-pub struct ChunkFactory {
-    t: ChunkerType,
-    s: Option<Box<str>>,
-}
-
-impl ChunkFactory {
-    pub fn new(chunkerType: ChunkerType, salt: Option<Box<str>>) -> Self {
-        Self {
-            t: chunkerType,
-            s: salt,
-        }
-    }
 
-    pub fn createChunker(&self) -> Box<dyn Chunker> {
-        match self.t {
-            ChunkerType::FILE => Box::new(FileChunker::new()),
+    pub fn getScheme(&self) -> ChunkingScheme {
+        match self {
+            ChunkerType::FILE => ChunkingScheme::FILE,
             ChunkerType::SC1K
             | ChunkerType::SC2K
             | ChunkerType::SC4K
             | ChunkerType::SC8K
             | ChunkerType::SC16K
             | ChunkerType::SC32K
-            | ChunkerType::SC64K => Box::new(StaticChunker::new(ChunkerType::getSize(&self.t))),
+            | ChunkerType::SC64K => ChunkingScheme::STATIC,
             ChunkerType::CDC1K
             | ChunkerType::CDC2K
             | ChunkerType::CDC4K
             | ChunkerType::CDC8K
             | ChunkerType::CDC16K
             | ChunkerType::CDC32K
-            | ChunkerType::CDC64K => Box::new(DynamicChunker::new(ChunkerType::getSize(&self.t))),
+            | ChunkerType::CDC64K => ChunkingScheme::CONTENT,
+        }
+    }
+}
+
+pub trait Chunker {
+    fn chunk<'a>(&self, mmap: &'a Mmap) -> Box<dyn Iterator<Item = &'a [u8]> + 'a>;
+}
+
+pub struct ChunkFactory {
+    t: ChunkerType,
+}
+
+impl ChunkFactory {
+    pub fn new(chunkerType: ChunkerType) -> Self {
+        Self { t: chunkerType }
+    }
+
+    pub fn createChunker(&self) -> Box<dyn Chunker> {
+        match self.t.getScheme() {
+            ChunkingScheme::FILE => Box::new(FileChunker::new()),
+            ChunkingScheme::STATIC => Box::new(StaticChunker::new(self.t.getSize())),
+            ChunkingScheme::CONTENT => Box::new(RabinChunker::<64>::new(self.t.getSize())),
         }
     }
 }

src/chunker/dynamicChunker.rs

@@ -1,6 +1,5 @@
 use crate::chunker::chunker::Chunker;
 use memmap2::Mmap;
-use std::slice::Chunks;
 
 pub struct FileChunker {}
 
@@ -11,8 +10,9 @@ impl FileChunker {
 }
 
 impl Chunker for FileChunker {
-    fn chunk<'a>(&self, mmap: &'a Mmap) -> Chunks<'a, u8> {
-        (&mmap[..]).chunks(mmap.len())
+    fn chunk<'a>(&self, mmap: &'a Mmap) -> Box<dyn Iterator<Item = &'a [u8]> + 'a> {
+        let fileSize = mmap.len();
+        Box::new(mmap.chunks(fileSize))
     }
 }
 
@@ -49,8 +49,9 @@ mod tests {
     fn testFileChunkerFactory() {
         let (data, mmap) = generateTestData();
 
-        let factory = ChunkFactory::new(ChunkerType::FILE, None);
+        let factory = ChunkFactory::new(ChunkerType::FILE);
         let chunker = factory.createChunker();
+
         let mut chunks = chunker.chunk(&mmap);
 
         assert_eq!(chunks.next(), Some(data.as_bytes()));

src/chunker/fileChunker.rs

@@ -1,4 +1,4 @@
 pub mod chunker;
-pub mod dynamicChunker;
 pub mod fileChunker;
+pub mod rabinChunker;
 pub mod staticChunker;