cratify file-store

Cargo.lock

@@ -1538,6 +1538,14 @@ version = "1.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a58880ee1c8f961b122908bb1a195e13cd31b81a28ed1136910bf881ce23bbe8"
 
+[[package]]
+name = "file_store"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "bincode",
+]
+
 [[package]]
 name = "filetime"
 version = "0.2.23"
@@ -4621,6 +4629,7 @@ dependencies = [
  "anyhow",
  "bincode",
  "bloom",
+ "file_store",
  "fst",
  "memmap",
  "serde",
@@ -4696,6 +4705,7 @@ dependencies = [
  "enum_dispatch",
  "eventsource-stream",
  "fend-core",
+ "file_store",
  "flate2",
  "fnv",
  "fst",

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "crates/client-wasm",
     "crates/client-wasm",
     "crates/core",
+    "crates/file-store",
     "crates/kuchiki",
     "crates/optics",
     "crates/speedy-kv",

assets/licenses.html

@@ -49,8 +49,8 @@
             <li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
             <li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (7)</li>
             <li><a href="#Unicode-DFS-2016">Unicode License Agreement - Data Files and Software (2016)</a> (5)</li>
+            <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (3)</li>
             <li><a href="#ISC">ISC License</a> (3)</li>
-            <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (2)</li>
             <li><a href="#CC0-1.0">Creative Commons Zero v1.0 Universal</a> (2)</li>
             <li><a href="#Zlib">zlib License</a> (2)</li>
             <li><a href="#BSL-1.0">Boost Software License 1.0</a> (1)</li>
@@ -69,6 +69,7 @@
                 <h4>Used by:</h4>
                 <ul class="license-used-by">
                     <li><a href=" https://crates.io/crates/bloom ">bloom 0.1.0</a></li>
+                    <li><a href=" https://crates.io/crates/file_store ">file_store 0.1.0</a></li>
                     <li><a href=" https://crates.io/crates/speedy_kv ">speedy_kv 0.1.0</a></li>
                 </ul>
                 <pre class="license-text">GNU AFFERO GENERAL PUBLIC LICENSE

crates/core/Cargo.toml

@@ -50,6 +50,7 @@ encoding_rs = { workspace = true }
 enum_dispatch = { workspace = true }
 eventsource-stream = { workspace = true }
 fend-core = { workspace = true }
+file_store = { path = "../file-store" }
 flate2 = { workspace = true }
 fnv = { workspace = true }
 fst = { workspace = true }

crates/core/src/lib.rs

@@ -51,7 +51,6 @@ mod executor;
 mod external_sort;
 mod fastfield_reader;
 pub mod feed;
-mod file_store;
 mod highlighted;
 mod human_website_annotations;
 pub mod hyperloglog;

crates/core/src/webgraph/store_writer.rs

@@ -24,11 +24,8 @@ use std::{
 
 use itertools::Itertools;
 
-use crate::file_store::{
-    self,
-    iterable::{IterableStoreReader, SortedIterableStoreReader},
-};
 use crate::Result;
+use file_store::iterable::{IterableStoreReader, SortedIterableStoreReader};
 
 use super::{store::EdgeStore, Compression, EdgeLabel, InnerEdge, NodeID};
 

crates/file-store/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "file_store"
+version = "0.1.0"
+edition = "2021"
+license = "AGPL-3.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow.workspace = true
+bincode.workspace = true

crates/core/src/file_store/mod.rs → crates/file-store/src/lib.rs

@@ -14,6 +14,10 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>
 
+//! A collection of simple disk-based data structures.
+
+pub type Result<T> = std::result::Result<T, anyhow::Error>;
+
 pub mod iterable;
 pub mod peekable;
 

crates/core/src/file_store/peekable.rs → crates/file-store/src/peekable.rs

@@ -14,8 +14,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/
 
-// TODO: Remove peekable from speedy_kv::segment and replace it with this
-
 /// An iterator that allows peeking at the next element.
 /// Unlike the standard library's `Peekable`, this implementation
 /// *is not* lazy and will always consume the next element when peeking.

crates/speedy-kv/Cargo.toml

@@ -10,6 +10,7 @@ license = "AGPL-3.0"
 anyhow.workspace = true
 bincode.workspace = true
 bloom = { path = "../bloom" }
+file_store = { path = "../file-store" }
 fst.workspace = true
 memmap.workspace = true
 serde.workspace = true

crates/speedy-kv/src/segment.rs

@@ -20,6 +20,8 @@ use std::{
     path::{Path, PathBuf},
 };
 
+use file_store::Peekable;
+
 use super::{
     blob_id_index::{BlobIdIndex, BlobIdIndexWriter},
     blob_index::{BlobIndex, BlobIndexWriter},
@@ -297,45 +299,6 @@ impl<K, V> Segment<K, V> {
     }
 }
 
-struct Peekable<I>
-where
-    I: Iterator,
-{
-    iter: I,
-    peeked: Option<I::Item>,
-}
-
-impl<I> Peekable<I>
-where
-    I: Iterator,
-{
-    fn new(iter: I) -> Self {
-        let mut iter = iter;
-        let peeked = iter.next();
-        Self { iter, peeked }
-    }
-
-    fn peek(&self) -> Option<&I::Item> {
-        self.peeked.as_ref()
-    }
-}
-
-impl<I> Iterator for Peekable<I>
-where
-    I: Iterator,
-{
-    type Item = I::Item;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let peeked = self.peeked.take();
-        if peeked.is_some() {
-            self.peeked = self.iter.next();
-        }
-
-        peeked
-    }
-}
-
 struct SortedSegments<I>
 where
     I: Iterator,