From e945c4b8e2dd4058cc6e40547454e5ea5b86c438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20DOUIN?= Date: Sat, 24 Feb 2024 09:37:55 +0100 Subject: [PATCH] replace sqlite by sled for id mapping storing --- CHANGELOG.md | 2 + Cargo.lock | 140 +++++++++++++++++++++------------------ Cargo.toml | 13 +--- flake.nix | 2 - src/backend/mod.rs | 22 ++----- src/cache/mod.rs | 160 +++++++++++++++++++-------------------------- 6 files changed, 156 insertions(+), 183 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06792da..59afe09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added account check-up command. - Added wizard warning about google passwords [#41]. ### Changed - Removed account configurations flatten level in order to improve diagnostic errors, due to a [bug](https://github.com/toml-rs/toml/issues/589#issuecomment-1872345017) in clap. **This means that accounts need to be prefixed by `accounts`: `[my-account]` becomes `[accounts.my-account]`**. It also opens doors for interface-specific configurations. - Rolled back cargo feature additions from the previous release. It was a mistake: the amount of features was too big, the code (both CLI and lib) was too hard to maintain. Cargo features kept: `imap`, `maildir`, `notmuch`, `smtp`, `sendmail`, `account-sync`, `account-discovery`, `pgp-gpg`, `pgp-commands` and `pgp-native`. +- Replaced id mapping database `SQLite` by `sled`, a pure key-val store written in Rust to improve portability of the tool. Therefore, id aliases are reset. - Improved pre and post edit choices interaction [#58]. - Improved account synchronization performances, making it 50% faster than `mbsync` and 370% faster than `OfflineIMAP`. - Changed `envelope.watch.{event}.{hook}`: hooks can now be cumulated. For example it is possible to send a system notification and execute a shell command when receiving a new envelope: diff --git a/Cargo.lock b/Cargo.lock index 6d6029d..4dfee17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1217,8 +1217,7 @@ dependencies = [ [[package]] name = "email-lib" version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e3535a72128056ee823c40edcf682d3d79c88fe6d86def04f5ae36ce229075" +source = "git+https://git.sr.ht/~soywod/pimalaya#9cdfca0f6729de2ebc0309883b8c78d0bbdbf31e" dependencies = [ "advisory-lock", "anyhow", @@ -1408,18 +1407,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastrand" version = "1.9.0" @@ -1459,7 +1446,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "windows-sys 0.52.0", ] @@ -1515,6 +1502,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures" version = "0.3.29" @@ -1632,6 +1629,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1753,15 +1759,6 @@ dependencies = [ "allocator-api2", ] -[[package]] -name = "hashlink" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" -dependencies = [ - "hashbrown", -] - [[package]] name = "heck" version = "0.4.1" @@ -1829,7 +1826,7 @@ dependencies = [ "ipconfig", "lru-cache", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "rand", "resolv-conf", "rustls 0.21.10", @@ -1867,11 +1864,11 @@ dependencies = [ "oauth-lib", "once_cell", "process-lib", - "rusqlite", "secret-lib", "serde", "serde_json", "shellexpand-utils", + "sled", "tempfile", "termcolor", "terminal_size", @@ -2287,18 +2284,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.1", "libc", - "redox_syscall", -] - -[[package]] -name = "libsqlite3-sys" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", + "redox_syscall 0.4.1", ] [[package]] @@ -2379,7 +2365,7 @@ dependencies = [ "lru-cache", "mail-builder", "mail-parser", - "parking_lot", + "parking_lot 0.12.1", "quick-xml", "ring 0.17.7", "rustls-pemfile", @@ -2930,6 +2916,17 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2937,7 +2934,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -2948,7 +2959,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -3332,6 +3343,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -3522,20 +3542,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rusqlite" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" -dependencies = [ - "bitflags 2.4.1", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -3957,6 +3963,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.11.2" @@ -4128,7 +4150,7 @@ checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand 2.0.1", - "redox_syscall", + "redox_syscall 0.4.1", "rustix 0.38.28", "windows-sys 0.48.0", ] @@ -4495,12 +4517,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version-compare" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index a2448da..07896ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,7 @@ secret-lib = "=0.3.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" shellexpand-utils = "=0.2.0" +sled = "=0.34.7" termcolor = "1.1" terminal_size = "0.1" tokio = { version = "1.23", default-features = false, features = ["macros", "rt-multi-thread"] } @@ -87,16 +88,8 @@ unicode-width = "0.1" url = "2.2" uuid = { version = "0.8", features = ["v4"] } -[target.'cfg(target_env = "musl")'.dependencies.rusqlite] -version = "0.29" -features = [] - -[target.'cfg(not(target_env = "musl"))'.dependencies.rusqlite] -version = "0.29" -features = ["bundled"] - [target.'cfg(not(windows))'.dependencies.coredump] version = "0.1" -# [patch.crates-io] -# email-lib = { git = "https://git.sr.ht/~soywod/pimalaya" } +[patch.crates-io] +email-lib = { git = "https://git.sr.ht/~soywod/pimalaya" } diff --git a/flake.nix b/flake.nix index 96df0e4..cdc801e 100644 --- a/flake.nix +++ b/flake.nix @@ -96,8 +96,6 @@ linux = mkPackage' null { }; linux-musl = mkPackage' "x86_64-unknown-linux-musl" (with pkgs.pkgsStatic; { CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static"; - SQLITE3_STATIC = 1; - SQLITE3_LIB_DIR = "${sqlite.out}/lib"; hardeningDisable = [ "all" ]; }); macos = mkPackage' null (with pkgs.darwin.apple_sdk.frameworks; { diff --git a/src/backend/mod.rs b/src/backend/mod.rs index de8b772..94ad6c7 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -619,32 +619,20 @@ impl Backend { match backend_kind { #[cfg(feature = "maildir")] Some(BackendKind::Maildir) => { - if let Some(mdir_config) = &self.toml_account_config.maildir { - id_mapper = IdMapper::new( - &self.backend.account_config, - folder, - mdir_config.root_dir.clone(), - )?; + if let Some(_) = &self.toml_account_config.maildir { + id_mapper = IdMapper::new(&self.backend.account_config, folder)?; } } #[cfg(feature = "account-sync")] Some(BackendKind::MaildirForSync) => { - id_mapper = IdMapper::new( - &self.backend.account_config, - folder, - self.backend.account_config.get_sync_dir()?, - )?; + id_mapper = IdMapper::new(&self.backend.account_config, folder)?; } #[cfg(feature = "notmuch")] Some(BackendKind::Notmuch) => { - if let Some(notmuch_config) = &self.toml_account_config.notmuch { - id_mapper = IdMapper::new( - &self.backend.account_config, - folder, - notmuch_config.get_maildir_path()?, - )?; + if let Some(_) = &self.toml_account_config.notmuch { + id_mapper = IdMapper::new(&self.backend.account_config, folder)?; } } _ => (), diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 0a4e67e..9190120 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -2,61 +2,34 @@ pub mod arg; pub mod args; use anyhow::{anyhow, Context, Result}; +use dirs::data_dir; use email::account::config::AccountConfig; -use log::{debug, trace}; -use std::path::{Path, PathBuf}; - -const ID_MAPPER_DB_FILE_NAME: &str = ".id-mapper.sqlite"; +use log::debug; +use sled::{Config, Db}; +use std::collections::HashSet; #[derive(Debug)] pub enum IdMapper { Dummy, - Mapper(String, rusqlite::Connection), + Mapper(Db), } impl IdMapper { - pub fn find_closest_db_path(dir: impl AsRef) -> PathBuf { - let mut db_path = dir.as_ref().join(ID_MAPPER_DB_FILE_NAME); - let mut db_parent_dir = dir.as_ref().parent(); + pub fn new(account_config: &AccountConfig, folder: &str) -> Result { + let digest = md5::compute(account_config.name.clone() + folder); + let db_path = data_dir() + .ok_or(anyhow!("cannot get XDG data directory"))? + .join("himalaya") + .join(".id-mappers") + .join(format!("{digest:x}")); - while !db_path.is_file() { - match db_parent_dir { - Some(dir) => { - db_path = dir.join(ID_MAPPER_DB_FILE_NAME); - db_parent_dir = dir.parent(); - } - None => { - db_path = dir.as_ref().join(ID_MAPPER_DB_FILE_NAME); - break; - } - } - } - - db_path - } - - pub fn new(account_config: &AccountConfig, folder: &str, db_path: PathBuf) -> Result { - let folder = account_config.get_folder_alias(folder); - let digest = md5::compute(account_config.name.clone() + &folder); - let table = format!("id_mapper_{digest:x}"); - debug!("creating id mapper table {table} at {db_path:?}…"); - - let db_path = Self::find_closest_db_path(db_path); - let conn = rusqlite::Connection::open(&db_path) + let conn = Config::new() + .path(&db_path) + .idgen_persist_interval(1) + .open() .with_context(|| format!("cannot open id mapper database at {db_path:?}"))?; - let query = format!( - "CREATE TABLE IF NOT EXISTS {table} ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - internal_id TEXT UNIQUE - )", - ); - trace!("create table query: {query:#?}"); - - conn.execute(&query, []) - .context("cannot create id mapper table")?; - - Ok(Self::Mapper(table, conn)) + Ok(Self::Mapper(conn)) } pub fn create_alias(&self, id: I) -> Result @@ -66,18 +39,18 @@ impl IdMapper { let id = id.as_ref(); match self { Self::Dummy => Ok(id.to_owned()), - Self::Mapper(table, conn) => { + Self::Mapper(conn) => { debug!("creating alias for id {id}…"); - let query = format!("INSERT OR IGNORE INTO {} (internal_id) VALUES (?)", table); - trace!("insert query: {query:#?}"); - - conn.execute(&query, [id]) - .with_context(|| format!("cannot create id alias for id {id}"))?; - - let alias = conn.last_insert_rowid().to_string(); + let alias = conn + .generate_id() + .with_context(|| format!("cannot create alias for id {id}"))? + .to_string(); debug!("created alias {alias} for id {id}"); + conn.insert(&id, alias.as_bytes()) + .with_context(|| format!("cannot insert alias {alias} for id {id}"))?; + Ok(alias) } } @@ -90,22 +63,16 @@ impl IdMapper { let id = id.as_ref(); match self { Self::Dummy => Ok(id.to_owned()), - Self::Mapper(table, conn) => { + Self::Mapper(conn) => { debug!("getting alias for id {id}…"); - let query = format!("SELECT id FROM {} WHERE internal_id = ?", table); - trace!("select query: {query:#?}"); + let alias = conn + .get(id) + .with_context(|| format!("cannot get alias for id {id}"))?; - let mut stmt = conn - .prepare(&query) - .with_context(|| format!("cannot get alias for id {id}"))?; - let aliases: Vec = stmt - .query_map([id], |row| row.get(0)) - .with_context(|| format!("cannot get alias for id {id}"))? - .collect::>() - .with_context(|| format!("cannot get alias for id {id}"))?; - let alias = match aliases.first() { + let alias = match alias { Some(alias) => { + let alias = String::from_utf8_lossy(alias.as_ref()); debug!("found alias {alias} for id {id}"); alias.to_string() } @@ -125,30 +92,24 @@ impl IdMapper { A: ToString, { let alias = alias.to_string(); - let alias = alias - .parse::() - .context(format!("cannot parse id mapper alias {alias}"))?; match self { Self::Dummy => Ok(alias.to_string()), - Self::Mapper(table, conn) => { + Self::Mapper(conn) => { debug!("getting id from alias {alias}…"); - let query = format!("SELECT internal_id FROM {} WHERE id = ?", table); - trace!("select query: {query:#?}"); - - let mut stmt = conn - .prepare(&query) - .with_context(|| format!("cannot get id from alias {alias}"))?; - let ids: Vec = stmt - .query_map([alias], |row| row.get(0)) - .with_context(|| format!("cannot get id from alias {alias}"))? - .collect::>() - .with_context(|| format!("cannot get id from alias {alias}"))?; - let id = ids - .first() - .ok_or_else(|| anyhow!("cannot get id from alias {alias}"))? - .to_owned(); + let id = conn + .iter() + .flat_map(|entry| entry) + .find_map(|(entry_id, entry_alias)| { + if entry_alias.as_ref() == alias.as_bytes() { + let entry_id = String::from_utf8_lossy(entry_id.as_ref()); + Some(entry_id.to_string()) + } else { + None + } + }) + .ok_or_else(|| anyhow!("cannot get id from alias {alias}"))?; debug!("found id {id} from alias {alias}"); Ok(id) @@ -156,14 +117,29 @@ impl IdMapper { } } - pub fn get_ids(&self, aliases: I) -> Result> - where - A: ToString, - I: IntoIterator, - { - aliases - .into_iter() - .map(|alias| self.get_id(alias)) - .collect() + pub fn get_ids(&self, aliases: impl IntoIterator) -> Result> { + let aliases: Vec = aliases.into_iter().map(|alias| alias.to_string()).collect(); + + match self { + Self::Dummy => Ok(aliases), + Self::Mapper(conn) => { + let aliases: HashSet<&str> = aliases.iter().map(|alias| alias.as_str()).collect(); + let ids: Vec = conn + .iter() + .flat_map(|entry| entry) + .filter_map(|(entry_id, entry_alias)| { + let alias = String::from_utf8_lossy(entry_alias.as_ref()); + if aliases.contains(alias.as_ref()) { + let entry_id = String::from_utf8_lossy(entry_id.as_ref()); + Some(entry_id.to_string()) + } else { + None + } + }) + .collect(); + + Ok(ids) + } + } } }