replace sqlite by sled for id mapping storing

CHANGELOG.md

@@ -9,12 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added account check-up command.
 - Added wizard warning about google passwords [#41].
 
 ### Changed
 
 - Removed account configurations flatten level in order to improve diagnostic errors, due to a [bug](https://github.com/toml-rs/toml/issues/589#issuecomment-1872345017) in clap. **This means that accounts need to be prefixed by `accounts`: `[my-account]` becomes `[accounts.my-account]`**. It also opens doors for interface-specific configurations.
 - Rolled back cargo feature additions from the previous release. It was a mistake: the amount of features was too big, the code (both CLI and lib) was too hard to maintain. Cargo features kept: `imap`, `maildir`, `notmuch`, `smtp`, `sendmail`, `account-sync`, `account-discovery`, `pgp-gpg`, `pgp-commands` and `pgp-native`.
+- Replaced id mapping database `SQLite` by `sled`, a pure key-val store written in Rust to improve portability of the tool. Therefore, id aliases are reset.
 - Improved pre and post edit choices interaction [#58].
 - Improved account synchronization performances, making it 50% faster than `mbsync` and 370% faster than `OfflineIMAP`.
 - Changed `envelope.watch.{event}.{hook}`: hooks can now be cumulated. For example it is possible to send a system notification and execute a shell command when receiving a new envelope:

Cargo.lock

@@ -1217,8 +1217,7 @@ dependencies = [
 [[package]]
 name = "email-lib"
 version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://git.sr.ht/~soywod/pimalaya#9cdfca0f6729de2ebc0309883b8c78d0bbdbf31e"
-checksum = "37e3535a72128056ee823c40edcf682d3d79c88fe6d86def04f5ae36ce229075"
 dependencies = [
  "advisory-lock",
  "anyhow",
@@ -1408,18 +1407,6 @@ dependencies = [
  "pin-project-lite",
 ]
 
-[[package]]
-name = "fallible-iterator"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
-
-[[package]]
-name = "fallible-streaming-iterator"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
-
 [[package]]
 name = "fastrand"
 version = "1.9.0"
@@ -1459,7 +1446,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.4.1",
  "windows-sys 0.52.0",
 ]
 
@@ -1515,6 +1502,16 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "fs2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "futures"
 version = "0.3.29"
@@ -1632,6 +1629,15 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -1753,15 +1759,6 @@ dependencies = [
  "allocator-api2",
 ]
 
-[[package]]
-name = "hashlink"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
-dependencies = [
- "hashbrown",
-]
-
 [[package]]
 name = "heck"
 version = "0.4.1"
@@ -1829,7 +1826,7 @@ dependencies = [
  "ipconfig",
  "lru-cache",
  "once_cell",
- "parking_lot",
+ "parking_lot 0.12.1",
  "rand",
  "resolv-conf",
  "rustls 0.21.10",
@@ -1867,11 +1864,11 @@ dependencies = [
  "oauth-lib",
  "once_cell",
  "process-lib",
- "rusqlite",
  "secret-lib",
  "serde",
  "serde_json",
  "shellexpand-utils",
+ "sled",
  "tempfile",
  "termcolor",
  "terminal_size",
@@ -2287,18 +2284,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
 dependencies = [
  "bitflags 2.4.1",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.4.1",
-]
-
-[[package]]
-name = "libsqlite3-sys"
-version = "0.26.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"
-dependencies = [
- "cc",
- "pkg-config",
- "vcpkg",
 ]
 
 [[package]]
@@ -2379,7 +2365,7 @@ dependencies = [
  "lru-cache",
  "mail-builder",
  "mail-parser",
- "parking_lot",
+ "parking_lot 0.12.1",
  "quick-xml",
  "ring 0.17.7",
  "rustls-pemfile",
@@ -2930,6 +2916,17 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
 
+[[package]]
+name = "parking_lot"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
+dependencies = [
+ "instant",
+ "lock_api",
+ "parking_lot_core 0.8.6",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -2937,7 +2934,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
 dependencies = [
  "lock_api",
- "parking_lot_core",
+ "parking_lot_core 0.9.9",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
+dependencies = [
+ "cfg-if",
+ "instant",
+ "libc",
+ "redox_syscall 0.2.16",
+ "smallvec",
+ "winapi",
 ]
 
 [[package]]
@@ -2948,7 +2959,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.4.1",
  "smallvec",
  "windows-targets 0.48.5",
 ]
@@ -3332,6 +3343,15 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.4.1"
@@ -3522,20 +3542,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "rusqlite"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"
-dependencies = [
- "bitflags 2.4.1",
- "fallible-iterator",
- "fallible-streaming-iterator",
- "hashlink",
- "libsqlite3-sys",
- "smallvec",
-]
-
 [[package]]
 name = "rustc-demangle"
 version = "0.1.23"
@@ -3957,6 +3963,22 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "sled"
+version = "0.34.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
+dependencies = [
+ "crc32fast",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "fs2",
+ "fxhash",
+ "libc",
+ "log",
+ "parking_lot 0.11.2",
+]
+
 [[package]]
 name = "smallvec"
 version = "1.11.2"
@@ -4128,7 +4150,7 @@ checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
 dependencies = [
  "cfg-if",
  "fastrand 2.0.1",
- "redox_syscall",
+ "redox_syscall 0.4.1",
  "rustix 0.38.28",
  "windows-sys 0.48.0",
 ]
@@ -4495,12 +4517,6 @@ dependencies = [
  "getrandom",
 ]
 
-[[package]]
-name = "vcpkg"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-
 [[package]]
 name = "version-compare"
 version = "0.1.1"

Cargo.toml

@@ -78,6 +78,7 @@ secret-lib = "=0.3.3"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 shellexpand-utils = "=0.2.0"
+sled = "=0.34.7"
 termcolor = "1.1"
 terminal_size = "0.1"
 tokio = { version = "1.23", default-features = false, features = ["macros", "rt-multi-thread"] }
@@ -87,16 +88,8 @@ unicode-width = "0.1"
 url = "2.2"
 uuid = { version = "0.8", features = ["v4"] }
 
-[target.'cfg(target_env = "musl")'.dependencies.rusqlite]
-version = "0.29"
-features = []
-
-[target.'cfg(not(target_env = "musl"))'.dependencies.rusqlite]
-version = "0.29"
-features = ["bundled"]
-
 [target.'cfg(not(windows))'.dependencies.coredump]
 version = "0.1"
 
-# [patch.crates-io]
+[patch.crates-io]
-# email-lib = { git = "https://git.sr.ht/~soywod/pimalaya" }
+email-lib = { git = "https://git.sr.ht/~soywod/pimalaya" }

flake.nix

@@ -96,8 +96,6 @@
           linux = mkPackage' null { };
           linux-musl = mkPackage' "x86_64-unknown-linux-musl" (with pkgs.pkgsStatic; {
             CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static";
-            SQLITE3_STATIC = 1;
-            SQLITE3_LIB_DIR = "${sqlite.out}/lib";
             hardeningDisable = [ "all" ];
           });
           macos = mkPackage' null (with pkgs.darwin.apple_sdk.frameworks; {

src/backend/mod.rs

@@ -619,32 +619,20 @@ impl Backend {
         match backend_kind {
             #[cfg(feature = "maildir")]
             Some(BackendKind::Maildir) => {
-                if let Some(mdir_config) = &self.toml_account_config.maildir {
+                if let Some(_) = &self.toml_account_config.maildir {
-                    id_mapper = IdMapper::new(
+                    id_mapper = IdMapper::new(&self.backend.account_config, folder)?;
-                        &self.backend.account_config,
-                        folder,
-                        mdir_config.root_dir.clone(),
-                    )?;
                 }
             }
 
             #[cfg(feature = "account-sync")]
             Some(BackendKind::MaildirForSync) => {
-                id_mapper = IdMapper::new(
+                id_mapper = IdMapper::new(&self.backend.account_config, folder)?;
-                    &self.backend.account_config,
-                    folder,
-                    self.backend.account_config.get_sync_dir()?,
-                )?;
             }
 
             #[cfg(feature = "notmuch")]
             Some(BackendKind::Notmuch) => {
-                if let Some(notmuch_config) = &self.toml_account_config.notmuch {
+                if let Some(_) = &self.toml_account_config.notmuch {
-                    id_mapper = IdMapper::new(
+                    id_mapper = IdMapper::new(&self.backend.account_config, folder)?;
-                        &self.backend.account_config,
-                        folder,
-                        notmuch_config.get_maildir_path()?,
-                    )?;
                 }
             }
             _ => (),

src/cache/mod.rs

@@ -2,61 +2,34 @@ pub mod arg;
 pub mod args;
 
 use anyhow::{anyhow, Context, Result};
+use dirs::data_dir;
 use email::account::config::AccountConfig;
-use log::{debug, trace};
+use log::debug;
-use std::path::{Path, PathBuf};
+use sled::{Config, Db};
-
+use std::collections::HashSet;
-const ID_MAPPER_DB_FILE_NAME: &str = ".id-mapper.sqlite";
 
 #[derive(Debug)]
 pub enum IdMapper {
     Dummy,
-    Mapper(String, rusqlite::Connection),
+    Mapper(Db),
 }
 
 impl IdMapper {
-    pub fn find_closest_db_path(dir: impl AsRef<Path>) -> PathBuf {
+    pub fn new(account_config: &AccountConfig, folder: &str) -> Result<Self> {
-        let mut db_path = dir.as_ref().join(ID_MAPPER_DB_FILE_NAME);
+        let digest = md5::compute(account_config.name.clone() + folder);
-        let mut db_parent_dir = dir.as_ref().parent();
+        let db_path = data_dir()
+            .ok_or(anyhow!("cannot get XDG data directory"))?
+            .join("himalaya")
+            .join(".id-mappers")
+            .join(format!("{digest:x}"));
 
-        while !db_path.is_file() {
+        let conn = Config::new()
-            match db_parent_dir {
+            .path(&db_path)
-                Some(dir) => {
+            .idgen_persist_interval(1)
-                    db_path = dir.join(ID_MAPPER_DB_FILE_NAME);
+            .open()
-                    db_parent_dir = dir.parent();
-                }
-                None => {
-                    db_path = dir.as_ref().join(ID_MAPPER_DB_FILE_NAME);
-                    break;
-                }
-            }
-        }
-
-        db_path
-    }
-
-    pub fn new(account_config: &AccountConfig, folder: &str, db_path: PathBuf) -> Result<Self> {
-        let folder = account_config.get_folder_alias(folder);
-        let digest = md5::compute(account_config.name.clone() + &folder);
-        let table = format!("id_mapper_{digest:x}");
-        debug!("creating id mapper table {table} at {db_path:?}…");
-
-        let db_path = Self::find_closest_db_path(db_path);
-        let conn = rusqlite::Connection::open(&db_path)
             .with_context(|| format!("cannot open id mapper database at {db_path:?}"))?;
 
-        let query = format!(
+        Ok(Self::Mapper(conn))
-            "CREATE TABLE IF NOT EXISTS {table} (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                internal_id TEXT UNIQUE
-            )",
-        );
-        trace!("create table query: {query:#?}");
-
-        conn.execute(&query, [])
-            .context("cannot create id mapper table")?;
-
-        Ok(Self::Mapper(table, conn))
     }
 
     pub fn create_alias<I>(&self, id: I) -> Result<String>
@@ -66,18 +39,18 @@ impl IdMapper {
         let id = id.as_ref();
         match self {
             Self::Dummy => Ok(id.to_owned()),
-            Self::Mapper(table, conn) => {
+            Self::Mapper(conn) => {
                 debug!("creating alias for id {id}…");
 
-                let query = format!("INSERT OR IGNORE INTO {} (internal_id) VALUES (?)", table);
+                let alias = conn
-                trace!("insert query: {query:#?}");
+                    .generate_id()
-
+                    .with_context(|| format!("cannot create alias for id {id}"))?
-                conn.execute(&query, [id])
+                    .to_string();
-                    .with_context(|| format!("cannot create id alias for id {id}"))?;
-
-                let alias = conn.last_insert_rowid().to_string();
                 debug!("created alias {alias} for id {id}");
 
+                conn.insert(&id, alias.as_bytes())
+                    .with_context(|| format!("cannot insert alias {alias} for id {id}"))?;
+
                 Ok(alias)
             }
         }
@@ -90,22 +63,16 @@ impl IdMapper {
         let id = id.as_ref();
         match self {
             Self::Dummy => Ok(id.to_owned()),
-            Self::Mapper(table, conn) => {
+            Self::Mapper(conn) => {
                 debug!("getting alias for id {id}…");
 
-                let query = format!("SELECT id FROM {} WHERE internal_id = ?", table);
+                let alias = conn
-                trace!("select query: {query:#?}");
+                    .get(id)
+                    .with_context(|| format!("cannot get alias for id {id}"))?;
 
-                let mut stmt = conn
+                let alias = match alias {
-                    .prepare(&query)
-                    .with_context(|| format!("cannot get alias for id {id}"))?;
-                let aliases: Vec<i64> = stmt
-                    .query_map([id], |row| row.get(0))
-                    .with_context(|| format!("cannot get alias for id {id}"))?
-                    .collect::<rusqlite::Result<_>>()
-                    .with_context(|| format!("cannot get alias for id {id}"))?;
-                let alias = match aliases.first() {
                     Some(alias) => {
+                        let alias = String::from_utf8_lossy(alias.as_ref());
                         debug!("found alias {alias} for id {id}");
                         alias.to_string()
                     }
@@ -125,30 +92,24 @@ impl IdMapper {
         A: ToString,
     {
         let alias = alias.to_string();
-        let alias = alias
-            .parse::<i64>()
-            .context(format!("cannot parse id mapper alias {alias}"))?;
 
         match self {
             Self::Dummy => Ok(alias.to_string()),
-            Self::Mapper(table, conn) => {
+            Self::Mapper(conn) => {
                 debug!("getting id from alias {alias}…");
 
-                let query = format!("SELECT internal_id FROM {} WHERE id = ?", table);
+                let id = conn
-                trace!("select query: {query:#?}");
+                    .iter()
-
+                    .flat_map(|entry| entry)
-                let mut stmt = conn
+                    .find_map(|(entry_id, entry_alias)| {
-                    .prepare(&query)
+                        if entry_alias.as_ref() == alias.as_bytes() {
-                    .with_context(|| format!("cannot get id from alias {alias}"))?;
+                            let entry_id = String::from_utf8_lossy(entry_id.as_ref());
-                let ids: Vec<String> = stmt
+                            Some(entry_id.to_string())
-                    .query_map([alias], |row| row.get(0))
+                        } else {
-                    .with_context(|| format!("cannot get id from alias {alias}"))?
+                            None
-                    .collect::<rusqlite::Result<_>>()
+                        }
-                    .with_context(|| format!("cannot get id from alias {alias}"))?;
+                    })
-                let id = ids
+                    .ok_or_else(|| anyhow!("cannot get id from alias {alias}"))?;
-                    .first()
-                    .ok_or_else(|| anyhow!("cannot get id from alias {alias}"))?
-                    .to_owned();
                 debug!("found id {id} from alias {alias}");
 
                 Ok(id)
@@ -156,14 +117,29 @@ impl IdMapper {
         }
     }
 
-    pub fn get_ids<A, I>(&self, aliases: I) -> Result<Vec<String>>
+    pub fn get_ids(&self, aliases: impl IntoIterator<Item = impl ToString>) -> Result<Vec<String>> {
-    where
+        let aliases: Vec<String> = aliases.into_iter().map(|alias| alias.to_string()).collect();
-        A: ToString,
+
-        I: IntoIterator<Item = A>,
+        match self {
-    {
+            Self::Dummy => Ok(aliases),
-        aliases
+            Self::Mapper(conn) => {
-            .into_iter()
+                let aliases: HashSet<&str> = aliases.iter().map(|alias| alias.as_str()).collect();
-            .map(|alias| self.get_id(alias))
+                let ids: Vec<String> = conn
-            .collect()
+                    .iter()
+                    .flat_map(|entry| entry)
+                    .filter_map(|(entry_id, entry_alias)| {
+                        let alias = String::from_utf8_lossy(entry_alias.as_ref());
+                        if aliases.contains(alias.as_ref()) {
+                            let entry_id = String::from_utf8_lossy(entry_id.as_ref());
+                            Some(entry_id.to_string())
+                        } else {
+                            None
+                        }
+                    })
+                    .collect();
+
+                Ok(ids)
+            }
+        }
     }
 }