Merge pull request #629 from neon-mmd/FEAT/427_export-import-settings-to-and-from-a-json-file-support-for-the-ui

✨ Export/Import settings to and from a `json` file support for the UI

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "websurfx"
-version = "1.18.0"
+version = "1.21.0"
 edition = "2021"
 description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
 repository = "https://github.com/neon-mmd/websurfx"
@@ -43,17 +43,17 @@ actix-cors = { version = "0.7.0", default-features = false }
 fake-useragent = { version = "0.1.3", default-features = false }
 env_logger = { version = "0.11.6", default-features = false }
 log = { version = "0.4.21", default-features = false }
-mlua = { version = "0.9.9", features = [
+mlua = { version = "0.10.2", features = [
     "luajit",
     "vendored",
 ], default-features = false }
-redis = { version = "0.27.5", features = [
+redis = { version = "0.28.1", features = [
     "tokio-comp",
     "connection-manager",
     "tcp_nodelay"
 ], default-features = false, optional = true }
-blake3 = { version = "1.5.5", default-features = false }
-error-stack = { version = "0.5.0", default-features = false, features = [
+blake3 = { version = "1.5.4", default-features = false }
+error-stack = { version = "0.4.0", default-features = false, features = [
     "std",
 ] }
 async-trait = { version = "0.1.80", default-features = false }
@@ -62,7 +62,7 @@ futures = { version = "0.3.31", default-features = false, features = ["alloc"] }
 dhat = { version = "0.3.2", optional = true, default-features = false }
 mimalloc = { version = "0.1.43", default-features = false }
 async-once-cell = { version = "0.5.4", default-features = false }
-actix-governor = { version = "0.7.0", default-features = false }
+actix-governor = { version = "0.8.0", default-features = false }
 moka = { version = "0.12.8", optional = true, default-features = false, features = [
     "future",
 ] }
@@ -87,6 +87,11 @@ stop-words = { version = "0.8.0", default-features = false, features = ["iso"] }
 thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [
     "moby",
 ]}
+
+actix-multipart = { version = "0.7.2", default-features = false, features = [
+    "derive",
+    "tempfile",
+]}
 itertools = {version = "0.14.0", default-features = false}
 
 [dev-dependencies]

public/static/themes/simple.css

@@ -5,7 +5,8 @@
   font-weight: 200 600;
   font-stretch: 0% 200%;
   font-display: swap;
-  src: url("https://fonts.gstatic.com/s/rubik/v28/iJWKBXyIfDnIV7nErXyi0A.woff2") format('woff2');
+  src: url('https://fonts.gstatic.com/s/rubik/v28/iJWKBXyIfDnIV7nErXyi0A.woff2')
+    format('woff2');
 }
 
 * {
@@ -97,7 +98,7 @@ button {
   outline-offset: 3px;
   outline: 2px solid transparent;
   border: none;
-  transition: .1s;
+  transition: 0.1s;
   gap: 0;
   background-color: var(--color-six);
   color: var(--background-color);
@@ -107,10 +108,10 @@ button {
 }
 
 .search_bar button img {
-  position:absolute;
-  left:50%;
-  top:50%;
-  transform:translate(-50%, -50%);
+  position: absolute;
+  left: 50%;
+  top: 50%;
+  transform: translate(-50%, -50%);
 }
 
 .search_bar button:active {
@@ -253,7 +254,6 @@ button {
 
 /* styles for the footer and header */
 
-
 header {
   width: 100%;
   background: var(--background-color);
@@ -341,7 +341,7 @@ footer div {
 .results_aggregated {
   display: flex;
   flex-direction: column;
-  justify-content: space-between; 
+  justify-content: space-between;
   margin: 2rem 0;
   content-visibility: auto;
 }
@@ -714,7 +714,8 @@ footer div {
 }
 
 .settings_container .user_interface select,
-.settings_container .general select {
+.settings_container .general select,
+.settings_container .general form input {
   margin: 0.7rem 0;
   width: 20rem;
   background-color: var(--color-one);
@@ -726,6 +727,38 @@ footer div {
   text-transform: capitalize;
 }
 
+.settings_container .general form input {
+  padding: 0;
+  width: 30rem;
+  text-align: center;
+  text-transform: none;
+}
+
+.settings_container .general form input::file-selector-button {
+  content: 'Browse';
+  padding: 1rem 2rem;
+  font-size: 1.5rem;
+  background: var(--color-three);
+  color: var(--background-color);
+  border-radius: 0.5rem;
+  border: 2px solid transparent;
+  font-weight: bold;
+  transition: all 0.1s ease-out;
+  cursor: pointer;
+  box-shadow: 5px 5px;
+  outline: none;
+  translate: -1rem 0;
+}
+
+.settings_container .general form input::file-selector-button:active {
+  box-shadow: none;
+  translate: 5px 5px;
+}
+
+.settings_container .general .export_btn {
+  margin-bottom: 1rem;
+}
+
 .settings_container .user_interface option:hover,
 .settings_container .general option:hover {
   background-color: var(--color-one);
@@ -798,7 +831,7 @@ footer div {
   left: 0.4rem;
   bottom: 0.4rem;
   background-color: var(--background-color);
-  transition: .2s;
+  transition: 0.2s;
 }
 
 input:checked + .slider {
@@ -822,7 +855,7 @@ input:checked + .slider::before {
   border-radius: 50%;
 }
 
-@media screen and (width <= 1136px) {
+@media screen and (width <=1136px) {
   .hero-text-container {
     width: unset;
   }
@@ -832,7 +865,7 @@ input:checked + .slider::before {
   }
 }
 
-@media screen and (width <= 706px) {
+@media screen and (width <=706px) {
   .about-container article .logo-container svg {
     width: clamp(200px, 290px, 815px);
   }
@@ -856,7 +889,7 @@ input:checked + .slider::before {
   .features {
     grid-template-columns: 1fr;
   }
-  
+
   .feature-list {
     padding: 35px 0;
   }

src/cache/cacher.rs

@@ -213,12 +213,10 @@ pub trait Cacher: Send + Sync {
     }
 
     /// A helper function that compresses or encrypts search results before they're inserted into a cache store
-
     /// # Arguments
     ///
     /// * `search_results` - A reference to the search_Results to process.
     ///
-
     ///
     /// # Error
     /// Returns a Vec of compressed or encrypted bytes on success otherwise it returns a CacheError

src/config/parser.rs

@@ -78,11 +78,11 @@ impl Config {
         lua.load(&fs::read_to_string(file_path(FileType::Config)?)?)
             .exec()?;
 
-        let parsed_threads: u8 = globals.get::<_, u8>("threads")?;
+        let parsed_threads: u8 = globals.get("threads")?;
 
-        let debug: bool = globals.get::<_, bool>("debug")?;
-        let logging: bool = globals.get::<_, bool>("logging")?;
-        let adaptive_window: bool = globals.get::<_, bool>("adaptive_window")?;
+        let debug: bool = globals.get("debug")?;
+        let logging: bool = globals.get("logging")?;
+        let adaptive_window: bool = globals.get("adaptive_window")?;
 
         if !logging_initialized {
             set_logging_level(debug, logging);
@@ -99,9 +99,9 @@ impl Config {
             parsed_threads
         };
 
-        let rate_limiter = globals.get::<_, HashMap<String, u8>>("rate_limiter")?;
+        let rate_limiter: HashMap<String, u8> = globals.get("rate_limiter")?;
 
-        let parsed_safe_search: u8 = globals.get::<_, u8>("safe_search")?;
+        let parsed_safe_search: u8 = globals.get::<_>("safe_search")?;
         let safe_search: u8 = match parsed_safe_search {
             0..=4 => parsed_safe_search,
             _ => {
@@ -112,7 +112,7 @@ impl Config {
         };
 
         #[cfg(any(feature = "redis-cache", feature = "memory-cache"))]
-        let parsed_cet = globals.get::<_, u16>("cache_expiry_time")?;
+        let parsed_cet = globals.get::<_>("cache_expiry_time")?;
         #[cfg(any(feature = "redis-cache", feature = "memory-cache"))]
         let cache_expiry_time = match parsed_cet {
             0..=59 => {
@@ -125,7 +125,7 @@ impl Config {
             _ => parsed_cet,
         };
 
-        let proxy_opt = globals.get::<_, Option<String>>("proxy")?;
+        let proxy_opt: Option<String> = globals.get::<_>("proxy")?;
         let proxy = proxy_opt.and_then(|proxy_str| {
             Proxy::all(proxy_str).ok().and_then(|_| {
                 log::error!("Invalid proxy url, defaulting to no proxy.");
@@ -135,30 +135,29 @@ impl Config {
 
         Ok(Config {
             operating_system_tls_certificates: globals
-                .get::<_, bool>("operating_system_tls_certificates")?,
-            port: globals.get::<_, u16>("port")?,
-            binding_ip: globals.get::<_, String>("binding_ip")?,
+                .get::<_>("operating_system_tls_certificates")?,
+            port: globals.get::<_>("port")?,
+            binding_ip: globals.get::<_>("binding_ip")?,
             style: Style::new(
-                globals.get::<_, String>("theme")?,
-                globals.get::<_, String>("colorscheme")?,
-                globals.get::<_, Option<String>>("animation")?,
+                globals.get::<_>("theme")?,
+                globals.get::<_>("colorscheme")?,
+                globals.get::<_>("animation")?,
             ),
             #[cfg(feature = "redis-cache")]
-            redis_url: globals.get::<_, String>("redis_url")?,
+            redis_url: globals.get::<_>("redis_url")?,
             aggregator: AggregatorConfig {
-                random_delay: globals.get::<_, bool>("production_use")?,
+                random_delay: globals.get::<_>("production_use")?,
             },
             logging,
             debug,
             adaptive_window,
-            upstream_search_engines: globals
-                .get::<_, HashMap<String, bool>>("upstream_search_engines")?,
-            request_timeout: globals.get::<_, u8>("request_timeout")?,
-            tcp_connection_keep_alive: globals.get::<_, u8>("tcp_connection_keep_alive")?,
-            pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?,
-            number_of_https_connections: globals.get::<_, u8>("number_of_https_connections")?,
+            upstream_search_engines: globals.get::<_>("upstream_search_engines")?,
+            request_timeout: globals.get::<_>("request_timeout")?,
+            tcp_connection_keep_alive: globals.get::<_>("tcp_connection_keep_alive")?,
+            pool_idle_connection_timeout: globals.get::<_>("pool_idle_connection_timeout")?,
+            number_of_https_connections: globals.get::<_>("number_of_https_connections")?,
             threads,
-            client_connection_keep_alive: globals.get::<_, u8>("client_connection_keep_alive")?,
+            client_connection_keep_alive: globals.get::<_>("client_connection_keep_alive")?,
             rate_limiter: RateLimiter {
                 number_of_requests: rate_limiter["number_of_requests"],
                 time_limit: rate_limiter["time_limit"],

src/lib.rs

@@ -110,6 +110,7 @@ pub fn run(
             .service(server::routes::search::search) // search page
             .service(router::about) // about page
             .service(router::settings) // settings page
+            .service(server::routes::export_import::download) // download page
             .default_service(web::route().to(router::not_found)) // error page
     })
     .workers(config.threads as usize)

src/models/parser_models.rs

@@ -10,7 +10,7 @@
 /// order to allow the deserializing the json back to struct in aggregate function in
 /// aggregator.rs and create a new struct out of it and then serialize it back to json and pass
 /// it to the template files.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct Style {
     /// It stores the parsed theme option used to set a theme for the website.
     pub theme: String,

src/models/server_models.rs

@@ -2,7 +2,7 @@
 //! engine website.
 use std::borrow::Cow;
 
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 use super::parser_models::Style;
 
@@ -22,16 +22,22 @@ pub struct SearchParams {
 
 /// A named struct which is used to deserialize the cookies fetched from the client side.
 #[allow(dead_code)]
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize)]
 pub struct Cookie<'a> {
+    #[serde(borrow)]
     /// It stores the theme name used in the website.
     pub theme: Cow<'a, str>,
+    #[serde(borrow)]
     /// It stores the colorscheme name used for the website theme.
     pub colorscheme: Cow<'a, str>,
+    #[serde(borrow)]
     /// It stores the user selected upstream search engines selected from the UI.
     pub engines: Cow<'a, [Cow<'a, str>]>,
     /// It stores the user selected safe search level from the UI.
     pub safe_search_level: u8,
+    #[serde(borrow)]
+    /// It stores the animation name used for the website theme.
+    pub animation: Option<Cow<'a, str>>,
 }
 
 impl<'a> Cookie<'a> {
@@ -43,6 +49,10 @@ impl<'a> Cookie<'a> {
             colorscheme: Cow::Borrowed(&style.colorscheme),
             engines: Cow::Owned(engines),
             safe_search_level,
+            animation: style
+                .animation
+                .as_ref()
+                .map(|str| Cow::Borrowed(str.as_str())),
         }
     }
 }

src/server/routes/export_import.rs

@@ -0,0 +1,194 @@
+//! This module handles the settings and download route of the search engine website.
+
+use crate::{
+    handler::{file_path, FileType},
+    models::{self, server_models},
+    Config,
+};
+use actix_multipart::form::{tempfile::TempFile, MultipartForm};
+use actix_web::{
+    cookie::{
+        time::{Duration, OffsetDateTime},
+        Cookie,
+    },
+    get, post, web, HttpRequest, HttpResponse,
+};
+use std::borrow::Cow;
+use std::io::Read;
+
+use tokio::fs::read_dir;
+
+/// A helper function that helps in building the list of all available colorscheme/theme/animation
+/// names present in the colorschemes, animations and themes folder respectively by excluding the
+/// ones that have already been selected via the config file.
+///
+/// # Arguments
+///
+/// * `style_type` - It takes the style type of the values `theme` and `colorscheme` as an
+/// argument.
+///
+/// # Error
+///
+/// Returns a list of colorscheme/theme names as a vector of tuple strings on success otherwise
+/// returns a standard error message.
+async fn style_option_list<'a>(
+    style_type: &'a str,
+) -> Result<Box<[Cow<'a, str>]>, Box<dyn std::error::Error>> {
+    let mut style_options = Vec::new();
+    let mut dir = read_dir(format!(
+        "{}static/{}/",
+        file_path(FileType::Theme)?,
+        style_type,
+    ))
+    .await?;
+    while let Some(file) = dir.next_entry().await? {
+        let style_name = file.file_name().to_str().unwrap().replace(".css", "");
+        style_options.push(Cow::Owned(style_name));
+    }
+
+    if style_type == "animations" {
+        style_options.push(Cow::default())
+    }
+
+    Ok(style_options.into_boxed_slice())
+}
+
+/// A helper function which santizes user provided json data from the input file.
+///
+/// # Arguments
+///
+/// * `config` - It takes the config struct as an argument.
+/// * `setting_value` - It takes the cookie struct as an argument.
+///
+/// # Error
+///
+/// returns a standard error message on failure otherwise it returns the unit type.
+async fn sanitize(
+    config: web::Data<&'static Config>,
+    setting_value: &mut models::server_models::Cookie<'_>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    // Check whether the theme, colorscheme and animation option is valid by matching it against
+    // the available option list. If the option provided by the user via the JSON file is invalid
+    // then replace the user provided by the default one used by the server via the config file.
+
+    if !style_option_list("themes")
+        .await?
+        .contains(&setting_value.theme)
+    {
+        setting_value.theme = Cow::Borrowed(&config.style.theme)
+    } else if !style_option_list("colorschemes")
+        .await?
+        .contains(&setting_value.colorscheme)
+    {
+        setting_value.colorscheme = Cow::Borrowed(&config.style.colorscheme)
+    } else if !style_option_list("animations")
+        .await?
+        .contains(setting_value.animation.as_ref().unwrap())
+    {
+        setting_value.animation = config
+            .style
+            .animation
+            .as_ref()
+            .map(|str| Cow::Borrowed(str.as_str()));
+    }
+
+    // Filters out any engines in the list that are invalid by matching each engine against the
+    // available engine list.
+    let engines: Vec<_> = setting_value
+        .engines
+        .iter()
+        .cloned()
+        .filter_map(|engine| {
+            config
+                .upstream_search_engines
+                .keys()
+                .cloned()
+                .any(|other_engine| *engine == other_engine)
+                .then_some(engine.clone())
+        })
+        .collect();
+    setting_value.engines = Cow::Owned(engines);
+
+    setting_value.safe_search_level = match setting_value.safe_search_level {
+        0..2 => setting_value.safe_search_level,
+        _ => u8::default(),
+    };
+
+    Ok(())
+}
+
+/// A multipart struct which stores user provided input file data in memory.
+#[derive(MultipartForm)]
+struct File {
+    /// It stores the input file data in memory.
+    file: TempFile,
+}
+
+/// Handles the route of the post settings page.
+#[post("/settings")]
+pub async fn set_settings(
+    config: web::Data<&'static Config>,
+    MultipartForm(mut form): MultipartForm<File>,
+) -> Result<HttpResponse, Box<dyn std::error::Error>> {
+    if let Some(file_name) = form.file.file_name {
+        let file_name_parts = file_name.split(".");
+        if let 2 = file_name_parts.clone().count() {
+            if let Some("json") = file_name_parts.last() {
+                if let 0 = form.file.size {
+                    return Ok(HttpResponse::BadRequest().finish());
+                } else {
+                    let mut data = String::new();
+                    form.file.file.read_to_string(&mut data).unwrap();
+
+                    let mut unsanitized_json_data: models::server_models::Cookie<'_> =
+                        serde_json::from_str(&data)?;
+
+                    sanitize(config, &mut unsanitized_json_data).await?;
+
+                    let sanitized_json_data: String =
+                        serde_json::json!(unsanitized_json_data).to_string();
+
+                    return Ok(HttpResponse::Ok()
+                        .cookie(
+                            Cookie::build("appCookie", sanitized_json_data)
+                                .expires(
+                                    OffsetDateTime::now_utc().saturating_add(Duration::weeks(52)),
+                                )
+                                .finish(),
+                        )
+                        .finish());
+                }
+            }
+        }
+    }
+    Ok(HttpResponse::Ok().finish())
+}
+
+/// Handles the route of the download page.
+#[get("/download")]
+pub async fn download(
+    config: web::Data<&'static Config>,
+    req: HttpRequest,
+) -> Result<HttpResponse, Box<dyn std::error::Error>> {
+    let cookie = req.cookie("appCookie");
+
+    // Get search settings using the user's cookie or from the server's config
+    let preferences: server_models::Cookie<'_> = cookie
+        .as_ref()
+        .and_then(|cookie_value| serde_json::from_str(cookie_value.value()).ok())
+        .unwrap_or_else(|| {
+            server_models::Cookie::build(
+                &config.style,
+                config
+                    .upstream_search_engines
+                    .iter()
+                    .filter_map(|(engine, enabled)| {
+                        enabled.then_some(Cow::Borrowed(engine.as_str()))
+                    })
+                    .collect(),
+                u8::default(),
+            )
+        });
+
+    Ok(HttpResponse::Ok().json(preferences))
+}

src/server/routes/mod.rs

@@ -1,3 +1,4 @@
 //! This module provides modules to handle various routes in the search engine website.
 
+pub mod export_import;
 pub mod search;

src/server/routes/search.rs

@@ -55,6 +55,7 @@ pub async fn search(
 
             // Get search settings using the user's cookie or from the server's config
             let mut search_settings: server_models::Cookie<'_> = cookie
+                .as_ref()
                 .and_then(|cookie_value| serde_json::from_str(cookie_value.value()).ok())
                 .unwrap_or_else(|| {
                     server_models::Cookie::build(

src/templates/partials/settings_tabs/general.rs

@@ -37,6 +37,21 @@ pub fn general(safe_search_level: u8) -> Markup {
                      option value=(SAFE_SEARCH_LEVELS[2].0){(SAFE_SEARCH_LEVELS[2].1)}
                }
            }
+           h3{"Restore preferences from file"}
+           p class="description"{
+               "Select a json file to restore preferences for the search engine."
+           }
+           form method="post" enctype="multipart/form-data" onsubmit="setTimeout(function () { window.location.reload(); }, 10)" action="/settings" target="_self"{
+                input type="file" name="file" accept="application/json";
+                button type="submit" {"Submit"}
+           }
+           h3{"Export preferences to a file"}
+           p class="description"{
+               "Export all the settings saved as a cookie as a json file."
+           }
+           a class="export_btn" href="./download" download="settings.json" {
+                button type="submit" {"Export"}
+           }
         }
     )
 }