From 36600e2f13a9ba5b334394a47096b57437cb984c Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 5 Nov 2023 00:48:26 +0530 Subject: [PATCH] feat: database methods to compute percentiles on analysis records --- .env_sample | 2 +- db/db-core/src/lib.rs | 8 +++ db/db-core/src/tests.rs | 67 ++++++++++++++++++- ...2a40f20cf812584aaf44418089bc7a51e07c4.json | 25 +++++++ ...c6712cbadb40f31bb12e160c9d333c7e3835c.json | 25 +++++++ db/db-sqlx-maria/src/lib.rs | 58 ++++++++++++++++ db/db-sqlx-maria/src/tests.rs | 40 +++++------ ...e79526a012b006ce9deb80bceb4e1a04c835d.json | 22 ++++++ ...106437d8e5034d3a40bf8face2ca7c12f2694.json | 23 +++++++ db/db-sqlx-postgres/src/lib.rs | 53 +++++++++++++++ db/db-sqlx-postgres/src/tests.rs | 42 +++++------- 11 files changed, 313 insertions(+), 52 deletions(-) create mode 100644 db/db-sqlx-maria/.sqlx/query-9bae79667a8cc631541879321e72a40f20cf812584aaf44418089bc7a51e07c4.json create mode 100644 db/db-sqlx-maria/.sqlx/query-c4d6ad934e38218931e74ae1c31c6712cbadb40f31bb12e160c9d333c7e3835c.json create mode 100644 db/db-sqlx-postgres/.sqlx/query-c08c1dd4bfcb6cbd0359c79cc3be79526a012b006ce9deb80bceb4e1a04c835d.json create mode 100644 db/db-sqlx-postgres/.sqlx/query-c67aec0c3d5786fb495b6ed60fa106437d8e5034d3a40bf8face2ca7c12f2694.json diff --git a/.env_sample b/.env_sample index 2833570f..1900d5e5 100644 --- a/.env_sample +++ b/.env_sample @@ -1,2 +1,2 @@ export POSTGRES_DATABASE_URL="postgres://postgres:password@localhost:5432/postgres" -export MARIA_DATABASE_URL="mysql://maria:password@localhost:3306/maria" +export MARIA_DATABASE_URL="mysql://root:password@localhost:3306/maria" diff --git a/db/db-core/src/lib.rs b/db/db-core/src/lib.rs index 1a51b31e..268f7f0e 100644 --- a/db/db-core/src/lib.rs +++ b/db/db-core/src/lib.rs @@ -307,6 +307,14 @@ pub trait MCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase { captcha_key: &str, difficulty_factor: u32, ) -> DBResult; + + /// Get number of analytics entries that are under a certain duration + async fn stats_get_num_logs_under_time(&self, duration: u32) -> DBResult; + + /// Get the entry at a location in the list of analytics entires under a certain time limit + /// and sorted in ascending order + async fn stats_get_entry_at_location_for_time_limit_asc(&self, duration: u32, location: u32) -> DBResult>; + } #[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] diff --git a/db/db-core/src/tests.rs b/db/db-core/src/tests.rs index fd59230c..d4a3a26a 100644 --- a/db/db-core/src/tests.rs +++ b/db/db-core/src/tests.rs @@ -7,6 +7,29 @@ use crate::errors::*; use crate::prelude::*; +/// easy traffic pattern +pub const TRAFFIC_PATTERN: TrafficPattern = TrafficPattern { + avg_traffic: 500, + peak_sustainable_traffic: 5_000, + broke_my_site_traffic: Some(10_000), + }; + +/// levels for complex captcha config +pub const LEVELS: [Level; 3] = [ + Level { + difficulty_factor: 1, + visitor_threshold: 1, + }, + Level { + difficulty_factor: 2, + visitor_threshold: 2, + }, + Level { + difficulty_factor: 3, + visitor_threshold: 3, + }, + ]; + /// test all database functions pub async fn database_works<'a, T: MCDatabase>( db: &T, @@ -250,7 +273,6 @@ pub async fn database_works<'a, T: MCDatabase>( db.record_confirm(c.key).await.unwrap(); // analytics start - db.analytics_create_psuedo_id_if_not_exists(c.key) .await .unwrap(); @@ -282,11 +304,17 @@ pub async fn database_works<'a, T: MCDatabase>( ); let analytics = CreatePerformanceAnalytics { - time: 0, - difficulty_factor: 0, + time: 1, + difficulty_factor: 1, worker_type: "wasm".into(), }; + + + assert_eq!(db.stats_get_num_logs_under_time(analytics.time).await.unwrap(), 0); + db.analysis_save(c.key, &analytics).await.unwrap(); + assert_eq!(db.stats_get_num_logs_under_time(analytics.time).await.unwrap(), 1); + assert_eq!(db.stats_get_num_logs_under_time(analytics.time - 1).await.unwrap(), 0); let limit = 50; let mut offset = 0; let a = db.analytics_fetch(c.key, limit, offset).await.unwrap(); @@ -305,6 +333,39 @@ pub async fn database_works<'a, T: MCDatabase>( .unwrap(); assert_eq!(db.analytics_fetch(c.key, 1000, 0).await.unwrap().len(), 0); assert!(!db.analytics_captcha_is_published(c.key).await.unwrap()); + + let rest_analytics= [ + CreatePerformanceAnalytics { + time: 2, + difficulty_factor: 2, + worker_type: "wasm".into(), + }, + CreatePerformanceAnalytics { + time: 3, + difficulty_factor: 3, + worker_type: "wasm".into(), + }, + + CreatePerformanceAnalytics { + time: 4, + difficulty_factor: 4, + worker_type: "wasm".into(), + }, + + CreatePerformanceAnalytics { + time: 5, + difficulty_factor: 5, + worker_type: "wasm".into(), + }, + ]; + for a in rest_analytics.iter() { + db.analysis_save(c.key, &a).await.unwrap(); + } + assert!(db.stats_get_entry_at_location_for_time_limit_asc(1, 2).await.unwrap().is_none()); + assert_eq!(db.stats_get_entry_at_location_for_time_limit_asc(2, 1).await.unwrap(), Some(2)); + assert_eq!(db.stats_get_entry_at_location_for_time_limit_asc(3, 2).await.unwrap(), Some(3)); + + db.analytics_delete_all_records_for_campaign(c.key) .await .unwrap(); diff --git a/db/db-sqlx-maria/.sqlx/query-9bae79667a8cc631541879321e72a40f20cf812584aaf44418089bc7a51e07c4.json b/db/db-sqlx-maria/.sqlx/query-9bae79667a8cc631541879321e72a40f20cf812584aaf44418089bc7a51e07c4.json new file mode 100644 index 00000000..e1379fc3 --- /dev/null +++ b/db/db-sqlx-maria/.sqlx/query-9bae79667a8cc631541879321e72a40f20cf812584aaf44418089bc7a51e07c4.json @@ -0,0 +1,25 @@ +{ + "db_name": "MySQL", + "query": "SELECT\n COUNT(difficulty_factor) AS count\n FROM\n mcaptcha_pow_analytics\n WHERE time <= ?;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": { + "type": "LongLong", + "flags": "NOT_NULL | BINARY", + "char_set": 63, + "max_size": 21 + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false + ] + }, + "hash": "9bae79667a8cc631541879321e72a40f20cf812584aaf44418089bc7a51e07c4" +} diff --git a/db/db-sqlx-maria/.sqlx/query-c4d6ad934e38218931e74ae1c31c6712cbadb40f31bb12e160c9d333c7e3835c.json b/db/db-sqlx-maria/.sqlx/query-c4d6ad934e38218931e74ae1c31c6712cbadb40f31bb12e160c9d333c7e3835c.json new file mode 100644 index 00000000..b01dbe2b --- /dev/null +++ b/db/db-sqlx-maria/.sqlx/query-c4d6ad934e38218931e74ae1c31c6712cbadb40f31bb12e160c9d333c7e3835c.json @@ -0,0 +1,25 @@ +{ + "db_name": "MySQL", + "query": "SELECT\n difficulty_factor\n FROM\n mcaptcha_pow_analytics\n WHERE\n time <= ?\n ORDER BY difficulty_factor ASC LIMIT 1 OFFSET ?;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "difficulty_factor", + "type_info": { + "type": "Long", + "flags": "NOT_NULL | NO_DEFAULT_VALUE", + "char_set": 63, + "max_size": 11 + } + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false + ] + }, + "hash": "c4d6ad934e38218931e74ae1c31c6712cbadb40f31bb12e160c9d333c7e3835c" +} diff --git a/db/db-sqlx-maria/src/lib.rs b/db/db-sqlx-maria/src/lib.rs index d4f4cc78..95b256ad 100644 --- a/db/db-sqlx-maria/src/lib.rs +++ b/db/db-sqlx-maria/src/lib.rs @@ -1219,6 +1219,64 @@ impl MCDatabase for Database { Ok(res.nonce as u32) } } + + + + /// Get number of analytics entries that are under a certain duration + async fn stats_get_num_logs_under_time(&self, duration: u32) -> DBResult { + + struct Count { + count: Option, + } + + //"SELECT COUNT(*) FROM (SELECT difficulty_factor FROM mcaptcha_pow_analytics WHERE time <= ?) as count", + let count = sqlx::query_as!( + Count, + "SELECT + COUNT(difficulty_factor) AS count + FROM + mcaptcha_pow_analytics + WHERE time <= ?;", + duration as i32, + ) + .fetch_one(&self.pool) + .await + .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?; + + Ok(count.count.unwrap_or_else(|| 0) as usize) + } + + /// Get the entry at a location in the list of analytics entires under a certain time limited + /// and sorted in ascending order + async fn stats_get_entry_at_location_for_time_limit_asc(&self, duration: u32, location: u32) -> DBResult> { + + + struct Difficulty { + difficulty_factor: Option, + } + + match sqlx::query_as!( + Difficulty, + "SELECT + difficulty_factor + FROM + mcaptcha_pow_analytics + WHERE + time <= ? + ORDER BY difficulty_factor ASC LIMIT 1 OFFSET ?;", + duration as i32, + location as i64 - 1, + ) + .fetch_one(&self.pool) + .await + { + Ok(res) => Ok(Some(res.difficulty_factor.unwrap() as usize)), + Err(sqlx::Error::RowNotFound) => Ok(None), + Err(e) => Err(map_row_not_found_err(e, DBError::CaptchaNotFound)) + + } + + } } #[derive(Clone)] diff --git a/db/db-sqlx-maria/src/tests.rs b/db/db-sqlx-maria/src/tests.rs index 2fee47ba..030b7eb8 100644 --- a/db/db-sqlx-maria/src/tests.rs +++ b/db/db-sqlx-maria/src/tests.rs @@ -5,9 +5,11 @@ #![cfg(test)] -use sqlx::mysql::MySqlPoolOptions; use std::env; +use sqlx::{mysql::MySqlPoolOptions, migrate::MigrateDatabase}; +use url::Url; + use crate::*; use db_core::tests::*; @@ -26,28 +28,6 @@ async fn everyting_works() { const HEADING: &str = "testing notifications get db mariadb"; const MESSAGE: &str = "testing notifications get message db mariadb"; - // easy traffic pattern - const TRAFFIC_PATTERN: TrafficPattern = TrafficPattern { - avg_traffic: 500, - peak_sustainable_traffic: 5_000, - broke_my_site_traffic: Some(10_000), - }; - - const LEVELS: [Level; 3] = [ - Level { - difficulty_factor: 1, - visitor_threshold: 1, - }, - Level { - difficulty_factor: 2, - visitor_threshold: 2, - }, - Level { - difficulty_factor: 3, - visitor_threshold: 3, - }, - ]; - const ADD_NOTIFICATION: AddNotification = AddNotification { from: NAME, to: NAME, @@ -56,10 +36,20 @@ async fn everyting_works() { }; let url = env::var("MARIA_DATABASE_URL").unwrap(); + + let mut parsed = Url::parse(&url).unwrap(); + parsed.set_path("db_maria_test"); + let url = parsed.to_string(); + + if sqlx::MySql::database_exists(&url).await.unwrap() { + sqlx::MySql::drop_database(&url).await.unwrap(); + } + sqlx::MySql::create_database(&url).await.unwrap(); + let pool_options = MySqlPoolOptions::new().max_connections(2); let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, - url, + url: url.clone(), disable_logging: false, }); let db = connection_options.connect().await.unwrap(); @@ -78,4 +68,6 @@ async fn everyting_works() { description: CAPTCHA_DESCRIPTION, }; database_works(&db, &p, &c, &LEVELS, &TRAFFIC_PATTERN, &ADD_NOTIFICATION).await; + drop(db); + sqlx::MySql::drop_database(&url).await.unwrap(); } diff --git a/db/db-sqlx-postgres/.sqlx/query-c08c1dd4bfcb6cbd0359c79cc3be79526a012b006ce9deb80bceb4e1a04c835d.json b/db/db-sqlx-postgres/.sqlx/query-c08c1dd4bfcb6cbd0359c79cc3be79526a012b006ce9deb80bceb4e1a04c835d.json new file mode 100644 index 00000000..337fd8a5 --- /dev/null +++ b/db/db-sqlx-postgres/.sqlx/query-c08c1dd4bfcb6cbd0359c79cc3be79526a012b006ce9deb80bceb4e1a04c835d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(difficulty_factor) FROM mcaptcha_pow_analytics WHERE time <= $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c08c1dd4bfcb6cbd0359c79cc3be79526a012b006ce9deb80bceb4e1a04c835d" +} diff --git a/db/db-sqlx-postgres/.sqlx/query-c67aec0c3d5786fb495b6ed60fa106437d8e5034d3a40bf8face2ca7c12f2694.json b/db/db-sqlx-postgres/.sqlx/query-c67aec0c3d5786fb495b6ed60fa106437d8e5034d3a40bf8face2ca7c12f2694.json new file mode 100644 index 00000000..38add3a6 --- /dev/null +++ b/db/db-sqlx-postgres/.sqlx/query-c67aec0c3d5786fb495b6ed60fa106437d8e5034d3a40bf8face2ca7c12f2694.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n difficulty_factor\n FROM\n mcaptcha_pow_analytics\n WHERE\n time <= $1\n ORDER BY difficulty_factor ASC LIMIT 1 OFFSET $2;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "difficulty_factor", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c67aec0c3d5786fb495b6ed60fa106437d8e5034d3a40bf8face2ca7c12f2694" +} diff --git a/db/db-sqlx-postgres/src/lib.rs b/db/db-sqlx-postgres/src/lib.rs index d29b0962..79cbbccf 100644 --- a/db/db-sqlx-postgres/src/lib.rs +++ b/db/db-sqlx-postgres/src/lib.rs @@ -1227,6 +1227,59 @@ impl MCDatabase for Database { Ok(res.nonce as u32) } } + + /// Get number of analytics entries that are under a certain duration + async fn stats_get_num_logs_under_time(&self, duration: u32) -> DBResult { + + struct Count { + count: Option, + } + + let count = sqlx::query_as!( + Count, + "SELECT COUNT(difficulty_factor) FROM mcaptcha_pow_analytics WHERE time <= $1;", + duration as i32, + ) + .fetch_one(&self.pool) + .await + .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?; + + Ok(count.count.unwrap_or_else(|| 0) as usize) + } + + /// Get the entry at a location in the list of analytics entires under a certain time limit + /// and sorted in ascending order + async fn stats_get_entry_at_location_for_time_limit_asc(&self, duration: u32, location: u32) -> DBResult> { + + + struct Difficulty { + difficulty_factor: Option, + } + + match sqlx::query_as!( + Difficulty, + "SELECT + difficulty_factor + FROM + mcaptcha_pow_analytics + WHERE + time <= $1 + ORDER BY difficulty_factor ASC LIMIT 1 OFFSET $2;", + duration as i32, + location as i64 - 1, + ) + .fetch_one(&self.pool) + .await + { + Ok(res) => Ok(Some(res.difficulty_factor.unwrap() as usize)), + Err(sqlx::Error::RowNotFound) => Ok(None), + Err(e) => Err(map_row_not_found_err(e, DBError::CaptchaNotFound)) + + } + + + } + } #[derive(Clone)] diff --git a/db/db-sqlx-postgres/src/tests.rs b/db/db-sqlx-postgres/src/tests.rs index a7bd4390..6957182e 100644 --- a/db/db-sqlx-postgres/src/tests.rs +++ b/db/db-sqlx-postgres/src/tests.rs @@ -5,9 +5,12 @@ #![cfg(test)] -use sqlx::postgres::PgPoolOptions; use std::env; +use sqlx::postgres::PgPoolOptions; +use sqlx::migrate::MigrateDatabase; +use url::Url; + use crate::*; use db_core::tests::*; @@ -26,28 +29,6 @@ async fn everyting_works() { const HEADING: &str = "testing notifications get db postgres"; const MESSAGE: &str = "testing notifications get message db postgres"; - // easy traffic pattern - const TRAFFIC_PATTERN: TrafficPattern = TrafficPattern { - avg_traffic: 500, - peak_sustainable_traffic: 5_000, - broke_my_site_traffic: Some(10_000), - }; - - const LEVELS: [Level; 3] = [ - Level { - difficulty_factor: 1, - visitor_threshold: 1, - }, - Level { - difficulty_factor: 2, - visitor_threshold: 2, - }, - Level { - difficulty_factor: 3, - visitor_threshold: 3, - }, - ]; - const ADD_NOTIFICATION: AddNotification = AddNotification { from: NAME, to: NAME, @@ -56,10 +37,21 @@ async fn everyting_works() { }; let url = env::var("POSTGRES_DATABASE_URL").unwrap(); + + let mut parsed = Url::parse(&url).unwrap(); + parsed.set_path("db_postgres_test"); + let url = parsed.to_string(); + + if sqlx::Postgres::database_exists(&url).await.unwrap() { + sqlx::Postgres::drop_database(&url).await.unwrap(); + } + sqlx::Postgres::create_database(&url).await.unwrap(); + + let pool_options = PgPoolOptions::new().max_connections(2); let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, - url, + url: url.clone(), disable_logging: false, }); let db = connection_options.connect().await.unwrap(); @@ -78,4 +70,6 @@ async fn everyting_works() { description: CAPTCHA_DESCRIPTION, }; database_works(&db, &p, &c, &LEVELS, &TRAFFIC_PATTERN, &ADD_NOTIFICATION).await; + drop(db); + sqlx::Postgres::drop_database(&url).await.unwrap(); }