From b60005329ff4f7f0f87b4f95016d08d0cf4eb3a0 Mon Sep 17 00:00:00 2001 From: Bulat Kurbanov Date: Mon, 22 May 2023 01:09:51 +0200 Subject: [PATCH] Optimize user activity update --- Cargo.lock | 360 ++++++++++++++++++- Cargo.toml | 1 + src/bots/approved_bot/mod.rs | 30 +- src/bots_manager/bot_manager_client.rs | 44 +++ src/{bots_manager.rs => bots_manager/mod.rs} | 55 +-- 5 files changed, 439 insertions(+), 51 deletions(-) create mode 100644 src/bots_manager/bot_manager_client.rs rename src/{bots_manager.rs => bots_manager/mod.rs} (84%) diff --git a/Cargo.lock b/Cargo.lock index f937954..7b307b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite", + "log", + "parking", + "polling", + "rustix", + "slab", + "socket2", + "waker-fn", +] + +[[package]] +name = "async-lock" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" +dependencies = [ + "event-listener", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -186,6 +215,7 @@ dependencies = [ "futures", "lazy_static", "log", + "moka", "pretty_env_logger", "regex", "reqwest", @@ -207,12 +237,49 @@ version = "3.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c6ed94e98ecff0c12dd1b04c15ec0d7d9458ca8fe806cea6f12954efe74c63b" +[[package]] +name = "bytecount" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" + [[package]] name = "bytes" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "camino" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c530edf18f37068ac2d977409ed5cd50d53d73bc653c7647b48eb78976ac9ae2" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + [[package]] name = "cc" version = "1.0.79" @@ -240,6 +307,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "concurrent-queue" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -262,6 +338,38 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + [[package]] name = "ctrlc" version = "3.2.5" @@ -410,6 +518,21 @@ dependencies = [ "libc", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "fastrand" version = "1.9.0" @@ -497,6 +620,21 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.28" @@ -555,6 +693,12 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "h2" version = "0.3.19" @@ -833,6 +977,16 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -842,6 +996,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -860,6 +1023,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -897,6 +1069,32 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "moka" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "934030d03f6191edbb4ba16835ccdb80d560788ac686570a8e2986a0fb59ded8" +dependencies = [ + "async-io", + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "futures-util", + "num_cpus", + "once_cell", + "parking_lot", + "quanta", + "rustc_version", + "scheduled-thread-pool", + "skeptic", + "smallvec", + "tagptr", + "thiserror", + "triomphe", + "uuid", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1021,6 +1219,35 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "windows-sys 0.45.0", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -1065,6 +1292,22 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1114,6 +1357,33 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pulldown-cmark" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "quanta" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cc73c42f9314c4bdce450c77e6f09ecbddefbeddb1b5979ded332a3913ded33" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1159,6 +1429,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags", +] + [[package]] name = "rc-box" version = "1.2.0" @@ -1168,6 +1447,15 @@ dependencies = [ "erasable", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -1275,6 +1563,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.21" @@ -1284,6 +1581,15 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -1318,6 +1624,9 @@ name = "semver" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +dependencies = [ + "serde", +] [[package]] name = "sentry" @@ -1472,6 +1781,21 @@ dependencies = [ "libc", ] +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark", + "tempfile", + "walkdir", +] + [[package]] name = "slab" version = "0.4.8" @@ -1481,6 +1805,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + [[package]] name = "smawk" version = "0.3.1" @@ -1556,6 +1886,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "take_mut" version = "0.2.2" @@ -1650,7 +1986,7 @@ checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", "windows-sys 0.45.0", ] @@ -1881,6 +2217,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "triomphe" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ee9bd9239c339d714d657fac840c6d2a4f9c45f4f9ec7b0975113458be78db" + [[package]] name = "try-lock" version = "0.2.4" @@ -1982,6 +2324,22 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 6cd60f9..7c25c0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,3 +26,4 @@ chrono = "0.4.22" dateparser = "0.1.7" sentry = "0.27.0" lazy_static = "1.4.0" +moka = { version = "0.11", features = ["future"] } diff --git a/src/bots/approved_bot/mod.rs b/src/bots/approved_bot/mod.rs index 80a214b..95a573a 100644 --- a/src/bots/approved_bot/mod.rs +++ b/src/bots/approved_bot/mod.rs @@ -2,9 +2,10 @@ pub mod modules; pub mod services; mod tools; +use moka::future::Cache; use teloxide::{prelude::*, types::BotCommand, adaptors::{Throttle, CacheMe}}; -use crate::bots::approved_bot::services::user_settings::create_or_update_user_settings; +use crate::{bots::approved_bot::services::user_settings::create_or_update_user_settings, bots_manager::AppState}; use self::{ modules::{ @@ -18,9 +19,15 @@ use self::{ use super::{ignore_channel_messages, BotCommands, BotHandler, bots_manager::get_manager_handler, ignore_chat_member_update}; -async fn _update_activity(me: teloxide::types::Me, user: teloxide::types::User) -> Option<()> { +async fn _update_activity(me: teloxide::types::Me, user: teloxide::types::User, cache: Cache) -> Option<()> { + if cache.contains_key(&user.id) { + return None; + } + tokio::spawn(async move { - if update_user_activity(user.id).await.is_err() { + let mut update_result = update_user_activity(user.id).await; + + if update_result.is_err() { let allowed_langs = get_user_or_default_lang_codes(user.id).await; if create_or_update_user_settings( @@ -32,12 +39,13 @@ async fn _update_activity(me: teloxide::types::Me, user: teloxide::types::User) allowed_langs, ).await.is_ok() { - #[allow(unused_must_use)] - { - update_user_activity(user.id).await; - } + update_result = update_user_activity(user.id).await; } } + + if update_result.is_ok() { + cache.insert(user.id, true).await; + } }); None @@ -47,15 +55,15 @@ fn update_user_activity_handler() -> BotHandler { dptree::entry() .branch( Update::filter_callback_query().chain(dptree::filter_map_async( - |cq: CallbackQuery, bot: CacheMe>| async move { - _update_activity(bot.get_me().await.unwrap(), cq.from).await + |cq: CallbackQuery, bot: CacheMe>, app_state: AppState| async move { + _update_activity(bot.get_me().await.unwrap(), cq.from, app_state.user_activity_cache).await }, )), ) .branch(Update::filter_message().chain(dptree::filter_map_async( - |message: Message, bot: CacheMe>| async move { + |message: Message, bot: CacheMe>, app_state: AppState| async move { match message.from() { - Some(user) => _update_activity(bot.get_me().await.unwrap(), user.clone()).await, + Some(user) => _update_activity(bot.get_me().await.unwrap(), user.clone(), app_state.user_activity_cache).await, None => None, } }, diff --git a/src/bots_manager/bot_manager_client.rs b/src/bots_manager/bot_manager_client.rs new file mode 100644 index 0000000..54a993b --- /dev/null +++ b/src/bots_manager/bot_manager_client.rs @@ -0,0 +1,44 @@ +use serde::Deserialize; + +use crate::config; + + +#[derive(Deserialize, Debug, PartialEq, Clone, Copy)] +pub enum BotStatus { + #[serde(rename = "pending")] + Pending, + #[serde(rename = "approved")] + Approved, + #[serde(rename = "blocked")] + Blocked, +} + +#[derive(Deserialize, Debug, PartialEq, Clone, Copy)] +pub enum BotCache { + #[serde(rename = "original")] + Original, + #[serde(rename = "no_cache")] + NoCache, +} + +#[derive(Deserialize, Debug)] +pub struct BotData { + pub id: u32, + pub token: String, + pub status: BotStatus, + pub cache: BotCache, +} + +pub async fn get_bots() -> Result, reqwest::Error> { + let client = reqwest::Client::new(); + let response = client + .get(&config::CONFIG.manager_url) + .header("Authorization", &config::CONFIG.manager_api_key) + .send() + .await; + + match response { + Ok(v) => v.json::>().await, + Err(err) => Err(err), + } +} diff --git a/src/bots_manager.rs b/src/bots_manager/mod.rs similarity index 84% rename from src/bots_manager.rs rename to src/bots_manager/mod.rs index e2d7706..a0f22b8 100644 --- a/src/bots_manager.rs +++ b/src/bots_manager/mod.rs @@ -1,3 +1,5 @@ +pub mod bot_manager_client; + use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -12,51 +14,20 @@ use teloxide::{ }; use url::Url; -use serde::Deserialize; +use moka::future::Cache; use crate::config; +pub use self::bot_manager_client::{BotStatus, BotCache, BotData}; +use self::bot_manager_client::get_bots; -#[derive(Deserialize, Debug, PartialEq, Clone, Copy)] -pub enum BotStatus { - #[serde(rename = "pending")] - Pending, - #[serde(rename = "approved")] - Approved, - #[serde(rename = "blocked")] - Blocked, -} -#[derive(Deserialize, Debug, PartialEq, Clone, Copy)] -pub enum BotCache { - #[serde(rename = "original")] - Original, - #[serde(rename = "no_cache")] - NoCache, -} - -#[derive(Deserialize, Debug)] -struct BotData { - id: u32, - token: String, - status: BotStatus, - cache: BotCache, -} - -async fn get_bots() -> Result, reqwest::Error> { - let client = reqwest::Client::new(); - let response = client - .get(&config::CONFIG.manager_url) - .header("Authorization", &config::CONFIG.manager_api_key) - .send() - .await; - - match response { - Ok(v) => v.json::>().await, - Err(err) => Err(err), - } +#[derive(Clone)] +pub struct AppState { + pub user_activity_cache: Cache, } pub struct BotsManager { + app_state: AppState, next_port: u16, bot_port_map: HashMap, bot_status_and_cache_map: HashMap, @@ -66,6 +37,12 @@ pub struct BotsManager { impl BotsManager { pub fn create() -> Self { BotsManager { + app_state: AppState { + user_activity_cache: Cache::builder() + .time_to_live(Duration::from_secs(5 * 60)) + .max_capacity(4096) + .build() + }, next_port: 8000, bot_port_map: HashMap::new(), bot_status_and_cache_map: HashMap::new(), @@ -120,7 +97,7 @@ impl BotsManager { } let mut dispatcher = Dispatcher::builder(bot, handler) - .dependencies(dptree::deps![bot_data.cache]) + .dependencies(dptree::deps![bot_data.cache, self.app_state.clone()]) .build(); let shutdown_token = dispatcher.shutdown_token();