Add deleting old archives

2025-12-06 06:15:37 +01:00 · 2023-09-24 00:09:43 +02:00
parent 3fdad853f0
commit 9722d44304
5 changed files with 156 additions and 25 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -100,7 +100,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -111,7 +111,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -228,6 +228,7 @@ dependencies = [
 "axum-prometheus",
 "base64",
 "bytes",
+ "chrono",
 "futures-core",
 "md5",
 "minio-rsc",
@@ -241,6 +242,7 @@ dependencies = [
 "smartstring",
 "tempfile",
 "tokio",
+ "tokio-cron-scheduler",
 "tower-http",
 "tracing",
 "tracing-subscriber",
@@ -369,8 +371,10 @@ checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
 dependencies = [
 "android-tzdata",
 "iana-time-zone",
+ "js-sys",
 "num-traits",
 "serde",
+ "wasm-bindgen",
 "windows-targets",
 ]

@@ -424,6 +428,17 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "cron"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ff76b51e4c068c52bfd2866e1567bee7c567ae8f24ada09fd4307019e25eab7"
+dependencies = [
+ "chrono",
+ "nom",
+ "once_cell",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.8"
@@ -652,7 +667,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -1065,7 +1080,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -1099,6 +1114,12 @@ dependencies = [
 "unicase",
 ]

+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "minio-rsc"
 version = "0.2.0"
@@ -1188,6 +1209,16 @@ dependencies = [
 "tempfile",
 ]

+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -1198,6 +1229,17 @@ dependencies = [
 "winapi",
 ]

+[[package]]
+name = "num-derive"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.16"
@@ -1255,7 +1297,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -1362,7 +1404,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -1793,7 +1835,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -1952,6 +1994,17 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"

+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.37"
@@ -2005,7 +2058,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -2080,6 +2133,21 @@ dependencies = [
 "windows-sys",
 ]

+[[package]]
+name = "tokio-cron-scheduler"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de2c1fd54a857b29c6cd1846f31903d0ae8e28175615c14a277aed45c58d8e27"
+dependencies = [
+ "chrono",
+ "cron",
+ "num-derive",
+ "num-traits",
+ "tokio",
+ "tracing",
+ "uuid",
+]
+
 [[package]]
 name = "tokio-macros"
 version = "2.1.0"
@@ -2088,7 +2156,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -2183,7 +2251,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 ]

 [[package]]
@@ -2393,7 +2461,7 @@ dependencies = [
 "once_cell",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 "wasm-bindgen-shared",
 ]

@@ -2427,7 +2495,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.37",
 "wasm-bindgen-backend",
 "wasm-bindgen-shared",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -41,3 +41,7 @@ async-stream = "0.3.5"
 translit = "0.5.0"

 sentry = { version = "0.31.7", features = ["debug-images"] }
+
+tokio-cron-scheduler = "0.9.4"
+
+chrono = "0.4.31"
--- a/src/main.rs
+++ b/src/main.rs
@@ -5,22 +5,13 @@ pub mod structures;

 use std::{net::SocketAddr, str::FromStr};
 use sentry::{ClientOptions, types::Dsn, integrations::debug_images::DebugImagesIntegration};
+use tokio_cron_scheduler::{JobScheduler, Job};
 use tracing::info;

-use crate::views::get_router;
+use crate::{views::get_router, services::files_cleaner::clean_files};


-#[tokio::main]
-async fn main() {
-    let options = ClientOptions {
-        dsn: Some(Dsn::from_str(&config::CONFIG.sentry_dsn).unwrap()),
-        default_integrations: false,
-        ..Default::default()
-    }
-    .add_integration(DebugImagesIntegration::new());
-
-    let _guard = sentry::init(options);
-
+async fn start_app() {
    tracing_subscriber::fmt()
        .with_target(false)
        .compact()
@@ -35,5 +26,46 @@ async fn main() {
        .serve(app.into_make_service())
        .await
        .unwrap();
-    info!("Webserver shutdown...")
+    info!("Webserver shutdown...");
+}
+
+
+async fn start_job_scheduler() {
+    let job_scheduler = JobScheduler::new().await.unwrap();
+
+    let clean_files_job = match Job::new_async("0 */5 * * * *", |_uuid, _l| Box::pin(async {
+        match clean_files().await {
+            Ok(_) => info!("Files cleaned!"),
+            Err(err) => info!("Clean files err: {:?}", err),
+        };
+    })) {
+        Ok(v) => v,
+        Err(err) => panic!("{:?}", err),
+    };
+
+    job_scheduler.add(clean_files_job).await.unwrap();
+
+    info!("Scheduler start...");
+    match job_scheduler.start().await {
+        Ok(v) => v,
+        Err(err) => panic!("{:?}", err),
+    };
+}
+
+
+#[tokio::main]
+async fn main() {
+    let options = ClientOptions {
+        dsn: Some(Dsn::from_str(&config::CONFIG.sentry_dsn).unwrap()),
+        default_integrations: false,
+        ..Default::default()
+    }
+    .add_integration(DebugImagesIntegration::new());
+
+    let _guard = sentry::init(options);
+
+    tokio::join![
+        start_app(),
+        start_job_scheduler()
+    ];
 }
--- a/src/services/files_cleaner.rs
+++ b/src/services/files_cleaner.rs
@@ -0,0 +1,26 @@
+use chrono::{DateTime, Utc, Duration};
+use minio_rsc::{client::ListObjectsArgs, datatype::Object};
+
+use super::minio::get_minio;
+use crate::config;
+
+
+pub async fn clean_files() -> Result<(), Box<dyn std::error::Error>> {
+    let minio_client = get_minio();
+
+    let objects = minio_client.list_objects(
+        &config::CONFIG.minio_bucket,
+        ListObjectsArgs::default()
+    ).await?;
+
+    let delete_before = Utc::now() - Duration::hours(3);
+    for Object { key, last_modified, .. } in objects.contents {
+        let last_modified_date: DateTime<Utc> = DateTime::parse_from_rfc3339(&last_modified)?.into();
+
+        if last_modified_date <= delete_before {
+            let _ = minio_client.remove_object(&config::CONFIG.minio_bucket, key).await;
+        }
+    }
+
+    Ok(())
+}
--- a/src/services/mod.rs
+++ b/src/services/mod.rs
@@ -3,3 +3,4 @@ pub mod library_client;
 pub mod utils;
 pub mod downloader;
 pub mod minio;
+pub mod files_cleaner;