mirror of
https://github.com/flibusta-apps/batch_downloader.git
synced 2025-12-06 06:15:37 +01:00
Fix
This commit is contained in:
199
Cargo.lock
generated
199
Cargo.lock
generated
@@ -84,15 +84,6 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-mutex"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e"
|
||||
dependencies = [
|
||||
"event-listener 2.5.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-stream"
|
||||
version = "0.3.5"
|
||||
@@ -112,7 +103,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -123,7 +114,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -151,6 +142,7 @@ dependencies = [
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"multer",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
@@ -247,8 +239,8 @@ dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"md5",
|
||||
"minio-rsc",
|
||||
"moka",
|
||||
"once_cell",
|
||||
"reqwest",
|
||||
@@ -259,7 +251,7 @@ dependencies = [
|
||||
"smartstring",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tokio-cron-scheduler",
|
||||
"tokio-util",
|
||||
"tower-http",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
@@ -354,7 +346,6 @@ dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"serde",
|
||||
"wasm-bindgen",
|
||||
"windows-targets 0.52.5",
|
||||
]
|
||||
@@ -442,17 +433,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cron"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f8c3e73077b4b4a6ab1ea5047c37c57aee77657bc8ecd6f29b0af082d0b0c07"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"nom",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.12"
|
||||
@@ -520,7 +500,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -559,12 +539,6 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "2.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "4.0.3"
|
||||
@@ -712,7 +686,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1201,40 +1175,6 @@ dependencies = [
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minimal-lexical"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "minio-rsc"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "febcba5252c6c5dc08498c12b48af40710bfee5297984be8bf41001f3adad36a"
|
||||
dependencies = [
|
||||
"async-mutex",
|
||||
"async-stream",
|
||||
"base64 0.22.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"crc32fast",
|
||||
"futures",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hmac",
|
||||
"hyper 1.3.1",
|
||||
"md5",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde-xml-rs",
|
||||
"sha2",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.7.2"
|
||||
@@ -1279,6 +1219,24 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multer"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a15d522be0a9c3e46fd2632e272d178f56387bdb5c9fbb3a36c649062e9b5219"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-util",
|
||||
"http 1.1.0",
|
||||
"httparse",
|
||||
"log",
|
||||
"memchr",
|
||||
"mime",
|
||||
"spin",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "native-tls"
|
||||
version = "0.2.11"
|
||||
@@ -1297,16 +1255,6 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.46.0"
|
||||
@@ -1323,17 +1271,6 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
|
||||
|
||||
[[package]]
|
||||
name = "num-derive"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.18"
|
||||
@@ -1391,7 +1328,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1491,7 +1428,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1918,18 +1855,6 @@ dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde-xml-rs"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb3aa78ecda1ebc9ec9847d5d3aba7d618823446a049ba2491940506da6e2782"
|
||||
dependencies = [
|
||||
"log",
|
||||
"serde",
|
||||
"thiserror",
|
||||
"xml-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.198"
|
||||
@@ -1938,7 +1863,7 @@ checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1985,17 +1910,6 @@ dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sharded-slab"
|
||||
version = "0.1.7"
|
||||
@@ -2066,6 +1980,12 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
@@ -2078,17 +1998,6 @@ version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.60"
|
||||
@@ -2168,7 +2077,7 @@ checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2246,21 +2155,6 @@ dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-cron-scheduler"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1215c91d6f74868e18a65bda99853e012cfb2a0e42f3438382e318277da76a0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"cron",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.2.0"
|
||||
@@ -2269,7 +2163,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2290,6 +2184,7 @@ checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
@@ -2361,7 +2256,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2497,12 +2392,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "urlencoding"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.8.0"
|
||||
@@ -2567,7 +2456,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@@ -2601,7 +2490,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
@@ -2825,12 +2714,6 @@ dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xml-rs"
|
||||
version = "0.8.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.32"
|
||||
@@ -2848,7 +2731,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -7,7 +7,9 @@ edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
tokio = { version = "1.37.0", features = ["full"] }
|
||||
tokio-util = { version = "0.7.10", features = ["compat", "io"] }
|
||||
futures-core = "0.3.30"
|
||||
futures-util = "0.3.30"
|
||||
|
||||
tracing = "0.1.40"
|
||||
tracing-subscriber = { version = "0.3.18", features = ["env-filter"]}
|
||||
@@ -15,7 +17,7 @@ tower-http = { version = "0.5.2", features = ["trace"] }
|
||||
|
||||
once_cell = "1.19.0"
|
||||
|
||||
axum = "0.7.5"
|
||||
axum = { version = "0.7.5", features = ["multipart"] }
|
||||
axum-prometheus = "0.6.1"
|
||||
|
||||
serde = { version = "1.0.198", features = ["derive"] }
|
||||
@@ -35,13 +37,10 @@ zip = "1.1.1"
|
||||
|
||||
base64 = "0.22.0"
|
||||
|
||||
minio-rsc = "0.2.2"
|
||||
async-stream = "0.3.5"
|
||||
|
||||
translit = "0.5.0"
|
||||
|
||||
sentry = { version = "0.32.3", features = ["debug-images"] }
|
||||
|
||||
tokio-cron-scheduler = "0.10.0"
|
||||
|
||||
chrono = "0.4.38"
|
||||
|
||||
@@ -7,14 +7,6 @@ fn get_env(env: &'static str) -> String {
|
||||
pub struct Config {
|
||||
pub api_key: String,
|
||||
|
||||
pub minio_host: String,
|
||||
pub internal_minio_host: String,
|
||||
pub minio_bucket: String,
|
||||
pub minio_access_key: String,
|
||||
pub minio_secret_key: String,
|
||||
|
||||
pub minio_share_books_bucket: String,
|
||||
|
||||
pub library_api_key: String,
|
||||
pub library_url: String,
|
||||
|
||||
@@ -29,14 +21,6 @@ impl Config {
|
||||
Config {
|
||||
api_key: get_env("API_KEY"),
|
||||
|
||||
minio_host: get_env("MINIO_HOST"),
|
||||
internal_minio_host: get_env("INTERNAL_MINIO_HOST"),
|
||||
minio_bucket: get_env("MINIO_BUCKET"),
|
||||
minio_access_key: get_env("MINIO_ACCESS_KEY"),
|
||||
minio_secret_key: get_env("MINIO_SECRET_KEY"),
|
||||
|
||||
minio_share_books_bucket: get_env("MINIO_SHARE_BOOKS_BUCKET"),
|
||||
|
||||
library_api_key: get_env("LIBRARY_API_KEY"),
|
||||
library_url: get_env("LIBRARY_URL"),
|
||||
|
||||
|
||||
34
src/main.rs
34
src/main.rs
@@ -5,10 +5,9 @@ pub mod views;
|
||||
|
||||
use sentry::{integrations::debug_images::DebugImagesIntegration, types::Dsn, ClientOptions};
|
||||
use std::{net::SocketAddr, str::FromStr};
|
||||
use tokio_cron_scheduler::{Job, JobScheduler};
|
||||
use tracing::info;
|
||||
|
||||
use crate::{services::files_cleaner::clean_files, views::get_router};
|
||||
use crate::views::get_router;
|
||||
|
||||
async fn start_app() {
|
||||
tracing_subscriber::fmt()
|
||||
@@ -26,35 +25,6 @@ async fn start_app() {
|
||||
info!("Webserver shutdown...");
|
||||
}
|
||||
|
||||
async fn start_job_scheduler() {
|
||||
let job_scheduler = JobScheduler::new().await.unwrap();
|
||||
|
||||
let clean_files_job = match Job::new_async("0 */5 * * * *", |_uuid, _l| {
|
||||
Box::pin(async {
|
||||
match clean_files(config::CONFIG.minio_bucket.clone()).await {
|
||||
Ok(_) => info!("Archive files cleaned!"),
|
||||
Err(err) => info!("Clean archive files err: {:?}", err),
|
||||
};
|
||||
|
||||
match clean_files(config::CONFIG.minio_share_books_bucket.clone()).await {
|
||||
Ok(_) => info!("Share files cleaned!"),
|
||||
Err(err) => info!("Clean share files err: {:?}", err),
|
||||
};
|
||||
})
|
||||
}) {
|
||||
Ok(v) => v,
|
||||
Err(err) => panic!("{:?}", err),
|
||||
};
|
||||
|
||||
job_scheduler.add(clean_files_job).await.unwrap();
|
||||
|
||||
info!("Scheduler start...");
|
||||
match job_scheduler.start().await {
|
||||
Ok(v) => v,
|
||||
Err(err) => panic!("{:?}", err),
|
||||
};
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let options = ClientOptions {
|
||||
@@ -66,5 +36,5 @@ async fn main() {
|
||||
|
||||
let _guard = sentry::init(options);
|
||||
|
||||
tokio::join![start_app(), start_job_scheduler()];
|
||||
start_app().await
|
||||
}
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use minio_rsc::{client::ListObjectsArgs, datatype::Object};
|
||||
|
||||
use super::minio::get_internal_minio;
|
||||
|
||||
pub async fn clean_files(bucket: String) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let minio_client = get_internal_minio();
|
||||
|
||||
let objects = minio_client
|
||||
.list_objects(&bucket, ListObjectsArgs::default())
|
||||
.await?;
|
||||
|
||||
let delete_before = Utc::now() - Duration::hours(3);
|
||||
for Object {
|
||||
key, last_modified, ..
|
||||
} in objects.contents
|
||||
{
|
||||
let last_modified_date: DateTime<Utc> =
|
||||
DateTime::parse_from_rfc3339(&last_modified)?.into();
|
||||
|
||||
if last_modified_date <= delete_before {
|
||||
let _ = minio_client.remove_object(&bucket, key).await;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
use minio_rsc::{provider::StaticProvider, Minio};
|
||||
|
||||
use crate::config;
|
||||
|
||||
pub fn get_minio() -> Minio {
|
||||
let provider = StaticProvider::new(
|
||||
&config::CONFIG.minio_access_key,
|
||||
&config::CONFIG.minio_secret_key,
|
||||
None,
|
||||
);
|
||||
|
||||
Minio::builder()
|
||||
.endpoint(&config::CONFIG.minio_host)
|
||||
.provider(provider)
|
||||
.secure(true)
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn get_internal_minio() -> Minio {
|
||||
let provider = StaticProvider::new(
|
||||
&config::CONFIG.minio_access_key,
|
||||
&config::CONFIG.minio_secret_key,
|
||||
None,
|
||||
);
|
||||
|
||||
Minio::builder()
|
||||
.endpoint(&config::CONFIG.internal_minio_host)
|
||||
.provider(provider)
|
||||
.secure(false)
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
@@ -1,6 +1,4 @@
|
||||
pub mod downloader;
|
||||
pub mod files_cleaner;
|
||||
pub mod library_client;
|
||||
pub mod minio;
|
||||
pub mod task_creator;
|
||||
pub mod utils;
|
||||
|
||||
@@ -1,26 +1,18 @@
|
||||
use std::io::Seek;
|
||||
use std::{fs::File, io::Seek};
|
||||
|
||||
use minio_rsc::client::PresignedArgs;
|
||||
use smallvec::SmallVec;
|
||||
use smartstring::alias::String as SmartString;
|
||||
use tempfile::SpooledTempFile;
|
||||
use tracing::log;
|
||||
use zip::write::FileOptions;
|
||||
|
||||
use crate::{
|
||||
config,
|
||||
services::{
|
||||
downloader::download,
|
||||
minio::get_minio,
|
||||
utils::{get_filename, get_stream},
|
||||
},
|
||||
services::{downloader::download, utils::get_filename},
|
||||
structures::{CreateTask, ObjectType, Task},
|
||||
views::TASK_RESULTS,
|
||||
};
|
||||
|
||||
use super::{
|
||||
library_client::{get_author_books, get_sequence_books, get_translator_books, Book, Page},
|
||||
minio::get_internal_minio,
|
||||
utils::get_key,
|
||||
};
|
||||
|
||||
@@ -71,8 +63,6 @@ pub async fn set_task_error(key: String, error_message: String) {
|
||||
status_description: "Ошибка!".to_string(),
|
||||
error_message: Some(error_message),
|
||||
result_filename: None,
|
||||
result_internal_link: None,
|
||||
result_link: None,
|
||||
content_size: None,
|
||||
};
|
||||
|
||||
@@ -86,96 +76,18 @@ pub async fn set_progress_description(key: String, description: String) {
|
||||
status_description: description,
|
||||
error_message: None,
|
||||
result_filename: None,
|
||||
result_internal_link: None,
|
||||
result_link: None,
|
||||
content_size: None,
|
||||
};
|
||||
|
||||
TASK_RESULTS.insert(key, task.clone()).await;
|
||||
}
|
||||
|
||||
pub async fn upload_to_minio(
|
||||
archive: SpooledTempFile,
|
||||
folder_name: String,
|
||||
filename: String,
|
||||
) -> Result<(String, String, u64), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let full_filename = format!("{}/{}", folder_name, filename);
|
||||
|
||||
let internal_minio = get_internal_minio();
|
||||
|
||||
let is_bucket_exist = match internal_minio
|
||||
.bucket_exists(&config::CONFIG.minio_bucket)
|
||||
.await
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(err) => return Err(Box::new(err)),
|
||||
};
|
||||
|
||||
if !is_bucket_exist {
|
||||
let _ = internal_minio
|
||||
.make_bucket(&config::CONFIG.minio_bucket, false)
|
||||
.await;
|
||||
}
|
||||
|
||||
let data_stream = get_stream(Box::new(archive));
|
||||
|
||||
if let Err(err) = internal_minio
|
||||
.put_object_stream(
|
||||
&config::CONFIG.minio_bucket,
|
||||
full_filename.clone(),
|
||||
Box::pin(data_stream),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Err(Box::new(err));
|
||||
}
|
||||
|
||||
let minio = get_minio();
|
||||
|
||||
let link = match minio
|
||||
.presigned_get_object(PresignedArgs::new(
|
||||
&config::CONFIG.minio_bucket,
|
||||
full_filename.clone(),
|
||||
))
|
||||
.await
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
return Err(Box::new(err));
|
||||
}
|
||||
};
|
||||
|
||||
let internal_link = match internal_minio
|
||||
.presigned_get_object(PresignedArgs::new(
|
||||
&config::CONFIG.minio_bucket,
|
||||
full_filename.clone(),
|
||||
))
|
||||
.await
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
return Err(Box::new(err));
|
||||
}
|
||||
};
|
||||
|
||||
let obj_size = match internal_minio
|
||||
.stat_object(&config::CONFIG.minio_bucket, full_filename.clone())
|
||||
.await
|
||||
{
|
||||
Ok(v) => v.unwrap().size().try_into().unwrap(),
|
||||
Err(_) => todo!(),
|
||||
};
|
||||
|
||||
Ok((link, internal_link, obj_size))
|
||||
}
|
||||
|
||||
pub async fn create_archive(
|
||||
key: String,
|
||||
books: Vec<Book>,
|
||||
file_format: SmartString,
|
||||
) -> Result<(SpooledTempFile, u64), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let output_file = tempfile::spooled_tempfile(5 * 1024 * 1024);
|
||||
) -> Result<(File, u64), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let output_file = File::create(format!("/tmp/{}", key))?;
|
||||
let mut archive = zip::ZipWriter::new(output_file);
|
||||
|
||||
let options: FileOptions<_> = FileOptions::default()
|
||||
@@ -290,31 +202,13 @@ pub async fn create_archive_task(key: String, data: CreateTask) {
|
||||
|
||||
set_progress_description(key.clone(), "Загрузка архива...".to_string()).await;
|
||||
|
||||
let folder_name = {
|
||||
let mut langs = data.allowed_langs.clone();
|
||||
langs.sort();
|
||||
langs.join("_")
|
||||
};
|
||||
|
||||
let (link, internal_link, content_size) =
|
||||
match upload_to_minio(archive_result, folder_name, final_filename.clone()).await {
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
set_task_error(key.clone(), "Failed uploading archive!".to_string()).await;
|
||||
log::error!("{}", err);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let task = Task {
|
||||
id: key.clone(),
|
||||
status: crate::structures::TaskStatus::Complete,
|
||||
status_description: "Архив готов! Ожидайте файл".to_string(),
|
||||
error_message: None,
|
||||
result_filename: Some(final_filename),
|
||||
result_internal_link: Some(internal_link),
|
||||
result_link: Some(link),
|
||||
content_size: Some(content_size),
|
||||
content_size: Some(archive_result.metadata().unwrap().len()),
|
||||
};
|
||||
|
||||
TASK_RESULTS.insert(key.clone(), task.clone()).await;
|
||||
@@ -329,8 +223,6 @@ pub async fn create_task(data: CreateTask) -> Task {
|
||||
status_description: "Подготовка".to_string(),
|
||||
error_message: None,
|
||||
result_filename: None,
|
||||
result_internal_link: None,
|
||||
result_link: None,
|
||||
content_size: None,
|
||||
};
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
use async_stream::stream;
|
||||
use bytes::{Buf, Bytes};
|
||||
use minio_rsc::error::Error;
|
||||
use reqwest::Response;
|
||||
use smartstring::alias::String as SmartString;
|
||||
use tempfile::SpooledTempFile;
|
||||
use translit::{gost779b_ru, CharsMapping, Transliterator};
|
||||
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::{
|
||||
error::Error,
|
||||
io::{Read, Seek, SeekFrom, Write},
|
||||
};
|
||||
|
||||
use crate::structures::{CreateTask, ObjectType};
|
||||
|
||||
@@ -56,7 +58,7 @@ pub async fn response_to_tempfile(res: &mut Response) -> Option<(SpooledTempFile
|
||||
|
||||
pub fn get_stream(
|
||||
mut temp_file: Box<dyn Read + Send + Sync>,
|
||||
) -> impl futures_core::Stream<Item = Result<Bytes, Error>> {
|
||||
) -> impl futures_core::Stream<Item = Result<Bytes, Box<dyn Error + Sync>>> {
|
||||
stream! {
|
||||
let mut buf = [0; 2048];
|
||||
|
||||
|
||||
@@ -33,8 +33,7 @@ pub struct Task {
|
||||
pub status: TaskStatus,
|
||||
pub status_description: String,
|
||||
pub error_message: Option<String>,
|
||||
|
||||
pub result_filename: Option<String>,
|
||||
pub result_link: Option<String>,
|
||||
pub result_internal_link: Option<String>,
|
||||
pub content_size: Option<u64>,
|
||||
}
|
||||
|
||||
20
src/views.rs
20
src/views.rs
@@ -1,6 +1,7 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
extract::Path,
|
||||
http::{self, Request, StatusCode},
|
||||
middleware::{self, Next},
|
||||
@@ -11,6 +12,8 @@ use axum::{
|
||||
use axum_prometheus::PrometheusMetricLayer;
|
||||
use moka::future::Cache;
|
||||
use once_cell::sync::Lazy;
|
||||
use tokio::fs::File;
|
||||
use tokio_util::io::ReaderStream;
|
||||
use tower_http::trace::{self, TraceLayer};
|
||||
|
||||
use tracing::Level;
|
||||
@@ -71,6 +74,22 @@ async fn auth(req: Request<axum::body::Body>, next: Next) -> Result<Response, St
|
||||
Ok(next.run(req).await)
|
||||
}
|
||||
|
||||
async fn download(Path(task_id): Path<String>) -> impl IntoResponse {
|
||||
let task = match TASK_RESULTS.get(&task_id).await {
|
||||
Some(result) => result,
|
||||
None => return StatusCode::NOT_FOUND.into_response(),
|
||||
};
|
||||
|
||||
let file = match File::open(format!("/tmp/{}", task.id)).await {
|
||||
Ok(v) => v,
|
||||
Err(_) => return StatusCode::NOT_FOUND.into_response(),
|
||||
};
|
||||
|
||||
let stream = ReaderStream::new(file);
|
||||
|
||||
Body::from_stream(stream).into_response()
|
||||
}
|
||||
|
||||
pub async fn get_router() -> Router {
|
||||
let (prometheus_layer, metric_handle) = PrometheusMetricLayer::pair();
|
||||
|
||||
@@ -80,6 +99,7 @@ pub async fn get_router() -> Router {
|
||||
"/api/check_archive/:task_id",
|
||||
get(check_archive_task_status),
|
||||
)
|
||||
.route("/api/download/:task_id", get(download))
|
||||
.layer(middleware::from_fn(auth))
|
||||
.layer(prometheus_layer);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user