Add pre-commit config

This commit is contained in:
2023-09-24 22:39:54 +02:00
parent dd79f3635d
commit 90d461025c
6 changed files with 108 additions and 161 deletions

7
.pre-commit-config.yaml Normal file
View File

@@ -0,0 +1,7 @@
repos:
- repo: https://github.com/doublify/pre-commit-rust
rev: v1.0
hooks:
- id: fmt
- id: cargo-check
- id: clippy

View File

@@ -6,7 +6,7 @@ pub enum Method {
#[serde(rename = "get")] #[serde(rename = "get")]
Get, Get,
#[serde(rename = "post")] #[serde(rename = "post")]
Post Post,
} }
#[derive(Deserialize, Clone)] #[derive(Deserialize, Clone)]

View File

@@ -3,19 +3,18 @@ extern crate lazy_static;
pub mod config; pub mod config;
pub mod types; pub mod types;
pub mod utils;
pub mod updater; pub mod updater;
pub mod utils;
use axum::{http::HeaderMap, routing::post, Router};
use sentry::{integrations::debug_images::DebugImagesIntegration, types::Dsn, ClientOptions};
use std::{net::SocketAddr, str::FromStr}; use std::{net::SocketAddr, str::FromStr};
use axum::{Router, routing::post, http::HeaderMap}; use tower_http::trace::{self, TraceLayer};
use sentry::{ClientOptions, types::Dsn, integrations::debug_images::DebugImagesIntegration};
use tracing::log; use tracing::log;
use tower_http::trace::{TraceLayer, self};
use tracing::Level; use tracing::Level;
use crate::updater::cron_jobs; use crate::updater::cron_jobs;
async fn update(headers: HeaderMap) -> &'static str { async fn update(headers: HeaderMap) -> &'static str {
let config_api_key = config::CONFIG.api_key.clone(); let config_api_key = config::CONFIG.api_key.clone();
@@ -25,7 +24,7 @@ async fn update(headers: HeaderMap) -> &'static str {
}; };
if config_api_key != api_key.to_str().unwrap() { if config_api_key != api_key.to_str().unwrap() {
return "Wrong api-key!" return "Wrong api-key!";
} }
tokio::spawn(async { tokio::spawn(async {
@@ -38,15 +37,12 @@ async fn update(headers: HeaderMap) -> &'static str {
"Update started" "Update started"
} }
async fn start_app() { async fn start_app() {
let app = Router::new() let app = Router::new().route("/update", post(update)).layer(
.route("/update", post(update)) TraceLayer::new_for_http()
.layer( .make_span_with(trace::DefaultMakeSpan::new().level(Level::INFO))
TraceLayer::new_for_http() .on_response(trace::DefaultOnResponse::new().level(Level::INFO)),
.make_span_with(trace::DefaultMakeSpan::new().level(Level::INFO)) );
.on_response(trace::DefaultOnResponse::new().level(Level::INFO)),
);
let addr = SocketAddr::from(([0, 0, 0, 0], 8080)); let addr = SocketAddr::from(([0, 0, 0, 0], 8080));
@@ -74,8 +70,5 @@ async fn main() {
let _guard = sentry::init(options); let _guard = sentry::init(options);
tokio::join![ tokio::join![cron_jobs(), start_app()];
cron_jobs(),
start_app()
];
} }

View File

@@ -11,9 +11,7 @@ pub trait FromVecExpression<T> {
#[async_trait] #[async_trait]
pub trait Update { pub trait Update {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>>;
client: &Client,
) -> Result<(), Box<tokio_postgres::Error>>;
async fn update( async fn update(
&self, &self,
@@ -21,9 +19,7 @@ pub trait Update {
source_id: i16, source_id: i16,
) -> Result<(), Box<tokio_postgres::Error>>; ) -> Result<(), Box<tokio_postgres::Error>>;
async fn after_update( async fn after_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>>;
client: &Client,
) -> Result<(), Box<tokio_postgres::Error>>;
} }
#[derive(Debug)] #[derive(Debug)]
@@ -59,9 +55,7 @@ impl FromVecExpression<Author> for Author {
#[async_trait] #[async_trait]
impl Update for Author { impl Update for Author {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_author( CREATE OR REPLACE FUNCTION update_author(
@@ -98,9 +92,7 @@ impl Update for Author {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -157,9 +149,7 @@ impl FromVecExpression<Book> for Book {
#[async_trait] #[async_trait]
impl Update for Book { impl Update for Book {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_book( CREATE OR REPLACE FUNCTION update_book(
@@ -198,13 +188,14 @@ impl Update for Book {
} }
} }
async fn after_update( async fn after_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client match client
) -> Result<(), Box<tokio_postgres::Error>> { .execute(
match client.execute( "UPDATE books SET is_deleted = 't' WHERE lang NOT IN ('ru', 'be', 'uk');",
"UPDATE books SET is_deleted = 't' WHERE lang NOT IN ('ru', 'be', 'uk');", &[],
&[] )
).await { .await
{
Ok(_) => Ok(()), Ok(_) => Ok(()),
Err(err) => Err(Box::new(err)), Err(err) => Err(Box::new(err)),
} }
@@ -235,9 +226,7 @@ impl FromVecExpression<BookAuthor> for BookAuthor {
#[async_trait] #[async_trait]
impl Update for BookAuthor { impl Update for BookAuthor {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_book_author(source_ smallint, book_ integer, author_ integer) RETURNS void AS $$ CREATE OR REPLACE FUNCTION update_book_author(source_ smallint, book_ integer, author_ integer) RETURNS void AS $$
@@ -283,9 +272,7 @@ impl Update for BookAuthor {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -318,9 +305,7 @@ impl FromVecExpression<Translator> for Translator {
#[async_trait] #[async_trait]
impl Update for Translator { impl Update for Translator {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_translation(source_ smallint, book_ integer, author_ integer, position_ smallint) RETURNS void AS $$ CREATE OR REPLACE FUNCTION update_translation(source_ smallint, book_ integer, author_ integer, position_ smallint) RETURNS void AS $$
@@ -372,9 +357,7 @@ impl Update for Translator {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -402,9 +385,7 @@ impl FromVecExpression<Sequence> for Sequence {
#[async_trait] #[async_trait]
impl Update for Sequence { impl Update for Sequence {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_sequences(source_ smallint, remote_id_ int, name_ varchar) RETURNS void AS $$ CREATE OR REPLACE FUNCTION update_sequences(source_ smallint, remote_id_ int, name_ varchar) RETURNS void AS $$
@@ -440,9 +421,7 @@ impl Update for Sequence {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -472,9 +451,7 @@ impl FromVecExpression<SequenceInfo> for SequenceInfo {
op_span: _, op_span: _,
operand, operand,
} => match (op, operand.as_ref()) { } => match (op, operand.as_ref()) {
(sql_parse::UnaryOperator::Minus, Expression::Integer(v)) => { (sql_parse::UnaryOperator::Minus, Expression::Integer(v)) => v.0,
v.0
}
(_, _) => panic!("SequenceInfo.position = {:?}", &value[2]), (_, _) => panic!("SequenceInfo.position = {:?}", &value[2]),
}, },
_ => panic!("SequenceInfo.position = {:?}", &value[2]), _ => panic!("SequenceInfo.position = {:?}", &value[2]),
@@ -485,9 +462,7 @@ impl FromVecExpression<SequenceInfo> for SequenceInfo {
#[async_trait] #[async_trait]
impl Update for SequenceInfo { impl Update for SequenceInfo {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_book_sequence(source_ smallint, book_ integer, sequence_ integer, position_ smallint) RETURNS void AS $$ CREATE OR REPLACE FUNCTION update_book_sequence(source_ smallint, book_ integer, sequence_ integer, position_ smallint) RETURNS void AS $$
@@ -543,9 +518,7 @@ impl Update for SequenceInfo {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -579,9 +552,7 @@ impl FromVecExpression<BookAnnotation> for BookAnnotation {
#[async_trait] #[async_trait]
impl Update for BookAnnotation { impl Update for BookAnnotation {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_book_annotation(source_ smallint, book_ integer, title_ varchar, text_ text) RETURNS void AS $$ CREATE OR REPLACE FUNCTION update_book_annotation(source_ smallint, book_ integer, title_ varchar, text_ text) RETURNS void AS $$
@@ -625,9 +596,7 @@ impl Update for BookAnnotation {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -655,9 +624,7 @@ impl FromVecExpression<BookAnnotationPic> for BookAnnotationPic {
#[async_trait] #[async_trait]
impl Update for BookAnnotationPic { impl Update for BookAnnotationPic {
async fn before_update( async fn before_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
@@ -683,9 +650,7 @@ WHERE book = books.id;\
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -719,9 +684,7 @@ impl FromVecExpression<AuthorAnnotation> for AuthorAnnotation {
#[async_trait] #[async_trait]
impl Update for AuthorAnnotation { impl Update for AuthorAnnotation {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_author_annotation(source_ smallint, author_ integer, title_ varchar, text_ text) RETURNS void AS $$ CREATE OR REPLACE FUNCTION update_author_annotation(source_ smallint, author_ integer, title_ varchar, text_ text) RETURNS void AS $$
@@ -765,9 +728,7 @@ impl Update for AuthorAnnotation {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -795,9 +756,7 @@ impl FromVecExpression<AuthorAnnotationPic> for AuthorAnnotationPic {
#[async_trait] #[async_trait]
impl Update for AuthorAnnotationPic { impl Update for AuthorAnnotationPic {
async fn before_update( async fn before_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
@@ -822,9 +781,7 @@ WHERE author = authors.id;",
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -862,9 +819,7 @@ impl FromVecExpression<Genre> for Genre {
#[async_trait] #[async_trait]
impl Update for Genre { impl Update for Genre {
async fn before_update( async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute( match client.execute(
" "
CREATE OR REPLACE FUNCTION update_book_sequence(source_ smallint, book_ integer, genre_ integer) RETURNS void AS $$ CREATE OR REPLACE FUNCTION update_book_sequence(source_ smallint, book_ integer, genre_ integer) RETURNS void AS $$
@@ -908,9 +863,7 @@ impl Update for Genre {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }
@@ -938,9 +891,7 @@ impl FromVecExpression<BookGenre> for BookGenre {
#[async_trait] #[async_trait]
impl Update for BookGenre { impl Update for BookGenre {
async fn before_update( async fn before_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
@@ -961,9 +912,7 @@ impl Update for BookGenre {
} }
} }
async fn after_update( async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
Ok(()) Ok(())
} }
} }

View File

@@ -1,31 +1,27 @@
use std::{ use std::{fmt::Debug, str::FromStr, sync::Arc};
fmt::Debug,
sync::Arc,
str::FromStr
};
use crate::config::{Webhook, self}; use crate::config::{self, Webhook};
use deadpool_postgres::{Config, CreatePoolError, ManagerConfig, Pool, RecyclingMethod, Runtime}; use deadpool_postgres::{Config, CreatePoolError, ManagerConfig, Pool, RecyclingMethod, Runtime};
use futures::{io::copy, TryStreamExt}; use futures::{io::copy, TryStreamExt};
use reqwest::header::{HeaderMap, HeaderValue, HeaderName}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use tokio::fs::{File, remove_file}; use tokio::fs::{remove_file, File};
use tokio::sync::Mutex; use tokio::sync::Mutex;
use tokio_cron_scheduler::{JobScheduler, Job}; use tokio_cron_scheduler::{Job, JobScheduler};
use tokio_postgres::NoTls; use tokio_postgres::NoTls;
use tracing::log; use tracing::log;
use async_compression::futures::bufread::GzipDecoder; use async_compression::futures::bufread::GzipDecoder;
use sql_parse::{
parse_statement, InsertReplace, InsertReplaceType, ParseOptions, SQLArguments, SQLDialect,
Statement,
};
use tokio_util::compat::TokioAsyncReadCompatExt;
use crate::types::{ use crate::types::{
Author, AuthorAnnotation, AuthorAnnotationPic, BookAnnotation, BookAnnotationPic, BookAuthor, Author, AuthorAnnotation, AuthorAnnotationPic, BookAnnotation, BookAnnotationPic, BookAuthor,
BookGenre, FromVecExpression, Genre, Sequence, SequenceInfo, Translator, Update, BookGenre, FromVecExpression, Genre, Sequence, SequenceInfo, Translator, Update,
}; };
use crate::utils::read_lines; use crate::utils::read_lines;
use sql_parse::{
parse_statement, InsertReplace, InsertReplaceType, ParseOptions, SQLArguments, SQLDialect,
Statement,
};
use tokio_util::compat::TokioAsyncReadCompatExt;
use crate::types::Book; use crate::types::Book;
@@ -53,8 +49,8 @@ async fn download_file(filename_str: &str) -> Result<(), Box<dyn std::error::Err
Ok(v) => v.compat(), Ok(v) => v.compat(),
Err(err) => { Err(err) => {
log::error!("Can't create {filename_str}: {:?}", err); log::error!("Can't create {filename_str}: {:?}", err);
return Err(Box::new(err)) return Err(Box::new(err));
}, }
}; };
let data = response let data = response
@@ -68,8 +64,8 @@ async fn download_file(filename_str: &str) -> Result<(), Box<dyn std::error::Err
Ok(_) => (), Ok(_) => (),
Err(err) => { Err(err) => {
log::error!("Can't write data {filename_str}: {}", err); log::error!("Can't write data {filename_str}: {}", err);
return Err(Box::new(err)) return Err(Box::new(err));
}, }
}; };
log::info!("{filename_str} downloaded!"); log::info!("{filename_str} downloaded!");
@@ -148,7 +144,8 @@ where
type_: InsertReplaceType::Insert(_), type_: InsertReplaceType::Insert(_),
.. ..
}, },
)) = ast { )) = ast
{
for value in i.values.into_iter() { for value in i.values.into_iter() {
for t_value in value.1.into_iter() { for t_value in value.1.into_iter() {
let value = T::from_vec_expression(&t_value); let value = T::from_vec_expression(&t_value);
@@ -160,8 +157,8 @@ where
} }
Err(err) => { Err(err) => {
log::error!("Update error: {:?} : {:?}", value, err); log::error!("Update error: {:?} : {:?}", value, err);
return Err(err) return Err(err);
}, }
} }
} }
} }
@@ -220,30 +217,33 @@ enum UpdateStatus {
async fn send_webhooks() -> Result<(), Box<reqwest::Error>> { async fn send_webhooks() -> Result<(), Box<reqwest::Error>> {
for webhook in config::CONFIG.webhooks.clone().into_iter() { for webhook in config::CONFIG.webhooks.clone().into_iter() {
let Webhook { method, url, headers } = webhook; let Webhook {
method,
url,
headers,
} = webhook;
let client = reqwest::Client::new(); let client = reqwest::Client::new();
let builder = match method { let builder = match method {
config::Method::Get => { config::Method::Get => client.get(url),
client.get(url) config::Method::Post => client.post(url),
},
config::Method::Post => {
client.post(url)
},
}; };
let t_headers: Vec<(HeaderName, HeaderValue)> = headers.into_iter().map(|(key, val)| { let t_headers: Vec<(HeaderName, HeaderValue)> = headers
let value = match val { .into_iter()
serde_json::Value::String(v) => v, .map(|(key, val)| {
_ => panic!("Header value not string!") let value = match val {
}; serde_json::Value::String(v) => v,
_ => panic!("Header value not string!"),
};
( (
HeaderName::from_str(key.as_ref()).unwrap(), HeaderName::from_str(key.as_ref()).unwrap(),
HeaderValue::from_str(&value).unwrap() HeaderValue::from_str(&value).unwrap(),
) )
}).collect(); })
.collect();
let headers = HeaderMap::from_iter(t_headers.into_iter()); let headers = HeaderMap::from_iter(t_headers.into_iter());
@@ -258,7 +258,7 @@ async fn send_webhooks() -> Result<(), Box<reqwest::Error>> {
Ok(_) => (), Ok(_) => (),
Err(err) => return Err(Box::new(err)), Err(err) => return Err(Box::new(err)),
}; };
}; }
Ok(()) Ok(())
} }
@@ -479,7 +479,7 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
author_annotation_process, author_annotation_process,
author_annotation_pics_process, author_annotation_pics_process,
genre_annotation_process, genre_annotation_process,
book_genre_process book_genre_process,
] { ] {
let process_result = match process.await { let process_result = match process.await {
Ok(v) => v, Ok(v) => v,
@@ -495,11 +495,11 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
match send_webhooks().await { match send_webhooks().await {
Ok(_) => { Ok(_) => {
log::info!("Webhooks sended!"); log::info!("Webhooks sended!");
}, }
Err(err) => { Err(err) => {
log::info!("Webhooks send failed : {err}"); log::info!("Webhooks send failed : {err}");
return Err(Box::new(err)) return Err(Box::new(err));
}, }
}; };
Ok(()) Ok(())
@@ -508,12 +508,14 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
pub async fn cron_jobs() { pub async fn cron_jobs() {
let job_scheduler = JobScheduler::new().await.unwrap(); let job_scheduler = JobScheduler::new().await.unwrap();
let update_job = match Job::new_async("0 0 3 * * *", |_uuid, _l| Box::pin(async { let update_job = match Job::new_async("0 0 3 * * *", |_uuid, _l| {
match update().await { Box::pin(async {
Ok(_) => log::info!("Updated"), match update().await {
Err(err) => log::info!("Update err: {:?}", err), Ok(_) => log::info!("Updated"),
}; Err(err) => log::info!("Update err: {:?}", err),
})) { };
})
}) {
Ok(v) => v, Ok(v) => v,
Err(err) => panic!("{:?}", err), Err(err) => panic!("{:?}", err),
}; };
@@ -525,4 +527,4 @@ pub async fn cron_jobs() {
Ok(v) => v, Ok(v) => v,
Err(err) => panic!("{:?}", err), Err(err) => panic!("{:?}", err),
}; };
} }

View File

@@ -1,8 +1,8 @@
use ammonia::Builder;
use maplit::hashset;
use std::fs::File; use std::fs::File;
use std::io::{self, BufRead}; use std::io::{self, BufRead};
use std::path::Path; use std::path::Path;
use ammonia::Builder;
use maplit::hashset;
pub fn read_lines<P>(filename: P) -> io::Result<io::Lines<io::BufReader<File>>> pub fn read_lines<P>(filename: P) -> io::Result<io::Lines<io::BufReader<File>>>
where where
@@ -35,13 +35,9 @@ pub fn fix_annotation_text(text: &str) -> String {
} }
let tags = hashset!["a"]; let tags = hashset!["a"];
Builder::new() Builder::new().tags(tags).clean(&temp_text).to_string()
.tags(tags)
.clean(&temp_text)
.to_string()
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::utils::fix_annotation_text; use crate::utils::fix_annotation_text;