flibusta-apps/library_updater
1
0
Fork 0
mirror of https://github.com/flibusta-apps/library_updater.git synced 2025-12-06 15:45:36 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add pre-commit config

Browse Source
This commit is contained in:
Bulat Kurbanov
2023-09-24 22:39:54 +02:00
parent dd79f3635d
commit 90d461025c
6 changed files with 108 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/config.rs
View File

@@ -6,7 +6,7 @@ pub enum Method {
#[serde(rename = "get")]
Get,
#[serde(rename = "post")]
Post
Post,
}
#[derive(Deserialize, Clone)]

29
src/main.rs
View File

@@ -3,19 +3,18 @@ extern crate lazy_static;
pub mod config;
pub mod types;
pub mod utils;
pub mod updater;
pub mod utils;
use axum::{http::HeaderMap, routing::post, Router};
use sentry::{integrations::debug_images::DebugImagesIntegration, types::Dsn, ClientOptions};
use std::{net::SocketAddr, str::FromStr};
use axum::{Router, routing::post, http::HeaderMap};
use sentry::{ClientOptions, types::Dsn, integrations::debug_images::DebugImagesIntegration};
use tower_http::trace::{self, TraceLayer};
use tracing::log;
use tower_http::trace::{TraceLayer, self};
use tracing::Level;
use crate::updater::cron_jobs;
async fn update(headers: HeaderMap) -> &'static str {
let config_api_key = config::CONFIG.api_key.clone();
@@ -25,7 +24,7 @@ async fn update(headers: HeaderMap) -> &'static str {
};
if config_api_key != api_key.to_str().unwrap() {
return "Wrong api-key!"
return "Wrong api-key!";
}
tokio::spawn(async {
@@ -38,15 +37,12 @@ async fn update(headers: HeaderMap) -> &'static str {
"Update started"
}
async fn start_app() {
let app = Router::new()
.route("/update", post(update))
.layer(
TraceLayer::new_for_http()
.make_span_with(trace::DefaultMakeSpan::new().level(Level::INFO))
.on_response(trace::DefaultOnResponse::new().level(Level::INFO)),
);
let app = Router::new().route("/update", post(update)).layer(
TraceLayer::new_for_http()
.make_span_with(trace::DefaultMakeSpan::new().level(Level::INFO))
.on_response(trace::DefaultOnResponse::new().level(Level::INFO)),
);
let addr = SocketAddr::from(([0, 0, 0, 0], 8080));
@@ -74,8 +70,5 @@ async fn main() {
let _guard = sentry::init(options);
tokio::join![
cron_jobs(),
start_app()
];
tokio::join![cron_jobs(), start_app()];
}

119
src/types.rs
View File

@@ -11,9 +11,7 @@ pub trait FromVecExpression<T> {
#[async_trait]
pub trait Update {
async fn before_update(
client: &Client,
) -> Result<(), Box<tokio_postgres::Error>>;
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>>;
async fn update(
&self,
@@ -21,9 +19,7 @@ pub trait Update {
source_id: i16,
) -> Result<(), Box<tokio_postgres::Error>>;
async fn after_update(
client: &Client,
) -> Result<(), Box<tokio_postgres::Error>>;
async fn after_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>>;
}
#[derive(Debug)]
@@ -59,9 +55,7 @@ impl FromVecExpression<Author> for Author {
#[async_trait]
impl Update for Author {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_author(
@@ -98,9 +92,7 @@ impl Update for Author {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -157,9 +149,7 @@ impl FromVecExpression<Book> for Book {
#[async_trait]
impl Update for Book {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_book(
@@ -198,13 +188,14 @@ impl Update for Book {
}
}
async fn after_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"UPDATE books SET is_deleted = 't' WHERE lang NOT IN ('ru', 'be', 'uk');",
&[]
).await {
async fn after_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client
.execute(
"UPDATE books SET is_deleted = 't' WHERE lang NOT IN ('ru', 'be', 'uk');",
&[],
)
.await
{
Ok(_) => Ok(()),
Err(err) => Err(Box::new(err)),
}
@@ -235,9 +226,7 @@ impl FromVecExpression<BookAuthor> for BookAuthor {
#[async_trait]
impl Update for BookAuthor {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_book_author(source_ smallint, book_ integer, author_ integer) RETURNS void AS $$
@@ -283,9 +272,7 @@ impl Update for BookAuthor {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -318,9 +305,7 @@ impl FromVecExpression<Translator> for Translator {
#[async_trait]
impl Update for Translator {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_translation(source_ smallint, book_ integer, author_ integer, position_ smallint) RETURNS void AS $$
@@ -372,9 +357,7 @@ impl Update for Translator {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -402,9 +385,7 @@ impl FromVecExpression<Sequence> for Sequence {
#[async_trait]
impl Update for Sequence {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_sequences(source_ smallint, remote_id_ int, name_ varchar) RETURNS void AS $$
@@ -440,9 +421,7 @@ impl Update for Sequence {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -472,9 +451,7 @@ impl FromVecExpression<SequenceInfo> for SequenceInfo {
op_span: _,
operand,
} => match (op, operand.as_ref()) {
(sql_parse::UnaryOperator::Minus, Expression::Integer(v)) => {
v.0
}
(sql_parse::UnaryOperator::Minus, Expression::Integer(v)) => v.0,
(_, _) => panic!("SequenceInfo.position = {:?}", &value[2]),
},
_ => panic!("SequenceInfo.position = {:?}", &value[2]),
@@ -485,9 +462,7 @@ impl FromVecExpression<SequenceInfo> for SequenceInfo {
#[async_trait]
impl Update for SequenceInfo {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_book_sequence(source_ smallint, book_ integer, sequence_ integer, position_ smallint) RETURNS void AS $$
@@ -543,9 +518,7 @@ impl Update for SequenceInfo {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -579,9 +552,7 @@ impl FromVecExpression<BookAnnotation> for BookAnnotation {
#[async_trait]
impl Update for BookAnnotation {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_book_annotation(source_ smallint, book_ integer, title_ varchar, text_ text) RETURNS void AS $$
@@ -625,9 +596,7 @@ impl Update for BookAnnotation {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -655,9 +624,7 @@ impl FromVecExpression<BookAnnotationPic> for BookAnnotationPic {
#[async_trait]
impl Update for BookAnnotationPic {
async fn before_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
@@ -683,9 +650,7 @@ WHERE book = books.id;\
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -719,9 +684,7 @@ impl FromVecExpression<AuthorAnnotation> for AuthorAnnotation {
#[async_trait]
impl Update for AuthorAnnotation {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_author_annotation(source_ smallint, author_ integer, title_ varchar, text_ text) RETURNS void AS $$
@@ -765,9 +728,7 @@ impl Update for AuthorAnnotation {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -795,9 +756,7 @@ impl FromVecExpression<AuthorAnnotationPic> for AuthorAnnotationPic {
#[async_trait]
impl Update for AuthorAnnotationPic {
async fn before_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
@@ -822,9 +781,7 @@ WHERE author = authors.id;",
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -862,9 +819,7 @@ impl FromVecExpression<Genre> for Genre {
#[async_trait]
impl Update for Genre {
async fn before_update(
client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
match client.execute(
"
CREATE OR REPLACE FUNCTION update_book_sequence(source_ smallint, book_ integer, genre_ integer) RETURNS void AS $$
@@ -908,9 +863,7 @@ impl Update for Genre {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}
@@ -938,9 +891,7 @@ impl FromVecExpression<BookGenre> for BookGenre {
#[async_trait]
impl Update for BookGenre {
async fn before_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn before_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
@@ -961,9 +912,7 @@ impl Update for BookGenre {
}
}
async fn after_update(
_client: &Client
) -> Result<(), Box<tokio_postgres::Error>> {
async fn after_update(_client: &Client) -> Result<(), Box<tokio_postgres::Error>> {
Ok(())
}
}

102
src/updater.rs
View File

@@ -1,31 +1,27 @@
use std::{
fmt::Debug,
sync::Arc,
str::FromStr
};
use std::{fmt::Debug, str::FromStr, sync::Arc};
use crate::config::{Webhook, self};
use crate::config::{self, Webhook};
use deadpool_postgres::{Config, CreatePoolError, ManagerConfig, Pool, RecyclingMethod, Runtime};
use futures::{io::copy, TryStreamExt};
use reqwest::header::{HeaderMap, HeaderValue, HeaderName};
use tokio::fs::{File, remove_file};
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use tokio::fs::{remove_file, File};
use tokio::sync::Mutex;
use tokio_cron_scheduler::{JobScheduler, Job};
use tokio_cron_scheduler::{Job, JobScheduler};
use tokio_postgres::NoTls;
use tracing::log;
use async_compression::futures::bufread::GzipDecoder;
use sql_parse::{
parse_statement, InsertReplace, InsertReplaceType, ParseOptions, SQLArguments, SQLDialect,
Statement,
};
use tokio_util::compat::TokioAsyncReadCompatExt;
use crate::types::{
Author, AuthorAnnotation, AuthorAnnotationPic, BookAnnotation, BookAnnotationPic, BookAuthor,
BookGenre, FromVecExpression, Genre, Sequence, SequenceInfo, Translator, Update,
};
use crate::utils::read_lines;
use sql_parse::{
parse_statement, InsertReplace, InsertReplaceType, ParseOptions, SQLArguments, SQLDialect,
Statement,
};
use tokio_util::compat::TokioAsyncReadCompatExt;
use crate::types::Book;
@@ -53,8 +49,8 @@ async fn download_file(filename_str: &str) -> Result<(), Box<dyn std::error::Err
Ok(v) => v.compat(),
Err(err) => {
log::error!("Can't create {filename_str}: {:?}", err);
return Err(Box::new(err))
},
return Err(Box::new(err));
}
};
let data = response
@@ -68,8 +64,8 @@ async fn download_file(filename_str: &str) -> Result<(), Box<dyn std::error::Err
Ok(_) => (),
Err(err) => {
log::error!("Can't write data {filename_str}: {}", err);
return Err(Box::new(err))
},
return Err(Box::new(err));
}
};
log::info!("{filename_str} downloaded!");
@@ -148,7 +144,8 @@ where
type_: InsertReplaceType::Insert(_),
..
},
)) = ast {
)) = ast
{
for value in i.values.into_iter() {
for t_value in value.1.into_iter() {
let value = T::from_vec_expression(&t_value);
@@ -160,8 +157,8 @@ where
}
Err(err) => {
log::error!("Update error: {:?} : {:?}", value, err);
return Err(err)
},
return Err(err);
}
}
}
}
@@ -220,30 +217,33 @@ enum UpdateStatus {
async fn send_webhooks() -> Result<(), Box<reqwest::Error>> {
for webhook in config::CONFIG.webhooks.clone().into_iter() {
let Webhook { method, url, headers } = webhook;
let Webhook {
method,
url,
headers,
} = webhook;
let client = reqwest::Client::new();
let builder = match method {
config::Method::Get => {
client.get(url)
},
config::Method::Post => {
client.post(url)
},
config::Method::Get => client.get(url),
config::Method::Post => client.post(url),
};
let t_headers: Vec<(HeaderName, HeaderValue)> = headers.into_iter().map(|(key, val)| {
let value = match val {
serde_json::Value::String(v) => v,
_ => panic!("Header value not string!")
};
let t_headers: Vec<(HeaderName, HeaderValue)> = headers
.into_iter()
.map(|(key, val)| {
let value = match val {
serde_json::Value::String(v) => v,
_ => panic!("Header value not string!"),
};
(
HeaderName::from_str(key.as_ref()).unwrap(),
HeaderValue::from_str(&value).unwrap()
)
}).collect();
(
HeaderName::from_str(key.as_ref()).unwrap(),
HeaderValue::from_str(&value).unwrap(),
)
})
.collect();
let headers = HeaderMap::from_iter(t_headers.into_iter());
@@ -258,7 +258,7 @@ async fn send_webhooks() -> Result<(), Box<reqwest::Error>> {
Ok(_) => (),
Err(err) => return Err(Box::new(err)),
};
};
}
Ok(())
}
@@ -479,7 +479,7 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
author_annotation_process,
author_annotation_pics_process,
genre_annotation_process,
book_genre_process
book_genre_process,
] {
let process_result = match process.await {
Ok(v) => v,
@@ -495,11 +495,11 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
match send_webhooks().await {
Ok(_) => {
log::info!("Webhooks sended!");
},
}
Err(err) => {
log::info!("Webhooks send failed : {err}");
return Err(Box::new(err))
},
return Err(Box::new(err));
}
};
Ok(())
@@ -508,12 +508,14 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
pub async fn cron_jobs() {
let job_scheduler = JobScheduler::new().await.unwrap();
let update_job = match Job::new_async("0 0 3 * * *", |_uuid, _l| Box::pin(async {
match update().await {
Ok(_) => log::info!("Updated"),
Err(err) => log::info!("Update err: {:?}", err),
};
})) {
let update_job = match Job::new_async("0 0 3 * * *", |_uuid, _l| {
Box::pin(async {
match update().await {
Ok(_) => log::info!("Updated"),
Err(err) => log::info!("Update err: {:?}", err),
};
})
}) {
Ok(v) => v,
Err(err) => panic!("{:?}", err),
};
@@ -525,4 +527,4 @@ pub async fn cron_jobs() {
Ok(v) => v,
Err(err) => panic!("{:?}", err),
};
}
}

10
src/utils.rs
View File

@@ -1,8 +1,8 @@
use ammonia::Builder;
use maplit::hashset;
use std::fs::File;
use std::io::{self, BufRead};
use std::path::Path;
use ammonia::Builder;
use maplit::hashset;
pub fn read_lines<P>(filename: P) -> io::Result<io::Lines<io::BufReader<File>>>
where
@@ -35,13 +35,9 @@ pub fn fix_annotation_text(text: &str) -> String {
}
let tags = hashset!["a"];
Builder::new()
.tags(tags)
.clean(&temp_text)
.to_string()
Builder::new().tags(tags).clean(&temp_text).to_string()
}
#[cfg(test)]
mod tests {
use crate::utils::fix_annotation_text;