Add pre-commit config

This commit is contained in:
2023-09-24 22:40:36 +02:00
parent cbab567692
commit 2390b43b93
13 changed files with 119 additions and 97 deletions

7
.pre-commit-config.yaml Normal file
View File

@@ -0,0 +1,7 @@
repos:
- repo: https://github.com/doublify/pre-commit-rust
rev: v1.0
hooks:
- id: fmt
- id: cargo-check
- id: clippy

View File

@@ -8,7 +8,7 @@ fn get_env(env: &'static str) -> String {
#[derive(Deserialize, Clone)] #[derive(Deserialize, Clone)]
pub struct SourceConfig { pub struct SourceConfig {
pub url: String, pub url: String,
pub proxy: Option<String> pub proxy: Option<String>,
} }
pub struct Config { pub struct Config {
@@ -22,7 +22,7 @@ pub struct Config {
pub converter_url: String, pub converter_url: String,
pub converter_api_key: String, pub converter_api_key: String,
pub sentry_dsn: String pub sentry_dsn: String,
} }
impl Config { impl Config {
@@ -38,11 +38,9 @@ impl Config {
converter_url: get_env("CONVERTER_URL"), converter_url: get_env("CONVERTER_URL"),
converter_api_key: get_env("CONVERTER_API_KEY"), converter_api_key: get_env("CONVERTER_API_KEY"),
sentry_dsn: get_env("SENTRY_DSN") sentry_dsn: get_env("SENTRY_DSN"),
} }
} }
} }
pub static CONFIG: Lazy<Config> = Lazy::new(|| { pub static CONFIG: Lazy<Config> = Lazy::new(Config::load);
Config::load()
});

View File

@@ -1,14 +1,13 @@
pub mod config; pub mod config;
pub mod views;
pub mod services; pub mod services;
pub mod views;
use sentry::{integrations::debug_images::DebugImagesIntegration, types::Dsn, ClientOptions};
use std::{net::SocketAddr, str::FromStr}; use std::{net::SocketAddr, str::FromStr};
use sentry::{ClientOptions, types::Dsn, integrations::debug_images::DebugImagesIntegration};
use tracing::info; use tracing::info;
use crate::views::get_router; use crate::views::get_router;
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
tracing_subscriber::fmt() tracing_subscriber::fmt()

View File

@@ -52,7 +52,12 @@ pub async fn get_remote_book(
source_id: u32, source_id: u32,
remote_id: u32, remote_id: u32,
) -> Result<types::BookWithRemote, Box<dyn std::error::Error + Send + Sync>> { ) -> Result<types::BookWithRemote, Box<dyn std::error::Error + Send + Sync>> {
match _make_request::<types::Book>(format!("/api/v1/books/remote/{source_id}/{remote_id}").as_ref(), vec![]).await { match _make_request::<types::Book>(
format!("/api/v1/books/remote/{source_id}/{remote_id}").as_ref(),
vec![],
)
.await
{
Ok(v) => Ok(types::BookWithRemote::from_book(v, remote_id)), Ok(v) => Ok(types::BookWithRemote::from_book(v, remote_id)),
Err(err) => Err(err), Err(err) => Err(err),
} }

View File

@@ -1,6 +1,5 @@
use serde::Deserialize; use serde::Deserialize;
#[derive(Deserialize, Debug, Clone)] #[derive(Deserialize, Debug, Clone)]
pub struct Source { pub struct Source {
// id: u32, // id: u32,
@@ -45,7 +44,7 @@ impl BookWithRemote {
lang: book.lang, lang: book.lang,
file_type: book.file_type, file_type: book.file_type,
uploaded: book.uploaded, uploaded: book.uploaded,
authors: book.authors authors: book.authors,
} }
} }
} }

View File

@@ -1,4 +1,4 @@
use reqwest::{Response, Body}; use reqwest::{Body, Response};
use tempfile::SpooledTempFile; use tempfile::SpooledTempFile;
use tokio_util::io::ReaderStream; use tokio_util::io::ReaderStream;
@@ -10,29 +10,20 @@ pub async fn convert_file(file: SpooledTempFile, file_type: String) -> Option<Re
let body = Body::wrap_stream(ReaderStream::new(SpooledTempAsyncRead::new(file))); let body = Body::wrap_stream(ReaderStream::new(SpooledTempAsyncRead::new(file)));
let response = reqwest::Client::new() let response = reqwest::Client::new()
.post( .post(format!("{}{}", config::CONFIG.converter_url, file_type))
format!(
"{}{}",
config::CONFIG.converter_url,
file_type
)
)
.body(body) .body(body)
.header("Authorization", &config::CONFIG.converter_api_key) .header("Authorization", &config::CONFIG.converter_api_key)
.send().await; .send()
.await;
let response = match response { let response = match response {
Ok(v) => v, Ok(v) => v,
Err(_) => { Err(_) => return None,
return None
},
}; };
let response = match response.error_for_status() { let response = match response.error_for_status() {
Ok(v) => v, Ok(v) => v,
Err(_) => { Err(_) => return None,
return None
},
}; };
Some(response) Some(response)

View File

@@ -7,7 +7,7 @@ use tokio::task::JoinSet;
use crate::config; use crate::config;
use self::types::{DownloadResult, Data, SpooledTempAsyncRead}; use self::types::{Data, DownloadResult, SpooledTempAsyncRead};
use self::utils::response_to_tempfile; use self::utils::response_to_tempfile;
use self::zip::{unzip, zip}; use self::zip::{unzip, zip};
@@ -15,7 +15,6 @@ use super::book_library::types::BookWithRemote;
use super::covert::convert_file; use super::covert::convert_file;
use super::{book_library::get_remote_book, filename_getter::get_filename_by_book}; use super::{book_library::get_remote_book, filename_getter::get_filename_by_book};
pub async fn download<'a>( pub async fn download<'a>(
book_id: &'a u32, book_id: &'a u32,
book_file_type: &'a str, book_file_type: &'a str,
@@ -63,8 +62,7 @@ pub async fn download<'a>(
return Some((response, false)); return Some((response, false));
} }
if content_type.contains("text/html") if content_type.contains("text/html") {
{
return None; return None;
} }
@@ -77,7 +75,7 @@ pub async fn download_chain<'a>(
book: BookWithRemote, book: BookWithRemote,
file_type: String, file_type: String,
source_config: config::SourceConfig, source_config: config::SourceConfig,
converting: bool converting: bool,
) -> Option<DownloadResult> { ) -> Option<DownloadResult> {
let final_need_zip = file_type == "fb2zip"; let final_need_zip = file_type == "fb2zip";
@@ -87,7 +85,8 @@ pub async fn download_chain<'a>(
file_type.clone() file_type.clone()
}; };
let (mut response, is_zip) = match download(&book.remote_id, &file_type_, &source_config).await { let (mut response, is_zip) = match download(&book.remote_id, &file_type_, &source_config).await
{
Some(v) => v, Some(v) => v,
None => return None, None => return None,
}; };
@@ -95,31 +94,41 @@ pub async fn download_chain<'a>(
if is_zip && book.file_type.to_lowercase() == "html" { if is_zip && book.file_type.to_lowercase() == "html" {
let filename = get_filename_by_book(&book, &file_type, true, false); let filename = get_filename_by_book(&book, &file_type, true, false);
let filename_ascii = get_filename_by_book(&book, &file_type, true, true); let filename_ascii = get_filename_by_book(&book, &file_type, true, true);
let data_size: usize = response.headers().get("Content-Length").unwrap().to_str().unwrap().parse().unwrap(); let data_size: usize = response
.headers()
.get("Content-Length")
.unwrap()
.to_str()
.unwrap()
.parse()
.unwrap();
return Some( return Some(DownloadResult::new(
DownloadResult::new( Data::Response(response),
Data::Response(response), filename,
filename, filename_ascii,
filename_ascii, data_size,
data_size ));
)
);
} }
if !is_zip && !final_need_zip && !converting { if !is_zip && !final_need_zip && !converting {
let filename = get_filename_by_book(&book, &book.file_type, false, false); let filename = get_filename_by_book(&book, &book.file_type, false, false);
let filename_ascii = get_filename_by_book(&book, &file_type, false, true); let filename_ascii = get_filename_by_book(&book, &file_type, false, true);
let data_size: usize = response.headers().get("Content-Length").unwrap().to_str().unwrap().parse().unwrap(); let data_size: usize = response
.headers()
.get("Content-Length")
.unwrap()
.to_str()
.unwrap()
.parse()
.unwrap();
return Some( return Some(DownloadResult::new(
DownloadResult::new( Data::Response(response),
Data::Response(response), filename,
filename, filename_ascii,
filename_ascii, data_size,
data_size, ));
)
);
}; };
let (unzipped_temp_file, data_size) = { let (unzipped_temp_file, data_size) = {
@@ -135,14 +144,11 @@ pub async fn download_chain<'a>(
} }
}; };
let (mut clean_file, data_size) = if converting { let (mut clean_file, data_size) = if converting {
match convert_file(unzipped_temp_file, file_type.to_string()).await { match convert_file(unzipped_temp_file, file_type.to_string()).await {
Some(mut response) => { Some(mut response) => match response_to_tempfile(&mut response).await {
match response_to_tempfile(&mut response).await { Some(v) => v,
Some(v) => v, None => return None,
None => return None,
}
}, },
None => return None, None => return None,
} }
@@ -155,17 +161,19 @@ pub async fn download_chain<'a>(
let filename = get_filename_by_book(&book, &file_type, false, false); let filename = get_filename_by_book(&book, &file_type, false, false);
let filename_ascii = get_filename_by_book(&book, &file_type, false, true); let filename_ascii = get_filename_by_book(&book, &file_type, false, true);
return Some( return Some(DownloadResult::new(
DownloadResult::new( Data::SpooledTempAsyncRead(t),
Data::SpooledTempAsyncRead(t), filename,
filename, filename_ascii,
filename_ascii, data_size,
data_size ));
)
);
}; };
let t_file_type = if file_type == "fb2zip" { "fb2" } else { &file_type }; let t_file_type = if file_type == "fb2zip" {
"fb2"
} else {
&file_type
};
let filename = get_filename_by_book(&book, t_file_type, false, false); let filename = get_filename_by_book(&book, t_file_type, false, false);
match zip(&mut clean_file, filename.as_str()) { match zip(&mut clean_file, filename.as_str()) {
Some((t_file, data_size)) => { Some((t_file, data_size)) => {
@@ -173,15 +181,13 @@ pub async fn download_chain<'a>(
let filename = get_filename_by_book(&book, &file_type, true, false); let filename = get_filename_by_book(&book, &file_type, true, false);
let filename_ascii = get_filename_by_book(&book, &file_type, true, true); let filename_ascii = get_filename_by_book(&book, &file_type, true, true);
Some( Some(DownloadResult::new(
DownloadResult::new( Data::SpooledTempAsyncRead(t),
Data::SpooledTempAsyncRead(t), filename,
filename, filename_ascii,
filename_ascii, data_size,
data_size ))
) }
)
},
None => None, None => None,
} }
} }
@@ -197,7 +203,7 @@ pub async fn start_download_futures(
book.clone(), book.clone(),
file_type.to_string(), file_type.to_string(),
source_config.clone(), source_config.clone(),
false false,
)); ));
if file_type == "epub" || file_type == "fb2" { if file_type == "epub" || file_type == "fb2" {
@@ -205,7 +211,7 @@ pub async fn start_download_futures(
book.clone(), book.clone(),
file_type.to_string(), file_type.to_string(),
source_config.clone(), source_config.clone(),
true true,
)); ));
} }
} }

View File

@@ -27,7 +27,12 @@ pub fn get_response_async_read(it: Response) -> impl AsyncRead {
impl DownloadResult { impl DownloadResult {
pub fn new(data: Data, filename: String, filename_ascii: String, data_size: usize) -> Self { pub fn new(data: Data, filename: String, filename_ascii: String, data_size: usize) -> Self {
Self { data, filename, filename_ascii, data_size } Self {
data,
filename,
filename_ascii,
data_size,
}
} }
pub fn get_async_read(self) -> Pin<Box<dyn AsyncRead + Send>> { pub fn get_async_read(self) -> Pin<Box<dyn AsyncRead + Send>> {
@@ -54,7 +59,8 @@ impl AsyncRead for SpooledTempAsyncRead {
_cx: &mut std::task::Context<'_>, _cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>, buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> { ) -> std::task::Poll<std::io::Result<()>> {
let result = match std::io::Read::read(&mut self.get_mut().file, buf.initialize_unfilled()) { let result = match std::io::Read::read(&mut self.get_mut().file, buf.initialize_unfilled())
{
Ok(v) => v, Ok(v) => v,
Err(err) => return std::task::Poll::Ready(Err(err)), Err(err) => return std::task::Poll::Ready(Err(err)),
}; };

View File

@@ -1,11 +1,9 @@
use bytes::Buf;
use reqwest::Response; use reqwest::Response;
use tempfile::SpooledTempFile; use tempfile::SpooledTempFile;
use bytes::Buf;
use std::io::{Seek, SeekFrom, Write}; use std::io::{Seek, SeekFrom, Write};
pub async fn response_to_tempfile(res: &mut Response) -> Option<(SpooledTempFile, usize)> { pub async fn response_to_tempfile(res: &mut Response) -> Option<(SpooledTempFile, usize)> {
let mut tmp_file = tempfile::spooled_tempfile(5 * 1024 * 1024); let mut tmp_file = tempfile::spooled_tempfile(5 * 1024 * 1024);

View File

@@ -3,7 +3,6 @@ use std::io::Seek;
use tempfile::SpooledTempFile; use tempfile::SpooledTempFile;
use zip::write::FileOptions; use zip::write::FileOptions;
pub fn unzip(tmp_file: SpooledTempFile, file_type: &str) -> Option<(SpooledTempFile, usize)> { pub fn unzip(tmp_file: SpooledTempFile, file_type: &str) -> Option<(SpooledTempFile, usize)> {
let mut archive = zip::ZipArchive::new(tmp_file).unwrap(); let mut archive = zip::ZipArchive::new(tmp_file).unwrap();
@@ -54,7 +53,11 @@ pub fn zip(tmp_file: &mut SpooledTempFile, filename: &str) -> Option<(SpooledTem
Err(_) => return None, Err(_) => return None,
}; };
let data_size: usize = archive_result.stream_position().unwrap().try_into().unwrap(); let data_size: usize = archive_result
.stream_position()
.unwrap()
.try_into()
.unwrap();
archive_result.rewind().unwrap(); archive_result.rewind().unwrap();

View File

@@ -22,7 +22,12 @@ pub fn get_author_short_name(author: BookAuthor) -> String {
parts.join(" ") parts.join(" ")
} }
pub fn get_filename_by_book(book: &BookWithRemote, file_type: &str, force_zip: bool, only_ascii: bool) -> String { pub fn get_filename_by_book(
book: &BookWithRemote,
file_type: &str,
force_zip: bool,
only_ascii: bool,
) -> String {
let book_id = book.remote_id; let book_id = book.remote_id;
let mut filename_parts: Vec<String> = vec![]; let mut filename_parts: Vec<String> = vec![];
@@ -67,7 +72,8 @@ pub fn get_filename_by_book(book: &BookWithRemote, file_type: &str, force_zip: b
("[", ""), ("[", ""),
("]", ""), ("]", ""),
("\"", ""), ("\"", ""),
].to_vec(); ]
.to_vec();
let replace_transliterator = Transliterator::new(replace_char_map); let replace_transliterator = Transliterator::new(replace_char_map);
let mut normal_filename = replace_transliterator.convert(&filename_without_type, false); let mut normal_filename = replace_transliterator.convert(&filename_without_type, false);
@@ -82,7 +88,14 @@ pub fn get_filename_by_book(book: &BookWithRemote, file_type: &str, force_zip: b
let left_part = if normal_filename_slice == normal_filename.len() - 1 { let left_part = if normal_filename_slice == normal_filename.len() - 1 {
&normal_filename &normal_filename
} else { } else {
normal_filename.get(..normal_filename_slice).unwrap_or_else(|| panic!("Can't slice left part: {:?} {:?}", normal_filename, normal_filename_slice)) normal_filename
.get(..normal_filename_slice)
.unwrap_or_else(|| {
panic!(
"Can't slice left part: {:?} {:?}",
normal_filename, normal_filename_slice
)
})
}; };
format!("{left_part}{right_part}") format!("{left_part}{right_part}")

View File

@@ -1,4 +1,4 @@
pub mod book_library; pub mod book_library;
pub mod filename_getter;
pub mod downloader;
pub mod covert; pub mod covert;
pub mod downloader;
pub mod filename_getter;

View File

@@ -25,7 +25,6 @@ use crate::{
}, },
}; };
pub async fn download( pub async fn download(
Path((source_id, remote_id, file_type)): Path<(u32, u32, String)>, Path((source_id, remote_id, file_type)): Path<(u32, u32, String)>,
) -> impl IntoResponse { ) -> impl IntoResponse {
@@ -54,10 +53,7 @@ pub async fn download(
header::CONTENT_DISPOSITION, header::CONTENT_DISPOSITION,
format!("attachment; filename={filename_ascii}"), format!("attachment; filename={filename_ascii}"),
), ),
( (header::CONTENT_LENGTH, format!("{file_size}")),
header::CONTENT_LENGTH,
format!("{file_size}")
),
( (
header::HeaderName::from_static("x-filename-b64-ascii"), header::HeaderName::from_static("x-filename-b64-ascii"),
encoder.encode(filename_ascii), encoder.encode(filename_ascii),
@@ -85,7 +81,8 @@ pub async fn get_filename(Path((book_id, file_type)): Path<(u32, String)>) -> im
json!({ json!({
"filename": filename, "filename": filename,
"filename_ascii": filename_ascii "filename_ascii": filename_ascii
}).to_string() })
.to_string(),
) )
} }