flibusta-apps/books_downloader
1
0
Fork 0
mirror of https://github.com/flibusta-apps/books_downloader.git synced 2025-12-06 15:05:37 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Add pre-commit config

Browse Source
This commit is contained in:
Bulat Kurbanov
2023-09-24 22:40:36 +02:00
parent cbab567692
commit 2390b43b93
13 changed files with 119 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/config.rs
View File

@@ -8,7 +8,7 @@ fn get_env(env: &'static str) -> String {
#[derive(Deserialize, Clone)]
pub struct SourceConfig {
pub url: String,
pub proxy: Option<String>
pub proxy: Option<String>,
}
pub struct Config {
@@ -22,7 +22,7 @@ pub struct Config {
pub converter_url: String,
pub converter_api_key: String,
pub sentry_dsn: String
pub sentry_dsn: String,
}
impl Config {
@@ -38,11 +38,9 @@ impl Config {
converter_url: get_env("CONVERTER_URL"),
converter_api_key: get_env("CONVERTER_API_KEY"),
sentry_dsn: get_env("SENTRY_DSN")
sentry_dsn: get_env("SENTRY_DSN"),
}
}
}
pub static CONFIG: Lazy<Config> = Lazy::new(|| {
Config::load()
});
pub static CONFIG: Lazy<Config> = Lazy::new(Config::load);

5
src/main.rs
View File

@@ -1,14 +1,13 @@
pub mod config;
pub mod views;
pub mod services;
pub mod views;
use sentry::{integrations::debug_images::DebugImagesIntegration, types::Dsn, ClientOptions};
use std::{net::SocketAddr, str::FromStr};
use sentry::{ClientOptions, types::Dsn, integrations::debug_images::DebugImagesIntegration};
use tracing::info;
use crate::views::get_router;
#[tokio::main]
async fn main() {
tracing_subscriber::fmt()

7
src/services/book_library/mod.rs
View File

@@ -52,7 +52,12 @@ pub async fn get_remote_book(
source_id: u32,
remote_id: u32,
) -> Result<types::BookWithRemote, Box<dyn std::error::Error + Send + Sync>> {
match _make_request::<types::Book>(format!("/api/v1/books/remote/{source_id}/{remote_id}").as_ref(), vec![]).await {
match _make_request::<types::Book>(
format!("/api/v1/books/remote/{source_id}/{remote_id}").as_ref(),
vec![],
)
.await
{
Ok(v) => Ok(types::BookWithRemote::from_book(v, remote_id)),
Err(err) => Err(err),
}

3
src/services/book_library/types.rs
View File

@@ -1,6 +1,5 @@
use serde::Deserialize;
#[derive(Deserialize, Debug, Clone)]
pub struct Source {
// id: u32,
@@ -45,7 +44,7 @@ impl BookWithRemote {
lang: book.lang,
file_type: book.file_type,
uploaded: book.uploaded,
authors: book.authors
authors: book.authors,
}
}
}

21
src/services/covert.rs
View File

@@ -1,4 +1,4 @@
use reqwest::{Response, Body};
use reqwest::{Body, Response};
use tempfile::SpooledTempFile;
use tokio_util::io::ReaderStream;
@@ -10,29 +10,20 @@ pub async fn convert_file(file: SpooledTempFile, file_type: String) -> Option<Re
let body = Body::wrap_stream(ReaderStream::new(SpooledTempAsyncRead::new(file)));
let response = reqwest::Client::new()
.post(
format!(
"{}{}",
config::CONFIG.converter_url,
file_type
)
)
.post(format!("{}{}", config::CONFIG.converter_url, file_type))
.body(body)
.header("Authorization", &config::CONFIG.converter_api_key)
.send().await;
.send()
.await;
let response = match response {
Ok(v) => v,
Err(_) => {
return None
},
Err(_) => return None,
};
let response = match response.error_for_status() {
Ok(v) => v,
Err(_) => {
return None
},
Err(_) => return None,
};
Some(response)

106
src/services/downloader/mod.rs
View File

@@ -7,7 +7,7 @@ use tokio::task::JoinSet;
use crate::config;
use self::types::{DownloadResult, Data, SpooledTempAsyncRead};
use self::types::{Data, DownloadResult, SpooledTempAsyncRead};
use self::utils::response_to_tempfile;
use self::zip::{unzip, zip};
@@ -15,7 +15,6 @@ use super::book_library::types::BookWithRemote;
use super::covert::convert_file;
use super::{book_library::get_remote_book, filename_getter::get_filename_by_book};
pub async fn download<'a>(
book_id: &'a u32,
book_file_type: &'a str,
@@ -63,8 +62,7 @@ pub async fn download<'a>(
return Some((response, false));
}
if content_type.contains("text/html")
{
if content_type.contains("text/html") {
return None;
}
@@ -77,7 +75,7 @@ pub async fn download_chain<'a>(
book: BookWithRemote,
file_type: String,
source_config: config::SourceConfig,
converting: bool
converting: bool,
) -> Option<DownloadResult> {
let final_need_zip = file_type == "fb2zip";
@@ -87,7 +85,8 @@ pub async fn download_chain<'a>(
file_type.clone()
};
let (mut response, is_zip) = match download(&book.remote_id, &file_type_, &source_config).await {
let (mut response, is_zip) = match download(&book.remote_id, &file_type_, &source_config).await
{
Some(v) => v,
None => return None,
};
@@ -95,31 +94,41 @@ pub async fn download_chain<'a>(
if is_zip && book.file_type.to_lowercase() == "html" {
let filename = get_filename_by_book(&book, &file_type, true, false);
let filename_ascii = get_filename_by_book(&book, &file_type, true, true);
let data_size: usize = response.headers().get("Content-Length").unwrap().to_str().unwrap().parse().unwrap();
let data_size: usize = response
.headers()
.get("Content-Length")
.unwrap()
.to_str()
.unwrap()
.parse()
.unwrap();
return Some(
DownloadResult::new(
Data::Response(response),
filename,
filename_ascii,
data_size
)
);
return Some(DownloadResult::new(
Data::Response(response),
filename,
filename_ascii,
data_size,
));
}
if !is_zip && !final_need_zip && !converting {
let filename = get_filename_by_book(&book, &book.file_type, false, false);
let filename_ascii = get_filename_by_book(&book, &file_type, false, true);
let data_size: usize = response.headers().get("Content-Length").unwrap().to_str().unwrap().parse().unwrap();
let data_size: usize = response
.headers()
.get("Content-Length")
.unwrap()
.to_str()
.unwrap()
.parse()
.unwrap();
return Some(
DownloadResult::new(
Data::Response(response),
filename,
filename_ascii,
data_size,
)
);
return Some(DownloadResult::new(
Data::Response(response),
filename,
filename_ascii,
data_size,
));
};
let (unzipped_temp_file, data_size) = {
@@ -135,14 +144,11 @@ pub async fn download_chain<'a>(
}
};
let (mut clean_file, data_size) = if converting {
match convert_file(unzipped_temp_file, file_type.to_string()).await {
Some(mut response) => {
match response_to_tempfile(&mut response).await {
Some(v) => v,
None => return None,
}
Some(mut response) => match response_to_tempfile(&mut response).await {
Some(v) => v,
None => return None,
},
None => return None,
}
@@ -155,17 +161,19 @@ pub async fn download_chain<'a>(
let filename = get_filename_by_book(&book, &file_type, false, false);
let filename_ascii = get_filename_by_book(&book, &file_type, false, true);
return Some(
DownloadResult::new(
Data::SpooledTempAsyncRead(t),
filename,
filename_ascii,
data_size
)
);
return Some(DownloadResult::new(
Data::SpooledTempAsyncRead(t),
filename,
filename_ascii,
data_size,
));
};
let t_file_type = if file_type == "fb2zip" { "fb2" } else { &file_type };
let t_file_type = if file_type == "fb2zip" {
"fb2"
} else {
&file_type
};
let filename = get_filename_by_book(&book, t_file_type, false, false);
match zip(&mut clean_file, filename.as_str()) {
Some((t_file, data_size)) => {
@@ -173,15 +181,13 @@ pub async fn download_chain<'a>(
let filename = get_filename_by_book(&book, &file_type, true, false);
let filename_ascii = get_filename_by_book(&book, &file_type, true, true);
Some(
DownloadResult::new(
Data::SpooledTempAsyncRead(t),
filename,
filename_ascii,
data_size
)
)
},
Some(DownloadResult::new(
Data::SpooledTempAsyncRead(t),
filename,
filename_ascii,
data_size,
))
}
None => None,
}
}
@@ -197,7 +203,7 @@ pub async fn start_download_futures(
book.clone(),
file_type.to_string(),
source_config.clone(),
false
false,
));
if file_type == "epub" || file_type == "fb2" {
@@ -205,7 +211,7 @@ pub async fn start_download_futures(
book.clone(),
file_type.to_string(),
source_config.clone(),
true
true,
));
}
}

10
src/services/downloader/types.rs
View File

@@ -27,7 +27,12 @@ pub fn get_response_async_read(it: Response) -> impl AsyncRead {
impl DownloadResult {
pub fn new(data: Data, filename: String, filename_ascii: String, data_size: usize) -> Self {
Self { data, filename, filename_ascii, data_size }
Self {
data,
filename,
filename_ascii,
data_size,
}
}
pub fn get_async_read(self) -> Pin<Box<dyn AsyncRead + Send>> {
@@ -54,7 +59,8 @@ impl AsyncRead for SpooledTempAsyncRead {
_cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
let result = match std::io::Read::read(&mut self.get_mut().file, buf.initialize_unfilled()) {
let result = match std::io::Read::read(&mut self.get_mut().file, buf.initialize_unfilled())
{
Ok(v) => v,
Err(err) => return std::task::Poll::Ready(Err(err)),
};

6
src/services/downloader/utils.rs
View File

@@ -1,11 +1,9 @@
use bytes::Buf;
use reqwest::Response;
use tempfile::SpooledTempFile;
use bytes::Buf;
use std::io::{Seek, SeekFrom, Write};
pub async fn response_to_tempfile(res: &mut Response) -> Option<(SpooledTempFile, usize)> {
let mut tmp_file = tempfile::spooled_tempfile(5 * 1024 * 1024);
@@ -37,4 +35,4 @@ pub async fn response_to_tempfile(res: &mut Response) -> Option<(SpooledTempFile
}
Some((tmp_file, data_size))
}
}

9
src/services/downloader/zip.rs
View File

@@ -3,7 +3,6 @@ use std::io::Seek;
use tempfile::SpooledTempFile;
use zip::write::FileOptions;
pub fn unzip(tmp_file: SpooledTempFile, file_type: &str) -> Option<(SpooledTempFile, usize)> {
let mut archive = zip::ZipArchive::new(tmp_file).unwrap();
@@ -54,9 +53,13 @@ pub fn zip(tmp_file: &mut SpooledTempFile, filename: &str) -> Option<(SpooledTem
Err(_) => return None,
};
let data_size: usize = archive_result.stream_position().unwrap().try_into().unwrap();
let data_size: usize = archive_result
.stream_position()
.unwrap()
.try_into()
.unwrap();
archive_result.rewind().unwrap();
Some((archive_result, data_size))
}
}

19
src/services/filename_getter.rs
View File

@@ -22,7 +22,12 @@ pub fn get_author_short_name(author: BookAuthor) -> String {
parts.join(" ")
}
pub fn get_filename_by_book(book: &BookWithRemote, file_type: &str, force_zip: bool, only_ascii: bool) -> String {
pub fn get_filename_by_book(
book: &BookWithRemote,
file_type: &str,
force_zip: bool,
only_ascii: bool,
) -> String {
let book_id = book.remote_id;
let mut filename_parts: Vec<String> = vec![];
@@ -67,7 +72,8 @@ pub fn get_filename_by_book(book: &BookWithRemote, file_type: &str, force_zip: b
("[", ""),
("]", ""),
("\"", ""),
].to_vec();
]
.to_vec();
let replace_transliterator = Transliterator::new(replace_char_map);
let mut normal_filename = replace_transliterator.convert(&filename_without_type, false);
@@ -82,7 +88,14 @@ pub fn get_filename_by_book(book: &BookWithRemote, file_type: &str, force_zip: b
let left_part = if normal_filename_slice == normal_filename.len() - 1 {
&normal_filename
} else {
normal_filename.get(..normal_filename_slice).unwrap_or_else(|| panic!("Can't slice left part: {:?} {:?}", normal_filename, normal_filename_slice))
normal_filename
.get(..normal_filename_slice)
.unwrap_or_else(|| {
panic!(
"Can't slice left part: {:?} {:?}",
normal_filename, normal_filename_slice
)
})
};
format!("{left_part}{right_part}")

4
src/services/mod.rs
View File

@@ -1,4 +1,4 @@
pub mod book_library;
pub mod filename_getter;
pub mod downloader;
pub mod covert;
pub mod downloader;
pub mod filename_getter;

9
src/views.rs
View File

@@ -25,7 +25,6 @@ use crate::{
},
};
pub async fn download(
Path((source_id, remote_id, file_type)): Path<(u32, u32, String)>,
) -> impl IntoResponse {
@@ -54,10 +53,7 @@ pub async fn download(
header::CONTENT_DISPOSITION,
format!("attachment; filename={filename_ascii}"),
),
(
header::CONTENT_LENGTH,
format!("{file_size}")
),
(header::CONTENT_LENGTH, format!("{file_size}")),
(
header::HeaderName::from_static("x-filename-b64-ascii"),
encoder.encode(filename_ascii),
@@ -85,7 +81,8 @@ pub async fn get_filename(Path((book_id, file_type)): Path<(u32, String)>) -> im
json!({
"filename": filename,
"filename_ascii": filename_ascii
}).to_string()
})
.to_string(),
)
}