mirror of
https://github.com/flibusta-apps/meilie_updater.git
synced 2025-12-06 07:05:37 +01:00
Add pre-commit config
This commit is contained in:
7
.pre-commit-config.yaml
Normal file
7
.pre-commit-config.yaml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
repos:
|
||||||
|
- repo: https://github.com/doublify/pre-commit-rust
|
||||||
|
rev: v1.0
|
||||||
|
hooks:
|
||||||
|
- id: fmt
|
||||||
|
- id: cargo-check
|
||||||
|
- id: clippy
|
||||||
@@ -10,7 +10,7 @@ pub struct Config {
|
|||||||
pub postgres_password: String,
|
pub postgres_password: String,
|
||||||
|
|
||||||
pub meili_host: String,
|
pub meili_host: String,
|
||||||
pub meili_master_key: String
|
pub meili_master_key: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_env(env: &'static str) -> String {
|
fn get_env(env: &'static str) -> String {
|
||||||
@@ -31,7 +31,7 @@ impl Config {
|
|||||||
postgres_password: get_env("POSTGRES_PASSWORD"),
|
postgres_password: get_env("POSTGRES_PASSWORD"),
|
||||||
|
|
||||||
meili_host: get_env("MEILI_HOST"),
|
meili_host: get_env("MEILI_HOST"),
|
||||||
meili_master_key: get_env("MEILI_MASTER_KEY")
|
meili_master_key: get_env("MEILI_MASTER_KEY"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,11 +2,11 @@
|
|||||||
extern crate lazy_static;
|
extern crate lazy_static;
|
||||||
|
|
||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod updater;
|
|
||||||
pub mod models;
|
pub mod models;
|
||||||
|
pub mod updater;
|
||||||
|
|
||||||
use axum::{http::HeaderMap, routing::post, Router};
|
use axum::{http::HeaderMap, routing::post, Router};
|
||||||
use sentry::{ClientOptions, types::Dsn, integrations::debug_images::DebugImagesIntegration};
|
use sentry::{integrations::debug_images::DebugImagesIntegration, types::Dsn, ClientOptions};
|
||||||
use std::{net::SocketAddr, str::FromStr};
|
use std::{net::SocketAddr, str::FromStr};
|
||||||
|
|
||||||
async fn update(headers: HeaderMap) -> &'static str {
|
async fn update(headers: HeaderMap) -> &'static str {
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ pub struct Book {
|
|||||||
pub id: i32,
|
pub id: i32,
|
||||||
pub title: String,
|
pub title: String,
|
||||||
pub lang: String,
|
pub lang: String,
|
||||||
pub genres: Vec<i32>
|
pub genres: Vec<i32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UpdateModel for Book {
|
impl UpdateModel for Book {
|
||||||
@@ -32,7 +32,7 @@ impl UpdateModel for Book {
|
|||||||
id: row.get(0),
|
id: row.get(0),
|
||||||
title: row.get(1),
|
title: row.get(1),
|
||||||
lang: row.get(2),
|
lang: row.get(2),
|
||||||
genres: row.get(3)
|
genres: row.get(3),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,10 +41,7 @@ impl UpdateModel for Book {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn get_filterable_attributes() -> Vec<String> {
|
fn get_filterable_attributes() -> Vec<String> {
|
||||||
vec![
|
vec!["lang".to_string(), "genres".to_string()]
|
||||||
"lang".to_string(),
|
|
||||||
"genres".to_string()
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_ranking_rules() -> Vec<String> {
|
fn get_ranking_rules() -> Vec<String> {
|
||||||
@@ -59,7 +56,6 @@ impl UpdateModel for Book {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
#[derive(Serialize, Deserialize, Debug)]
|
||||||
pub struct Author {
|
pub struct Author {
|
||||||
pub id: i32,
|
pub id: i32,
|
||||||
@@ -134,18 +130,17 @@ impl UpdateModel for Author {
|
|||||||
"attribute".to_string(),
|
"attribute".to_string(),
|
||||||
"sort".to_string(),
|
"sort".to_string(),
|
||||||
"exactness".to_string(),
|
"exactness".to_string(),
|
||||||
"books_count:desc".to_string()
|
"books_count:desc".to_string(),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
#[derive(Serialize, Deserialize, Debug)]
|
||||||
pub struct Sequence {
|
pub struct Sequence {
|
||||||
pub id: i32,
|
pub id: i32,
|
||||||
pub name: String,
|
pub name: String,
|
||||||
pub langs: Vec<String>,
|
pub langs: Vec<String>,
|
||||||
pub books_count: i64
|
pub books_count: i64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UpdateModel for Sequence {
|
impl UpdateModel for Sequence {
|
||||||
@@ -167,7 +162,8 @@ impl UpdateModel for Sequence {
|
|||||||
WHERE sequences.id = book_sequences.sequence
|
WHERE sequences.id = book_sequences.sequence
|
||||||
AND books.is_deleted = 'f') as books_count
|
AND books.is_deleted = 'f') as books_count
|
||||||
FROM sequences;
|
FROM sequences;
|
||||||
".to_string()
|
"
|
||||||
|
.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn from_row(row: Row) -> Self {
|
fn from_row(row: Row) -> Self {
|
||||||
@@ -175,7 +171,7 @@ impl UpdateModel for Sequence {
|
|||||||
id: row.get(0),
|
id: row.get(0),
|
||||||
name: row.get(1),
|
name: row.get(1),
|
||||||
langs: row.get(2),
|
langs: row.get(2),
|
||||||
books_count: row.get(3)
|
books_count: row.get(3),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -195,12 +191,11 @@ impl UpdateModel for Sequence {
|
|||||||
"attribute".to_string(),
|
"attribute".to_string(),
|
||||||
"sort".to_string(),
|
"sort".to_string(),
|
||||||
"exactness".to_string(),
|
"exactness".to_string(),
|
||||||
"books_count:desc".to_string()
|
"books_count:desc".to_string(),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
#[derive(Serialize, Deserialize, Debug)]
|
||||||
pub struct Genre {
|
pub struct Genre {
|
||||||
pub id: i32,
|
pub id: i32,
|
||||||
@@ -231,7 +226,8 @@ impl UpdateModel for Genre {
|
|||||||
AND books.is_deleted = 'f'
|
AND books.is_deleted = 'f'
|
||||||
) as books_count
|
) as books_count
|
||||||
FROM genres;
|
FROM genres;
|
||||||
".to_string()
|
"
|
||||||
|
.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn from_row(row: Row) -> Self {
|
fn from_row(row: Row) -> Self {
|
||||||
@@ -240,7 +236,7 @@ impl UpdateModel for Genre {
|
|||||||
description: row.get(1),
|
description: row.get(1),
|
||||||
meta: row.get(2),
|
meta: row.get(2),
|
||||||
langs: row.get(3),
|
langs: row.get(3),
|
||||||
books_count: row.get(4)
|
books_count: row.get(4),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -260,7 +256,7 @@ impl UpdateModel for Genre {
|
|||||||
"attribute".to_string(),
|
"attribute".to_string(),
|
||||||
"sort".to_string(),
|
"sort".to_string(),
|
||||||
"exactness".to_string(),
|
"exactness".to_string(),
|
||||||
"books_count:desc".to_string()
|
"books_count:desc".to_string(),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
use deadpool_postgres::{Config, CreatePoolError, ManagerConfig, Pool, RecyclingMethod, Runtime};
|
use deadpool_postgres::{Config, CreatePoolError, ManagerConfig, Pool, RecyclingMethod, Runtime};
|
||||||
use tokio_postgres::NoTls;
|
use futures::{pin_mut, StreamExt};
|
||||||
use futures::{StreamExt, pin_mut};
|
|
||||||
use meilisearch_sdk::client::*;
|
use meilisearch_sdk::client::*;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
use tokio_postgres::NoTls;
|
||||||
|
|
||||||
use crate::{config, models::{Book, UpdateModel, Author, Sequence, Genre}};
|
use crate::{
|
||||||
|
config,
|
||||||
|
models::{Author, Book, Genre, Sequence, UpdateModel},
|
||||||
|
};
|
||||||
|
|
||||||
async fn get_postgres_pool() -> Result<Pool, CreatePoolError> {
|
async fn get_postgres_pool() -> Result<Pool, CreatePoolError> {
|
||||||
let mut config = Config::new();
|
let mut config = Config::new();
|
||||||
@@ -26,14 +29,15 @@ async fn get_postgres_pool() -> Result<Pool, CreatePoolError> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn get_meili_client() -> Client {
|
fn get_meili_client() -> Client {
|
||||||
Client::new(config::CONFIG.meili_host.clone(), Some(config::CONFIG.meili_master_key.clone()))
|
Client::new(
|
||||||
|
config::CONFIG.meili_host.clone(),
|
||||||
|
Some(config::CONFIG.meili_master_key.clone()),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn update_model<T>(
|
async fn update_model<T>(pool: Pool) -> Result<(), Box<dyn std::error::Error + Send>>
|
||||||
pool: Pool,
|
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send>>
|
|
||||||
where
|
where
|
||||||
T: UpdateModel + Serialize
|
T: UpdateModel + Serialize,
|
||||||
{
|
{
|
||||||
let client = pool.get().await.unwrap();
|
let client = pool.get().await.unwrap();
|
||||||
|
|
||||||
@@ -42,9 +46,7 @@ where
|
|||||||
let index = meili_client.index(T::get_index());
|
let index = meili_client.index(T::get_index());
|
||||||
|
|
||||||
let params: Vec<String> = vec![];
|
let params: Vec<String> = vec![];
|
||||||
let stream = match client.query_raw(
|
let stream = match client.query_raw(&T::get_query(), params).await {
|
||||||
&T::get_query(), params
|
|
||||||
).await {
|
|
||||||
Ok(stream) => stream,
|
Ok(stream) => stream,
|
||||||
Err(err) => return Err(Box::new(err)),
|
Err(err) => return Err(Box::new(err)),
|
||||||
};
|
};
|
||||||
@@ -55,11 +57,9 @@ where
|
|||||||
while let Some(chunk) = chunks.next().await {
|
while let Some(chunk) = chunks.next().await {
|
||||||
let items: Vec<T> = chunk
|
let items: Vec<T> = chunk
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|result| {
|
.map(|result| match result {
|
||||||
match result {
|
Ok(v) => T::from_row(v),
|
||||||
Ok(v) => T::from_row(v),
|
Err(err) => panic!("{}", err),
|
||||||
Err(err) => panic!("{}", err),
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
@@ -68,11 +68,17 @@ where
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(err) = index.set_searchable_attributes(T::get_searchable_attributes()).await {
|
if let Err(err) = index
|
||||||
|
.set_searchable_attributes(T::get_searchable_attributes())
|
||||||
|
.await
|
||||||
|
{
|
||||||
return Err(Box::new(err));
|
return Err(Box::new(err));
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Err(err) = index.set_filterable_attributes(T::get_filterable_attributes()).await {
|
if let Err(err) = index
|
||||||
|
.set_filterable_attributes(T::get_filterable_attributes())
|
||||||
|
.await
|
||||||
|
{
|
||||||
return Err(Box::new(err));
|
return Err(Box::new(err));
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -109,7 +115,7 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
|
|
||||||
let pool_clone = pool.clone();
|
let pool_clone = pool.clone();
|
||||||
let update_sequences_process = tokio::spawn(async move {
|
let update_sequences_process = tokio::spawn(async move {
|
||||||
match update_model::<Sequence>(pool_clone).await {
|
match update_model::<Sequence>(pool_clone).await {
|
||||||
Ok(_) => (),
|
Ok(_) => (),
|
||||||
Err(err) => panic!("{}", err),
|
Err(err) => panic!("{}", err),
|
||||||
}
|
}
|
||||||
@@ -117,7 +123,7 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
|
|
||||||
let pool_clone = pool.clone();
|
let pool_clone = pool.clone();
|
||||||
let update_genres_process = tokio::spawn(async move {
|
let update_genres_process = tokio::spawn(async move {
|
||||||
match update_model::<Genre>(pool_clone).await {
|
match update_model::<Genre>(pool_clone).await {
|
||||||
Ok(_) => (),
|
Ok(_) => (),
|
||||||
Err(err) => panic!("{}", err),
|
Err(err) => panic!("{}", err),
|
||||||
}
|
}
|
||||||
@@ -127,7 +133,7 @@ pub async fn update() -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
update_books_process,
|
update_books_process,
|
||||||
update_authors_process,
|
update_authors_process,
|
||||||
update_sequences_process,
|
update_sequences_process,
|
||||||
update_genres_process
|
update_genres_process,
|
||||||
] {
|
] {
|
||||||
match process.await {
|
match process.await {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
|
|||||||
Reference in New Issue
Block a user