Add linters configs

This commit is contained in:
2022-01-01 20:13:14 +03:00
parent 3b54f80892
commit 19f6ed111b
18 changed files with 911 additions and 78 deletions

View File

@@ -1,18 +1,21 @@
from alembic import context
import sys, os
import os
import sys
from alembic import context
from sqlalchemy.engine import create_engine
from core.db import DATABASE_URL
myPath = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, myPath + '/../../')
sys.path.insert(0, myPath + "/../../")
config = context.config
from app.models import BaseMeta
target_metadata = BaseMeta.metadata

View File

@@ -10,7 +10,7 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '9b7cfb422191'
revision = "9b7cfb422191"
down_revision = None
branch_labels = None
depends_on = None
@@ -18,18 +18,21 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('cached_files',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('object_id', sa.Integer(), nullable=False),
sa.Column('object_type', sa.String(length=8), nullable=False),
sa.Column('data', sa.JSON(), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('object_id', 'object_type', name='uc_cached_files_object_id_object_type')
op.create_table(
"cached_files",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("object_id", sa.Integer(), nullable=False),
sa.Column("object_type", sa.String(length=8), nullable=False),
sa.Column("data", sa.JSON(), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"object_id", "object_type", name="uc_cached_files_object_id_object_type"
),
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('cached_files')
op.drop_table("cached_files")
# ### end Alembic commands ###

View File

@@ -6,4 +6,6 @@ from core.config import env_config
async def check_token(api_key: str = Security(default_security)):
if api_key != env_config.API_KEY:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!")
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!"
)

View File

@@ -11,10 +11,8 @@ class BaseMeta(ormar.ModelMeta):
class CachedFile(ormar.Model):
class Meta(BaseMeta):
tablename = "cached_files"
constraints = [
ormar.UniqueColumns('object_id','object_type')
]
constraints = [ormar.UniqueColumns("object_id", "object_type")]
id: int = ormar.Integer(primary_key=True) # type: ignore
object_id: int = ormar.Integer() # type: ignore
object_type: str = ormar.String(max_length=8) # type: ignore

View File

@@ -1,12 +1,11 @@
import asyncio
from typing import Optional
import asyncio
from app.services.library_client import get_books, get_book, Book
from app.models import CachedFile
from app.services.caption_getter import get_caption
from app.services.downloader import download
from app.services.files_uploader import upload_file
from app.services.caption_getter import get_caption
from app.models import CachedFile
from app.services.library_client import get_books, get_book, Book
PAGE_SIZE = 50
@@ -38,7 +37,7 @@ class CacheUpdater:
for book in page.items:
await self._check_book(book)
self.all_books_checked = True
@classmethod
@@ -55,9 +54,7 @@ class CacheUpdater:
upload_data = await upload_file(content, filename, caption)
return await CachedFile.objects.create(
object_id=book.id,
object_type=file_type,
data=upload_data.data
object_id=book.id, object_type=file_type, data=upload_data.data
)
async def _start_worker(self):
@@ -69,14 +66,12 @@ class CacheUpdater:
except asyncio.QueueEmpty:
await asyncio.sleep(0.1)
continue
await self._cache_file(book, file_type)
async def _update(self):
await asyncio.gather(
self._start_producer(),
self._start_worker(),
self._start_worker()
self._start_producer(), self._start_worker(), self._start_worker()
)
@classmethod

View File

@@ -6,10 +6,10 @@ def get_author_string(author: BookAuthor) -> str:
if author.last_name:
author_parts.append(author.last_name)
if author.first_name:
author_parts.append(author.first_name)
if author.middle_name:
author_parts.append(author.middle_name)
@@ -21,13 +21,11 @@ def get_caption(book: Book) -> str:
caption_authors_parts = []
for author in book.authors:
caption_authors_parts.append(
f"👤 {get_author_string(author)}"
)
caption_authors_parts.append(f"👤 {get_author_string(author)}")
if not caption_authors_parts:
return caption_title
caption_authors = "\n".join(caption_authors_parts)
return caption_title + "\n\n" + caption_authors

View File

@@ -5,17 +5,23 @@ import httpx
from core.config import env_config
async def download(source_id: int, remote_id: int, file_type: str) -> Optional[tuple[bytes, str]]:
async def download(
source_id: int, remote_id: int, file_type: str
) -> Optional[tuple[bytes, str]]:
headers = {"Authorization": env_config.DOWNLOADER_API_KEY}
async with httpx.AsyncClient() as client:
response = await client.get(f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}", headers=headers, timeout=5 * 60)
response = await client.get(
f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}",
headers=headers,
timeout=5 * 60,
)
if response.status_code != 200:
return None
content_disposition = response.headers['Content-Disposition']
content_disposition = response.headers["Content-Disposition"]
name = content_disposition.replace('attachment; filename=', '')
name = content_disposition.replace("attachment; filename=", "")
return response.content, name

View File

@@ -17,9 +17,15 @@ async def upload_file(content: bytes, filename: str, caption: str) -> UploadedFi
headers = {"Authorization": env_config.FILES_SERVER_API_KEY}
async with httpx.AsyncClient() as client:
form = {'caption': caption}
files = {'file': (filename, content)}
form = {"caption": caption}
files = {"file": (filename, content)}
response = await client.post(f"{env_config.FILES_SERVER_URL}/api/v1/files/upload/", data=form, files=files, headers=headers, timeout=5 * 60)
response = await client.post(
f"{env_config.FILES_SERVER_URL}/api/v1/files/upload/",
data=form,
files=files,
headers=headers,
timeout=5 * 60,
)
return UploadedFile.parse_obj(response.json())

View File

@@ -1,13 +1,13 @@
from typing import Generic, TypeVar
from pydantic import BaseModel
from datetime import date
from typing import Generic, TypeVar
import httpx
from pydantic import BaseModel
from core.config import env_config
import httpx
T = TypeVar('T')
T = TypeVar("T")
class Page(BaseModel, Generic[T]):
@@ -49,14 +49,22 @@ AUTH_HEADERS = {"Authorization": env_config.LIBRARY_API_KEY}
async def get_book(book_id: int) -> BookDetail:
async with httpx.AsyncClient() as client:
response = await client.get(f"{env_config.LIBRARY_URL}/api/v1/books/{book_id}", headers=AUTH_HEADERS)
response = await client.get(
f"{env_config.LIBRARY_URL}/api/v1/books/{book_id}", headers=AUTH_HEADERS
)
return BookDetail.parse_obj(response.json())
async def get_books(page: int, page_size: int) -> Page[Book]:
async with httpx.AsyncClient() as client:
response = await client.get(f"{env_config.LIBRARY_URL}/api/v1/books/?page={page}&size={page_size}&is_deleted=false", headers=AUTH_HEADERS)
response = await client.get(
(
f"{env_config.LIBRARY_URL}/api/v1/books/"
f"?page={page}&size={page_size}&is_deleted=false"
),
headers=AUTH_HEADERS,
)
data = response.json()

View File

@@ -7,48 +7,42 @@ from app.services.cache_updater import CacheUpdater
router = APIRouter(
prefix="/api/v1",
tags=["files"],
dependencies=[Depends(check_token)]
prefix="/api/v1", tags=["files"], dependencies=[Depends(check_token)]
)
@router.get("/{object_id}/{object_type}", response_model=CachedFile)
async def get_cached_file(object_id: int, object_type: str):
cached_file = await CachedFileDB.objects.get_or_none(
object_id=object_id,
object_type=object_type
object_id=object_id, object_type=object_type
)
if not cached_file:
cached_file = await CacheUpdater.cache_file(object_id, object_type)
if not cached_file:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
return cached_file
@router.delete("/{object_id}/{object_type}", response_model=CachedFile)
async def delete_cached_file(object_id: int, object_type: str):
cached_file = await CachedFileDB.objects.get(
object_id=object_id,
object_type=object_type
object_id=object_id, object_type=object_type
)
if not cached_file:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
await cached_file.delete()
return cached_file
@router.post("/", response_model=CachedFile)
async def create_cached_file(data: CreateCachedFile):
return await CachedFileDB.objects.create(
**data.dict()
)
return await CachedFileDB.objects.create(**data.dict())
@router.post("/update_cache")

View File

@@ -1,7 +1,7 @@
from fastapi import FastAPI
from core.db import database
from app.views import router
from core.db import database
def start_app() -> FastAPI:
@@ -11,13 +11,13 @@ def start_app() -> FastAPI:
app.include_router(router)
@app.on_event('startup')
@app.on_event("startup")
async def startup() -> None:
database_ = app.state.database
if not database_.is_connected:
await database_.connect()
@app.on_event('shutdown')
@app.on_event("shutdown")
async def shutdown() -> None:
database_ = app.state.database
if database_.is_connected:

View File

@@ -1,6 +1,6 @@
from urllib.parse import quote
from databases import Database
from databases import Database
from sqlalchemy import MetaData
from core.config import env_config