mirror of
https://github.com/flibusta-apps/telegram_files_cache_server.git
synced 2025-12-06 14:45:36 +01:00
Add linters configs
This commit is contained in:
@@ -1,18 +1,21 @@
|
||||
from alembic import context
|
||||
import sys, os
|
||||
import os
|
||||
import sys
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy.engine import create_engine
|
||||
|
||||
from core.db import DATABASE_URL
|
||||
|
||||
|
||||
myPath = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, myPath + '/../../')
|
||||
sys.path.insert(0, myPath + "/../../")
|
||||
|
||||
config = context.config
|
||||
|
||||
|
||||
from app.models import BaseMeta
|
||||
|
||||
|
||||
target_metadata = BaseMeta.metadata
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '9b7cfb422191'
|
||||
revision = "9b7cfb422191"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
@@ -18,18 +18,21 @@ depends_on = None
|
||||
|
||||
def upgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('cached_files',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('object_id', sa.Integer(), nullable=False),
|
||||
sa.Column('object_type', sa.String(length=8), nullable=False),
|
||||
sa.Column('data', sa.JSON(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('object_id', 'object_type', name='uc_cached_files_object_id_object_type')
|
||||
op.create_table(
|
||||
"cached_files",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("object_id", sa.Integer(), nullable=False),
|
||||
sa.Column("object_type", sa.String(length=8), nullable=False),
|
||||
sa.Column("data", sa.JSON(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"object_id", "object_type", name="uc_cached_files_object_id_object_type"
|
||||
),
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('cached_files')
|
||||
op.drop_table("cached_files")
|
||||
# ### end Alembic commands ###
|
||||
|
||||
@@ -6,4 +6,6 @@ from core.config import env_config
|
||||
|
||||
async def check_token(api_key: str = Security(default_security)):
|
||||
if api_key != env_config.API_KEY:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!"
|
||||
)
|
||||
|
||||
@@ -11,10 +11,8 @@ class BaseMeta(ormar.ModelMeta):
|
||||
class CachedFile(ormar.Model):
|
||||
class Meta(BaseMeta):
|
||||
tablename = "cached_files"
|
||||
constraints = [
|
||||
ormar.UniqueColumns('object_id','object_type')
|
||||
]
|
||||
|
||||
constraints = [ormar.UniqueColumns("object_id", "object_type")]
|
||||
|
||||
id: int = ormar.Integer(primary_key=True) # type: ignore
|
||||
object_id: int = ormar.Integer() # type: ignore
|
||||
object_type: str = ormar.String(max_length=8) # type: ignore
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
import asyncio
|
||||
|
||||
from app.services.library_client import get_books, get_book, Book
|
||||
from app.models import CachedFile
|
||||
from app.services.caption_getter import get_caption
|
||||
from app.services.downloader import download
|
||||
from app.services.files_uploader import upload_file
|
||||
from app.services.caption_getter import get_caption
|
||||
from app.models import CachedFile
|
||||
from app.services.library_client import get_books, get_book, Book
|
||||
|
||||
|
||||
PAGE_SIZE = 50
|
||||
@@ -38,7 +37,7 @@ class CacheUpdater:
|
||||
|
||||
for book in page.items:
|
||||
await self._check_book(book)
|
||||
|
||||
|
||||
self.all_books_checked = True
|
||||
|
||||
@classmethod
|
||||
@@ -55,9 +54,7 @@ class CacheUpdater:
|
||||
upload_data = await upload_file(content, filename, caption)
|
||||
|
||||
return await CachedFile.objects.create(
|
||||
object_id=book.id,
|
||||
object_type=file_type,
|
||||
data=upload_data.data
|
||||
object_id=book.id, object_type=file_type, data=upload_data.data
|
||||
)
|
||||
|
||||
async def _start_worker(self):
|
||||
@@ -69,14 +66,12 @@ class CacheUpdater:
|
||||
except asyncio.QueueEmpty:
|
||||
await asyncio.sleep(0.1)
|
||||
continue
|
||||
|
||||
|
||||
await self._cache_file(book, file_type)
|
||||
|
||||
async def _update(self):
|
||||
await asyncio.gather(
|
||||
self._start_producer(),
|
||||
self._start_worker(),
|
||||
self._start_worker()
|
||||
self._start_producer(), self._start_worker(), self._start_worker()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -6,10 +6,10 @@ def get_author_string(author: BookAuthor) -> str:
|
||||
|
||||
if author.last_name:
|
||||
author_parts.append(author.last_name)
|
||||
|
||||
|
||||
if author.first_name:
|
||||
author_parts.append(author.first_name)
|
||||
|
||||
|
||||
if author.middle_name:
|
||||
author_parts.append(author.middle_name)
|
||||
|
||||
@@ -21,13 +21,11 @@ def get_caption(book: Book) -> str:
|
||||
|
||||
caption_authors_parts = []
|
||||
for author in book.authors:
|
||||
caption_authors_parts.append(
|
||||
f"👤 {get_author_string(author)}"
|
||||
)
|
||||
|
||||
caption_authors_parts.append(f"👤 {get_author_string(author)}")
|
||||
|
||||
if not caption_authors_parts:
|
||||
return caption_title
|
||||
|
||||
|
||||
caption_authors = "\n".join(caption_authors_parts)
|
||||
|
||||
return caption_title + "\n\n" + caption_authors
|
||||
|
||||
@@ -5,17 +5,23 @@ import httpx
|
||||
from core.config import env_config
|
||||
|
||||
|
||||
async def download(source_id: int, remote_id: int, file_type: str) -> Optional[tuple[bytes, str]]:
|
||||
async def download(
|
||||
source_id: int, remote_id: int, file_type: str
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
headers = {"Authorization": env_config.DOWNLOADER_API_KEY}
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}", headers=headers, timeout=5 * 60)
|
||||
response = await client.get(
|
||||
f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}",
|
||||
headers=headers,
|
||||
timeout=5 * 60,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
return None
|
||||
|
||||
content_disposition = response.headers['Content-Disposition']
|
||||
content_disposition = response.headers["Content-Disposition"]
|
||||
|
||||
name = content_disposition.replace('attachment; filename=', '')
|
||||
name = content_disposition.replace("attachment; filename=", "")
|
||||
|
||||
return response.content, name
|
||||
|
||||
@@ -17,9 +17,15 @@ async def upload_file(content: bytes, filename: str, caption: str) -> UploadedFi
|
||||
headers = {"Authorization": env_config.FILES_SERVER_API_KEY}
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
form = {'caption': caption}
|
||||
files = {'file': (filename, content)}
|
||||
form = {"caption": caption}
|
||||
files = {"file": (filename, content)}
|
||||
|
||||
response = await client.post(f"{env_config.FILES_SERVER_URL}/api/v1/files/upload/", data=form, files=files, headers=headers, timeout=5 * 60)
|
||||
response = await client.post(
|
||||
f"{env_config.FILES_SERVER_URL}/api/v1/files/upload/",
|
||||
data=form,
|
||||
files=files,
|
||||
headers=headers,
|
||||
timeout=5 * 60,
|
||||
)
|
||||
|
||||
return UploadedFile.parse_obj(response.json())
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
from typing import Generic, TypeVar
|
||||
from pydantic import BaseModel
|
||||
from datetime import date
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.config import env_config
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
T = TypeVar('T')
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Page(BaseModel, Generic[T]):
|
||||
@@ -49,14 +49,22 @@ AUTH_HEADERS = {"Authorization": env_config.LIBRARY_API_KEY}
|
||||
|
||||
async def get_book(book_id: int) -> BookDetail:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"{env_config.LIBRARY_URL}/api/v1/books/{book_id}", headers=AUTH_HEADERS)
|
||||
response = await client.get(
|
||||
f"{env_config.LIBRARY_URL}/api/v1/books/{book_id}", headers=AUTH_HEADERS
|
||||
)
|
||||
|
||||
return BookDetail.parse_obj(response.json())
|
||||
|
||||
|
||||
async def get_books(page: int, page_size: int) -> Page[Book]:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"{env_config.LIBRARY_URL}/api/v1/books/?page={page}&size={page_size}&is_deleted=false", headers=AUTH_HEADERS)
|
||||
response = await client.get(
|
||||
(
|
||||
f"{env_config.LIBRARY_URL}/api/v1/books/"
|
||||
f"?page={page}&size={page_size}&is_deleted=false"
|
||||
),
|
||||
headers=AUTH_HEADERS,
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
|
||||
|
||||
@@ -7,48 +7,42 @@ from app.services.cache_updater import CacheUpdater
|
||||
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1",
|
||||
tags=["files"],
|
||||
dependencies=[Depends(check_token)]
|
||||
prefix="/api/v1", tags=["files"], dependencies=[Depends(check_token)]
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{object_id}/{object_type}", response_model=CachedFile)
|
||||
async def get_cached_file(object_id: int, object_type: str):
|
||||
cached_file = await CachedFileDB.objects.get_or_none(
|
||||
object_id=object_id,
|
||||
object_type=object_type
|
||||
object_id=object_id, object_type=object_type
|
||||
)
|
||||
|
||||
if not cached_file:
|
||||
cached_file = await CacheUpdater.cache_file(object_id, object_type)
|
||||
|
||||
|
||||
if not cached_file:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
|
||||
return cached_file
|
||||
|
||||
|
||||
@router.delete("/{object_id}/{object_type}", response_model=CachedFile)
|
||||
async def delete_cached_file(object_id: int, object_type: str):
|
||||
cached_file = await CachedFileDB.objects.get(
|
||||
object_id=object_id,
|
||||
object_type=object_type
|
||||
object_id=object_id, object_type=object_type
|
||||
)
|
||||
|
||||
if not cached_file:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
await cached_file.delete()
|
||||
|
||||
|
||||
return cached_file
|
||||
|
||||
|
||||
@router.post("/", response_model=CachedFile)
|
||||
async def create_cached_file(data: CreateCachedFile):
|
||||
return await CachedFileDB.objects.create(
|
||||
**data.dict()
|
||||
)
|
||||
return await CachedFileDB.objects.create(**data.dict())
|
||||
|
||||
|
||||
@router.post("/update_cache")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from fastapi import FastAPI
|
||||
|
||||
from core.db import database
|
||||
from app.views import router
|
||||
from core.db import database
|
||||
|
||||
|
||||
def start_app() -> FastAPI:
|
||||
@@ -11,13 +11,13 @@ def start_app() -> FastAPI:
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
@app.on_event('startup')
|
||||
@app.on_event("startup")
|
||||
async def startup() -> None:
|
||||
database_ = app.state.database
|
||||
if not database_.is_connected:
|
||||
await database_.connect()
|
||||
|
||||
@app.on_event('shutdown')
|
||||
@app.on_event("shutdown")
|
||||
async def shutdown() -> None:
|
||||
database_ = app.state.database
|
||||
if database_.is_connected:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from urllib.parse import quote
|
||||
from databases import Database
|
||||
|
||||
from databases import Database
|
||||
from sqlalchemy import MetaData
|
||||
|
||||
from core.config import env_config
|
||||
|
||||
Reference in New Issue
Block a user