From 778d6429d57c7871782b520764b9ad73245847db Mon Sep 17 00:00:00 2001 From: Bulat Kurbanov Date: Sun, 22 Jan 2023 23:03:08 +0100 Subject: [PATCH] Fix downloading --- src/app/services/downloader.py | 23 ++++++++++------- src/app/services/library_client.py | 12 ++++++--- src/app/utils.py | 22 ++++++++++++++++ src/app/views.py | 40 ++++++++++++------------------ 4 files changed, 61 insertions(+), 36 deletions(-) create mode 100644 src/app/utils.py diff --git a/src/app/services/downloader.py b/src/app/services/downloader.py index b9d4179..cf9f1bd 100644 --- a/src/app/services/downloader.py +++ b/src/app/services/downloader.py @@ -2,6 +2,7 @@ from base64 import b64decode from typing import Optional import httpx +from sentry_sdk import capture_exception from core.config import env_config @@ -37,14 +38,18 @@ async def download( async def get_filename(book_id: int, file_type: str) -> Optional[str]: headers = {"Authorization": env_config.DOWNLOADER_API_KEY} - async with httpx.AsyncClient() as client: - response = await client.get( - f"{env_config.DOWNLOADER_URL}/filename/{book_id}/{file_type}", - headers=headers, - timeout=5 * 60, - ) + try: + async with httpx.AsyncClient() as client: + response = await client.get( + f"{env_config.DOWNLOADER_URL}/filename/{book_id}/{file_type}", + headers=headers, + timeout=5 * 60, + ) - if response.status_code != 200: - return None + if response.status_code != 200: + return None - return response.text + return response.text + except httpx.HTTPError as e: + capture_exception(e) + return None diff --git a/src/app/services/library_client.py b/src/app/services/library_client.py index c3bd2c5..bbf5d4d 100644 --- a/src/app/services/library_client.py +++ b/src/app/services/library_client.py @@ -3,6 +3,7 @@ from typing import Generic, Optional, TypeVar import httpx from pydantic import BaseModel +from sentry_sdk import capture_exception from core.config import env_config @@ -47,8 +48,13 @@ class BookDetail(Book): AUTH_HEADERS = {"Authorization": env_config.LIBRARY_API_KEY} -async def get_book(book_id: int, retry: int = 3) -> Optional[BookDetail]: +async def get_book( + book_id: int, retry: int = 3, last_exp: Exception | None = None +) -> Optional[BookDetail]: if retry == 0: + if last_exp: + capture_exception(last_exp) + return None try: @@ -61,8 +67,8 @@ async def get_book(book_id: int, retry: int = 3) -> Optional[BookDetail]: return None return BookDetail.parse_obj(response.json()) - except (httpx.ConnectError, httpx.ReadError, httpx.ReadTimeout): - return await get_book(book_id, retry=retry - 1) + except httpx.HTTPError as e: + return await get_book(book_id, retry=retry - 1, last_exp=e) async def get_books(page: int, page_size: int) -> Page[Book]: diff --git a/src/app/utils.py b/src/app/utils.py new file mode 100644 index 0000000..6e7ea03 --- /dev/null +++ b/src/app/utils.py @@ -0,0 +1,22 @@ +from fastapi import HTTPException, Request, status + +from app.models import CachedFile as CachedFileDB +from app.services.cache_updater import cache_file_by_book_id + + +async def get_cached_file_or_cache( + request: Request, object_id: int, object_type: str +) -> CachedFileDB: + cached_file = await CachedFileDB.objects.get_or_none( + object_id=object_id, object_type=object_type + ) + + if not cached_file: + cached_file = await cache_file_by_book_id( + {"redis": request.app.state.redis_client}, object_id, object_type + ) + + if not cached_file: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + + return cached_file diff --git a/src/app/views.py b/src/app/views.py index 5f6d5e4..ea38773 100644 --- a/src/app/views.py +++ b/src/app/views.py @@ -1,4 +1,3 @@ -import asyncio from base64 import b64encode from arq.connections import ArqRedis @@ -14,6 +13,7 @@ from app.services.caption_getter import get_caption from app.services.downloader import get_filename from app.services.files_client import download_file as download_file_from_cache from app.services.library_client import get_book +from app.utils import get_cached_file_or_cache router = APIRouter( prefix="/api/v1", tags=["files"], dependencies=[Depends(check_token)] @@ -39,34 +39,29 @@ async def get_cached_file(request: Request, object_id: int, object_type: str): @router.get("/download/{object_id}/{object_type}") async def download_cached_file(request: Request, object_id: int, object_type: str): - cached_file = await CachedFileDB.objects.get_or_none( - object_id=object_id, object_type=object_type - ) - - if not cached_file: - cached_file = await cache_file_by_book_id( - {"redis": request.app.state.redis_client}, object_id, object_type - ) - - if not cached_file: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) - + cached_file = await get_cached_file_or_cache(request, object_id, object_type) cache_data: dict = cached_file.data # type: ignore - data, filename, book = await asyncio.gather( - download_file_from_cache(cache_data["chat_id"], cache_data["message_id"]), - get_filename(object_id, object_type), - get_book(object_id), + data = await download_file_from_cache( + cache_data["chat_id"], cache_data["message_id"] ) - if data is None: await CachedFileDB.objects.filter(id=cached_file.id).delete() - arq_pool: ArqRedis = request.app.state.arq_pool - await arq_pool.enqueue_job( - "cache_file_by_book_id", object_id, object_type, by_request=False + cached_file = await get_cached_file_or_cache(request, object_id, object_type) + cache_data: dict = cached_file.data # type: ignore + + data = await download_file_from_cache( + cache_data["chat_id"], cache_data["message_id"] ) + if data is None: + raise HTTPException(status_code=status.HTTP_204_NO_CONTENT) + + if (filename := await get_filename(object_id, object_type)) is None: + raise HTTPException(status_code=status.HTTP_204_NO_CONTENT) + + if (book := await get_book(object_id)) is None: raise HTTPException(status_code=status.HTTP_204_NO_CONTENT) response, client = data @@ -75,9 +70,6 @@ async def download_cached_file(request: Request, object_id: int, object_type: st await response.aclose() await client.aclose() - assert book - assert filename - filename_ascii = filename.encode("ascii", "ignore").decode("ascii") return StreamingResponse(