From 3680b370c21c7a7f99f359cf2fb65d77fd9d0d8b Mon Sep 17 00:00:00 2001 From: Bulat Kurbanov Date: Wed, 1 Feb 2023 01:36:25 +0100 Subject: [PATCH] Update getting books when caching --- src/app/services/cache_updater.py | 10 ++++------ src/app/services/library_client.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/app/services/cache_updater.py b/src/app/services/cache_updater.py index f8f2afd..50bbe4c 100644 --- a/src/app/services/cache_updater.py +++ b/src/app/services/cache_updater.py @@ -16,7 +16,7 @@ from app.models import CachedFile from app.services.caption_getter import get_caption from app.services.downloader import download from app.services.files_client import upload_file -from app.services.library_client import Book, get_book, get_books +from app.services.library_client import Book, get_book, get_books, get_last_book_id logger = logging.getLogger("telegram_channel_files_manager") @@ -56,12 +56,10 @@ async def check_books_page(ctx, page_number: int) -> None: async def check_books(ctx: dict, *args, **kwargs) -> None: # NOSONAR arq_pool: ArqRedis = ctx["arc_pool"] - try: - books_page = await get_books(1, PAGE_SIZE) - except httpx.ConnectError: - raise Retry(defer=15) # noqa: B904 - for page_number in range(books_page.total_pages, 0, -1): + last_book_id = await get_last_book_id() + + for page_number in range(0, last_book_id // 100 + 1): await arq_pool.enqueue_job( "check_books_page", page_number, diff --git a/src/app/services/library_client.py b/src/app/services/library_client.py index bbf5d4d..396a1c8 100644 --- a/src/app/services/library_client.py +++ b/src/app/services/library_client.py @@ -72,11 +72,14 @@ async def get_book( async def get_books(page: int, page_size: int) -> Page[Book]: + id_lte = page * page_size + id_gte = (page + 1) * page_size - 1 + async with httpx.AsyncClient(timeout=5 * 60) as client: response = await client.get( ( f"{env_config.LIBRARY_URL}/api/v1/books/" - f"?page={page}&size={page_size}&is_deleted=false" + f"?is_deleted=false&id_gte={id_gte}&id_lte={id_lte}" ), headers=AUTH_HEADERS, ) @@ -87,3 +90,12 @@ async def get_books(page: int, page_size: int) -> Page[Book]: page_data.items = [Book.parse_obj(item) for item in page_data.items] return page_data + + +async def get_last_book_id() -> int: + async with httpx.AsyncClient() as client: + response = await client.get( + f"{env_config.LIBRARY_URL}/api/v1/books/last", headers=AUTH_HEADERS + ) + + return int(response.text)