Add revers order caching

This commit is contained in:
2022-03-23 11:57:02 +03:00
parent 8705be1f6f
commit 8c0624725f
2 changed files with 11 additions and 11 deletions

View File

@@ -1,9 +1,10 @@
import logging
from typing import Optional, cast
from tempfile import SpooledTemporaryFile
from typing import Optional, cast
from fastapi import UploadFile
from arq.connections import ArqRedis
from fastapi import UploadFile
from app.models import CachedFile
from app.services.caption_getter import get_caption
@@ -46,12 +47,11 @@ async def check_books(ctx: dict, *args, **kwargs) -> None:
arq_pool: ArqRedis = ctx["arc_pool"]
books_page = await get_books(1, PAGE_SIZE)
for page_number in range(1, books_page.total_pages + 1):
for page_number in range(books_page.total_pages, 0, -1):
await arq_pool.enqueue_job("check_books_page", page_number)
async def cache_file(book: Book, file_type) -> Optional[CachedFile]:
logger.info(f"Cache {book.id} {file_type}...")
data = await download(book.source.id, book.remote_id, file_type)
if data is None:
@@ -68,7 +68,9 @@ async def cache_file(book: Book, file_type) -> Optional[CachedFile]:
await response.aclose()
await client.aclose()
upload_data = await upload_file(cast(SpooledTemporaryFile, temp_file.file), filename, caption)
upload_data = await upload_file(
cast(SpooledTemporaryFile, temp_file.file), filename, caption
)
if upload_data is None:
return None
@@ -79,7 +81,7 @@ async def cache_file(book: Book, file_type) -> Optional[CachedFile]:
async def cache_file_by_book_id(
ctx, book_id: int, file_type: str
ctx: dict, book_id: int, file_type: str
) -> Optional[CachedFile]:
book = await get_book(book_id)

View File

@@ -22,12 +22,10 @@ async def shutdown(ctx):
class WorkerSettings:
functions = [check_books, cache_file_by_book_id, check_books_page]
functions = [check_books, check_books_page, cache_file_by_book_id]
on_startup = startup
on_shutdown = shutdown
redis_settings = get_redis_settings()
max_jobs = 1
job_timeout = 3 * 60
cron_jobs = [
cron(check_books, hour={7}, minute=0)
]
cron_jobs = [cron(check_books, hour={7}, minute=0)]