flibusta-apps/batch_downloader
1
0
Fork 0
mirror of https://github.com/flibusta-apps/batch_downloader.git synced 2025-12-06 14:25:36 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix concurrent downloading

Browse Source
This commit is contained in:
Bulat Kurbanov
2023-07-23 02:15:44 +02:00
parent db15bdff4e
commit c67e675883
2 changed files with 15 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/app/services/downloader.py
View File

@@ -81,7 +81,15 @@ async def download_file_to_file(link: str, output: BytesIO) -> bool:
@broker.task() @broker.task()
async def download(task_id: str, book_id: int, file_type: str) -> str | None: async def download(
task_id: str, book_id: int, file_type: str, prev_task_id: str | None = None
) -> str | None:
if prev_task_id:
prev_task = AsyncTaskiqTask(prev_task_id, result_backend)
while not (await prev_task.is_ready()):
await asyncio.sleep(0.1)
try: try:
with tempfile.SpooledTemporaryFile() as temp_file: with tempfile.SpooledTemporaryFile() as temp_file:
data = await _download_to_tmpfile(book_id, file_type, temp_file) data = await _download_to_tmpfile(book_id, file_type, temp_file)

7
src/app/services/task_creator.py
View File

@@ -60,11 +60,16 @@ class TaskCreator:
task_ids: list[str] = [] task_ids: list[str] = []
prev_task_id = None
for book in books: for book in books:
if file_format not in book.available_types: if file_format not in book.available_types:
continue continue
task = await download.kiq(str(task_id), book.id, file_format) task = await download.kiq(
str(task_id), book.id, file_format, prev_task_id=prev_task_id
)
prev_task_id = task.task_id
task_ids.append(task.task_id) task_ids.append(task.task_id)
if len(task_ids) == 0: if len(task_ids) == 0: