Add linters configs

This commit is contained in:
2022-01-01 20:59:14 +03:00
parent 2a70687df0
commit 2a31f27f44
14 changed files with 532 additions and 76 deletions

View File

@@ -1,13 +1,11 @@
from asyncio.exceptions import CancelledError
from typing import Optional, cast
import asyncio
from typing import Optional, cast
import httpx
from app.services.base import BaseDownloader
from app.services.utils import zip, unzip, get_filename, process_pool_executor
from app.services.book_library import BookLibraryClient, Book
from app.services.utils import zip, unzip, get_filename, process_pool_executor
from core.config import env_config, SourceConfig
@@ -40,17 +38,19 @@ class FLDownloader(BaseDownloader):
await asyncio.wait_for(self.get_book_data_task, None)
if self.book is None:
raise ValueError('Book is None!')
raise ValueError("Book is None!")
return get_filename(self.book, self.file_type)
async def get_final_filename(self) -> str:
if self.need_zip:
return (await self.get_filename()) + '.zip'
return (await self.get_filename()) + ".zip"
return await self.get_filename()
async def _download_from_source(self, source_config: SourceConfig, file_type: str = None) -> tuple[bytes, bool]:
async def _download_from_source(
self, source_config: SourceConfig, file_type: str = None
) -> tuple[bytes, bool]:
basic_url: str = source_config.URL
proxy: Optional[str] = source_config.PROXY
@@ -63,16 +63,14 @@ class FLDownloader(BaseDownloader):
httpx_proxy = None
if proxy is not None:
httpx_proxy = httpx.Proxy(
url=proxy
)
httpx_proxy = httpx.Proxy(url=proxy)
async with httpx.AsyncClient(proxies=httpx_proxy) as client:
response = await client.get(url, follow_redirects=True, timeout=10 * 60)
content_type = response.headers.get("Content-Type")
if response.status_code != 200:
raise NotSuccess(f'Status code is {response.status_code}!')
raise NotSuccess(f"Status code is {response.status_code}!")
if "text/html" in content_type:
raise ReceivedHTML()
@@ -82,11 +80,15 @@ class FLDownloader(BaseDownloader):
return response.content, False
async def _wait_until_some_done(self, tasks: set[asyncio.Task]) -> Optional[tuple[bytes, bool]]:
async def _wait_until_some_done(
self, tasks: set[asyncio.Task]
) -> Optional[tuple[bytes, bool]]:
tasks_ = tasks
while tasks_:
done, pending = await asyncio.wait(tasks_, return_when=asyncio.FIRST_COMPLETED)
done, pending = await asyncio.wait(
tasks_, return_when=asyncio.FIRST_COMPLETED
)
for task in done:
try:
@@ -100,7 +102,7 @@ class FLDownloader(BaseDownloader):
continue
tasks_ = pending
return None
async def _download_with_converting(self) -> tuple[bytes, bool]:
@@ -108,9 +110,7 @@ class FLDownloader(BaseDownloader):
for source in env_config.FL_SOURCES:
tasks.add(
asyncio.create_task(
self._download_from_source(source, file_type='fb2')
)
asyncio.create_task(self._download_from_source(source, file_type="fb2"))
)
data = await self._wait_until_some_done(tasks)
@@ -122,13 +122,15 @@ class FLDownloader(BaseDownloader):
if is_zip:
content = await asyncio.get_event_loop().run_in_executor(
process_pool_executor, unzip, content, 'fb2'
process_pool_executor, unzip, content, "fb2"
)
async with httpx.AsyncClient() as client:
form = {'format': self.file_type}
files = {'file': content}
response = await client.post(env_config.CONVERTER_URL, data=form, files=files, timeout=2 * 60)
form = {"format": self.file_type}
files = {"file": content}
response = await client.post(
env_config.CONVERTER_URL, data=form, files=files, timeout=2 * 60
)
if response.status_code != 200:
raise ValueError
@@ -143,20 +145,12 @@ class FLDownloader(BaseDownloader):
async def _get_content(self) -> tuple[bytes, str]:
tasks = set()
if self.file_type in ['epub', 'mobi']:
tasks.add(
asyncio.create_task(
self._download_with_converting()
)
)
if self.file_type in ["epub", "mobi"]:
tasks.add(asyncio.create_task(self._download_with_converting()))
for source in env_config.FL_SOURCES:
tasks.add(
asyncio.create_task(
self._download_from_source(source)
)
)
tasks.add(asyncio.create_task(self._download_from_source(source)))
data = await self._wait_until_some_done(tasks)
if data is None:
@@ -192,6 +186,8 @@ class FLDownloader(BaseDownloader):
return tasks[0].result()
@classmethod
async def download(cls, remote_id: int, file_type: str, source_id: int) -> tuple[bytes, str]:
async def download(
cls, remote_id: int, file_type: str, source_id: int
) -> tuple[bytes, str]:
downloader = cls(remote_id, file_type, source_id)
return await downloader._download()