flibusta-apps/library_updater
1
0
Fork 0
mirror of https://github.com/flibusta-apps/library_updater.git synced 2025-12-06 15:45:36 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix updating

Browse Source
This commit is contained in:
Kurbanov Bulat
2022-01-28 12:25:39 +03:00
parent 1fceb8b472
commit f210420a46
1 changed files with 25 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/app/services/updaters/fl_updater.py
View File

@@ -1,5 +1,6 @@
import asyncio import asyncio
import logging import logging
import re
from typing import Optional from typing import Optional
import aiomysql import aiomysql
@@ -35,22 +36,21 @@ def remove_dots(s: str):
return s.replace(".", "") return s.replace(".", "")
tags_regexp = re.compile(r"<.*?>")
def fix_annotation_text(text: str) -> str: def fix_annotation_text(text: str) -> str:
replace_map = { replace_map = {
"<p class=book>": "",
'<p class="book">': "",
"<p>": "",
"</p>": "",
"&nbsp;": "", "&nbsp;": "",
"[b]": "", "[b]": "",
"[/b]": "", "[/b]": "",
"[hr]": "", "[hr]": "",
} }
t = text t = tags_regexp.sub("", text)
for key, value in replace_map: for key in replace_map:
t = t.replace(key, value) t = t.replace(key, replace_map[key])
return t return t
@@ -236,7 +236,7 @@ class FlUpdater:
for offset in range(0, rows_count, 4096): for offset in range(0, rows_count, 4096):
await cursor.execute( await cursor.execute(
"SELECT BookId, Title, Lang, FileType, Time, Deleted FROM libbook LIMIT 4096 OFFSET {offset};".format( "SELECT BookId, Title, Lang, FileType, Time, Deleted FROM libbook ORDER BY BookId LIMIT 4096 OFFSET {offset};".format(
offset=offset offset=offset
) )
) )
@@ -280,13 +280,13 @@ class FlUpdater:
async with self.mysql_pool.acquire() as conn: async with self.mysql_pool.acquire() as conn:
async with conn.cursor() as cursor: async with conn.cursor() as cursor:
await cursor.execute("SELECT COUNT(*) FROM libavtorname;") await cursor.execute("SELECT COUNT(*) FROM libavtor;")
(rows_count,) = await cursor.fetchone() (rows_count,) = await cursor.fetchone()
for offset in range(0, rows_count, 4096): for offset in range(0, rows_count, 4096):
await cursor.execute( await cursor.execute(
"SELECT BookId, AvtorId FROM libavtor LIMIT 4096 OFFSET {offset};".format( "SELECT BookId, AvtorId FROM libavtor ORDER BY BookId, AvtorId LIMIT 4096 OFFSET {offset};".format(
offset=offset offset=offset
) )
) )
@@ -298,7 +298,7 @@ class FlUpdater:
[(self.SOURCE, *row) for row in rows], [(self.SOURCE, *row) for row in rows],
) )
logger.info("Books_authors updated!") logger.info("Books authors updated!")
async def _update_translations(self): async def _update_translations(self):
await self.books_updated_event.wait() await self.books_updated_event.wait()
@@ -456,7 +456,7 @@ class FlUpdater:
[[self.SOURCE, *row] for row in rows], [[self.SOURCE, *row] for row in rows],
) )
logger.info("Book_sequences updated!") logger.info("Book sequences updated!")
async def _update_book_annotations(self): async def _update_book_annotations(self):
await self.books_updated_event.wait() await self.books_updated_event.wait()
@@ -520,6 +520,9 @@ class FlUpdater:
async def _update_book_annotations_pic(self): async def _update_book_annotations_pic(self):
logger.info("Update book_annotations_pic...") logger.info("Update book_annotations_pic...")
def fix_link(row):
return [self.SOURCE, row[0], f"{env_config.FL_BASE_URL}/i/{row[1]}"]
async with self.mysql_pool.acquire() as conn: async with self.mysql_pool.acquire() as conn:
async with conn.cursor() as cursor: async with conn.cursor() as cursor:
await cursor.execute("SELECT COUNT(*) FROM libbpics;") await cursor.execute("SELECT COUNT(*) FROM libbpics;")
@@ -540,10 +543,10 @@ class FlUpdater:
"SET file = cast($3 as varchar) " "SET file = cast($3 as varchar) "
"FROM (SELECT id FROM books WHERE source = $1 AND remote_id = $2) as books " "FROM (SELECT id FROM books WHERE source = $1 AND remote_id = $2) as books "
"WHERE book = books.id;", "WHERE book = books.id;",
[[self.SOURCE, *row] for row in rows], [fix_link(row) for row in rows],
) )
logger.info("Book_annotation_pics updated!") logger.info("Book annotation pics updated!")
async def _update_author_annotations(self): async def _update_author_annotations(self):
await self.authors_updated_event.wait() await self.authors_updated_event.wait()
@@ -597,13 +600,16 @@ class FlUpdater:
[fix_annotation(row) for row in rows], [fix_annotation(row) for row in rows],
) )
logger.info("Author_annotation_updated!") logger.info("Author annotation updated!")
await self._update_author_annotations_pics() await self._update_author_annotations_pics()
async def _update_author_annotations_pics(self): async def _update_author_annotations_pics(self):
logger.info("Update author_annotations_pic...") logger.info("Update author_annotations_pic...")
def fix_link(row):
return [self.SOURCE, row[0], f"{env_config.FL_BASE_URL}/ia/{row[1]}"]
async with self.mysql_pool.acquire() as conn: async with self.mysql_pool.acquire() as conn:
async with conn.cursor() as cursor: async with conn.cursor() as cursor:
await cursor.execute("SELECT COUNT(*) FROM libapics;") await cursor.execute("SELECT COUNT(*) FROM libapics;")
@@ -624,10 +630,10 @@ class FlUpdater:
"SET file = cast($3 as varchar) " "SET file = cast($3 as varchar) "
"FROM (SELECT id FROM authors WHERE source = $1 AND remote_id = $2) as authors " "FROM (SELECT id FROM authors WHERE source = $1 AND remote_id = $2) as authors "
"WHERE author = authors.id;", "WHERE author = authors.id;",
[[self.SOURCE, *row] for row in rows], [fix_link(row) for row in rows],
) )
logger.info("Author_annotatioins_pic updated!") logger.info("Author annotatioins pic updated!")
async def _update_genres(self): async def _update_genres(self):
logger.info("Update genres...") logger.info("Update genres...")
@@ -670,6 +676,8 @@ class FlUpdater:
[[self.SOURCE, *row] for row in rows], [[self.SOURCE, *row] for row in rows],
) )
self.genres_updated_event.set()
logger.info("Genres updated!") logger.info("Genres updated!")
async def _update_books_genres(self): async def _update_books_genres(self):