From 18df7c12f463f91e3ba54b8d51c10ae27d9e6149 Mon Sep 17 00:00:00 2001 From: Kurbanov Bulat Date: Sun, 21 Nov 2021 22:32:25 +0300 Subject: [PATCH] Init --- .github/workflows/build_docker_image.yml | 49 ++++++++++++ .gitignore | 5 ++ docker/production.dockerfile | 35 ++++++++ requirements.txt | 8 ++ scripts/start_production.sh | 3 + src/app/alembic.ini | 98 +++++++++++++++++++++++ src/app/alembic/README | 1 + src/app/alembic/env.py | 64 +++++++++++++++ src/app/alembic/script.py.mako | 24 ++++++ src/app/alembic/versions/9b7cfb422191_.py | 35 ++++++++ src/app/depends.py | 9 +++ src/app/models.py | 21 +++++ src/app/serializers.py | 14 ++++ src/app/services/cache_updater.py | 76 ++++++++++++++++++ src/app/services/downloader.py | 21 +++++ src/app/services/files_uploader.py | 25 ++++++ src/app/services/library_client.py | 54 +++++++++++++ src/app/views.py | 53 ++++++++++++ src/core/app.py | 26 ++++++ src/core/auth.py | 4 + src/core/config.py | 23 ++++++ src/core/db.py | 15 ++++ src/main.py | 4 + 23 files changed, 667 insertions(+) create mode 100644 .github/workflows/build_docker_image.yml create mode 100644 .gitignore create mode 100644 docker/production.dockerfile create mode 100644 requirements.txt create mode 100644 scripts/start_production.sh create mode 100644 src/app/alembic.ini create mode 100644 src/app/alembic/README create mode 100644 src/app/alembic/env.py create mode 100644 src/app/alembic/script.py.mako create mode 100644 src/app/alembic/versions/9b7cfb422191_.py create mode 100644 src/app/depends.py create mode 100644 src/app/models.py create mode 100644 src/app/serializers.py create mode 100644 src/app/services/cache_updater.py create mode 100644 src/app/services/downloader.py create mode 100644 src/app/services/files_uploader.py create mode 100644 src/app/services/library_client.py create mode 100644 src/app/views.py create mode 100644 src/core/app.py create mode 100644 src/core/auth.py create mode 100644 src/core/config.py create mode 100644 src/core/db.py create mode 100644 src/main.py diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml new file mode 100644 index 0000000..fd94750 --- /dev/null +++ b/.github/workflows/build_docker_image.yml @@ -0,0 +1,49 @@ +name: Build docker image + +on: + push: + branches: + - 'main' + +jobs: + Build-Docker-Image: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - id: repository_name + uses: ASzc/change-string-case-action@v1 + with: + string: ${{ github.repository }} + + - + name: Login to ghcr.io + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + env: + IMAGE: ${{ steps.repository_name.outputs.lowercase }} + with: + push: true + tags: ghcr.io/${{ env.IMAGE }}:latest + context: . + file: ./docker/production.dockerfile + + - + name: Invoke deployment hook + uses: joelwmale/webhook-action@master + with: + url: ${{ secrets.WEBHOOK_URL }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..684f953 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.vscode + +__pycache__ + +venv diff --git a/docker/production.dockerfile b/docker/production.dockerfile new file mode 100644 index 0000000..21e3701 --- /dev/null +++ b/docker/production.dockerfile @@ -0,0 +1,35 @@ +FROM python:3.10-slim as build-image + +RUN apt-get update \ + && apt-get install --no-install-recommends -y gcc build-essential python3-dev libpq-dev libffi-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /root/venv +COPY ./requirements.txt /root/venv/ + +ENV VENV_PATH=/opt/venv + +RUN python -m venv $VENV_PATH \ + && . /opt/venv/bin/activate \ + && pip install -r requirements.txt --no-cache-dir + + +FROM python:3.10-slim as runtime-image + +RUN apt-get update \ + && apt-get install --no-install-recommends -y python3-dev libpq-dev libffi-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY ./src/ /app/ + +ENV VENV_PATH=/opt/venv +COPY --from=build-image $VENV_PATH $VENV_PATH +ENV PATH="$VENV_PATH/bin:$PATH" + +COPY ./scripts/start_production.sh /root/ + +EXPOSE 8080 + +CMD bash /root/start_production.sh diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0fad2bf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +fastapi +pydantic +asyncpg +psycopg2 +httpx +ormar +alembic +uvicorn[standart] diff --git a/scripts/start_production.sh b/scripts/start_production.sh new file mode 100644 index 0000000..4b02d41 --- /dev/null +++ b/scripts/start_production.sh @@ -0,0 +1,3 @@ +cd /app +alembic -c ./app/alembic.ini upgrade head +uvicorn main:app --host="0.0.0.0" --port="8080" diff --git a/src/app/alembic.ini b/src/app/alembic.ini new file mode 100644 index 0000000..01b575f --- /dev/null +++ b/src/app/alembic.ini @@ -0,0 +1,98 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = ./app/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. Valid values are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # default: use os.pathsep + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/src/app/alembic/README b/src/app/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/src/app/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/src/app/alembic/env.py b/src/app/alembic/env.py new file mode 100644 index 0000000..a791adb --- /dev/null +++ b/src/app/alembic/env.py @@ -0,0 +1,64 @@ +from alembic import context +import sys, os + +from sqlalchemy.engine import create_engine + +from core.db import DATABASE_URL + + +myPath = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, myPath + '/../../') + +config = context.config + + +from app.models import BaseMeta +target_metadata = BaseMeta.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = create_engine(DATABASE_URL) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, compare_type=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/src/app/alembic/script.py.mako b/src/app/alembic/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/src/app/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/src/app/alembic/versions/9b7cfb422191_.py b/src/app/alembic/versions/9b7cfb422191_.py new file mode 100644 index 0000000..2805730 --- /dev/null +++ b/src/app/alembic/versions/9b7cfb422191_.py @@ -0,0 +1,35 @@ +"""empty message + +Revision ID: 9b7cfb422191 +Revises: +Create Date: 2021-11-21 14:09:17.478532 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9b7cfb422191' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('cached_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('object_id', sa.Integer(), nullable=False), + sa.Column('object_type', sa.String(length=8), nullable=False), + sa.Column('data', sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('object_id', 'object_type', name='uc_cached_files_object_id_object_type') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('cached_files') + # ### end Alembic commands ### diff --git a/src/app/depends.py b/src/app/depends.py new file mode 100644 index 0000000..b99768e --- /dev/null +++ b/src/app/depends.py @@ -0,0 +1,9 @@ +from fastapi import Security, HTTPException, status + +from core.auth import default_security +from core.config import env_config + + +async def check_token(api_key: str = Security(default_security)): + if api_key != env_config.API_KEY: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!") diff --git a/src/app/models.py b/src/app/models.py new file mode 100644 index 0000000..c109589 --- /dev/null +++ b/src/app/models.py @@ -0,0 +1,21 @@ +import ormar + +from core.db import metadata, database + + +class BaseMeta(ormar.ModelMeta): + metadata = metadata + database = database + + +class CachedFile(ormar.Model): + class Meta(BaseMeta): + tablename = "cached_files" + constraints = [ + ormar.UniqueColumns('object_id','object_type') + ] + + id: int = ormar.Integer(primary_key=True) # type: ignore + object_id: int = ormar.Integer() # type: ignore + object_type: str = ormar.String(max_length=8) # type: ignore + data = ormar.JSON() diff --git a/src/app/serializers.py b/src/app/serializers.py new file mode 100644 index 0000000..2714c24 --- /dev/null +++ b/src/app/serializers.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel, constr + + +class CachedFile(BaseModel): + id: int + object_id: int + object_type: str + data: dict + + +class CreateCachedFile(BaseModel): + object_id: int + object_type: constr(max_length=8) # type: ignore + data: dict diff --git a/src/app/services/cache_updater.py b/src/app/services/cache_updater.py new file mode 100644 index 0000000..fb2f9e9 --- /dev/null +++ b/src/app/services/cache_updater.py @@ -0,0 +1,76 @@ +import asyncio + +from app.services.library_client import get_books, Book +from app.services.downloader import download +from app.services.files_uploader import upload_file +from app.models import CachedFile + + +PAGE_SIZE = 50 + + +class CacheUpdater: + def __init__(self): + self.queue = asyncio.Queue(maxsize=10) + self.all_books_checked = False + + async def _check_book(self, book: Book): + for file_type in book.available_types: + cached_file = await CachedFile.objects.get_or_none( + object_id=book.id, object_type=file_type + ) + + if cached_file is None: + await self.queue.put((book, file_type)) + + async def _start_producer(self): + books_page = await get_books(1, 1) + + page_count = books_page.total // PAGE_SIZE + if books_page.total % PAGE_SIZE != 0: + page_count += 1 + + for page_number in range(1, page_count + 1): + page = await get_books(page_number, PAGE_SIZE) + + for book in page.items: + await self._check_book(book) + + self.all_books_checked = True + + async def _start_worker(self): + while not self.all_books_checked or not self.queue.empty(): + try: + task = self.queue.get_nowait() + book: Book = task[0] + file_type: str = task[1] + except asyncio.QueueEmpty: + await asyncio.sleep(0.1) + continue + + data = await download(book.source.id, book.remote_id, file_type) + + if data is None: + return + + content, filename = data + + upload_data = await upload_file(content, filename, 'Test') + + await CachedFile.objects.create( + object_id=book.id, + object_type=file_type, + data=upload_data.data + ) + + async def _update(self): + await asyncio.gather( + self._start_producer(), + self._start_worker(), + self._start_worker() + ) + + @classmethod + async def update(cls): + updater = cls() + return await updater._update() diff --git a/src/app/services/downloader.py b/src/app/services/downloader.py new file mode 100644 index 0000000..e0d0ab0 --- /dev/null +++ b/src/app/services/downloader.py @@ -0,0 +1,21 @@ +from typing import Optional + +import httpx + +from core.config import env_config + + +async def download(source_id: int, remote_id: int, file_type: str) -> Optional[tuple[bytes, str]]: + headers = {"Authorization": env_config.DOWNLOADER_API_KEY} + + async with httpx.AsyncClient() as client: + response = await client.get(f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}", headers=headers, timeout=5 * 60) + + if response.status_code != 200: + return None + + content_disposition = response.headers['Content-Disposition'] + + name = content_disposition.replace('attachment; filename=', '') + + return response.content, name diff --git a/src/app/services/files_uploader.py b/src/app/services/files_uploader.py new file mode 100644 index 0000000..07cbeab --- /dev/null +++ b/src/app/services/files_uploader.py @@ -0,0 +1,25 @@ +from datetime import datetime + +import httpx +from pydantic import BaseModel + +from core.config import env_config + + +class UploadedFile(BaseModel): + id: int + backend: str + data: dict + upload_time: datetime + + +async def upload_file(content: bytes, filename: str, caption: str) -> UploadedFile: + headers = {"Authorization": env_config.FILES_SERVER_API_KEY} + + async with httpx.AsyncClient() as client: + form = {'caption': caption} + files = {'file': (filename, content)} + + response = await client.post(f"{env_config.FILES_SERVER_URL}/api/v1/files/upload/", data=form, files=files, headers=headers, timeout=5 * 60) + + return UploadedFile.parse_obj(response.json()) diff --git a/src/app/services/library_client.py b/src/app/services/library_client.py new file mode 100644 index 0000000..2568cdc --- /dev/null +++ b/src/app/services/library_client.py @@ -0,0 +1,54 @@ +from typing import Generic, TypeVar +from pydantic import BaseModel +from datetime import date + +from core.config import env_config + +import httpx + + +T = TypeVar('T') + + +class Page(BaseModel, Generic[T]): + items: list[T] + total: int + page: int + size: int + + +class BookAuthor(BaseModel): + id: int + first_name: str + last_name: str + middle_name: str + + +class BookSource(BaseModel): + id: int + name: str + + +class Book(BaseModel): + id: int + title: str + file_type: str + available_types: list[str] + source: BookSource + remote_id: int + uploaded: date + authors: list[BookAuthor] + + +async def get_books(page: int, page_size: int) -> Page[Book]: + headers = {"Authorization": env_config.LIBRARY_API_KEY} + + async with httpx.AsyncClient() as client: + response = await client.get(f"{env_config.LIBRARY_URL}/api/v1/books/?page={page}&size={page_size}&is_deleted=false", headers=headers) + + data = response.json() + + page_data = Page[Book].parse_obj(data) + page_data.items = [Book.parse_obj(item) for item in page_data.items] + + return page_data diff --git a/src/app/views.py b/src/app/views.py new file mode 100644 index 0000000..58b5c9d --- /dev/null +++ b/src/app/views.py @@ -0,0 +1,53 @@ +from fastapi import APIRouter, Depends, HTTPException, status + +from app.depends import check_token +from app.models import CachedFile as CachedFileDB +from app.serializers import CachedFile, CreateCachedFile +from app.services.cache_updater import CacheUpdater + + +router = APIRouter( + prefix="/api/v1", + tags=["files"], + dependencies=[Depends(check_token)] +) + + +@router.get("/{object_id}/{object_type}", response_model=CachedFile) +async def get_cached_file(object_id: int, object_type: str): + cached_file = await CachedFileDB.objects.get( + object_id=object_id, + object_type=object_type + ) + + if not cached_file: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + + return cached_file + + +@router.delete("/{object_id}/{object_type}", response_model=CachedFile) +async def delete_cached_file(object_id: int, object_type: str): + cached_file = await CachedFileDB.objects.get( + object_id=object_id, + object_type=object_type + ) + + if not cached_file: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) + + await cached_file.delete() + + return cached_file + + +@router.post("/", response_model=CachedFile) +async def create_cached_file(data: CreateCachedFile): + return await CachedFileDB.objects.create( + *data.dict() + ) + + +@router.post("/update_cache") +async def update_cache(): + await CacheUpdater.update() diff --git a/src/core/app.py b/src/core/app.py new file mode 100644 index 0000000..200cfda --- /dev/null +++ b/src/core/app.py @@ -0,0 +1,26 @@ +from fastapi import FastAPI + +from core.db import database +from app.views import router + + +def start_app() -> FastAPI: + app = FastAPI() + + app.state.database = database + + app.include_router(router) + + @app.on_event('startup') + async def startup() -> None: + database_ = app.state.database + if not database_.is_connected: + await database_.connect() + + @app.on_event('shutdown') + async def shutdown() -> None: + database_ = app.state.database + if database_.is_connected: + await database_.disconnect() + + return app diff --git a/src/core/auth.py b/src/core/auth.py new file mode 100644 index 0000000..7cc07b5 --- /dev/null +++ b/src/core/auth.py @@ -0,0 +1,4 @@ +from fastapi.security import APIKeyHeader + + +default_security = APIKeyHeader(name="Authorization") diff --git a/src/core/config.py b/src/core/config.py new file mode 100644 index 0000000..8ae15f0 --- /dev/null +++ b/src/core/config.py @@ -0,0 +1,23 @@ +from pydantic import BaseSettings + + +class EnvConfig(BaseSettings): + API_KEY: str + + POSTGRES_USER: str + POSTGRES_PASSWORD: str + POSTGRES_HOST: str + POSTGRES_PORT: int + POSTGRES_DB: str + + DOWNLOADER_API_KEY: str + DOWNLOADER_URL: str + + LIBRARY_API_KEY: str + LIBRARY_URL: str + + FILES_SERVER_API_KEY: str + FILES_SERVER_URL: str + + +env_config = EnvConfig() diff --git a/src/core/db.py b/src/core/db.py new file mode 100644 index 0000000..6037ba3 --- /dev/null +++ b/src/core/db.py @@ -0,0 +1,15 @@ +from urllib.parse import quote +from databases import Database + +from sqlalchemy import MetaData + +from core.config import env_config + + +DATABASE_URL = ( + f"postgresql://{env_config.POSTGRES_USER}:{quote(env_config.POSTGRES_PASSWORD)}@" + f"{env_config.POSTGRES_HOST}:{env_config.POSTGRES_PORT}/{env_config.POSTGRES_DB}" +) + +metadata = MetaData() +database = Database(DATABASE_URL) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..0a4385b --- /dev/null +++ b/src/main.py @@ -0,0 +1,4 @@ +from core.app import start_app + + +app = start_app()