From 19fe00335e3548b9005f2a15c4c8fd118e521f2f Mon Sep 17 00:00:00 2001 From: Kurbanov Bulat Date: Wed, 9 Feb 2022 01:03:54 +0300 Subject: [PATCH] Add redis cache --- poetry.lock | 77 ++++++++- pyproject.toml | 2 + .../allowed_langs_updater.py} | 8 +- src/app/services/users_data_manager.py | 157 ++++++++++++++++++ src/app/views.py | 74 ++------- src/core/app.py | 9 + src/core/config.py | 7 + 7 files changed, 265 insertions(+), 69 deletions(-) rename src/app/{services.py => services/allowed_langs_updater.py} (85%) create mode 100644 src/app/services/users_data_manager.py diff --git a/poetry.lock b/poetry.lock index f10d52b..e51631a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,18 @@ +[[package]] +name = "aioredis" +version = "2.0.1" +description = "asyncio (PEP 3156) Redis support" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +async-timeout = "*" +typing-extensions = "*" + +[package.extras] +hiredis = ["hiredis (>=1.0)"] + [[package]] name = "aiosqlite" version = "0.17.0" @@ -52,6 +67,14 @@ python-versions = ">=3.6" [package.extras] tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"] +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "asyncpg" version = "0.25.0" @@ -153,13 +176,13 @@ pydantic = ">=1.7.2" [package.extras] gino = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)"] -all = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)", "databases[sqlite,mysql,postgresql] (>=0.4.0)", "orm (>=0.1.5)", "tortoise-orm[aiosqlite,aiomysql,asyncpg] (>=0.16.18,<0.18.0)", "asyncpg (>=0.24.0)", "ormar (>=0.10.5)", "Django (<3.3.0)", "piccolo (>=0.29,<0.35)", "motor (>=2.5.1,<3.0.0)"] +all = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)", "databases[mysql,postgresql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "tortoise-orm[aiosqlite,asyncpg,aiomysql] (>=0.16.18,<0.18.0)", "asyncpg (>=0.24.0)", "ormar (>=0.10.5)", "Django (<3.3.0)", "piccolo (>=0.29,<0.35)", "motor (>=2.5.1,<3.0.0)"] sqlalchemy = ["SQLAlchemy (>=1.3.20)"] asyncpg = ["SQLAlchemy (>=1.3.20)", "asyncpg (>=0.24.0)"] -databases = ["databases[sqlite,mysql,postgresql] (>=0.4.0)"] -orm = ["databases[sqlite,mysql,postgresql] (>=0.4.0)", "orm (>=0.1.5)", "typesystem (>=0.2.0,<0.3.0)"] -django = ["databases[sqlite,mysql,postgresql] (>=0.4.0)", "Django (<3.3.0)"] -tortoise = ["tortoise-orm[aiosqlite,aiomysql,asyncpg] (>=0.16.18,<0.18.0)"] +databases = ["databases[mysql,postgresql,sqlite] (>=0.4.0)"] +orm = ["databases[mysql,postgresql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "typesystem (>=0.2.0,<0.3.0)"] +django = ["databases[mysql,postgresql,sqlite] (>=0.4.0)", "Django (<3.3.0)"] +tortoise = ["tortoise-orm[aiosqlite,asyncpg,aiomysql] (>=0.16.18,<0.18.0)"] ormar = ["ormar (>=0.10.5)"] piccolo = ["piccolo (>=0.29,<0.35)"] motor = ["motor (>=2.5.1,<3.0.0)"] @@ -253,6 +276,14 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "orjson" +version = "3.6.6" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "ormar" version = "0.10.24" @@ -395,9 +426,13 @@ standard = ["httptools (>=0.2.0,<0.4.0)", "watchgod (>=0.6)", "python-dotenv (>= [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "cd0bcc4a26806b876c187a737d47cdbc955e7b569c01811b902732daa0fb46e7" +content-hash = "8de594deadf42d15c00004f93df1ef2bde459d0cdca4503e2798ac5a345baa6e" [metadata.files] +aioredis = [ + {file = "aioredis-2.0.1-py3-none-any.whl", hash = "sha256:9ac0d0b3b485d293b8ca1987e6de8658d7dafcca1cddfcd1d506cae8cdebfdd6"}, + {file = "aioredis-2.0.1.tar.gz", hash = "sha256:eaa51aaf993f2d71f54b70527c440437ba65340588afeb786cd87c55c89cd98e"}, +] aiosqlite = [ {file = "aiosqlite-0.17.0-py3-none-any.whl", hash = "sha256:6c49dc6d3405929b1d08eeccc72306d3677503cc5e5e43771efc1e00232e8231"}, {file = "aiosqlite-0.17.0.tar.gz", hash = "sha256:f0e6acc24bc4864149267ac82fb46dfb3be4455f99fe21df82609cc6e6baee51"}, @@ -414,6 +449,10 @@ asgiref = [ {file = "asgiref-3.4.1-py3-none-any.whl", hash = "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"}, {file = "asgiref-3.4.1.tar.gz", hash = "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9"}, ] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] asyncpg = [ {file = "asyncpg-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf5e3408a14a17d480f36ebaf0401a12ff6ae5457fdf45e4e2775c51cc9517d3"}, {file = "asyncpg-0.25.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2bc197fc4aca2fd24f60241057998124012469d2e414aed3f992579db0c88e3a"}, @@ -598,6 +637,32 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, ] +orjson = [ + {file = "orjson-3.6.6-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:e4a7cad6c63306318453980d302c7c0b74c0cc290dd1f433bbd7d31a5af90cf1"}, + {file = "orjson-3.6.6-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:e533941dca4a0530a876de32e54bf2fd3269cdec3751aebde7bfb5b5eba98e74"}, + {file = "orjson-3.6.6-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:9adf63be386eaa34278967512b83ff8fc4bed036a246391ae236f68d23c47452"}, + {file = "orjson-3.6.6-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:3b636753ae34d4619b11ea7d664a2f1e87e55e9738e5123e12bcce22acae9d13"}, + {file = "orjson-3.6.6-cp310-none-win_amd64.whl", hash = "sha256:78a10295ed048fd916c6584d6d27c232eae805a43e7c14be56e3745f784f0eb6"}, + {file = "orjson-3.6.6-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:82b4f9fb2af7799b52932a62eac484083f930d5519560d6f64b24d66a368d03f"}, + {file = "orjson-3.6.6-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:a0033d07309cc7d8b8c4bc5d42f0dd4422b53ceb91dee9f4086bb2afa70b7772"}, + {file = "orjson-3.6.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b321f99473116ab7c7c028377372f7b4adba4029aaca19cd567e83898f55579"}, + {file = "orjson-3.6.6-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:b9c98ed94f1688cc11b5c61b8eea39d854a1a2f09f71d8a5af005461b14994ed"}, + {file = "orjson-3.6.6-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:00b333a41392bd07a8603c42670547dbedf9b291485d773f90c6470eff435608"}, + {file = "orjson-3.6.6-cp37-none-win_amd64.whl", hash = "sha256:8d4fd3bdee65a81f2b79c50937d4b3c054e1e6bfa3fc72ed018a97c0c7c3d521"}, + {file = "orjson-3.6.6-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:954c9f8547247cd7a8c91094ff39c9fe314b5eaeaec90b7bfb7384a4108f416f"}, + {file = "orjson-3.6.6-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:74e5aed657ed0b91ef05d44d6a26d3e3e12ce4d2d71f75df41a477b05878c4a9"}, + {file = "orjson-3.6.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4008a5130e6e9c33abaa95e939e0e755175da10745740aa6968461b2f16830e2"}, + {file = "orjson-3.6.6-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:012761d5f3d186deb4f6238f15e9ea7c1aac6deebc8f5b741ba3b4fafe017460"}, + {file = "orjson-3.6.6-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:b464546718a940b48d095a98df4c04808bfa6c8706fe751fc3f9390bc2f82643"}, + {file = "orjson-3.6.6-cp38-none-win_amd64.whl", hash = "sha256:f10a800f4e5a4aab52076d4628e9e4dab9370bdd9d8ea254ebfde846b653ab25"}, + {file = "orjson-3.6.6-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:8010d2610cfab721725ef14d578c7071e946bbdae63322d8f7b49061cf3fde8d"}, + {file = "orjson-3.6.6-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:8dca67a4855e1e0f9a2ea0386e8db892708522e1171dc0ddf456932288fbae63"}, + {file = "orjson-3.6.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af065d60523139b99bd35b839c7a2d8c5da55df8a8c4402d2eb6cdc07fa7a624"}, + {file = "orjson-3.6.6-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:fa1f389cc9f766ae0cf7ba3533d5089836b01a5ccb3f8d904297f1fcf3d9dc34"}, + {file = "orjson-3.6.6-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:ec1221ad78f94d27b162a1d35672b62ef86f27f0e4c2b65051edb480cc86b286"}, + {file = "orjson-3.6.6-cp39-none-win_amd64.whl", hash = "sha256:afed2af55eeda1de6b3f1cbc93431981b19d380fcc04f6ed86e74c1913070304"}, + {file = "orjson-3.6.6.tar.gz", hash = "sha256:55dd988400fa7fbe0e31407c683f5aaab013b5bd967167b8fe058186773c4d6c"}, +] ormar = [ {file = "ormar-0.10.24-py3-none-any.whl", hash = "sha256:0ac7765bc14237cb4ed828c823cae3a4a9f5dea6daa402e0999c80b36662c410"}, {file = "ormar-0.10.24.tar.gz", hash = "sha256:908eba2cb7350c5ef0c8e7d9653d061f357e2c7706b78298bd446e0848000762"}, diff --git a/pyproject.toml b/pyproject.toml index 3131c44..a9ccaf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,8 @@ alembic = "^1.7.5" ormar = {extras = ["postgresql"], version = "^0.10.23"} uvicorn = {extras = ["standart"], version = "^0.16.0"} httpx = "^0.22.0" +aioredis = "^2.0.1" +orjson = "^3.6.6" [tool.poetry.dev-dependencies] diff --git a/src/app/services.py b/src/app/services/allowed_langs_updater.py similarity index 85% rename from src/app/services.py rename to src/app/services/allowed_langs_updater.py index df36ee9..159e6cf 100644 --- a/src/app/services.py +++ b/src/app/services/allowed_langs_updater.py @@ -3,7 +3,7 @@ from typing import cast from app.models import User, Language -async def update_user_allowed_langs(user: User, new_allowed_langs: list[str]): +async def update_user_allowed_langs(user: User, new_allowed_langs: list[str]) -> bool: user_allowed_langs = cast(list[Language], user.allowed_langs) exists_langs = set(lang.code for lang in user_allowed_langs) @@ -16,9 +16,15 @@ async def update_user_allowed_langs(user: User, new_allowed_langs: list[str]): langs = await Language.objects.filter(code__in=all_process_langs).all() + updated = False + for lang in langs: if lang.code in to_delete: await user.allowed_langs.remove(lang) + updated = True if lang.code in to_add: await user.allowed_langs.add(lang) + updated = True + + return updated diff --git a/src/app/services/users_data_manager.py b/src/app/services/users_data_manager.py new file mode 100644 index 0000000..84a5b47 --- /dev/null +++ b/src/app/services/users_data_manager.py @@ -0,0 +1,157 @@ +from typing import Optional, Union + +from fastapi import HTTPException, status + +import aioredis +import orjson + +from app.models import User +from app.serializers import UserCreateOrUpdate, UserDetail, UserUpdate +from app.services.allowed_langs_updater import update_user_allowed_langs + + +class UsersDataManager: + @classmethod + async def _get_user_from_db(cls, user_id: int) -> Optional[User]: + return await User.objects.select_related("allowed_langs").get_or_none( + user_id=user_id + ) + + @classmethod + def _get_cache_key(cls, user_id: int) -> str: + return f"user_{user_id}" + + @classmethod + async def _get_user_from_cache( + cls, user_id: int, redis: aioredis.Redis + ) -> Optional[UserDetail]: + try: + key = cls._get_cache_key(user_id) + data = await redis.get(key) + + if data is None: + return None + + return UserDetail.parse_obj(orjson.loads(data)) + + except aioredis.RedisError: + return None + + @classmethod + async def _cache_user(cls, user: User, redis: aioredis.Redis) -> bool: + try: + key = cls._get_cache_key(user.id) + data = orjson.dumps(user.dict()) + await redis.set(key, data) + return True + except aioredis.RedisError: + return False + + @classmethod + async def get_user( + cls, user_id: int, redis: aioredis.Redis + ) -> Optional[UserDetail]: + if cached_user := await cls._get_user_from_cache(user_id, redis): + return cached_user + + user = await cls._get_user_from_db(user_id) + + if not user: + return None + + await cls._cache_user(user, redis) + return user # type: ignore + + @classmethod + def _is_has_data_to_update(cls, new_user: UserUpdate) -> bool: + data_dict = new_user.dict() + + update_data = {} + for key in data_dict: + if data_dict[key] is not None: + update_data[key] = data_dict[key] + + return bool(update_data) + + @classmethod + async def _create(cls, data: UserCreateOrUpdate): + data_dict = data.dict() + allowed_langs = data_dict.pop("allowed_langs", None) or ["ru", "be", "uk"] + + user_obj = await User.objects.select_related("allowed_langs").create( + **data_dict + ) + await update_user_allowed_langs(user_obj, allowed_langs) + + return user_obj + + @classmethod + async def _update( + cls, user_id: int, update_data: dict, redis: aioredis.Redis + ) -> User: + user_obj = await cls._get_user_from_db(user_id) + assert user_obj is not None + + if allowed_langs := update_data.pop("allowed_langs", None): + await update_user_allowed_langs(user_obj, allowed_langs) + + if update_data: + user_obj.update_from_dict(update_data) + await user_obj.update() + + await cls._cache_user(user_obj, redis) + + return user_obj + + @classmethod + async def create_or_update_user( + cls, data: UserCreateOrUpdate, redis: aioredis.Redis + ): + user = await cls.get_user(data.user_id, redis) + + if user is None: + new_user = await cls._create(data) + await cls._cache_user(new_user, redis) + return new_user + + if not cls._is_need_update(user, data): + return user + + return await cls._update(user.user_id, data.dict(), redis) + + @classmethod + def _is_need_update( + cls, old_user: UserDetail, new_user: Union[UserUpdate, UserCreateOrUpdate] + ) -> bool: + old_data = old_user.dict() + new_data = new_user.dict() + + allowed_langs = new_data.pop("allowed_lang", None) + + for key in new_data: + if new_data[key] != old_data[key]: + return True + + if allowed_langs and set(allowed_langs) != set( + [lang.code for lang in old_user.allowed_langs] + ): + return True + + return False + + @classmethod + async def update_user( + cls, user_id: int, user_data: UserUpdate, redis: aioredis.Redis + ) -> Union[UserDetail, User]: + user = await cls.get_user(user_id, redis) + + if user is None: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) + + if not cls._is_has_data_to_update(user_data): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) + + if not cls._is_need_update(user, user_data): + return user + + return await cls._update(user.user_id, user_data.dict(), redis) diff --git a/src/app/views.py b/src/app/views.py index 9a1e60e..e66b12b 100644 --- a/src/app/views.py +++ b/src/app/views.py @@ -1,5 +1,6 @@ -from fastapi import APIRouter, HTTPException, status, Depends +from fastapi import APIRouter, HTTPException, status, Depends, Request +import aioredis from fastapi_pagination import Page, Params from fastapi_pagination.ext.ormar import paginate @@ -12,10 +13,7 @@ from app.serializers import ( CreateLanguage, LanguageDetail, ) -from app.services import update_user_allowed_langs - - -# TODO: add redis cache +from app.services.users_data_manager import UsersDataManager users_router = APIRouter( @@ -29,10 +27,9 @@ async def get_users(): @users_router.get("/{user_id}", response_model=UserDetail) -async def get_user(user_id: int): - user_data = await User.objects.select_related("allowed_langs").get_or_none( - user_id=user_id - ) +async def get_user(request: Request, user_id: int): + redis: aioredis.Redis = request.app.state.redis + user_data = await UsersDataManager.get_user(user_id, redis) if user_data is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) @@ -41,62 +38,15 @@ async def get_user(user_id: int): @users_router.post("/", response_model=UserDetail) -async def create_or_update_user(data: UserCreateOrUpdate): - data_dict = data.dict() - - user_data = await User.objects.select_related("allowed_langs").get_or_none( - user_id=data_dict["user_id"] - ) - - allowed_langs = data_dict.pop("allowed_langs") - - if user_data is None: - user_data = await User.objects.select_related("allowed_langs").create( - **data_dict - ) - if allowed_langs is None: - allowed_langs = ["ru", "be", "uk"] - else: - data_dict.pop("user_id") - user_data.update_from_dict(data_dict) - - if allowed_langs: - await update_user_allowed_langs(user_data, allowed_langs) - - return user_data +async def create_or_update_user(request: Request, data: UserCreateOrUpdate): + redis: aioredis.Redis = request.app.state.redis + return await UsersDataManager.create_or_update_user(data, redis) @users_router.patch("/{user_id}", response_model=UserDetail) -async def update_user(user_id: int, data: UserUpdate): - user_data = await User.objects.select_related("allowed_langs").get_or_none( - user_id=user_id - ) - - if user_data is None: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) - - data_dict = data.dict() - - update_data = {} - for key in data_dict: - if data_dict[key] is not None: - update_data[key] = data_dict[key] - - if not update_data: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) - - allowed_langs = update_data.pop("allowed_langs", None) - - if update_data: - user_data.update_from_dict(update_data) - await user_data.update() - - if not allowed_langs: - return user_data - - await update_user_allowed_langs(user_data, allowed_langs) - - return user_data +async def update_user(request: Request, user_id: int, data: UserUpdate): + redis: aioredis.Redis = request.app.state.redis + return await UsersDataManager.update_user(user_id, data, redis) languages_router = APIRouter( diff --git a/src/core/app.py b/src/core/app.py index 9b30cc0..6922927 100644 --- a/src/core/app.py +++ b/src/core/app.py @@ -1,8 +1,10 @@ from fastapi import FastAPI +import aioredis from fastapi_pagination import add_pagination from app.views import users_router, languages_router, healthcheck_router +from core.config import env_config from core.db import database @@ -15,6 +17,13 @@ def start_app() -> FastAPI: app.state.database = database + app.state.redis = aioredis.Redis( + host=env_config.REDIS_HOST, + port=env_config.REDIS_PORT, + db=env_config.REDIS_DB, + password=env_config.REDIS_PASSWORD, + ) + add_pagination(app) @app.on_event("startup") diff --git a/src/core/config.py b/src/core/config.py index 56ae79a..9f8f58d 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -1,3 +1,5 @@ +from typing import Optional + from pydantic import BaseSettings @@ -10,5 +12,10 @@ class EnvConfig(BaseSettings): POSTGRES_PORT: int POSTGRES_DB: str + REDIS_HOST: str + REDIS_PORT: int + REDIS_DB: int + REDIS_PASSWORD: Optional[str] + env_config = EnvConfig()