Add meilisearch

fastapi_book_server/app/services/author.py

@@ -1,5 +1,5 @@
 from app.models import Author
-from app.services.common import TRGMSearchService, GetRandomService
+from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
 
 
 GET_OBJECT_IDS_QUERY = """
@@ -66,3 +66,12 @@ ORDER BY RANDOM() LIMIT 1;
 class GetRandomAuthorService(GetRandomService):
     MODEL_CLASS = Author
     GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
+
+
+class AuthorMeiliSearchService(MeiliSearchService):
+    MODEL_CLASS = Author
+    SELECT_RELATED = ["source"]
+    PREFETCH_RELATED = ["annotations"]
+
+    MS_INDEX_NAME = "authors"
+    MS_INDEX_LANG_KEY = "author_langs"

fastapi_book_server/app/services/book.py

@@ -5,7 +5,7 @@ from fastapi import HTTPException, status
 from app.models import Author as AuthorDB
 from app.models import Book as BookDB
 from app.serializers.book import CreateBook, CreateRemoteBook
-from app.services.common import TRGMSearchService, GetRandomService
+from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
 
 
 GET_OBJECT_IDS_QUERY = """
@@ -91,3 +91,12 @@ ORDER BY RANDOM() LIMIT 1;
 class GetRandomBookService(GetRandomService):
     MODEL_CLASS = BookDB
     GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
+
+
+class BookMeiliSearchService(MeiliSearchService):
+    MODEL_CLASS = BookDB
+    SELECT_RELATED = ["source"]
+    PREFETCH_RELATED = ["authors", "translators", "annotations"]
+
+    MS_INDEX_NAME = "books"
+    MS_INDEX_LANG_KEY = "lang"

fastapi_book_server/app/services/common.py

@@ -1,24 +1,28 @@
+import abc
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
 from typing import Optional, Generic, TypeVar, Union
 
 import aioredis
 from databases import Database
 from fastapi_pagination.api import resolve_params
 from fastapi_pagination.bases import AbstractParams, RawParams
+import meilisearch
 import orjson
 from ormar import Model, QuerySet
 from sqlalchemy import Table
 
 from app.utils.pagination import Page, CustomPage
+from core.config import env_config
 
 
 T = TypeVar("T", bound=Model)
 
 
-class TRGMSearchService(Generic[T]):
+class BaseSearchService(Generic[T], abc.ABC):
     MODEL_CLASS: Optional[T] = None
     SELECT_RELATED: Optional[Union[list[str], str]] = None
     PREFETCH_RELATED: Optional[Union[list[str], str]] = None
-    GET_OBJECT_IDS_QUERY: Optional[str] = None
     CUSTOM_CACHE_PREFIX: Optional[str] = None
     CACHE_TTL = 60 * 60
 
@@ -48,29 +52,14 @@ class TRGMSearchService(Generic[T]):
 
     @classmethod
     @property
-    def object_ids_query(cls) -> str:
-        assert (
-            cls.GET_OBJECT_IDS_QUERY is not None
-        ), f"GET_OBJECT_IDS_QUERY in {cls.__name__} don't set!"
-        return cls.GET_OBJECT_IDS_QUERY
+    def cache_prefix(cls) -> str:
+        return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename
 
     @classmethod
     async def _get_object_ids(
         cls, query_data: str, allowed_langs: list[str]
     ) -> list[int]:
-        row = await cls.database.fetch_one(
-            cls.object_ids_query, {"query": query_data, "langs": allowed_langs}
-        )
-
-        if row is None:
-            raise ValueError("Something is wrong!")
-
-        return row["array"]
-
-    @classmethod
-    @property
-    def cache_prefix(cls) -> str:
-        return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename
+        ...
 
     @classmethod
     def get_cache_key(cls, query_data: str, allowed_langs: list[str]) -> str:
@@ -151,6 +140,92 @@ class TRGMSearchService(Generic[T]):
         return CustomPage.create(items=objects, total=total, params=params)
 
 
+class TRGMSearchService(BaseSearchService[T]):
+    GET_OBJECT_IDS_QUERY: Optional[str] = None
+
+    @classmethod
+    @property
+    def object_ids_query(cls) -> str:
+        assert (
+            cls.GET_OBJECT_IDS_QUERY is not None
+        ), f"GET_OBJECT_IDS_QUERY in {cls.__name__} don't set!"
+        return cls.GET_OBJECT_IDS_QUERY
+
+    @classmethod
+    async def _get_object_ids(
+        cls, query_data: str, allowed_langs: list[str]
+    ) -> list[int]:
+        row = await cls.database.fetch_one(
+            cls.object_ids_query, {"query": query_data, "langs": allowed_langs}
+        )
+
+        if row is None:
+            raise ValueError("Something is wrong!")
+
+        return row["array"]
+
+
+class MeiliSearchService(BaseSearchService[T]):
+    MS_INDEX_NAME: Optional[str] = None
+    MS_INDEX_LANG_KEY: Optional[str] = None
+
+    _executor = ThreadPoolExecutor(4)
+
+    @classmethod
+    @property
+    def lang_key(cls) -> str:
+        assert cls.MS_INDEX_LANG_KEY is not None, f"MODEL in {cls.__name__} don't set!"
+        return cls.MS_INDEX_LANG_KEY
+
+    @classmethod
+    @property
+    def index_name(cls) -> str:
+        assert cls.MS_INDEX_NAME is not None, f"MODEL in {cls.__name__} don't set!"
+        return cls.MS_INDEX_NAME
+
+    @classmethod
+    def get_allowed_langs_filter(cls, allowed_langs: list[str]) -> list[list[str]]:
+        return [[f"{cls.lang_key} = {lang}" for lang in allowed_langs]]
+
+    @classmethod
+    def make_request(
+        cls, query: str, allowed_langs_filter: list[list[str]], offset: int
+    ):
+        client = meilisearch.Client(env_config.MEILI_HOST, env_config.MEILI_MASTER_KEY)
+        index = client.index(cls.index_name)
+
+        result = index.search(
+            query,
+            {
+                "filter": allowed_langs_filter,
+                "offset": offset,
+                "limit": 630,
+                "attributesToRetrieve": ["id"],
+            },
+        )
+
+        total: int = result["nbHits"]
+        ids: list[int] = [r["id"] for r in result["hits"][:total]]
+
+        return ids
+
+    @classmethod
+    async def _get_object_ids(
+        cls, query_data: str, allowed_langs: list[str]
+    ) -> list[int]:
+        params = cls.get_raw_params()
+
+        allowed_langs_filter = cls.get_allowed_langs_filter(allowed_langs)
+
+        return await asyncio.get_event_loop().run_in_executor(
+            cls._executor,
+            cls.make_request,
+            query_data,
+            allowed_langs_filter,
+            params.offset,
+        )
+
+
 class GetRandomService(Generic[T]):
     MODEL_CLASS: Optional[T] = None
     GET_RANDOM_OBJECT_ID_QUERY: Optional[str] = None

fastapi_book_server/app/services/sequence.py

@@ -1,5 +1,5 @@
 from app.models import Sequence
-from app.services.common import TRGMSearchService, GetRandomService
+from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
 
 
 GET_OBJECT_IDS_QUERY = """
@@ -56,3 +56,11 @@ ORDER BY RANDOM() LIMIT 1;
 class GetRandomSequenceService(GetRandomService):
     MODEL_CLASS = Sequence
     GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
+
+
+class SequenceMeiliSearchService(MeiliSearchService):
+    MODEL_CLASS = Sequence
+    SELECT_RELATED = ["source"]
+
+    MS_INDEX_NAME = "sequences"
+    MS_INDEX_LANG_KEY = "langs"

fastapi_book_server/app/services/translator.py

@@ -1,5 +1,5 @@
 from app.models import Author
-from app.services.common import TRGMSearchService
+from app.services.common import TRGMSearchService, MeiliSearchService
 
 
 GET_OBJECT_IDS_QUERY = """
@@ -47,3 +47,13 @@ class TranslatorTGRMSearchService(TRGMSearchService):
     SELECT_RELATED = ["source"]
     PREFETCH_RELATED = ["annotations"]
     GET_OBJECT_IDS_QUERY = GET_OBJECT_IDS_QUERY
+
+
+class TranslatorMeiliSearchService(MeiliSearchService):
+    MODEL_CLASS = Author
+    CUSTOM_CACHE_PREFIX = "translator"
+    SELECT_RELATED = ["source"]
+    PREFETCH_RELATED = ["annotations"]
+
+    MS_INDEX_NAME = "authors"
+    MS_INDEX_LANG_KEY = "translator_langs"

fastapi_book_server/app/views/author.py

@@ -15,8 +15,8 @@ from app.serializers.author import (
     TranslatedBook,
 )
 from app.serializers.author_annotation import AuthorAnnotation
-from app.services.author import AuthorTGRMSearchService, GetRandomAuthorService
-from app.services.translator import TranslatorTGRMSearchService
+from app.services.author import AuthorMeiliSearchService, GetRandomAuthorService
+from app.services.translator import TranslatorMeiliSearchService
 from app.utils.pagination import CustomPage
 
 
@@ -120,7 +120,7 @@ async def get_author_books(
 async def search_authors(
     query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
 ):
-    return await AuthorTGRMSearchService.get(
+    return await AuthorMeiliSearchService.get(
         query, request.app.state.redis, allowed_langs
     )
 
@@ -153,6 +153,6 @@ async def get_translated_books(
 async def search_translators(
     query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
 ):
-    return await TranslatorTGRMSearchService.get(
+    return await TranslatorMeiliSearchService.get(
         query, request.app.state.redis, allowed_langs
     )

fastapi_book_server/app/views/book.py

@@ -19,7 +19,7 @@ from app.serializers.book import (
     CreateRemoteBook,
 )
 from app.serializers.book_annotation import BookAnnotation
-from app.services.book import BookTGRMSearchService, GetRandomBookService, BookCreator
+from app.services.book import BookMeiliSearchService, GetRandomBookService, BookCreator
 from app.utils.pagination import CustomPage
 
 
@@ -137,6 +137,6 @@ async def get_book_annotation(id: int):
 async def search_books(
     query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
 ):
-    return await BookTGRMSearchService.get(
+    return await BookMeiliSearchService.get(
         query, request.app.state.redis, allowed_langs
     )

fastapi_book_server/app/views/sequence.py

@@ -8,7 +8,7 @@ from app.models import Book as BookDB
 from app.models import Sequence as SequenceDB
 from app.serializers.sequence import Book as SequenceBook
 from app.serializers.sequence import Sequence, CreateSequence
-from app.services.sequence import SequenceTGRMSearchService, GetRandomSequenceService
+from app.services.sequence import SequenceMeiliSearchService, GetRandomSequenceService
 from app.utils.pagination import CustomPage
 
 
@@ -67,6 +67,6 @@ async def create_sequence(data: CreateSequence):
 async def search_sequences(
     query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
 ):
-    return await SequenceTGRMSearchService.get(
+    return await SequenceMeiliSearchService.get(
         query, request.app.state.redis, allowed_langs
     )

fastapi_book_server/core/config.py

@@ -17,6 +17,9 @@ class EnvConfig(BaseSettings):
     REDIS_DB: int
     REDIS_PASSWORD: Optional[str]
 
+    MEILI_HOST: str
+    MEILI_MASTER_KEY: str
+
     class Config:
         env_file = ".env"
         env_file_encoding = "utf-8"