Add meilisearch

fastapi_book_server/app/services/author.py

@@ -1,5 +1,5 @@
 from app.models import Author
-from app.services.common import TRGMSearchService, GetRandomService
+from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
 
 
 GET_OBJECT_IDS_QUERY = """
@@ -66,3 +66,12 @@ ORDER BY RANDOM() LIMIT 1;
 class GetRandomAuthorService(GetRandomService):
     MODEL_CLASS = Author
     GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
+
+
+class AuthorMeiliSearchService(MeiliSearchService):
+    MODEL_CLASS = Author
+    SELECT_RELATED = ["source"]
+    PREFETCH_RELATED = ["annotations"]
+
+    MS_INDEX_NAME = "authors"
+    MS_INDEX_LANG_KEY = "author_langs"

fastapi_book_server/app/services/book.py

@@ -5,7 +5,7 @@ from fastapi import HTTPException, status
 from app.models import Author as AuthorDB
 from app.models import Book as BookDB
 from app.serializers.book import CreateBook, CreateRemoteBook
-from app.services.common import TRGMSearchService, GetRandomService
+from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
 
 
 GET_OBJECT_IDS_QUERY = """
@@ -91,3 +91,12 @@ ORDER BY RANDOM() LIMIT 1;
 class GetRandomBookService(GetRandomService):
     MODEL_CLASS = BookDB
     GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
+
+
+class BookMeiliSearchService(MeiliSearchService):
+    MODEL_CLASS = BookDB
+    SELECT_RELATED = ["source"]
+    PREFETCH_RELATED = ["authors", "translators", "annotations"]
+
+    MS_INDEX_NAME = "books"
+    MS_INDEX_LANG_KEY = "lang"

fastapi_book_server/app/services/common.py

@@ -1,24 +1,28 @@
+import abc
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
 from typing import Optional, Generic, TypeVar, Union
 
 import aioredis
 from databases import Database
 from fastapi_pagination.api import resolve_params
 from fastapi_pagination.bases import AbstractParams, RawParams
+import meilisearch
 import orjson
 from ormar import Model, QuerySet
 from sqlalchemy import Table
 
 from app.utils.pagination import Page, CustomPage
+from core.config import env_config
 
 
 T = TypeVar("T", bound=Model)
 
 
-class TRGMSearchService(Generic[T]):
+class BaseSearchService(Generic[T], abc.ABC):
     MODEL_CLASS: Optional[T] = None
     SELECT_RELATED: Optional[Union[list[str], str]] = None
     PREFETCH_RELATED: Optional[Union[list[str], str]] = None
-    GET_OBJECT_IDS_QUERY: Optional[str] = None
     CUSTOM_CACHE_PREFIX: Optional[str] = None
     CACHE_TTL = 60 * 60
 
@@ -48,29 +52,14 @@ class TRGMSearchService(Generic[T]):
 
     @classmethod
     @property
-    def object_ids_query(cls) -> str:
+    def cache_prefix(cls) -> str:
-        assert (
+        return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename
-            cls.GET_OBJECT_IDS_QUERY is not None
-        ), f"GET_OBJECT_IDS_QUERY in {cls.__name__} don't set!"
-        return cls.GET_OBJECT_IDS_QUERY
 
     @classmethod
     async def _get_object_ids(
         cls, query_data: str, allowed_langs: list[str]
     ) -> list[int]:
-        row = await cls.database.fetch_one(
+        ...
-            cls.object_ids_query, {"query": query_data, "langs": allowed_langs}
-        )
-
-        if row is None:
-            raise ValueError("Something is wrong!")
-
-        return row["array"]
-
-    @classmethod
-    @property
-    def cache_prefix(cls) -> str:
-        return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename
 
     @classmethod
     def get_cache_key(cls, query_data: str, allowed_langs: list[str]) -> str:
@@ -151,6 +140,92 @@ class TRGMSearchService(Generic[T]):
         return CustomPage.create(items=objects, total=total, params=params)
 
 
+class TRGMSearchService(BaseSearchService[T]):
+    GET_OBJECT_IDS_QUERY: Optional[str] = None
+
+    @classmethod
+    @property
+    def object_ids_query(cls) -> str:
+        assert (
+            cls.GET_OBJECT_IDS_QUERY is not None
+        ), f"GET_OBJECT_IDS_QUERY in {cls.__name__} don't set!"
+        return cls.GET_OBJECT_IDS_QUERY
+
+    @classmethod
+    async def _get_object_ids(
+        cls, query_data: str, allowed_langs: list[str]
+    ) -> list[int]:
+        row = await cls.database.fetch_one(
+            cls.object_ids_query, {"query": query_data, "langs": allowed_langs}
+        )
+
+        if row is None:
+            raise ValueError("Something is wrong!")
+
+        return row["array"]
+
+
+class MeiliSearchService(BaseSearchService[T]):
+    MS_INDEX_NAME: Optional[str] = None
+    MS_INDEX_LANG_KEY: Optional[str] = None
+
+    _executor = ThreadPoolExecutor(4)
+
+    @classmethod
+    @property
+    def lang_key(cls) -> str:
+        assert cls.MS_INDEX_LANG_KEY is not None, f"MODEL in {cls.__name__} don't set!"
+        return cls.MS_INDEX_LANG_KEY
+
+    @classmethod
+    @property
+    def index_name(cls) -> str:
+        assert cls.MS_INDEX_NAME is not None, f"MODEL in {cls.__name__} don't set!"
+        return cls.MS_INDEX_NAME
+
+    @classmethod
+    def get_allowed_langs_filter(cls, allowed_langs: list[str]) -> list[list[str]]:
+        return [[f"{cls.lang_key} = {lang}" for lang in allowed_langs]]
+
+    @classmethod
+    def make_request(
+        cls, query: str, allowed_langs_filter: list[list[str]], offset: int
+    ):
+        client = meilisearch.Client(env_config.MEILI_HOST, env_config.MEILI_MASTER_KEY)
+        index = client.index(cls.index_name)
+
+        result = index.search(
+            query,
+            {
+                "filter": allowed_langs_filter,
+                "offset": offset,
+                "limit": 630,
+                "attributesToRetrieve": ["id"],
+            },
+        )
+
+        total: int = result["nbHits"]
+        ids: list[int] = [r["id"] for r in result["hits"][:total]]
+
+        return ids
+
+    @classmethod
+    async def _get_object_ids(
+        cls, query_data: str, allowed_langs: list[str]
+    ) -> list[int]:
+        params = cls.get_raw_params()
+
+        allowed_langs_filter = cls.get_allowed_langs_filter(allowed_langs)
+
+        return await asyncio.get_event_loop().run_in_executor(
+            cls._executor,
+            cls.make_request,
+            query_data,
+            allowed_langs_filter,
+            params.offset,
+        )
+
+
 class GetRandomService(Generic[T]):
     MODEL_CLASS: Optional[T] = None
     GET_RANDOM_OBJECT_ID_QUERY: Optional[str] = None

fastapi_book_server/app/services/sequence.py

@@ -1,5 +1,5 @@
 from app.models import Sequence
-from app.services.common import TRGMSearchService, GetRandomService
+from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
 
 
 GET_OBJECT_IDS_QUERY = """
@@ -56,3 +56,11 @@ ORDER BY RANDOM() LIMIT 1;
 class GetRandomSequenceService(GetRandomService):
     MODEL_CLASS = Sequence
     GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
+
+
+class SequenceMeiliSearchService(MeiliSearchService):
+    MODEL_CLASS = Sequence
+    SELECT_RELATED = ["source"]
+
+    MS_INDEX_NAME = "sequences"
+    MS_INDEX_LANG_KEY = "langs"

fastapi_book_server/app/services/translator.py

@@ -1,5 +1,5 @@
 from app.models import Author
-from app.services.common import TRGMSearchService
+from app.services.common import TRGMSearchService, MeiliSearchService
 
 
 GET_OBJECT_IDS_QUERY = """
@@ -47,3 +47,13 @@ class TranslatorTGRMSearchService(TRGMSearchService):
     SELECT_RELATED = ["source"]
     PREFETCH_RELATED = ["annotations"]
     GET_OBJECT_IDS_QUERY = GET_OBJECT_IDS_QUERY
+
+
+class TranslatorMeiliSearchService(MeiliSearchService):
+    MODEL_CLASS = Author
+    CUSTOM_CACHE_PREFIX = "translator"
+    SELECT_RELATED = ["source"]
+    PREFETCH_RELATED = ["annotations"]
+
+    MS_INDEX_NAME = "authors"
+    MS_INDEX_LANG_KEY = "translator_langs"

fastapi_book_server/app/views/author.py

@@ -15,8 +15,8 @@ from app.serializers.author import (
     TranslatedBook,
 )
 from app.serializers.author_annotation import AuthorAnnotation
-from app.services.author import AuthorTGRMSearchService, GetRandomAuthorService
+from app.services.author import AuthorMeiliSearchService, GetRandomAuthorService
-from app.services.translator import TranslatorTGRMSearchService
+from app.services.translator import TranslatorMeiliSearchService
 from app.utils.pagination import CustomPage
 
 
@@ -120,7 +120,7 @@ async def get_author_books(
 async def search_authors(
     query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
 ):
-    return await AuthorTGRMSearchService.get(
+    return await AuthorMeiliSearchService.get(
         query, request.app.state.redis, allowed_langs
     )
 
@@ -153,6 +153,6 @@ async def get_translated_books(
 async def search_translators(
     query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
 ):
-    return await TranslatorTGRMSearchService.get(
+    return await TranslatorMeiliSearchService.get(
         query, request.app.state.redis, allowed_langs
     )

fastapi_book_server/app/views/book.py

@@ -19,7 +19,7 @@ from app.serializers.book import (
     CreateRemoteBook,
 )
 from app.serializers.book_annotation import BookAnnotation
-from app.services.book import BookTGRMSearchService, GetRandomBookService, BookCreator
+from app.services.book import BookMeiliSearchService, GetRandomBookService, BookCreator
 from app.utils.pagination import CustomPage
 
 
@@ -137,6 +137,6 @@ async def get_book_annotation(id: int):
 async def search_books(
     query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
 ):
-    return await BookTGRMSearchService.get(
+    return await BookMeiliSearchService.get(
         query, request.app.state.redis, allowed_langs
     )

fastapi_book_server/app/views/sequence.py

@@ -8,7 +8,7 @@ from app.models import Book as BookDB
 from app.models import Sequence as SequenceDB
 from app.serializers.sequence import Book as SequenceBook
 from app.serializers.sequence import Sequence, CreateSequence
-from app.services.sequence import SequenceTGRMSearchService, GetRandomSequenceService
+from app.services.sequence import SequenceMeiliSearchService, GetRandomSequenceService
 from app.utils.pagination import CustomPage
 
 
@@ -67,6 +67,6 @@ async def create_sequence(data: CreateSequence):
 async def search_sequences(
     query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
 ):
-    return await SequenceTGRMSearchService.get(
+    return await SequenceMeiliSearchService.get(
         query, request.app.state.redis, allowed_langs
     )

fastapi_book_server/core/config.py

@@ -17,6 +17,9 @@ class EnvConfig(BaseSettings):
     REDIS_DB: int
     REDIS_PASSWORD: Optional[str]
 
+    MEILI_HOST: str
+    MEILI_MASTER_KEY: str
+
     class Config:
         env_file = ".env"
         env_file_encoding = "utf-8"

poetry.lock

@@ -212,13 +212,13 @@ pydantic = ">=1.7.2"
 
 [package.extras]
 gino = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)"]
-all = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)", "databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "tortoise-orm[aiosqlite,asyncpg,aiomysql] (>=0.16.18,<0.18.0)", "asyncpg (>=0.24.0)", "ormar (>=0.10.5)", "Django (<3.3.0)", "piccolo (>=0.29,<0.35)", "motor (>=2.5.1,<3.0.0)"]
+all = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)", "databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "tortoise-orm[aiosqlite,aiomysql,asyncpg] (>=0.16.18,<0.18.0)", "asyncpg (>=0.24.0)", "ormar (>=0.10.5)", "Django (<3.3.0)", "piccolo (>=0.29,<0.35)", "motor (>=2.5.1,<3.0.0)"]
 sqlalchemy = ["SQLAlchemy (>=1.3.20)"]
 asyncpg = ["SQLAlchemy (>=1.3.20)", "asyncpg (>=0.24.0)"]
 databases = ["databases[postgresql,mysql,sqlite] (>=0.4.0)"]
 orm = ["databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "typesystem (>=0.2.0,<0.3.0)"]
 django = ["databases[postgresql,mysql,sqlite] (>=0.4.0)", "Django (<3.3.0)"]
-tortoise = ["tortoise-orm[aiosqlite,asyncpg,aiomysql] (>=0.16.18,<0.18.0)"]
+tortoise = ["tortoise-orm[aiosqlite,aiomysql,asyncpg] (>=0.16.18,<0.18.0)"]
 ormar = ["ormar (>=0.10.5)"]
 piccolo = ["piccolo (>=0.29,<0.35)"]
 motor = ["motor (>=2.5.1,<3.0.0)"]
@@ -312,6 +312,17 @@ category = "main"
 optional = false
 python-versions = ">=3.6"
 
+[[package]]
+name = "meilisearch"
+version = "0.18.0"
+description = "The python client for MeiliSearch API."
+category = "main"
+optional = false
+python-versions = ">=3"
+
+[package.dependencies]
+requests = "*"
+
 [[package]]
 name = "more-itertools"
 version = "8.10.0"
@@ -445,6 +456,24 @@ python-versions = ">=3.5"
 [package.extras]
 cli = ["click (>=5.0)"]
 
+[[package]]
+name = "requests"
+version = "2.27.1"
+description = "Python HTTP for Humans."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+
+[package.dependencies]
+certifi = ">=2017.4.17"
+charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""}
+idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""}
+urllib3 = ">=1.21.1,<1.27"
+
+[package.extras]
+socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
+use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
+
 [[package]]
 name = "rfc3986"
 version = "1.5.0"
@@ -521,6 +550,19 @@ category = "main"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "urllib3"
+version = "1.26.8"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
+
+[package.extras]
+brotli = ["brotlipy (>=0.6.0)"]
+secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
 [[package]]
 name = "uvicorn"
 version = "0.15.0"
@@ -548,7 +590,7 @@ python-versions = "*"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "c6b178194a961ae863f1db7834a3ee79a29311871268547446e5776c2ccebfa3"
+content-hash = "477f705b7a7d3a78d1e3751b80d5fe9b04c94a5ea44bbad3bf3b81185e83238a"
 
 [metadata.files]
 aiologger = [
@@ -757,6 +799,10 @@ markupsafe = [
     {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"},
     {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"},
 ]
+meilisearch = [
+    {file = "meilisearch-0.18.0-py3-none-any.whl", hash = "sha256:b3f327cd01d851d3128bac6949f85a0b0c1476229b0e5d711dd76346da1f0fe2"},
+    {file = "meilisearch-0.18.0.tar.gz", hash = "sha256:f697a48845f693b07c63c3462312b209b8c03ba0572da9644c6777849589d18d"},
+]
 more-itertools = [
     {file = "more-itertools-8.10.0.tar.gz", hash = "sha256:1debcabeb1df793814859d64a81ad7cb10504c24349368ccf214c664c474f41f"},
     {file = "more_itertools-8.10.0-py3-none-any.whl", hash = "sha256:56ddac45541718ba332db05f464bebfb0768110111affd27f66e0051f276fa43"},
@@ -849,6 +895,10 @@ python-dotenv = [
     {file = "python-dotenv-0.19.1.tar.gz", hash = "sha256:14f8185cc8d494662683e6914addcb7e95374771e707601dfc70166946b4c4b8"},
     {file = "python_dotenv-0.19.1-py2.py3-none-any.whl", hash = "sha256:bbd3da593fc49c249397cbfbcc449cf36cb02e75afc8157fcc6a81df6fb7750a"},
 ]
+requests = [
+    {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
+    {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
+]
 rfc3986 = [
     {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"},
     {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"},
@@ -898,6 +948,10 @@ typing-extensions = [
     {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"},
     {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"},
 ]
+urllib3 = [
+    {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"},
+    {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"},
+]
 uvicorn = [
     {file = "uvicorn-0.15.0-py3-none-any.whl", hash = "sha256:17f898c64c71a2640514d4089da2689e5db1ce5d4086c2d53699bf99513421c1"},
     {file = "uvicorn-0.15.0.tar.gz", hash = "sha256:d9a3c0dd1ca86728d3e235182683b4cf94cd53a867c288eaeca80ee781b2caff"},

pyproject.toml

@@ -18,6 +18,7 @@ aiologger = "^0.6.1"
 orjson = "^3.6.4"
 aioredis = "^2.0.0"
 httpx = "^0.22.0"
+meilisearch = "^0.18.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"