From df7db6a01f0c37ab7f005ec6cc932590416727fd Mon Sep 17 00:00:00 2001 From: Kurbanov Bulat Date: Tue, 8 Feb 2022 21:11:54 +0300 Subject: [PATCH] Add meilisearch --- fastapi_book_server/app/services/author.py | 11 +- fastapi_book_server/app/services/book.py | 11 +- fastapi_book_server/app/services/common.py | 115 +++++++++++++++--- fastapi_book_server/app/services/sequence.py | 10 +- .../app/services/translator.py | 12 +- fastapi_book_server/app/views/author.py | 8 +- fastapi_book_server/app/views/book.py | 4 +- fastapi_book_server/app/views/sequence.py | 4 +- fastapi_book_server/core/config.py | 3 + poetry.lock | 60 ++++++++- pyproject.toml | 1 + 11 files changed, 204 insertions(+), 35 deletions(-) diff --git a/fastapi_book_server/app/services/author.py b/fastapi_book_server/app/services/author.py index 6edd3f8..0c9984a 100644 --- a/fastapi_book_server/app/services/author.py +++ b/fastapi_book_server/app/services/author.py @@ -1,5 +1,5 @@ from app.models import Author -from app.services.common import TRGMSearchService, GetRandomService +from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService GET_OBJECT_IDS_QUERY = """ @@ -66,3 +66,12 @@ ORDER BY RANDOM() LIMIT 1; class GetRandomAuthorService(GetRandomService): MODEL_CLASS = Author GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY + + +class AuthorMeiliSearchService(MeiliSearchService): + MODEL_CLASS = Author + SELECT_RELATED = ["source"] + PREFETCH_RELATED = ["annotations"] + + MS_INDEX_NAME = "authors" + MS_INDEX_LANG_KEY = "author_langs" diff --git a/fastapi_book_server/app/services/book.py b/fastapi_book_server/app/services/book.py index c4ac534..9e31596 100644 --- a/fastapi_book_server/app/services/book.py +++ b/fastapi_book_server/app/services/book.py @@ -5,7 +5,7 @@ from fastapi import HTTPException, status from app.models import Author as AuthorDB from app.models import Book as BookDB from app.serializers.book import CreateBook, CreateRemoteBook -from app.services.common import TRGMSearchService, GetRandomService +from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService GET_OBJECT_IDS_QUERY = """ @@ -91,3 +91,12 @@ ORDER BY RANDOM() LIMIT 1; class GetRandomBookService(GetRandomService): MODEL_CLASS = BookDB GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY + + +class BookMeiliSearchService(MeiliSearchService): + MODEL_CLASS = BookDB + SELECT_RELATED = ["source"] + PREFETCH_RELATED = ["authors", "translators", "annotations"] + + MS_INDEX_NAME = "books" + MS_INDEX_LANG_KEY = "lang" diff --git a/fastapi_book_server/app/services/common.py b/fastapi_book_server/app/services/common.py index e7b948c..1f7d217 100644 --- a/fastapi_book_server/app/services/common.py +++ b/fastapi_book_server/app/services/common.py @@ -1,24 +1,28 @@ +import abc +import asyncio +from concurrent.futures import ThreadPoolExecutor from typing import Optional, Generic, TypeVar, Union import aioredis from databases import Database from fastapi_pagination.api import resolve_params from fastapi_pagination.bases import AbstractParams, RawParams +import meilisearch import orjson from ormar import Model, QuerySet from sqlalchemy import Table from app.utils.pagination import Page, CustomPage +from core.config import env_config T = TypeVar("T", bound=Model) -class TRGMSearchService(Generic[T]): +class BaseSearchService(Generic[T], abc.ABC): MODEL_CLASS: Optional[T] = None SELECT_RELATED: Optional[Union[list[str], str]] = None PREFETCH_RELATED: Optional[Union[list[str], str]] = None - GET_OBJECT_IDS_QUERY: Optional[str] = None CUSTOM_CACHE_PREFIX: Optional[str] = None CACHE_TTL = 60 * 60 @@ -48,29 +52,14 @@ class TRGMSearchService(Generic[T]): @classmethod @property - def object_ids_query(cls) -> str: - assert ( - cls.GET_OBJECT_IDS_QUERY is not None - ), f"GET_OBJECT_IDS_QUERY in {cls.__name__} don't set!" - return cls.GET_OBJECT_IDS_QUERY + def cache_prefix(cls) -> str: + return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename @classmethod async def _get_object_ids( cls, query_data: str, allowed_langs: list[str] ) -> list[int]: - row = await cls.database.fetch_one( - cls.object_ids_query, {"query": query_data, "langs": allowed_langs} - ) - - if row is None: - raise ValueError("Something is wrong!") - - return row["array"] - - @classmethod - @property - def cache_prefix(cls) -> str: - return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename + ... @classmethod def get_cache_key(cls, query_data: str, allowed_langs: list[str]) -> str: @@ -151,6 +140,92 @@ class TRGMSearchService(Generic[T]): return CustomPage.create(items=objects, total=total, params=params) +class TRGMSearchService(BaseSearchService[T]): + GET_OBJECT_IDS_QUERY: Optional[str] = None + + @classmethod + @property + def object_ids_query(cls) -> str: + assert ( + cls.GET_OBJECT_IDS_QUERY is not None + ), f"GET_OBJECT_IDS_QUERY in {cls.__name__} don't set!" + return cls.GET_OBJECT_IDS_QUERY + + @classmethod + async def _get_object_ids( + cls, query_data: str, allowed_langs: list[str] + ) -> list[int]: + row = await cls.database.fetch_one( + cls.object_ids_query, {"query": query_data, "langs": allowed_langs} + ) + + if row is None: + raise ValueError("Something is wrong!") + + return row["array"] + + +class MeiliSearchService(BaseSearchService[T]): + MS_INDEX_NAME: Optional[str] = None + MS_INDEX_LANG_KEY: Optional[str] = None + + _executor = ThreadPoolExecutor(4) + + @classmethod + @property + def lang_key(cls) -> str: + assert cls.MS_INDEX_LANG_KEY is not None, f"MODEL in {cls.__name__} don't set!" + return cls.MS_INDEX_LANG_KEY + + @classmethod + @property + def index_name(cls) -> str: + assert cls.MS_INDEX_NAME is not None, f"MODEL in {cls.__name__} don't set!" + return cls.MS_INDEX_NAME + + @classmethod + def get_allowed_langs_filter(cls, allowed_langs: list[str]) -> list[list[str]]: + return [[f"{cls.lang_key} = {lang}" for lang in allowed_langs]] + + @classmethod + def make_request( + cls, query: str, allowed_langs_filter: list[list[str]], offset: int + ): + client = meilisearch.Client(env_config.MEILI_HOST, env_config.MEILI_MASTER_KEY) + index = client.index(cls.index_name) + + result = index.search( + query, + { + "filter": allowed_langs_filter, + "offset": offset, + "limit": 630, + "attributesToRetrieve": ["id"], + }, + ) + + total: int = result["nbHits"] + ids: list[int] = [r["id"] for r in result["hits"][:total]] + + return ids + + @classmethod + async def _get_object_ids( + cls, query_data: str, allowed_langs: list[str] + ) -> list[int]: + params = cls.get_raw_params() + + allowed_langs_filter = cls.get_allowed_langs_filter(allowed_langs) + + return await asyncio.get_event_loop().run_in_executor( + cls._executor, + cls.make_request, + query_data, + allowed_langs_filter, + params.offset, + ) + + class GetRandomService(Generic[T]): MODEL_CLASS: Optional[T] = None GET_RANDOM_OBJECT_ID_QUERY: Optional[str] = None diff --git a/fastapi_book_server/app/services/sequence.py b/fastapi_book_server/app/services/sequence.py index 46116ac..e12f9d0 100644 --- a/fastapi_book_server/app/services/sequence.py +++ b/fastapi_book_server/app/services/sequence.py @@ -1,5 +1,5 @@ from app.models import Sequence -from app.services.common import TRGMSearchService, GetRandomService +from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService GET_OBJECT_IDS_QUERY = """ @@ -56,3 +56,11 @@ ORDER BY RANDOM() LIMIT 1; class GetRandomSequenceService(GetRandomService): MODEL_CLASS = Sequence GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY + + +class SequenceMeiliSearchService(MeiliSearchService): + MODEL_CLASS = Sequence + SELECT_RELATED = ["source"] + + MS_INDEX_NAME = "sequences" + MS_INDEX_LANG_KEY = "langs" diff --git a/fastapi_book_server/app/services/translator.py b/fastapi_book_server/app/services/translator.py index 8f4141c..5893345 100644 --- a/fastapi_book_server/app/services/translator.py +++ b/fastapi_book_server/app/services/translator.py @@ -1,5 +1,5 @@ from app.models import Author -from app.services.common import TRGMSearchService +from app.services.common import TRGMSearchService, MeiliSearchService GET_OBJECT_IDS_QUERY = """ @@ -47,3 +47,13 @@ class TranslatorTGRMSearchService(TRGMSearchService): SELECT_RELATED = ["source"] PREFETCH_RELATED = ["annotations"] GET_OBJECT_IDS_QUERY = GET_OBJECT_IDS_QUERY + + +class TranslatorMeiliSearchService(MeiliSearchService): + MODEL_CLASS = Author + CUSTOM_CACHE_PREFIX = "translator" + SELECT_RELATED = ["source"] + PREFETCH_RELATED = ["annotations"] + + MS_INDEX_NAME = "authors" + MS_INDEX_LANG_KEY = "translator_langs" diff --git a/fastapi_book_server/app/views/author.py b/fastapi_book_server/app/views/author.py index 757a023..0ebbda2 100644 --- a/fastapi_book_server/app/views/author.py +++ b/fastapi_book_server/app/views/author.py @@ -15,8 +15,8 @@ from app.serializers.author import ( TranslatedBook, ) from app.serializers.author_annotation import AuthorAnnotation -from app.services.author import AuthorTGRMSearchService, GetRandomAuthorService -from app.services.translator import TranslatorTGRMSearchService +from app.services.author import AuthorMeiliSearchService, GetRandomAuthorService +from app.services.translator import TranslatorMeiliSearchService from app.utils.pagination import CustomPage @@ -120,7 +120,7 @@ async def get_author_books( async def search_authors( query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs) ): - return await AuthorTGRMSearchService.get( + return await AuthorMeiliSearchService.get( query, request.app.state.redis, allowed_langs ) @@ -153,6 +153,6 @@ async def get_translated_books( async def search_translators( query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs) ): - return await TranslatorTGRMSearchService.get( + return await TranslatorMeiliSearchService.get( query, request.app.state.redis, allowed_langs ) diff --git a/fastapi_book_server/app/views/book.py b/fastapi_book_server/app/views/book.py index 70938de..754d9e2 100644 --- a/fastapi_book_server/app/views/book.py +++ b/fastapi_book_server/app/views/book.py @@ -19,7 +19,7 @@ from app.serializers.book import ( CreateRemoteBook, ) from app.serializers.book_annotation import BookAnnotation -from app.services.book import BookTGRMSearchService, GetRandomBookService, BookCreator +from app.services.book import BookMeiliSearchService, GetRandomBookService, BookCreator from app.utils.pagination import CustomPage @@ -137,6 +137,6 @@ async def get_book_annotation(id: int): async def search_books( query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs) ): - return await BookTGRMSearchService.get( + return await BookMeiliSearchService.get( query, request.app.state.redis, allowed_langs ) diff --git a/fastapi_book_server/app/views/sequence.py b/fastapi_book_server/app/views/sequence.py index 1ae9ab0..80aac9f 100644 --- a/fastapi_book_server/app/views/sequence.py +++ b/fastapi_book_server/app/views/sequence.py @@ -8,7 +8,7 @@ from app.models import Book as BookDB from app.models import Sequence as SequenceDB from app.serializers.sequence import Book as SequenceBook from app.serializers.sequence import Sequence, CreateSequence -from app.services.sequence import SequenceTGRMSearchService, GetRandomSequenceService +from app.services.sequence import SequenceMeiliSearchService, GetRandomSequenceService from app.utils.pagination import CustomPage @@ -67,6 +67,6 @@ async def create_sequence(data: CreateSequence): async def search_sequences( query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs) ): - return await SequenceTGRMSearchService.get( + return await SequenceMeiliSearchService.get( query, request.app.state.redis, allowed_langs ) diff --git a/fastapi_book_server/core/config.py b/fastapi_book_server/core/config.py index 9016a9e..df2dfdf 100644 --- a/fastapi_book_server/core/config.py +++ b/fastapi_book_server/core/config.py @@ -17,6 +17,9 @@ class EnvConfig(BaseSettings): REDIS_DB: int REDIS_PASSWORD: Optional[str] + MEILI_HOST: str + MEILI_MASTER_KEY: str + class Config: env_file = ".env" env_file_encoding = "utf-8" diff --git a/poetry.lock b/poetry.lock index f85fd53..b78596f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -212,13 +212,13 @@ pydantic = ">=1.7.2" [package.extras] gino = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)"] -all = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)", "databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "tortoise-orm[aiosqlite,asyncpg,aiomysql] (>=0.16.18,<0.18.0)", "asyncpg (>=0.24.0)", "ormar (>=0.10.5)", "Django (<3.3.0)", "piccolo (>=0.29,<0.35)", "motor (>=2.5.1,<3.0.0)"] +all = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)", "databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "tortoise-orm[aiosqlite,aiomysql,asyncpg] (>=0.16.18,<0.18.0)", "asyncpg (>=0.24.0)", "ormar (>=0.10.5)", "Django (<3.3.0)", "piccolo (>=0.29,<0.35)", "motor (>=2.5.1,<3.0.0)"] sqlalchemy = ["SQLAlchemy (>=1.3.20)"] asyncpg = ["SQLAlchemy (>=1.3.20)", "asyncpg (>=0.24.0)"] databases = ["databases[postgresql,mysql,sqlite] (>=0.4.0)"] orm = ["databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "typesystem (>=0.2.0,<0.3.0)"] django = ["databases[postgresql,mysql,sqlite] (>=0.4.0)", "Django (<3.3.0)"] -tortoise = ["tortoise-orm[aiosqlite,asyncpg,aiomysql] (>=0.16.18,<0.18.0)"] +tortoise = ["tortoise-orm[aiosqlite,aiomysql,asyncpg] (>=0.16.18,<0.18.0)"] ormar = ["ormar (>=0.10.5)"] piccolo = ["piccolo (>=0.29,<0.35)"] motor = ["motor (>=2.5.1,<3.0.0)"] @@ -312,6 +312,17 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "meilisearch" +version = "0.18.0" +description = "The python client for MeiliSearch API." +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +requests = "*" + [[package]] name = "more-itertools" version = "8.10.0" @@ -445,6 +456,24 @@ python-versions = ">=3.5" [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "requests" +version = "2.27.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + [[package]] name = "rfc3986" version = "1.5.0" @@ -521,6 +550,19 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "urllib3" +version = "1.26.8" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "uvicorn" version = "0.15.0" @@ -548,7 +590,7 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "c6b178194a961ae863f1db7834a3ee79a29311871268547446e5776c2ccebfa3" +content-hash = "477f705b7a7d3a78d1e3751b80d5fe9b04c94a5ea44bbad3bf3b81185e83238a" [metadata.files] aiologger = [ @@ -757,6 +799,10 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, ] +meilisearch = [ + {file = "meilisearch-0.18.0-py3-none-any.whl", hash = "sha256:b3f327cd01d851d3128bac6949f85a0b0c1476229b0e5d711dd76346da1f0fe2"}, + {file = "meilisearch-0.18.0.tar.gz", hash = "sha256:f697a48845f693b07c63c3462312b209b8c03ba0572da9644c6777849589d18d"}, +] more-itertools = [ {file = "more-itertools-8.10.0.tar.gz", hash = "sha256:1debcabeb1df793814859d64a81ad7cb10504c24349368ccf214c664c474f41f"}, {file = "more_itertools-8.10.0-py3-none-any.whl", hash = "sha256:56ddac45541718ba332db05f464bebfb0768110111affd27f66e0051f276fa43"}, @@ -849,6 +895,10 @@ python-dotenv = [ {file = "python-dotenv-0.19.1.tar.gz", hash = "sha256:14f8185cc8d494662683e6914addcb7e95374771e707601dfc70166946b4c4b8"}, {file = "python_dotenv-0.19.1-py2.py3-none-any.whl", hash = "sha256:bbd3da593fc49c249397cbfbcc449cf36cb02e75afc8157fcc6a81df6fb7750a"}, ] +requests = [ + {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, + {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, +] rfc3986 = [ {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, @@ -898,6 +948,10 @@ typing-extensions = [ {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, ] +urllib3 = [ + {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, + {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, +] uvicorn = [ {file = "uvicorn-0.15.0-py3-none-any.whl", hash = "sha256:17f898c64c71a2640514d4089da2689e5db1ce5d4086c2d53699bf99513421c1"}, {file = "uvicorn-0.15.0.tar.gz", hash = "sha256:d9a3c0dd1ca86728d3e235182683b4cf94cd53a867c288eaeca80ee781b2caff"}, diff --git a/pyproject.toml b/pyproject.toml index 6dd6061..7acd279 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ aiologger = "^0.6.1" orjson = "^3.6.4" aioredis = "^2.0.0" httpx = "^0.22.0" +meilisearch = "^0.18.0" [tool.poetry.dev-dependencies] pytest = "^5.2"