flibusta-apps/book_library_server
1
0
Fork 0
mirror of https://github.com/flibusta-apps/book_library_server.git synced 2025-12-06 07:05:36 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Add meilisearch

Browse Source
This commit is contained in:
Kurbanov Bulat
2022-02-08 21:11:54 +03:00
parent c759e0edc0
commit df7db6a01f
11 changed files with 204 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fastapi_book_server/app/services/author.py
View File

@@ -1,5 +1,5 @@
from app.models import Author
from app.services.common import TRGMSearchService, GetRandomService
from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
GET_OBJECT_IDS_QUERY = """
@@ -66,3 +66,12 @@ ORDER BY RANDOM() LIMIT 1;
class GetRandomAuthorService(GetRandomService):
MODEL_CLASS = Author
GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
class AuthorMeiliSearchService(MeiliSearchService):
MODEL_CLASS = Author
SELECT_RELATED = ["source"]
PREFETCH_RELATED = ["annotations"]
MS_INDEX_NAME = "authors"
MS_INDEX_LANG_KEY = "author_langs"

11
fastapi_book_server/app/services/book.py
View File

@@ -5,7 +5,7 @@ from fastapi import HTTPException, status
from app.models import Author as AuthorDB
from app.models import Book as BookDB
from app.serializers.book import CreateBook, CreateRemoteBook
from app.services.common import TRGMSearchService, GetRandomService
from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
GET_OBJECT_IDS_QUERY = """
@@ -91,3 +91,12 @@ ORDER BY RANDOM() LIMIT 1;
class GetRandomBookService(GetRandomService):
MODEL_CLASS = BookDB
GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
class BookMeiliSearchService(MeiliSearchService):
MODEL_CLASS = BookDB
SELECT_RELATED = ["source"]
PREFETCH_RELATED = ["authors", "translators", "annotations"]
MS_INDEX_NAME = "books"
MS_INDEX_LANG_KEY = "lang"

115
fastapi_book_server/app/services/common.py
View File

@@ -1,24 +1,28 @@
import abc
import asyncio
from concurrent.futures import ThreadPoolExecutor
from typing import Optional, Generic, TypeVar, Union
import aioredis
from databases import Database
from fastapi_pagination.api import resolve_params
from fastapi_pagination.bases import AbstractParams, RawParams
import meilisearch
import orjson
from ormar import Model, QuerySet
from sqlalchemy import Table
from app.utils.pagination import Page, CustomPage
from core.config import env_config
T = TypeVar("T", bound=Model)
class TRGMSearchService(Generic[T]):
class BaseSearchService(Generic[T], abc.ABC):
MODEL_CLASS: Optional[T] = None
SELECT_RELATED: Optional[Union[list[str], str]] = None
PREFETCH_RELATED: Optional[Union[list[str], str]] = None
GET_OBJECT_IDS_QUERY: Optional[str] = None
CUSTOM_CACHE_PREFIX: Optional[str] = None
CACHE_TTL = 60 * 60
@@ -48,29 +52,14 @@ class TRGMSearchService(Generic[T]):
@classmethod
@property
def object_ids_query(cls) -> str:
assert (
cls.GET_OBJECT_IDS_QUERY is not None
), f"GET_OBJECT_IDS_QUERY in {cls.__name__} don't set!"
return cls.GET_OBJECT_IDS_QUERY
def cache_prefix(cls) -> str:
return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename
@classmethod
async def _get_object_ids(
cls, query_data: str, allowed_langs: list[str]
) -> list[int]:
row = await cls.database.fetch_one(
cls.object_ids_query, {"query": query_data, "langs": allowed_langs}
)
if row is None:
raise ValueError("Something is wrong!")
return row["array"]
@classmethod
@property
def cache_prefix(cls) -> str:
return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename
...
@classmethod
def get_cache_key(cls, query_data: str, allowed_langs: list[str]) -> str:
@@ -151,6 +140,92 @@ class TRGMSearchService(Generic[T]):
return CustomPage.create(items=objects, total=total, params=params)
class TRGMSearchService(BaseSearchService[T]):
GET_OBJECT_IDS_QUERY: Optional[str] = None
@classmethod
@property
def object_ids_query(cls) -> str:
assert (
cls.GET_OBJECT_IDS_QUERY is not None
), f"GET_OBJECT_IDS_QUERY in {cls.__name__} don't set!"
return cls.GET_OBJECT_IDS_QUERY
@classmethod
async def _get_object_ids(
cls, query_data: str, allowed_langs: list[str]
) -> list[int]:
row = await cls.database.fetch_one(
cls.object_ids_query, {"query": query_data, "langs": allowed_langs}
)
if row is None:
raise ValueError("Something is wrong!")
return row["array"]
class MeiliSearchService(BaseSearchService[T]):
MS_INDEX_NAME: Optional[str] = None
MS_INDEX_LANG_KEY: Optional[str] = None
_executor = ThreadPoolExecutor(4)
@classmethod
@property
def lang_key(cls) -> str:
assert cls.MS_INDEX_LANG_KEY is not None, f"MODEL in {cls.__name__} don't set!"
return cls.MS_INDEX_LANG_KEY
@classmethod
@property
def index_name(cls) -> str:
assert cls.MS_INDEX_NAME is not None, f"MODEL in {cls.__name__} don't set!"
return cls.MS_INDEX_NAME
@classmethod
def get_allowed_langs_filter(cls, allowed_langs: list[str]) -> list[list[str]]:
return [[f"{cls.lang_key} = {lang}" for lang in allowed_langs]]
@classmethod
def make_request(
cls, query: str, allowed_langs_filter: list[list[str]], offset: int
):
client = meilisearch.Client(env_config.MEILI_HOST, env_config.MEILI_MASTER_KEY)
index = client.index(cls.index_name)
result = index.search(
query,
{
"filter": allowed_langs_filter,
"offset": offset,
"limit": 630,
"attributesToRetrieve": ["id"],
},
)
total: int = result["nbHits"]
ids: list[int] = [r["id"] for r in result["hits"][:total]]
return ids
@classmethod
async def _get_object_ids(
cls, query_data: str, allowed_langs: list[str]
) -> list[int]:
params = cls.get_raw_params()
allowed_langs_filter = cls.get_allowed_langs_filter(allowed_langs)
return await asyncio.get_event_loop().run_in_executor(
cls._executor,
cls.make_request,
query_data,
allowed_langs_filter,
params.offset,
)
class GetRandomService(Generic[T]):
MODEL_CLASS: Optional[T] = None
GET_RANDOM_OBJECT_ID_QUERY: Optional[str] = None

10
fastapi_book_server/app/services/sequence.py
View File

@@ -1,5 +1,5 @@
from app.models import Sequence
from app.services.common import TRGMSearchService, GetRandomService
from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService
GET_OBJECT_IDS_QUERY = """
@@ -56,3 +56,11 @@ ORDER BY RANDOM() LIMIT 1;
class GetRandomSequenceService(GetRandomService):
MODEL_CLASS = Sequence
GET_RANDOM_OBJECT_ID_QUERY = GET_RANDOM_OBJECT_ID_QUERY
class SequenceMeiliSearchService(MeiliSearchService):
MODEL_CLASS = Sequence
SELECT_RELATED = ["source"]
MS_INDEX_NAME = "sequences"
MS_INDEX_LANG_KEY = "langs"

12
fastapi_book_server/app/services/translator.py
View File

@@ -1,5 +1,5 @@
from app.models import Author
from app.services.common import TRGMSearchService
from app.services.common import TRGMSearchService, MeiliSearchService
GET_OBJECT_IDS_QUERY = """
@@ -47,3 +47,13 @@ class TranslatorTGRMSearchService(TRGMSearchService):
SELECT_RELATED = ["source"]
PREFETCH_RELATED = ["annotations"]
GET_OBJECT_IDS_QUERY = GET_OBJECT_IDS_QUERY
class TranslatorMeiliSearchService(MeiliSearchService):
MODEL_CLASS = Author
CUSTOM_CACHE_PREFIX = "translator"
SELECT_RELATED = ["source"]
PREFETCH_RELATED = ["annotations"]
MS_INDEX_NAME = "authors"
MS_INDEX_LANG_KEY = "translator_langs"

8
fastapi_book_server/app/views/author.py
View File

@@ -15,8 +15,8 @@ from app.serializers.author import (
TranslatedBook,
)
from app.serializers.author_annotation import AuthorAnnotation
from app.services.author import AuthorTGRMSearchService, GetRandomAuthorService
from app.services.translator import TranslatorTGRMSearchService
from app.services.author import AuthorMeiliSearchService, GetRandomAuthorService
from app.services.translator import TranslatorMeiliSearchService
from app.utils.pagination import CustomPage
@@ -120,7 +120,7 @@ async def get_author_books(
async def search_authors(
query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
):
return await AuthorTGRMSearchService.get(
return await AuthorMeiliSearchService.get(
query, request.app.state.redis, allowed_langs
)
@@ -153,6 +153,6 @@ async def get_translated_books(
async def search_translators(
query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
):
return await TranslatorTGRMSearchService.get(
return await TranslatorMeiliSearchService.get(
query, request.app.state.redis, allowed_langs
)

4
fastapi_book_server/app/views/book.py
View File

@@ -19,7 +19,7 @@ from app.serializers.book import (
CreateRemoteBook,
)
from app.serializers.book_annotation import BookAnnotation
from app.services.book import BookTGRMSearchService, GetRandomBookService, BookCreator
from app.services.book import BookMeiliSearchService, GetRandomBookService, BookCreator
from app.utils.pagination import CustomPage
@@ -137,6 +137,6 @@ async def get_book_annotation(id: int):
async def search_books(
query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
):
return await BookTGRMSearchService.get(
return await BookMeiliSearchService.get(
query, request.app.state.redis, allowed_langs
)

4
fastapi_book_server/app/views/sequence.py
View File

@@ -8,7 +8,7 @@ from app.models import Book as BookDB
from app.models import Sequence as SequenceDB
from app.serializers.sequence import Book as SequenceBook
from app.serializers.sequence import Sequence, CreateSequence
from app.services.sequence import SequenceTGRMSearchService, GetRandomSequenceService
from app.services.sequence import SequenceMeiliSearchService, GetRandomSequenceService
from app.utils.pagination import CustomPage
@@ -67,6 +67,6 @@ async def create_sequence(data: CreateSequence):
async def search_sequences(
query: str, request: Request, allowed_langs: list[str] = Depends(get_allowed_langs)
):
return await SequenceTGRMSearchService.get(
return await SequenceMeiliSearchService.get(
query, request.app.state.redis, allowed_langs
)

3
fastapi_book_server/core/config.py
View File

@@ -17,6 +17,9 @@ class EnvConfig(BaseSettings):
REDIS_DB: int
REDIS_PASSWORD: Optional[str]
MEILI_HOST: str
MEILI_MASTER_KEY: str
class Config:
env_file = ".env"
env_file_encoding = "utf-8"

60
poetry.lock generated
View File

@@ -212,13 +212,13 @@ pydantic = ">=1.7.2"
[package.extras]
gino = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)"]
all = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)", "databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "tortoise-orm[aiosqlite,asyncpg,aiomysql] (>=0.16.18,<0.18.0)", "asyncpg (>=0.24.0)", "ormar (>=0.10.5)", "Django (<3.3.0)", "piccolo (>=0.29,<0.35)", "motor (>=2.5.1,<3.0.0)"]
all = ["gino[starlette] (>=1.0.1)", "SQLAlchemy (>=1.3.20)", "databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "tortoise-orm[aiosqlite,aiomysql,asyncpg] (>=0.16.18,<0.18.0)", "asyncpg (>=0.24.0)", "ormar (>=0.10.5)", "Django (<3.3.0)", "piccolo (>=0.29,<0.35)", "motor (>=2.5.1,<3.0.0)"]
sqlalchemy = ["SQLAlchemy (>=1.3.20)"]
asyncpg = ["SQLAlchemy (>=1.3.20)", "asyncpg (>=0.24.0)"]
databases = ["databases[postgresql,mysql,sqlite] (>=0.4.0)"]
orm = ["databases[postgresql,mysql,sqlite] (>=0.4.0)", "orm (>=0.1.5)", "typesystem (>=0.2.0,<0.3.0)"]
django = ["databases[postgresql,mysql,sqlite] (>=0.4.0)", "Django (<3.3.0)"]
tortoise = ["tortoise-orm[aiosqlite,asyncpg,aiomysql] (>=0.16.18,<0.18.0)"]
tortoise = ["tortoise-orm[aiosqlite,aiomysql,asyncpg] (>=0.16.18,<0.18.0)"]
ormar = ["ormar (>=0.10.5)"]
piccolo = ["piccolo (>=0.29,<0.35)"]
motor = ["motor (>=2.5.1,<3.0.0)"]
@@ -312,6 +312,17 @@ category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "meilisearch"
version = "0.18.0"
description = "The python client for MeiliSearch API."
category = "main"
optional = false
python-versions = ">=3"
[package.dependencies]
requests = "*"
[[package]]
name = "more-itertools"
version = "8.10.0"
@@ -445,6 +456,24 @@ python-versions = ">=3.5"
[package.extras]
cli = ["click (>=5.0)"]
[[package]]
name = "requests"
version = "2.27.1"
description = "Python HTTP for Humans."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
[package.dependencies]
certifi = ">=2017.4.17"
charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""}
idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""}
urllib3 = ">=1.21.1,<1.27"
[package.extras]
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
[[package]]
name = "rfc3986"
version = "1.5.0"
@@ -521,6 +550,19 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "urllib3"
version = "1.26.8"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[[package]]
name = "uvicorn"
version = "0.15.0"
@@ -548,7 +590,7 @@ python-versions = "*"
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "c6b178194a961ae863f1db7834a3ee79a29311871268547446e5776c2ccebfa3"
content-hash = "477f705b7a7d3a78d1e3751b80d5fe9b04c94a5ea44bbad3bf3b81185e83238a"
[metadata.files]
aiologger = [
@@ -757,6 +799,10 @@ markupsafe = [
{file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"},
{file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"},
]
meilisearch = [
{file = "meilisearch-0.18.0-py3-none-any.whl", hash = "sha256:b3f327cd01d851d3128bac6949f85a0b0c1476229b0e5d711dd76346da1f0fe2"},
{file = "meilisearch-0.18.0.tar.gz", hash = "sha256:f697a48845f693b07c63c3462312b209b8c03ba0572da9644c6777849589d18d"},
]
more-itertools = [
{file = "more-itertools-8.10.0.tar.gz", hash = "sha256:1debcabeb1df793814859d64a81ad7cb10504c24349368ccf214c664c474f41f"},
{file = "more_itertools-8.10.0-py3-none-any.whl", hash = "sha256:56ddac45541718ba332db05f464bebfb0768110111affd27f66e0051f276fa43"},
@@ -849,6 +895,10 @@ python-dotenv = [
{file = "python-dotenv-0.19.1.tar.gz", hash = "sha256:14f8185cc8d494662683e6914addcb7e95374771e707601dfc70166946b4c4b8"},
{file = "python_dotenv-0.19.1-py2.py3-none-any.whl", hash = "sha256:bbd3da593fc49c249397cbfbcc449cf36cb02e75afc8157fcc6a81df6fb7750a"},
]
requests = [
{file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
{file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
]
rfc3986 = [
{file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"},
{file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"},
@@ -898,6 +948,10 @@ typing-extensions = [
{file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"},
{file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"},
]
urllib3 = [
{file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"},
{file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"},
]
uvicorn = [
{file = "uvicorn-0.15.0-py3-none-any.whl", hash = "sha256:17f898c64c71a2640514d4089da2689e5db1ce5d4086c2d53699bf99513421c1"},
{file = "uvicorn-0.15.0.tar.gz", hash = "sha256:d9a3c0dd1ca86728d3e235182683b4cf94cd53a867c288eaeca80ee781b2caff"},

1
pyproject.toml
View File

@@ -18,6 +18,7 @@ aiologger = "^0.6.1"
orjson = "^3.6.4"
aioredis = "^2.0.0"
httpx = "^0.22.0"
meilisearch = "^0.18.0"
[tool.poetry.dev-dependencies]
pytest = "^5.2"