From e0aec69240882ea1c7332400a11044ea96015948 Mon Sep 17 00:00:00 2001 From: Bulat Kurbanov Date: Tue, 19 Jul 2022 21:59:32 +0300 Subject: [PATCH] Refactor services --- fastapi_book_server/app/services/author.py | 14 +- fastapi_book_server/app/services/book.py | 14 +- fastapi_book_server/app/services/common.py | 168 +++++++----------- fastapi_book_server/app/services/sequence.py | 13 +- .../app/services/translator.py | 1 + fastapi_book_server/app/views/author.py | 2 +- fastapi_book_server/app/views/book.py | 2 +- fastapi_book_server/app/views/sequence.py | 2 +- 8 files changed, 103 insertions(+), 113 deletions(-) diff --git a/fastapi_book_server/app/services/author.py b/fastapi_book_server/app/services/author.py index 42a1881..2e5e080 100644 --- a/fastapi_book_server/app/services/author.py +++ b/fastapi_book_server/app/services/author.py @@ -1,3 +1,5 @@ +from typing import TypedDict + from app.models import Author from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService @@ -45,6 +47,7 @@ class AuthorTGRMSearchService(TRGMSearchService): MODEL_CLASS = Author PREFETCH_RELATED = ["source"] SELECT_RELATED = ["annotations"] + GET_OBJECT_IDS_QUERY = GET_OBJECT_IDS_QUERY @@ -62,8 +65,15 @@ SELECT id FROM filtered_authors; """ -class GetRandomAuthorService(GetRandomService): - MODEL_CLASS = Author +class RandomAuthorServiceQuery(TypedDict): + allowed_langs: frozenset[str] + + +class GetRandomAuthorService(GetRandomService[Author, RandomAuthorServiceQuery]): + MODEL_CLASS = Author # type: ignore + PREFETCH_RELATED = ["source"] + SELECT_RELATED = ["annotations"] + GET_OBJECTS_ID_QUERY = GET_OBJECTS_ID_QUERY diff --git a/fastapi_book_server/app/services/book.py b/fastapi_book_server/app/services/book.py index 6f0b9dd..ae1a0b7 100644 --- a/fastapi_book_server/app/services/book.py +++ b/fastapi_book_server/app/services/book.py @@ -1,3 +1,5 @@ +from typing import TypedDict + from app.models import Book as BookDB from app.services.common import ( TRGMSearchService, @@ -25,6 +27,7 @@ class BookTGRMSearchService(TRGMSearchService): MODEL_CLASS = BookDB PREFETCH_RELATED = ["source"] SELECT_RELATED = ["authors", "translators", "annotations"] + GET_OBJECT_IDS_QUERY = GET_OBJECT_IDS_QUERY @@ -43,8 +46,15 @@ SELECT id FROM filtered_books; """ -class GetRandomBookService(GetRandomService): - MODEL_CLASS = BookDB +class RandomBookServiceQuery(TypedDict): + allowed_langs: frozenset[str] + + +class GetRandomBookService(GetRandomService[BookDB, RandomBookServiceQuery]): + MODEL_CLASS = BookDB # type: ignore + PREFETCH_RELATED = ["source"] + SELECT_RELATED = ["authors", "translators", "annotations"] + GET_OBJECTS_ID_QUERY = GET_OBJECTS_ID_QUERY diff --git a/fastapi_book_server/app/services/common.py b/fastapi_book_server/app/services/common.py index 936490e..0519949 100644 --- a/fastapi_book_server/app/services/common.py +++ b/fastapi_book_server/app/services/common.py @@ -20,21 +20,12 @@ MODEL = TypeVar("MODEL", bound=Model) QUERY = TypeVar("QUERY", bound=TypedDict) -class BaseSearchService(Generic[MODEL, QUERY], abc.ABC): +class BaseService(Generic[MODEL, QUERY], abc.ABC): MODEL_CLASS: Optional[MODEL] = None - SELECT_RELATED: Optional[Union[list[str], str]] = None - PREFETCH_RELATED: Optional[Union[list[str], str]] = None - CUSTOM_CACHE_PREFIX: Optional[str] = None + CACHE_PREFIX: str = "" + CUSTOM_MODEL_CACHE_NAME: Optional[str] = None CACHE_TTL = 6 * 60 * 60 - @classmethod - def get_params(cls) -> AbstractParams: - return resolve_params() - - @classmethod - def get_raw_params(cls) -> RawParams: - return resolve_params().to_raw_params() - @classmethod @property def model(cls) -> MODEL: @@ -54,45 +45,18 @@ class BaseSearchService(Generic[MODEL, QUERY], abc.ABC): @classmethod @property def cache_prefix(cls) -> str: - return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename + return cls.CUSTOM_MODEL_CACHE_NAME or cls.model.Meta.tablename @staticmethod - def _get_query_hash(query: QUERY): + def _get_query_hash(query: QUERY) -> int: return hash(frozenset(query.items())) - @classmethod - async def _get_object_ids(cls, query: QUERY) -> list[int]: - ... - @classmethod def get_cache_key(cls, query: QUERY) -> str: model_class_name = cls.cache_prefix query_hash = cls._get_query_hash(query) - return f"{model_class_name}_{query_hash}" - - @classmethod - async def get_cached_ids( - cls, - query: QUERY, - redis: aioredis.Redis, - params: RawParams, - ) -> Optional[tuple[int, list[int]]]: - try: - key = cls.get_cache_key(query) - active_key = f"{key}_active" - - if not await redis.exists(active_key): - return None - - objects_count, objects = await asyncio.gather( - redis.llen(key), - redis.lrange(key, params.offset, params.offset + params.limit), - ) - - return objects_count, [int(item.decode()) for item in objects] - except aioredis.RedisError as e: - print(e) - return None + cache_key = f"{model_class_name}_{query_hash}" + return f"{cls.CACHE_PREFIX}_{cache_key}" if cls.CACHE_PREFIX else cache_key @classmethod async def cache_object_ids( @@ -118,6 +82,47 @@ class BaseSearchService(Generic[MODEL, QUERY], abc.ABC): print(e) return False + +class BaseSearchService(Generic[MODEL, QUERY], BaseService[MODEL, QUERY]): + SELECT_RELATED: Optional[Union[list[str], str]] = None + PREFETCH_RELATED: Optional[Union[list[str], str]] = None + + @classmethod + def get_params(cls) -> AbstractParams: + return resolve_params() + + @classmethod + def get_raw_params(cls) -> RawParams: + return resolve_params().to_raw_params() + + @classmethod + async def _get_object_ids(cls, query: QUERY) -> list[int]: + ... + + @classmethod + async def get_cached_ids( + cls, + query: QUERY, + redis: aioredis.Redis, + params: RawParams, + ) -> Optional[tuple[int, list[int]]]: + try: + key = cls.get_cache_key(query) + active_key = f"{key}_active" + + if not await redis.exists(active_key): + return None + + objects_count, objects = await asyncio.gather( + redis.llen(key), + redis.lrange(key, params.offset, params.offset + params.limit), + ) + + return objects_count, [int(item.decode()) for item in objects] + except aioredis.RedisError as e: + print(e) + return None + @classmethod async def get_object_ids( cls, query: QUERY, redis: aioredis.Redis @@ -201,13 +206,17 @@ class MeiliSearchService(Generic[MODEL], BaseSearchService[MODEL, SearchQuery]): @classmethod @property def lang_key(cls) -> str: - assert cls.MS_INDEX_LANG_KEY is not None, f"MODEL in {cls.__name__} don't set!" + assert ( + cls.MS_INDEX_LANG_KEY is not None + ), f"MS_INDEX_LANG_KEY in {cls.__name__} don't set!" return cls.MS_INDEX_LANG_KEY @classmethod @property def index_name(cls) -> str: - assert cls.MS_INDEX_NAME is not None, f"MODEL in {cls.__name__} don't set!" + assert ( + cls.MS_INDEX_NAME is not None + ), f"MS_INDEX_NAME in {cls.__name__} don't set!" return cls.MS_INDEX_NAME @classmethod @@ -251,37 +260,9 @@ class MeiliSearchService(Generic[MODEL], BaseSearchService[MODEL, SearchQuery]): ) -class GetRandomService(Generic[MODEL]): - MODEL_CLASS: Optional[MODEL] = None +class GetRandomService(Generic[MODEL, QUERY], BaseService[MODEL, QUERY]): GET_OBJECTS_ID_QUERY: Optional[str] = None - CUSTOM_CACHE_PREFIX: Optional[str] = None - CACHE_TTL = 6 * 60 * 60 - - @classmethod - @property - def model(cls) -> MODEL: - assert cls.MODEL_CLASS is not None, f"MODEL in {cls.__name__} don't set!" - return cls.MODEL_CLASS - - @classmethod - @property - def database(cls) -> Database: - return cls.model.Meta.database - - @classmethod - @property - def cache_prefix(cls) -> str: - return cls.CUSTOM_CACHE_PREFIX or cls.model.Meta.tablename - - @staticmethod - def _get_query_hash(query: frozenset[str]): - return hash(query) - - @classmethod - def get_cache_key(cls, query: frozenset[str]) -> str: - model_class_name = cls.cache_prefix - query_hash = cls._get_query_hash(query) - return f"random_{model_class_name}_{query_hash}" + CACHE_PREFIX: str = "random" @classmethod @property @@ -292,18 +273,18 @@ class GetRandomService(Generic[MODEL]): return cls.GET_OBJECTS_ID_QUERY @classmethod - async def _get_objects_from_db(cls, allowed_langs: frozenset[str]) -> list[int]: + async def _get_objects_from_db(cls, query: QUERY) -> list[int]: objects = await cls.database.fetch_all( - cls.objects_id_query, {"langs": allowed_langs} + cls.objects_id_query, {"langs": query["allowed_langs"]} ) return [obj["id"] for obj in objects] @classmethod async def _get_random_object_from_cache( - cls, allowed_langs: frozenset[str], redis: aioredis.Redis + cls, query: QUERY, redis: aioredis.Redis ) -> Optional[int]: try: - key = cls.get_cache_key(allowed_langs) + key = cls.get_cache_key(query) active_key = f"{key}_active" if not await redis.exists(active_key): @@ -316,41 +297,20 @@ class GetRandomService(Generic[MODEL]): print(e) return None - @classmethod - async def _cache_object_ids( - cls, object_ids: list[int], allowed_langs: frozenset[str], redis: aioredis.Redis - ) -> bool: - try: - key = cls.get_cache_key(allowed_langs) - active_key = f"{key}_active" - - p = redis.pipeline() - - await p.set(active_key, 1, ex=cls.CACHE_TTL) - await p.delete(key) - await p.sadd(key, *object_ids) - - await p.execute() - - return True - except aioredis.RedisError as e: - print(e) - return False - @classmethod async def get_random_id( cls, - allowed_langs: frozenset[str], + query: QUERY, redis: aioredis.Redis, ) -> int: - cached_object_id = await cls._get_random_object_from_cache(allowed_langs, redis) + cached_object_id = await cls._get_random_object_from_cache(query, redis) if cached_object_id is not None: return cached_object_id - object_ids = await cls._get_objects_from_db(allowed_langs) + object_ids = await cls._get_objects_from_db(query) - await cls._cache_object_ids(object_ids, allowed_langs, redis) + await cls.cache_object_ids(query, object_ids, redis) return choice(object_ids) diff --git a/fastapi_book_server/app/services/sequence.py b/fastapi_book_server/app/services/sequence.py index cc7be33..6f95a85 100644 --- a/fastapi_book_server/app/services/sequence.py +++ b/fastapi_book_server/app/services/sequence.py @@ -1,3 +1,5 @@ +from typing import TypedDict + from app.models import Sequence from app.services.common import TRGMSearchService, MeiliSearchService, GetRandomService @@ -33,6 +35,7 @@ SELECT ARRAY ( class SequenceTGRMSearchService(TRGMSearchService): MODEL_CLASS = Sequence PREFETCH_RELATED = ["source"] + GET_OBJECT_IDS_QUERY = GET_OBJECT_IDS_QUERY @@ -53,8 +56,14 @@ ORDER BY RANDOM() LIMIT 1; """ -class GetRandomSequenceService(GetRandomService): - MODEL_CLASS = Sequence +class RandomSequenceServiceQuery(TypedDict): + allowed_langs: frozenset[str] + + +class GetRandomSequenceService(GetRandomService[Sequence, RandomSequenceServiceQuery]): + MODEL_CLASS = Sequence # type: ignore + PREFETCH_RELATED = ["source"] + GET_OBJECTS_ID_QUERY = GET_OBJECTS_ID_QUERY diff --git a/fastapi_book_server/app/services/translator.py b/fastapi_book_server/app/services/translator.py index fdf2a19..0dd9d23 100644 --- a/fastapi_book_server/app/services/translator.py +++ b/fastapi_book_server/app/services/translator.py @@ -46,6 +46,7 @@ class TranslatorTGRMSearchService(TRGMSearchService): CUSTOM_CACHE_PREFIX = "translator" PREFETCH_RELATED = ["source"] SELECT_RELATED = ["annotations"] + GET_OBJECT_IDS_QUERY = GET_OBJECT_IDS_QUERY diff --git a/fastapi_book_server/app/views/author.py b/fastapi_book_server/app/views/author.py index 1b48966..e0bcaa1 100644 --- a/fastapi_book_server/app/views/author.py +++ b/fastapi_book_server/app/views/author.py @@ -42,7 +42,7 @@ async def get_random_author( allowed_langs: frozenset[str] = Depends(get_allowed_langs), ): author_id = await GetRandomAuthorService.get_random_id( - allowed_langs, request.app.state.redis + {"allowed_langs": allowed_langs}, request.app.state.redis ) return ( diff --git a/fastapi_book_server/app/views/book.py b/fastapi_book_server/app/views/book.py index a3132d0..c662685 100644 --- a/fastapi_book_server/app/views/book.py +++ b/fastapi_book_server/app/views/book.py @@ -44,7 +44,7 @@ async def get_random_book( allowed_langs: frozenset[str] = Depends(get_allowed_langs), ): book_id = await GetRandomBookService.get_random_id( - allowed_langs, request.app.state.redis + {"allowed_langs": allowed_langs}, request.app.state.redis ) book = ( diff --git a/fastapi_book_server/app/views/sequence.py b/fastapi_book_server/app/views/sequence.py index f51dccb..d3f94bd 100644 --- a/fastapi_book_server/app/views/sequence.py +++ b/fastapi_book_server/app/views/sequence.py @@ -32,7 +32,7 @@ async def get_random_sequence( allowed_langs: frozenset[str] = Depends(get_allowed_langs), ): sequence_id = await GetRandomSequenceService.get_random_id( - allowed_langs, + {"allowed_langs": allowed_langs}, request.app.state.redis, )