diff --git a/src/app/services/cache_updater.py b/src/app/services/cache_updater.py index b25d406..ef1ed0e 100644 --- a/src/app/services/cache_updater.py +++ b/src/app/services/cache_updater.py @@ -1,3 +1,4 @@ +import collections import logging from tempfile import SpooledTemporaryFile from typing import Optional, cast @@ -24,23 +25,23 @@ class FileTypeNotAllowed(Exception): super().__init__(message) -async def check_book(book: Book, arq_pool: ArqRedis) -> None: - for file_type in book.available_types: - exists = await CachedFile.objects.filter( - object_id=book.id, object_type=file_type - ).exists() - - if not exists: - await arq_pool.enqueue_job("cache_file_by_book_id", book.id, file_type) - - async def check_books_page(ctx, page_number: int) -> None: arq_pool: ArqRedis = ctx["arc_pool"] page = await get_books(page_number, PAGE_SIZE) + object_ids = [book.id for book in page.items] + + cached_files = await CachedFile.objects.filter(object_id__in=object_ids).all() + + cached_files_map = collections.defaultdict(set) + for cached_file in cached_files: + cached_files_map[cached_file.object_id].add(cached_file.object_type) + for book in page.items: - await check_book(book, arq_pool) + for file_type in book.available_types: + if file_type not in cached_files_map[book.id]: + await arq_pool.enqueue_job("cache_file_by_book_id", book.id, file_type) async def check_books(ctx: dict, *args, **kwargs) -> None: diff --git a/src/core/setup_arq.py b/src/core/setup_arq.py index 77f97f9..ba51097 100644 --- a/src/core/setup_arq.py +++ b/src/core/setup_arq.py @@ -26,6 +26,6 @@ class WorkerSettings: on_startup = startup on_shutdown = shutdown redis_settings = get_redis_settings() - max_jobs = 1 + max_jobs = 2 job_timeout = 3 * 60 cron_jobs = [cron(check_books, hour={7}, minute=0)]