From 8c0624725f64235da484241afce3877439ce616a Mon Sep 17 00:00:00 2001 From: Bulat Kurbanov Date: Wed, 23 Mar 2022 11:57:02 +0300 Subject: [PATCH] Add revers order caching --- src/app/services/cache_updater.py | 16 +++++++++------- src/core/setup_arq.py | 6 ++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/app/services/cache_updater.py b/src/app/services/cache_updater.py index 567e2ad..b25d406 100644 --- a/src/app/services/cache_updater.py +++ b/src/app/services/cache_updater.py @@ -1,9 +1,10 @@ import logging -from typing import Optional, cast from tempfile import SpooledTemporaryFile +from typing import Optional, cast + +from fastapi import UploadFile from arq.connections import ArqRedis -from fastapi import UploadFile from app.models import CachedFile from app.services.caption_getter import get_caption @@ -46,12 +47,11 @@ async def check_books(ctx: dict, *args, **kwargs) -> None: arq_pool: ArqRedis = ctx["arc_pool"] books_page = await get_books(1, PAGE_SIZE) - for page_number in range(1, books_page.total_pages + 1): + for page_number in range(books_page.total_pages, 0, -1): await arq_pool.enqueue_job("check_books_page", page_number) - + async def cache_file(book: Book, file_type) -> Optional[CachedFile]: - logger.info(f"Cache {book.id} {file_type}...") data = await download(book.source.id, book.remote_id, file_type) if data is None: @@ -68,7 +68,9 @@ async def cache_file(book: Book, file_type) -> Optional[CachedFile]: await response.aclose() await client.aclose() - upload_data = await upload_file(cast(SpooledTemporaryFile, temp_file.file), filename, caption) + upload_data = await upload_file( + cast(SpooledTemporaryFile, temp_file.file), filename, caption + ) if upload_data is None: return None @@ -79,7 +81,7 @@ async def cache_file(book: Book, file_type) -> Optional[CachedFile]: async def cache_file_by_book_id( - ctx, book_id: int, file_type: str + ctx: dict, book_id: int, file_type: str ) -> Optional[CachedFile]: book = await get_book(book_id) diff --git a/src/core/setup_arq.py b/src/core/setup_arq.py index 5ac0e85..77f97f9 100644 --- a/src/core/setup_arq.py +++ b/src/core/setup_arq.py @@ -22,12 +22,10 @@ async def shutdown(ctx): class WorkerSettings: - functions = [check_books, cache_file_by_book_id, check_books_page] + functions = [check_books, check_books_page, cache_file_by_book_id] on_startup = startup on_shutdown = shutdown redis_settings = get_redis_settings() max_jobs = 1 job_timeout = 3 * 60 - cron_jobs = [ - cron(check_books, hour={7}, minute=0) - ] + cron_jobs = [cron(check_books, hour={7}, minute=0)]