diff --git a/poetry.lock b/poetry.lock index 8ece333..fd999c0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -272,7 +272,7 @@ python-versions = ">=3.6" [[package]] name = "ormar" -version = "0.10.24" +version = "0.10.25" description = "A simple async ORM with fastapi in mind and pydantic validation." category = "main" optional = false @@ -281,18 +281,18 @@ python-versions = ">=3.6.2,<4.0.0" [package.dependencies] aiosqlite = ">=0.17.0,<0.18.0" asyncpg = {version = ">=0.24,<0.26", optional = true, markers = "extra == \"postgresql\" or extra == \"postgres\" or extra == \"dev\""} -databases = ">=0.3.2,<0.5.0 || >0.5.0,<0.5.1 || >0.5.1,<0.5.2 || >0.5.2,<0.5.3 || >0.5.3,<0.5.5" +databases = ">=0.3.2,<0.5.0 || >0.5.0,<0.5.1 || >0.5.1,<0.5.2 || >0.5.2,<0.5.3 || >0.5.3,<=0.5.5" psycopg2-binary = {version = ">=2.9.1,<3.0.0", optional = true, markers = "extra == \"postgresql\" or extra == \"postgres\" or extra == \"dev\""} pydantic = ">=1.6.1,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<=1.9.1" -SQLAlchemy = ">=1.3.18,<=1.4.29" +SQLAlchemy = ">=1.3.18,<=1.4.31" [package.extras] postgresql = ["asyncpg (>=0.24,<0.26)", "psycopg2-binary (>=2.9.1,<3.0.0)"] postgres = ["asyncpg (>=0.24,<0.26)", "psycopg2-binary (>=2.9.1,<3.0.0)"] -dev = ["asyncpg (>=0.24,<0.26)", "psycopg2-binary (>=2.9.1,<3.0.0)", "aiomysql (>=0.0.21,<0.0.23)", "cryptography (>=35,<37)", "orjson (>=3.6.4,<4.0.0)"] +dev = ["asyncpg (>=0.24,<0.26)", "psycopg2-binary (>=2.9.1,<3.0.0)", "aiomysql (>=0.0.21,<0.0.23)", "cryptography (>=35,<37)", "orjson (>=3.6.4)"] mysql = ["aiomysql (>=0.0.21,<0.0.23)"] crypto = ["cryptography (>=35,<37)"] -orjson = ["orjson (>=3.6.4,<4.0.0)"] +orjson = ["orjson (>=3.6.4)"] [[package]] name = "prometheus-client" @@ -710,8 +710,8 @@ markupsafe = [ {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, ] ormar = [ - {file = "ormar-0.10.24-py3-none-any.whl", hash = "sha256:0ac7765bc14237cb4ed828c823cae3a4a9f5dea6daa402e0999c80b36662c410"}, - {file = "ormar-0.10.24.tar.gz", hash = "sha256:908eba2cb7350c5ef0c8e7d9653d061f357e2c7706b78298bd446e0848000762"}, + {file = "ormar-0.10.25-py3-none-any.whl", hash = "sha256:8b68833bc037ae746080d0dce9a1b29c8944f8afb15d84580aa0b696ad9bb265"}, + {file = "ormar-0.10.25.tar.gz", hash = "sha256:27c771581417814f6933bc9ae38e1147b1435b1435b6c3f08b90301e9dc77de4"}, ] prometheus-client = [ {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"}, diff --git a/src/app/services/cache_updater.py b/src/app/services/cache_updater.py index 897383c..de612e7 100644 --- a/src/app/services/cache_updater.py +++ b/src/app/services/cache_updater.py @@ -1,7 +1,9 @@ import logging -from typing import Optional +from typing import Optional, cast +from tempfile import SpooledTemporaryFile from arq.connections import ArqRedis +from fastapi import UploadFile from app.models import CachedFile from app.services.caption_getter import get_caption @@ -46,7 +48,7 @@ async def check_books(ctx) -> None: for page_number in range(1, books_page.total_pages + 1): await arq_pool.enqueue_job("check_books_page", page_number) - + async def cache_file(book: Book, file_type) -> Optional[CachedFile]: logger.info(f"Cache {book.id} {file_type}...") @@ -55,11 +57,18 @@ async def cache_file(book: Book, file_type) -> Optional[CachedFile]: if data is None: return None - content, filename = data - + response, client, filename = data caption = get_caption(book) - upload_data = await upload_file(content, filename, caption) + temp_file = UploadFile(filename) + async for chunk in response.aiter_bytes(2048): + await temp_file.write(chunk) + await temp_file.seek(0) + + upload_data = await upload_file(cast(SpooledTemporaryFile, temp_file.file), filename, caption) + + await response.aclose() + await client.aclose() return await CachedFile.objects.create( object_id=book.id, object_type=file_type, data=upload_data.data diff --git a/src/app/services/downloader.py b/src/app/services/downloader.py index 62e5216..f159a28 100644 --- a/src/app/services/downloader.py +++ b/src/app/services/downloader.py @@ -7,24 +7,24 @@ from core.config import env_config async def download( source_id: int, remote_id: int, file_type: str -) -> Optional[tuple[bytes, str]]: +) -> Optional[tuple[httpx.Response, httpx.AsyncClient, str]]: headers = {"Authorization": env_config.DOWNLOADER_API_KEY} - async with httpx.AsyncClient() as client: - response = await client.get( - f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}", - headers=headers, - timeout=5 * 60, - ) + client = httpx.AsyncClient(timeout=120) + request = client.build_request( + "GET", + f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}", + headers=headers, + ) + response = await client.send(request, stream=True) - if response.status_code != 200: - return None + if response.status_code != 200: + return None - content_disposition = response.headers["Content-Disposition"] + content_disposition = response.headers["Content-Disposition"] + name = content_disposition.replace("attachment; filename=", "") - name = content_disposition.replace("attachment; filename=", "") - - return response.content, name + return response, client, name async def get_filename(book_id: int, file_type: str) -> Optional[str]: diff --git a/src/app/services/files_client.py b/src/app/services/files_client.py index ddc61a4..bc45130 100644 --- a/src/app/services/files_client.py +++ b/src/app/services/files_client.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import Optional +from tempfile import SpooledTemporaryFile import httpx from pydantic import BaseModel @@ -14,7 +15,7 @@ class UploadedFile(BaseModel): upload_time: datetime -async def upload_file(content: bytes, filename: str, caption: str) -> UploadedFile: +async def upload_file(content: SpooledTemporaryFile, filename: str, caption: str) -> UploadedFile: headers = {"Authorization": env_config.FILES_SERVER_API_KEY} async with httpx.AsyncClient() as client: