Fix RAM usage

poetry.lock

@@ -272,7 +272,7 @@ python-versions = ">=3.6"
 
 [[package]]
 name = "ormar"
-version = "0.10.24"
+version = "0.10.25"
 description = "A simple async ORM with fastapi in mind and pydantic validation."
 category = "main"
 optional = false
@@ -281,18 +281,18 @@ python-versions = ">=3.6.2,<4.0.0"
 [package.dependencies]
 aiosqlite = ">=0.17.0,<0.18.0"
 asyncpg = {version = ">=0.24,<0.26", optional = true, markers = "extra == \"postgresql\" or extra == \"postgres\" or extra == \"dev\""}
-databases = ">=0.3.2,<0.5.0 || >0.5.0,<0.5.1 || >0.5.1,<0.5.2 || >0.5.2,<0.5.3 || >0.5.3,<0.5.5"
+databases = ">=0.3.2,<0.5.0 || >0.5.0,<0.5.1 || >0.5.1,<0.5.2 || >0.5.2,<0.5.3 || >0.5.3,<=0.5.5"
 psycopg2-binary = {version = ">=2.9.1,<3.0.0", optional = true, markers = "extra == \"postgresql\" or extra == \"postgres\" or extra == \"dev\""}
 pydantic = ">=1.6.1,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<=1.9.1"
-SQLAlchemy = ">=1.3.18,<=1.4.29"
+SQLAlchemy = ">=1.3.18,<=1.4.31"
 
 [package.extras]
 postgresql = ["asyncpg (>=0.24,<0.26)", "psycopg2-binary (>=2.9.1,<3.0.0)"]
 postgres = ["asyncpg (>=0.24,<0.26)", "psycopg2-binary (>=2.9.1,<3.0.0)"]
-dev = ["asyncpg (>=0.24,<0.26)", "psycopg2-binary (>=2.9.1,<3.0.0)", "aiomysql (>=0.0.21,<0.0.23)", "cryptography (>=35,<37)", "orjson (>=3.6.4,<4.0.0)"]
+dev = ["asyncpg (>=0.24,<0.26)", "psycopg2-binary (>=2.9.1,<3.0.0)", "aiomysql (>=0.0.21,<0.0.23)", "cryptography (>=35,<37)", "orjson (>=3.6.4)"]
 mysql = ["aiomysql (>=0.0.21,<0.0.23)"]
 crypto = ["cryptography (>=35,<37)"]
-orjson = ["orjson (>=3.6.4,<4.0.0)"]
+orjson = ["orjson (>=3.6.4)"]
 
 [[package]]
 name = "prometheus-client"
@@ -710,8 +710,8 @@ markupsafe = [
     {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"},
 ]
 ormar = [
-    {file = "ormar-0.10.24-py3-none-any.whl", hash = "sha256:0ac7765bc14237cb4ed828c823cae3a4a9f5dea6daa402e0999c80b36662c410"},
+    {file = "ormar-0.10.25-py3-none-any.whl", hash = "sha256:8b68833bc037ae746080d0dce9a1b29c8944f8afb15d84580aa0b696ad9bb265"},
-    {file = "ormar-0.10.24.tar.gz", hash = "sha256:908eba2cb7350c5ef0c8e7d9653d061f357e2c7706b78298bd446e0848000762"},
+    {file = "ormar-0.10.25.tar.gz", hash = "sha256:27c771581417814f6933bc9ae38e1147b1435b1435b6c3f08b90301e9dc77de4"},
 ]
 prometheus-client = [
     {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"},

src/app/services/cache_updater.py

@@ -1,7 +1,9 @@
 import logging
-from typing import Optional
+from typing import Optional, cast
+from tempfile import SpooledTemporaryFile
 
 from arq.connections import ArqRedis
+from fastapi import UploadFile
 
 from app.models import CachedFile
 from app.services.caption_getter import get_caption
@@ -46,7 +48,7 @@ async def check_books(ctx) -> None:
 
     for page_number in range(1, books_page.total_pages + 1):
         await arq_pool.enqueue_job("check_books_page", page_number)
-
+    
 
 async def cache_file(book: Book, file_type) -> Optional[CachedFile]:
     logger.info(f"Cache {book.id} {file_type}...")
@@ -55,11 +57,18 @@ async def cache_file(book: Book, file_type) -> Optional[CachedFile]:
     if data is None:
         return None
 
-    content, filename = data
+    response, client, filename = data
-
     caption = get_caption(book)
 
-    upload_data = await upload_file(content, filename, caption)
+    temp_file = UploadFile(filename)
+    async for chunk in response.aiter_bytes(2048):
+        await temp_file.write(chunk)
+    await temp_file.seek(0)
+
+    upload_data = await upload_file(cast(SpooledTemporaryFile, temp_file.file), filename, caption)
+
+    await response.aclose()
+    await client.aclose()
 
     return await CachedFile.objects.create(
         object_id=book.id, object_type=file_type, data=upload_data.data

src/app/services/downloader.py

@@ -7,24 +7,24 @@ from core.config import env_config
 
 async def download(
     source_id: int, remote_id: int, file_type: str
-) -> Optional[tuple[bytes, str]]:
+) -> Optional[tuple[httpx.Response, httpx.AsyncClient, str]]:
     headers = {"Authorization": env_config.DOWNLOADER_API_KEY}
 
-    async with httpx.AsyncClient() as client:
+    client = httpx.AsyncClient(timeout=120)
-        response = await client.get(
+    request = client.build_request(
-            f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}",
+        "GET",
-            headers=headers,
+        f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}",
-            timeout=5 * 60,
+        headers=headers,
-        )
+    )
+    response = await client.send(request, stream=True)
 
-        if response.status_code != 200:
+    if response.status_code != 200:
-            return None
+        return None
 
-        content_disposition = response.headers["Content-Disposition"]
+    content_disposition = response.headers["Content-Disposition"]
+    name = content_disposition.replace("attachment; filename=", "")
 
-        name = content_disposition.replace("attachment; filename=", "")
+    return response, client, name
-
-        return response.content, name
 
 
 async def get_filename(book_id: int, file_type: str) -> Optional[str]:

src/app/services/files_client.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Optional
+from tempfile import SpooledTemporaryFile
 
 import httpx
 from pydantic import BaseModel
@@ -14,7 +15,7 @@ class UploadedFile(BaseModel):
     upload_time: datetime
 
 
-async def upload_file(content: bytes, filename: str, caption: str) -> UploadedFile:
+async def upload_file(content: SpooledTemporaryFile, filename: str, caption: str) -> UploadedFile:
     headers = {"Authorization": env_config.FILES_SERVER_API_KEY}
 
     async with httpx.AsyncClient() as client: