Fix RAM usage

src/app/services/cache_updater.py

@@ -1,7 +1,9 @@
 import logging
-from typing import Optional
+from typing import Optional, cast
+from tempfile import SpooledTemporaryFile
 
 from arq.connections import ArqRedis
+from fastapi import UploadFile
 
 from app.models import CachedFile
 from app.services.caption_getter import get_caption
@@ -46,7 +48,7 @@ async def check_books(ctx) -> None:
 
     for page_number in range(1, books_page.total_pages + 1):
         await arq_pool.enqueue_job("check_books_page", page_number)
-
+    
 
 async def cache_file(book: Book, file_type) -> Optional[CachedFile]:
     logger.info(f"Cache {book.id} {file_type}...")
@@ -55,11 +57,18 @@ async def cache_file(book: Book, file_type) -> Optional[CachedFile]:
     if data is None:
         return None
 
-    content, filename = data
-
+    response, client, filename = data
     caption = get_caption(book)
 
-    upload_data = await upload_file(content, filename, caption)
+    temp_file = UploadFile(filename)
+    async for chunk in response.aiter_bytes(2048):
+        await temp_file.write(chunk)
+    await temp_file.seek(0)
+
+    upload_data = await upload_file(cast(SpooledTemporaryFile, temp_file.file), filename, caption)
+
+    await response.aclose()
+    await client.aclose()
 
     return await CachedFile.objects.create(
         object_id=book.id, object_type=file_type, data=upload_data.data

src/app/services/downloader.py

@@ -7,24 +7,24 @@ from core.config import env_config
 
 async def download(
     source_id: int, remote_id: int, file_type: str
-) -> Optional[tuple[bytes, str]]:
+) -> Optional[tuple[httpx.Response, httpx.AsyncClient, str]]:
     headers = {"Authorization": env_config.DOWNLOADER_API_KEY}
 
-    async with httpx.AsyncClient() as client:
-        response = await client.get(
-            f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}",
-            headers=headers,
-            timeout=5 * 60,
-        )
+    client = httpx.AsyncClient(timeout=120)
+    request = client.build_request(
+        "GET",
+        f"{env_config.DOWNLOADER_URL}/download/{source_id}/{remote_id}/{file_type}",
+        headers=headers,
+    )
+    response = await client.send(request, stream=True)
 
-        if response.status_code != 200:
-            return None
+    if response.status_code != 200:
+        return None
 
-        content_disposition = response.headers["Content-Disposition"]
+    content_disposition = response.headers["Content-Disposition"]
+    name = content_disposition.replace("attachment; filename=", "")
 
-        name = content_disposition.replace("attachment; filename=", "")
-
-        return response.content, name
+    return response, client, name
 
 
 async def get_filename(book_id: int, file_type: str) -> Optional[str]:

src/app/services/files_client.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Optional
+from tempfile import SpooledTemporaryFile
 
 import httpx
 from pydantic import BaseModel
@@ -14,7 +15,7 @@ class UploadedFile(BaseModel):
     upload_time: datetime
 
 
-async def upload_file(content: bytes, filename: str, caption: str) -> UploadedFile:
+async def upload_file(content: SpooledTemporaryFile, filename: str, caption: str) -> UploadedFile:
     headers = {"Authorization": env_config.FILES_SERVER_API_KEY}
 
     async with httpx.AsyncClient() as client: