flibusta-apps/books_downloader
1
0
Fork 0
mirror of https://github.com/flibusta-apps/books_downloader.git synced 2025-12-06 15:05:37 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Add linters configs

Browse Source
This commit is contained in:
Kurbanov Bulat
2022-01-01 20:59:14 +03:00
parent 2a70687df0
commit 2a31f27f44
14 changed files with 532 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/app/depends.py
View File

@@ -6,4 +6,6 @@ from core.config import env_config
async def check_token(api_key: str = Security(default_security)):
if api_key != env_config.API_KEY:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!")
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!"
)

4
src/app/services/base.py
View File

@@ -3,5 +3,7 @@ from typing import Protocol
class BaseDownloader(Protocol):
@classmethod
async def download(cls, remote_id: int, file_type: str, source_id: int) -> tuple[bytes, str]:
async def download(
cls, remote_id: int, file_type: str, source_id: int
) -> tuple[bytes, str]:
...

12
src/app/services/book_library.py
View File

@@ -1,15 +1,13 @@
from datetime import date
from typing import Generic, TypeVar
import json
import httpx
from datetime import date
from pydantic import BaseModel
from core.config import env_config
T = TypeVar('T')
T = TypeVar("T")
class Page(BaseModel, Generic[T]):
@@ -47,7 +45,7 @@ class BookLibraryClient:
@classmethod
@property
def auth_headers(cls):
return {'Authorization': cls.API_KEY}
return {"Authorization": cls.API_KEY}
@classmethod
async def _make_request(cls, url) -> dict:
@@ -65,6 +63,8 @@ class BookLibraryClient:
@classmethod
async def get_remote_book(cls, source_id: int, book_id: int) -> Book:
data = await cls._make_request(f"{cls.BASE_URL}/api/v1/books/remote/{source_id}/{book_id}")
data = await cls._make_request(
f"{cls.BASE_URL}/api/v1/books/remote/{source_id}/{book_id}"
)
return Book.parse_obj(data)

5
src/app/services/dowloaders_manager.py
View File

@@ -1,13 +1,12 @@
from app.services.base import BaseDownloader
from app.services.fl_downloader import FLDownloader
from app.services.book_library import BookLibraryClient
from app.services.fl_downloader import FLDownloader
class DownloadersManager:
SOURCES_TABLE: dict[int, str] = {}
DOWNLOADERS_TABLE: dict[str, type[BaseDownloader]] = {
'flibusta': FLDownloader,
"flibusta": FLDownloader,
}
PREPARED = False

66
src/app/services/fl_downloader.py
View File

@@ -1,13 +1,11 @@
from asyncio.exceptions import CancelledError
from typing import Optional, cast
import asyncio
from typing import Optional, cast
import httpx
from app.services.base import BaseDownloader
from app.services.utils import zip, unzip, get_filename, process_pool_executor
from app.services.book_library import BookLibraryClient, Book
from app.services.utils import zip, unzip, get_filename, process_pool_executor
from core.config import env_config, SourceConfig
@@ -40,17 +38,19 @@ class FLDownloader(BaseDownloader):
await asyncio.wait_for(self.get_book_data_task, None)
if self.book is None:
raise ValueError('Book is None!')
raise ValueError("Book is None!")
return get_filename(self.book, self.file_type)
async def get_final_filename(self) -> str:
if self.need_zip:
return (await self.get_filename()) + '.zip'
return (await self.get_filename()) + ".zip"
return await self.get_filename()
async def _download_from_source(self, source_config: SourceConfig, file_type: str = None) -> tuple[bytes, bool]:
async def _download_from_source(
self, source_config: SourceConfig, file_type: str = None
) -> tuple[bytes, bool]:
basic_url: str = source_config.URL
proxy: Optional[str] = source_config.PROXY
@@ -63,16 +63,14 @@ class FLDownloader(BaseDownloader):
httpx_proxy = None
if proxy is not None:
httpx_proxy = httpx.Proxy(
url=proxy
)
httpx_proxy = httpx.Proxy(url=proxy)
async with httpx.AsyncClient(proxies=httpx_proxy) as client:
response = await client.get(url, follow_redirects=True, timeout=10 * 60)
content_type = response.headers.get("Content-Type")
if response.status_code != 200:
raise NotSuccess(f'Status code is {response.status_code}!')
raise NotSuccess(f"Status code is {response.status_code}!")
if "text/html" in content_type:
raise ReceivedHTML()
@@ -82,11 +80,15 @@ class FLDownloader(BaseDownloader):
return response.content, False
async def _wait_until_some_done(self, tasks: set[asyncio.Task]) -> Optional[tuple[bytes, bool]]:
async def _wait_until_some_done(
self, tasks: set[asyncio.Task]
) -> Optional[tuple[bytes, bool]]:
tasks_ = tasks
while tasks_:
done, pending = await asyncio.wait(tasks_, return_when=asyncio.FIRST_COMPLETED)
done, pending = await asyncio.wait(
tasks_, return_when=asyncio.FIRST_COMPLETED
)
for task in done:
try:
@@ -100,7 +102,7 @@ class FLDownloader(BaseDownloader):
continue
tasks_ = pending
return None
async def _download_with_converting(self) -> tuple[bytes, bool]:
@@ -108,9 +110,7 @@ class FLDownloader(BaseDownloader):
for source in env_config.FL_SOURCES:
tasks.add(
asyncio.create_task(
self._download_from_source(source, file_type='fb2')
)
asyncio.create_task(self._download_from_source(source, file_type="fb2"))
)
data = await self._wait_until_some_done(tasks)
@@ -122,13 +122,15 @@ class FLDownloader(BaseDownloader):
if is_zip:
content = await asyncio.get_event_loop().run_in_executor(
process_pool_executor, unzip, content, 'fb2'
process_pool_executor, unzip, content, "fb2"
)
async with httpx.AsyncClient() as client:
form = {'format': self.file_type}
files = {'file': content}
response = await client.post(env_config.CONVERTER_URL, data=form, files=files, timeout=2 * 60)
form = {"format": self.file_type}
files = {"file": content}
response = await client.post(
env_config.CONVERTER_URL, data=form, files=files, timeout=2 * 60
)
if response.status_code != 200:
raise ValueError
@@ -143,20 +145,12 @@ class FLDownloader(BaseDownloader):
async def _get_content(self) -> tuple[bytes, str]:
tasks = set()
if self.file_type in ['epub', 'mobi']:
tasks.add(
asyncio.create_task(
self._download_with_converting()
)
)
if self.file_type in ["epub", "mobi"]:
tasks.add(asyncio.create_task(self._download_with_converting()))
for source in env_config.FL_SOURCES:
tasks.add(
asyncio.create_task(
self._download_from_source(source)
)
)
tasks.add(asyncio.create_task(self._download_from_source(source)))
data = await self._wait_until_some_done(tasks)
if data is None:
@@ -192,6 +186,8 @@ class FLDownloader(BaseDownloader):
return tasks[0].result()
@classmethod
async def download(cls, remote_id: int, file_type: str, source_id: int) -> tuple[bytes, str]:
async def download(
cls, remote_id: int, file_type: str, source_id: int
) -> tuple[bytes, str]:
downloader = cls(remote_id, file_type, source_id)
return await downloader._download()

37
src/app/services/utils.py
View File

@@ -1,8 +1,7 @@
from concurrent.futures.process import ProcessPoolExecutor
import io
import zipfile
from concurrent.futures.process import ProcessPoolExecutor
import transliterate
from app.services.book_library import Book, BookAuthor
@@ -23,10 +22,10 @@ def zip(filename, content):
buffer = io.BytesIO()
zip_file = zipfile.ZipFile(
file=buffer,
mode='w',
mode="w",
compression=zipfile.ZIP_DEFLATED,
allowZip64=False,
compresslevel=9
compresslevel=9,
)
zip_file.writestr(filename, content)
@@ -60,29 +59,33 @@ def get_filename(book: Book, file_type: str) -> str:
if book.authors:
filename_parts.append(
'_'.join([get_short_name(a) for a in book.authors]) + '_-_'
"_".join([get_short_name(a) for a in book.authors]) + "_-_"
)
if book.title.startswith(" "):
filename_parts.append(
book.title[1:]
)
filename_parts.append(book.title[1:])
else:
filename_parts.append(
book.title
)
filename_parts.append(book.title)
filename = "".join(filename_parts)
if book.lang in ['ru']:
filename = transliterate.translit(filename, 'ru', reversed=True)
if book.lang in ["ru"]:
filename = transliterate.translit(filename, "ru", reversed=True)
for c in "(),….’!\"?»«':":
filename = filename.replace(c, '')
filename = filename.replace(c, "")
for c, r in (('—', '-'), ('/', '_'), ('№', 'N'), (' ', '_'), ('–', '-'), ('á', 'a'), (' ', '_')):
for c, r in (
("—", "-"),
("/", "_"),
("№", "N"),
(" ", "_"),
("–", "-"),
("á", "a"),
(" ", "_"),
):
filename = filename.replace(c, r)
right_part = f'.{book.id}.{file_type}'
right_part = f".{book.id}.{file_type}"
return filename[:64 - len(right_part)] + right_part
return filename[: 64 - len(right_part)] + right_part

8
src/app/views.py
View File

@@ -1,9 +1,8 @@
from fastapi import APIRouter, Depends
from fastapi.responses import Response
from app.services.dowloaders_manager import DownloadersManager
from app.depends import check_token
from app.services.dowloaders_manager import DownloadersManager
router = APIRouter(
@@ -19,8 +18,5 @@ async def download(source_id: int, remote_id: int, file_type: str):
content, filename = await downloader.download(remote_id, file_type, source_id)
return Response(
content,
headers={
"Content-Disposition": f"attachment; filename={filename}"
}
content, headers={"Content-Disposition": f"attachment; filename={filename}"}
)

1
src/main.py
View File

@@ -1,3 +1,4 @@
from core.app import start_app
app = start_app()