mirror of
https://github.com/flibusta-apps/books_downloader.git
synced 2025-12-06 06:55:37 +01:00
Add rust implementation
This commit is contained in:
3
.dockerignore
Normal file
3
.dockerignore
Normal file
@@ -0,0 +1,3 @@
|
||||
.github
|
||||
.vscode
|
||||
target
|
||||
2
.github/workflows/build_docker_image.yml
vendored
2
.github/workflows/build_docker_image.yml
vendored
@@ -44,7 +44,7 @@ jobs:
|
||||
IMAGE: ${{ steps.repository_name.outputs.lowercase }}
|
||||
with:
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
platforms: linux/amd64
|
||||
tags: ghcr.io/${{ env.IMAGE }}:latest
|
||||
context: .
|
||||
file: ./docker/build.dockerfile
|
||||
|
||||
35
.github/workflows/codeql-analysis.yml
vendored
35
.github/workflows/codeql-analysis.yml
vendored
@@ -1,35 +0,0 @@
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
schedule:
|
||||
- cron: '0 12 * * *'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'python' ]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
||||
35
.github/workflows/linters.yaml
vendored
35
.github/workflows/linters.yaml
vendored
@@ -1,35 +0,0 @@
|
||||
name: Linters
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
jobs:
|
||||
Run-Pre-Commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 32
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.9
|
||||
|
||||
- name: Install pre-commit
|
||||
run: pip3 install pre-commit
|
||||
|
||||
- name: Pre-commit (Push)
|
||||
env:
|
||||
SETUPTOOLS_USE_DISTUTILS: stdlib
|
||||
if: ${{ github.event_name == 'push' }}
|
||||
run: pre-commit run --source ${{ github.event.before }} --origin ${{ github.event.after }} --show-diff-on-failure
|
||||
|
||||
- name: Pre-commit (Pull-Request)
|
||||
env:
|
||||
SETUPTOOLS_USE_DISTUTILS: stdlib
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: pre-commit run --source ${{ github.event.pull_request.base.sha }} --origin ${{ github.event.pull_request.head.sha }} --show-diff-on-failure
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -4,3 +4,5 @@
|
||||
__pycache__
|
||||
|
||||
venv
|
||||
|
||||
target
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
exclude: 'docs|node_modules|migrations|.git|.tox'
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/ambv/black
|
||||
rev: 22.10.0
|
||||
hooks:
|
||||
- id: black
|
||||
language_version: python3.11
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.10.1
|
||||
hooks:
|
||||
- id: isort
|
||||
- repo: https://github.com/csachs/pyproject-flake8
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
- id: pyproject-flake8
|
||||
additional_dependencies: [
|
||||
'-e', 'git+https://github.com/pycqa/pyflakes@1911c20#egg=pyflakes',
|
||||
'-e', 'git+https://github.com/pycqa/pycodestyle@d219c68#egg=pycodestyle',
|
||||
]
|
||||
1701
Cargo.lock
generated
Normal file
1701
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
22
Cargo.toml
Normal file
22
Cargo.toml
Normal file
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "books_downloader"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
tokio = { version = "1.21.1", features = ["full"] }
|
||||
tokio-util = { version = "0.7.4", features = ["compat"] }
|
||||
futures = "0.3.25"
|
||||
reqwest = { version = "0.11.13", features = ["json", "stream", "multipart"] }
|
||||
log = "0.4"
|
||||
env_logger = "0.9.0"
|
||||
lazy_static = "1.4.0"
|
||||
serde = { version = "1.0.144", features = ["derive"] }
|
||||
serde_json = "1.0.85"
|
||||
axum = "0.5.16"
|
||||
translit = "0.5.0"
|
||||
zip = "0.6.3"
|
||||
tempfile = "3.3.0"
|
||||
bytes = "1.3.0"
|
||||
21
docker/build-dev.dockerfile
Normal file
21
docker/build-dev.dockerfile
Normal file
@@ -0,0 +1,21 @@
|
||||
FROM rust:bullseye AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN cargo build --bin books_downloader
|
||||
|
||||
|
||||
FROM debian:bullseye-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y openssl ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN update-ca-certificates
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /app/target/debug/books_downloader /usr/local/bin
|
||||
ENTRYPOINT ["/usr/local/bin/books_downloader"]
|
||||
@@ -1,26 +1,21 @@
|
||||
FROM ghcr.io/flibusta-apps/base_docker_images:3.11-poetry-buildtime as build-image
|
||||
FROM rust:bullseye AS builder
|
||||
|
||||
WORKDIR /root/poetry
|
||||
COPY pyproject.toml poetry.lock /root/poetry/
|
||||
WORKDIR /app
|
||||
|
||||
ENV VENV_PATH=/opt/venv
|
||||
COPY . .
|
||||
|
||||
RUN poetry export --without-hashes > requirements.txt \
|
||||
&& . /opt/venv/bin/activate \
|
||||
&& pip install -r requirements.txt --no-cache-dir
|
||||
RUN cargo build --release --bin books_downloader
|
||||
|
||||
|
||||
FROM python:3.11-slim as runtime-image
|
||||
FROM debian:bullseye-slim
|
||||
|
||||
ENV VENV_PATH=/opt/venv
|
||||
ENV PATH="$VENV_PATH/bin:$PATH"
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y openssl ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY ./src/ /app/
|
||||
COPY ./scripts/* /root/
|
||||
COPY --from=build-image $VENV_PATH $VENV_PATH
|
||||
RUN update-ca-certificates
|
||||
|
||||
EXPOSE 8080
|
||||
WORKDIR /app
|
||||
|
||||
WORKDIR /app/
|
||||
|
||||
CMD bash /root/start.sh
|
||||
COPY --from=builder /app/target/release/books_downloader /usr/local/bin
|
||||
ENTRYPOINT ["/usr/local/bin/books_downloader"]
|
||||
|
||||
512
poetry.lock
generated
512
poetry.lock
generated
@@ -1,512 +0,0 @@
|
||||
[[package]]
|
||||
name = "aiofiles"
|
||||
version = "0.8.0"
|
||||
description = "File support for asyncio."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6,<4.0"
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "3.6.1"
|
||||
description = "High level compatibility layer for multiple asynchronous event loop implementations"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6.2"
|
||||
|
||||
[package.dependencies]
|
||||
idna = ">=2.8"
|
||||
sniffio = ">=1.1"
|
||||
|
||||
[package.extras]
|
||||
doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
|
||||
test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"]
|
||||
trio = ["trio (>=0.16)"]
|
||||
|
||||
[[package]]
|
||||
name = "asynctempfile"
|
||||
version = "0.5.0"
|
||||
description = "Async version of tempfile"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.dependencies]
|
||||
aiofiles = ">=0.6.0"
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2022.12.7"
|
||||
description = "Python package for providing Mozilla's CA Bundle."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[[package]]
|
||||
name = "click"
|
||||
version = "8.1.3"
|
||||
description = "Composable command line interface toolkit"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
colorama = {version = "*", markers = "platform_system == \"Windows\""}
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.5"
|
||||
description = "Cross-platform colored terminal text."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
[[package]]
|
||||
name = "fastapi"
|
||||
version = "0.85.1"
|
||||
description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0"
|
||||
starlette = "0.20.4"
|
||||
|
||||
[package.extras]
|
||||
all = ["email-validator (>=1.1.1,<2.0.0)", "itsdangerous (>=1.1.0,<3.0.0)", "jinja2 (>=2.11.2,<4.0.0)", "orjson (>=3.2.1,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)", "uvicorn[standard] (>=0.12.0,<0.19.0)"]
|
||||
dev = ["autoflake (>=1.4.0,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "pre-commit (>=2.17.0,<3.0.0)", "uvicorn[standard] (>=0.12.0,<0.19.0)"]
|
||||
doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pyyaml (>=5.3.1,<7.0.0)", "typer (>=0.4.1,<0.7.0)"]
|
||||
test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==22.8.0)", "databases[sqlite] (>=0.3.2,<0.7.0)", "email-validator (>=1.1.1,<2.0.0)", "flake8 (>=3.8.3,<6.0.0)", "flask (>=1.1.2,<3.0.0)", "httpx (>=0.23.0,<0.24.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.971)", "orjson (>=3.2.1,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=7.1.3,<8.0.0)", "pytest-cov (>=2.12.0,<4.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<7.0.0)", "requests (>=2.24.0,<3.0.0)", "sqlalchemy (>=1.3.18,<1.5.0)", "types-orjson (==3.6.2)", "types-ujson (==5.4.0)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "gunicorn"
|
||||
version = "20.1.0"
|
||||
description = "WSGI HTTP Server for UNIX"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.5"
|
||||
|
||||
[package.dependencies]
|
||||
setuptools = ">=3.0"
|
||||
|
||||
[package.extras]
|
||||
eventlet = ["eventlet (>=0.24.1)"]
|
||||
gevent = ["gevent (>=1.4.0)"]
|
||||
setproctitle = ["setproctitle"]
|
||||
tornado = ["tornado (>=0.2)"]
|
||||
|
||||
[[package]]
|
||||
name = "h11"
|
||||
version = "0.12.0"
|
||||
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[[package]]
|
||||
name = "httpcore"
|
||||
version = "0.15.0"
|
||||
description = "A minimal low-level HTTP client."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
anyio = ">=3.0.0,<4.0.0"
|
||||
certifi = "*"
|
||||
h11 = ">=0.11,<0.13"
|
||||
sniffio = ">=1.0.0,<2.0.0"
|
||||
|
||||
[package.extras]
|
||||
http2 = ["h2 (>=3,<5)"]
|
||||
socks = ["socksio (>=1.0.0,<2.0.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "httpx"
|
||||
version = "0.23.0"
|
||||
description = "The next generation HTTP client."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
certifi = "*"
|
||||
httpcore = ">=0.15.0,<0.16.0"
|
||||
rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]}
|
||||
sniffio = "*"
|
||||
|
||||
[package.extras]
|
||||
brotli = ["brotli", "brotlicffi"]
|
||||
cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"]
|
||||
http2 = ["h2 (>=3,<5)"]
|
||||
socks = ["socksio (>=1.0.0,<2.0.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.3"
|
||||
description = "Internationalized Domain Names in Applications (IDNA)"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.5"
|
||||
|
||||
[[package]]
|
||||
name = "prometheus-client"
|
||||
version = "0.14.1"
|
||||
description = "Python client for the Prometheus monitoring system."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[package.extras]
|
||||
twisted = ["twisted"]
|
||||
|
||||
[[package]]
|
||||
name = "prometheus-fastapi-instrumentator"
|
||||
version = "5.9.1"
|
||||
description = "Instrument your FastAPI with Prometheus metrics"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7.0,<4.0.0"
|
||||
|
||||
[package.dependencies]
|
||||
fastapi = ">=0.38.1,<1.0.0"
|
||||
prometheus-client = ">=0.8.0,<1.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "pydantic"
|
||||
version = "1.10.2"
|
||||
description = "Data validation and settings management using python type hints"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
typing-extensions = ">=4.1.0"
|
||||
|
||||
[package.extras]
|
||||
dotenv = ["python-dotenv (>=0.10.4)"]
|
||||
email = ["email-validator (>=1.0.3)"]
|
||||
|
||||
[[package]]
|
||||
name = "rfc3986"
|
||||
version = "1.5.0"
|
||||
description = "Validating URI References per RFC 3986"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.dependencies]
|
||||
idna = {version = "*", optional = true, markers = "extra == \"idna2008\""}
|
||||
|
||||
[package.extras]
|
||||
idna2008 = ["idna"]
|
||||
|
||||
[[package]]
|
||||
name = "sentry-sdk"
|
||||
version = "1.10.1"
|
||||
description = "Python client for Sentry (https://sentry.io)"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.dependencies]
|
||||
certifi = "*"
|
||||
urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""}
|
||||
|
||||
[package.extras]
|
||||
aiohttp = ["aiohttp (>=3.5)"]
|
||||
beam = ["apache-beam (>=2.12)"]
|
||||
bottle = ["bottle (>=0.12.13)"]
|
||||
celery = ["celery (>=3)"]
|
||||
chalice = ["chalice (>=1.16.0)"]
|
||||
django = ["django (>=1.8)"]
|
||||
falcon = ["falcon (>=1.4)"]
|
||||
fastapi = ["fastapi (>=0.79.0)"]
|
||||
flask = ["blinker (>=1.1)", "flask (>=0.11)"]
|
||||
httpx = ["httpx (>=0.16.0)"]
|
||||
pure-eval = ["asttokens", "executing", "pure-eval"]
|
||||
pyspark = ["pyspark (>=2.4.4)"]
|
||||
quart = ["blinker (>=1.1)", "quart (>=0.16.1)"]
|
||||
rq = ["rq (>=0.6)"]
|
||||
sanic = ["sanic (>=0.8)"]
|
||||
sqlalchemy = ["sqlalchemy (>=1.2)"]
|
||||
starlette = ["starlette (>=0.19.1)"]
|
||||
tornado = ["tornado (>=5)"]
|
||||
|
||||
[[package]]
|
||||
name = "setuptools"
|
||||
version = "65.5.0"
|
||||
description = "Easily download, build, install, upgrade, and uninstall Python packages"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.extras]
|
||||
docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
|
||||
testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
|
||||
testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
|
||||
|
||||
[[package]]
|
||||
name = "six"
|
||||
version = "1.16.0"
|
||||
description = "Python 2 and 3 compatibility utilities"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
|
||||
|
||||
[[package]]
|
||||
name = "sniffio"
|
||||
version = "1.2.0"
|
||||
description = "Sniff out which async library your code is running under"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.5"
|
||||
|
||||
[[package]]
|
||||
name = "starlette"
|
||||
version = "0.20.4"
|
||||
description = "The little ASGI library that shines."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
anyio = ">=3.4.0,<5"
|
||||
typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""}
|
||||
|
||||
[package.extras]
|
||||
full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"]
|
||||
|
||||
[[package]]
|
||||
name = "transliterate"
|
||||
version = "1.10.2"
|
||||
description = "Bi-directional transliterator for Python"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.dependencies]
|
||||
six = ">=1.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.3.0"
|
||||
description = "Backported and Experimental Type Hints for Python 3.7+"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[[package]]
|
||||
name = "urllib3"
|
||||
version = "1.26.11"
|
||||
description = "HTTP library with thread-safe connection pooling, file post, and more."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4"
|
||||
|
||||
[package.extras]
|
||||
brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
|
||||
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)"]
|
||||
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "uvicorn"
|
||||
version = "0.19.0"
|
||||
description = "The lightning-fast ASGI server."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
click = ">=7.0"
|
||||
h11 = ">=0.8"
|
||||
|
||||
[package.extras]
|
||||
standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "uvloop"
|
||||
version = "0.17.0"
|
||||
description = "Fast implementation of asyncio event loop on top of libuv"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.extras]
|
||||
dev = ["Cython (>=0.29.32,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
|
||||
docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
|
||||
test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)"]
|
||||
|
||||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.9"
|
||||
content-hash = "170ac90550be01e631e312180b9f7e67585fc2b907181b7e082aa1aaaf0cfaa5"
|
||||
|
||||
[metadata.files]
|
||||
aiofiles = [
|
||||
{file = "aiofiles-0.8.0-py3-none-any.whl", hash = "sha256:7a973fc22b29e9962d0897805ace5856e6a566ab1f0c8e5c91ff6c866519c937"},
|
||||
{file = "aiofiles-0.8.0.tar.gz", hash = "sha256:8334f23235248a3b2e83b2c3a78a22674f39969b96397126cc93664d9a901e59"},
|
||||
]
|
||||
anyio = [
|
||||
{file = "anyio-3.6.1-py3-none-any.whl", hash = "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"},
|
||||
{file = "anyio-3.6.1.tar.gz", hash = "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b"},
|
||||
]
|
||||
asynctempfile = [
|
||||
{file = "asynctempfile-0.5.0-py3-none-any.whl", hash = "sha256:cec59bdb71c850e3de9bb4415f88998165c364709696240eea9ec5204a7439af"},
|
||||
{file = "asynctempfile-0.5.0.tar.gz", hash = "sha256:4a647c747357e8827397baadbdfe87f3095d30923fa789e797111eb02160884a"},
|
||||
]
|
||||
certifi = [
|
||||
{file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"},
|
||||
{file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"},
|
||||
]
|
||||
click = [
|
||||
{file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"},
|
||||
{file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"},
|
||||
]
|
||||
colorama = [
|
||||
{file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"},
|
||||
{file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"},
|
||||
]
|
||||
fastapi = [
|
||||
{file = "fastapi-0.85.1-py3-none-any.whl", hash = "sha256:de3166b6b1163dc22da4dc4ebdc3192fcbac7700dd1870a1afa44de636a636b5"},
|
||||
{file = "fastapi-0.85.1.tar.gz", hash = "sha256:1facd097189682a4ff11cbd01334a992e51b56be663b2bd50c2c09523624f144"},
|
||||
]
|
||||
gunicorn = [
|
||||
{file = "gunicorn-20.1.0-py3-none-any.whl", hash = "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e"},
|
||||
{file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"},
|
||||
]
|
||||
h11 = [
|
||||
{file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"},
|
||||
{file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"},
|
||||
]
|
||||
httpcore = [
|
||||
{file = "httpcore-0.15.0-py3-none-any.whl", hash = "sha256:1105b8b73c025f23ff7c36468e4432226cbb959176eab66864b8e31c4ee27fa6"},
|
||||
{file = "httpcore-0.15.0.tar.gz", hash = "sha256:18b68ab86a3ccf3e7dc0f43598eaddcf472b602aba29f9aa6ab85fe2ada3980b"},
|
||||
]
|
||||
httpx = [
|
||||
{file = "httpx-0.23.0-py3-none-any.whl", hash = "sha256:42974f577483e1e932c3cdc3cd2303e883cbfba17fe228b0f63589764d7b9c4b"},
|
||||
{file = "httpx-0.23.0.tar.gz", hash = "sha256:f28eac771ec9eb4866d3fb4ab65abd42d38c424739e80c08d8d20570de60b0ef"},
|
||||
]
|
||||
idna = [
|
||||
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
|
||||
{file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
|
||||
]
|
||||
prometheus-client = [
|
||||
{file = "prometheus_client-0.14.1-py3-none-any.whl", hash = "sha256:522fded625282822a89e2773452f42df14b5a8e84a86433e3f8a189c1d54dc01"},
|
||||
{file = "prometheus_client-0.14.1.tar.gz", hash = "sha256:5459c427624961076277fdc6dc50540e2bacb98eebde99886e59ec55ed92093a"},
|
||||
]
|
||||
prometheus-fastapi-instrumentator = [
|
||||
{file = "prometheus-fastapi-instrumentator-5.9.1.tar.gz", hash = "sha256:3651a72f73359a28e8afb0d370ebe3774147323ee2285e21236b229ce79172fc"},
|
||||
{file = "prometheus_fastapi_instrumentator-5.9.1-py3-none-any.whl", hash = "sha256:b5206ea9aa6975a0b07f3bf7376932b8a1b2983164b5abb04878e75ba336d9ed"},
|
||||
]
|
||||
pydantic = [
|
||||
{file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"},
|
||||
{file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"},
|
||||
{file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"},
|
||||
{file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"},
|
||||
{file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"},
|
||||
{file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"},
|
||||
{file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"},
|
||||
{file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"},
|
||||
{file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"},
|
||||
{file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"},
|
||||
{file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"},
|
||||
{file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"},
|
||||
{file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"},
|
||||
{file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"},
|
||||
{file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"},
|
||||
{file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"},
|
||||
{file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"},
|
||||
{file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"},
|
||||
{file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"},
|
||||
{file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"},
|
||||
{file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"},
|
||||
{file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"},
|
||||
{file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"},
|
||||
{file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"},
|
||||
{file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"},
|
||||
{file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"},
|
||||
{file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"},
|
||||
{file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"},
|
||||
{file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"},
|
||||
{file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"},
|
||||
{file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"},
|
||||
{file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"},
|
||||
{file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"},
|
||||
{file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"},
|
||||
{file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"},
|
||||
{file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"},
|
||||
]
|
||||
rfc3986 = [
|
||||
{file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"},
|
||||
{file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"},
|
||||
]
|
||||
sentry-sdk = [
|
||||
{file = "sentry-sdk-1.10.1.tar.gz", hash = "sha256:105faf7bd7b7fa25653404619ee261527266b14103fe1389e0ce077bd23a9691"},
|
||||
{file = "sentry_sdk-1.10.1-py2.py3-none-any.whl", hash = "sha256:06c0fa9ccfdc80d7e3b5d2021978d6eb9351fa49db9b5847cf4d1f2a473414ad"},
|
||||
]
|
||||
setuptools = [
|
||||
{file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"},
|
||||
{file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"},
|
||||
]
|
||||
six = [
|
||||
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
|
||||
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
|
||||
]
|
||||
sniffio = [
|
||||
{file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"},
|
||||
{file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"},
|
||||
]
|
||||
starlette = [
|
||||
{file = "starlette-0.20.4-py3-none-any.whl", hash = "sha256:c0414d5a56297d37f3db96a84034d61ce29889b9eaccf65eb98a0b39441fcaa3"},
|
||||
{file = "starlette-0.20.4.tar.gz", hash = "sha256:42fcf3122f998fefce3e2c5ad7e5edbf0f02cf685d646a83a08d404726af5084"},
|
||||
]
|
||||
transliterate = [
|
||||
{file = "transliterate-1.10.2-py2.py3-none-any.whl", hash = "sha256:010a5021bf6021689c4fade0985f3f7b3db1f2f16a48a09a56797f171c08ed42"},
|
||||
{file = "transliterate-1.10.2.tar.gz", hash = "sha256:bc608e0d48e687db9c2b1d7ea7c381afe0d1849cad216087d8e03d8d06a57c85"},
|
||||
]
|
||||
typing-extensions = [
|
||||
{file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"},
|
||||
{file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"},
|
||||
]
|
||||
urllib3 = [
|
||||
{file = "urllib3-1.26.11-py2.py3-none-any.whl", hash = "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc"},
|
||||
{file = "urllib3-1.26.11.tar.gz", hash = "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"},
|
||||
]
|
||||
uvicorn = [
|
||||
{file = "uvicorn-0.19.0-py3-none-any.whl", hash = "sha256:cc277f7e73435748e69e075a721841f7c4a95dba06d12a72fe9874acced16f6f"},
|
||||
{file = "uvicorn-0.19.0.tar.gz", hash = "sha256:cf538f3018536edb1f4a826311137ab4944ed741d52aeb98846f52215de57f25"},
|
||||
]
|
||||
uvloop = [
|
||||
{file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded"},
|
||||
{file = "uvloop-0.17.0.tar.gz", hash = "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1"},
|
||||
]
|
||||
@@ -1,65 +0,0 @@
|
||||
[tool.poetry]
|
||||
name = "books_downloader"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Kurbanov Bulat <kurbanovbul@gmail.com>"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.9"
|
||||
fastapi = "^0.85.1"
|
||||
httpx = "^0.23.0"
|
||||
transliterate = "^1.10.2"
|
||||
uvicorn = {extras = ["standart"], version = "^0.19.0"}
|
||||
prometheus-fastapi-instrumentator = "^5.9.1"
|
||||
uvloop = "^0.17.0"
|
||||
gunicorn = "^20.1.0"
|
||||
sentry-sdk = "^1.10.1"
|
||||
asynctempfile = "^0.5.0"
|
||||
pydantic = "1.10.2"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.black]
|
||||
include = '\.pyi?$'
|
||||
exclude = '''
|
||||
/(
|
||||
\.git
|
||||
| \.vscode
|
||||
| \venv
|
||||
| alembic
|
||||
)/
|
||||
'''
|
||||
|
||||
[tool.flake8]
|
||||
ignore = [
|
||||
# Whitespace before ':' ( https://www.flake8rules.com/rules/E203.html )
|
||||
"E203",
|
||||
"W503"
|
||||
]
|
||||
max-line-length=88
|
||||
max-complexity = 15
|
||||
select = "B,C,E,F,W,T4,B9"
|
||||
exclude = [
|
||||
# No need to traverse our git directory
|
||||
".git",
|
||||
# There's no value in checking cache directories
|
||||
"__pycache__",
|
||||
# The conf file is mostly autogenerated, ignore it
|
||||
"src/app/alembic/*",
|
||||
# The old directory contains Flake8 2.0
|
||||
]
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
only_sections = true
|
||||
force_sort_within_sections = true
|
||||
lines_after_imports = 2
|
||||
lexicographical = true
|
||||
sections = ["FUTURE", "STDLIB", "BASEFRAMEWORK", "FRAMEWORKEXT", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
|
||||
known_baseframework = ["fastapi",]
|
||||
known_frameworkext = ["starlette",]
|
||||
src_paths = ["src"]
|
||||
@@ -1,8 +0,0 @@
|
||||
import httpx
|
||||
|
||||
|
||||
response = httpx.get(
|
||||
"http://localhost:8080/healthcheck"
|
||||
)
|
||||
print(f"HEALTHCHECK STATUS: {response.status_code}")
|
||||
exit(0 if response.status_code == 200 else 1)
|
||||
@@ -1,3 +0,0 @@
|
||||
cd /app
|
||||
|
||||
gunicorn -k uvicorn.workers.UvicornWorker main:app --bind 0.0.0.0:8080 --timeout 600
|
||||
@@ -1,11 +0,0 @@
|
||||
from fastapi import Security, HTTPException, status
|
||||
|
||||
from core.auth import default_security
|
||||
from core.config import env_config
|
||||
|
||||
|
||||
async def check_token(api_key: str = Security(default_security)):
|
||||
if api_key != env_config.API_KEY:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!"
|
||||
)
|
||||
@@ -1,9 +0,0 @@
|
||||
from typing import Protocol, Optional, AsyncIterator
|
||||
|
||||
|
||||
class BaseDownloader(Protocol):
|
||||
@classmethod
|
||||
async def download(
|
||||
cls, remote_id: int, file_type: str, source_id: int
|
||||
) -> Optional[tuple[AsyncIterator[bytes], str]]:
|
||||
...
|
||||
@@ -1,86 +0,0 @@
|
||||
from datetime import date
|
||||
from typing import Generic, TypeVar, Optional
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.config import env_config
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Page(BaseModel, Generic[T]):
|
||||
items: list[T]
|
||||
total: int
|
||||
page: int
|
||||
size: int
|
||||
|
||||
|
||||
class Source(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
|
||||
|
||||
class BookAuthor(BaseModel):
|
||||
id: int
|
||||
first_name: str
|
||||
last_name: str
|
||||
middle_name: str
|
||||
|
||||
|
||||
class Book(BaseModel):
|
||||
id: int
|
||||
title: str
|
||||
lang: str
|
||||
file_type: str
|
||||
uploaded: date
|
||||
authors: list[BookAuthor]
|
||||
|
||||
|
||||
class BookDetail(Book):
|
||||
remote_id: int
|
||||
|
||||
|
||||
class BookLibraryClient:
|
||||
API_KEY = env_config.BOOK_LIBRARY_API_KEY
|
||||
BASE_URL = env_config.BOOK_LIBRARY_URL
|
||||
|
||||
_sources_cache: Optional[list[Source]] = None
|
||||
|
||||
@classmethod
|
||||
@property
|
||||
def auth_headers(cls):
|
||||
return {"Authorization": cls.API_KEY}
|
||||
|
||||
@classmethod
|
||||
async def _make_request(cls, url) -> dict:
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
return (await client.get(url, headers=cls.auth_headers)).json()
|
||||
|
||||
@classmethod
|
||||
async def get_sources(cls) -> list[Source]:
|
||||
if cls._sources_cache:
|
||||
return cls._sources_cache
|
||||
|
||||
data = await cls._make_request(f"{cls.BASE_URL}/api/v1/sources")
|
||||
|
||||
page = Page[Source].parse_obj(data)
|
||||
|
||||
sources = [Source.parse_obj(item) for item in page.items]
|
||||
cls._sources_cache = sources
|
||||
return sources
|
||||
|
||||
@classmethod
|
||||
async def get_book(cls, book_id: int) -> BookDetail:
|
||||
data = await cls._make_request(f"{cls.BASE_URL}/api/v1/books/{book_id}")
|
||||
|
||||
return BookDetail.parse_obj(data)
|
||||
|
||||
@classmethod
|
||||
async def get_remote_book(cls, source_id: int, book_id: int) -> Book:
|
||||
data = await cls._make_request(
|
||||
f"{cls.BASE_URL}/api/v1/books/remote/{source_id}/{book_id}"
|
||||
)
|
||||
|
||||
return Book.parse_obj(data)
|
||||
@@ -1,28 +0,0 @@
|
||||
from app.services.base import BaseDownloader
|
||||
from app.services.book_library import BookLibraryClient
|
||||
from app.services.fl_downloader import FLDownloader
|
||||
|
||||
|
||||
class DownloadersManager:
|
||||
SOURCES_TABLE: dict[int, str] = {}
|
||||
DOWNLOADERS_TABLE: dict[str, type[BaseDownloader]] = {
|
||||
"flibusta": FLDownloader,
|
||||
}
|
||||
|
||||
PREPARED = False
|
||||
|
||||
@classmethod
|
||||
async def _prepare(cls):
|
||||
sources = await BookLibraryClient.get_sources()
|
||||
|
||||
for source in sources:
|
||||
cls.SOURCES_TABLE[source.id] = source.name
|
||||
|
||||
@classmethod
|
||||
async def get_downloader(cls, source_id: int):
|
||||
if not cls.PREPARED:
|
||||
await cls._prepare()
|
||||
|
||||
name = cls.SOURCES_TABLE[source_id]
|
||||
|
||||
return cls.DOWNLOADERS_TABLE[name]
|
||||
@@ -1,10 +0,0 @@
|
||||
class NotSuccess(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ReceivedHTML(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ConvertationError(Exception):
|
||||
pass
|
||||
@@ -1,314 +0,0 @@
|
||||
import asyncio
|
||||
from typing import Optional, AsyncIterator, cast
|
||||
import zipfile
|
||||
|
||||
import aiofiles
|
||||
import aiofiles.os
|
||||
import asynctempfile
|
||||
import httpx
|
||||
|
||||
from app.services.base import BaseDownloader
|
||||
from app.services.book_library import BookLibraryClient
|
||||
from app.services.exceptions import NotSuccess, ReceivedHTML, ConvertationError
|
||||
from app.services.utils import (
|
||||
zip,
|
||||
unzip,
|
||||
get_filename,
|
||||
process_pool_executor,
|
||||
async_retry,
|
||||
)
|
||||
from core.config import env_config, SourceConfig
|
||||
|
||||
|
||||
class FLDownloader(BaseDownloader):
|
||||
EXCLUDE_UNZIP = ["html"]
|
||||
|
||||
def __init__(self, book_id: int, file_type: str, source_id: int):
|
||||
self.book_id = book_id
|
||||
self.original_file_type = file_type
|
||||
self.source_id = source_id
|
||||
|
||||
self.get_book_data_task = asyncio.create_task(self._get_book_data())
|
||||
self.get_content_task = asyncio.create_task(self._get_content())
|
||||
|
||||
@property
|
||||
def file_type(self):
|
||||
return self.original_file_type.replace("zip", "")
|
||||
|
||||
@property
|
||||
def need_zip(self):
|
||||
return "zip" in self.original_file_type
|
||||
|
||||
async def get_filename(self) -> str:
|
||||
if not self.get_book_data_task.done():
|
||||
await asyncio.wait_for(self.get_book_data_task, None)
|
||||
|
||||
book = self.get_book_data_task.result()
|
||||
if book is None:
|
||||
raise ValueError("Book is None!")
|
||||
|
||||
return get_filename(self.book_id, book, self.file_type)
|
||||
|
||||
async def get_final_filename(self, force_zip: bool = False) -> str:
|
||||
if self.need_zip or force_zip:
|
||||
return (await self.get_filename()) + ".zip"
|
||||
|
||||
return await self.get_filename()
|
||||
|
||||
@async_retry(NotSuccess, times=5, delay=10)
|
||||
async def _download_from_source(
|
||||
self, source_config: SourceConfig, file_type: Optional[str] = None
|
||||
) -> tuple[httpx.AsyncClient, httpx.Response, bool]:
|
||||
basic_url: str = source_config.URL
|
||||
proxy: Optional[str] = source_config.PROXY
|
||||
|
||||
file_type_ = file_type or self.file_type
|
||||
|
||||
if self.file_type in ("fb2", "epub", "mobi"):
|
||||
url = basic_url + f"/b/{self.book_id}/{file_type_}"
|
||||
else:
|
||||
url = basic_url + f"/b/{self.book_id}/download"
|
||||
|
||||
client_kwargs = {
|
||||
"timeout": httpx.Timeout(10 * 60, connect=15, read=60),
|
||||
"follow_redirects": True,
|
||||
}
|
||||
|
||||
if proxy is not None:
|
||||
client = httpx.AsyncClient(proxies=httpx.Proxy(url=proxy), **client_kwargs)
|
||||
else:
|
||||
client = httpx.AsyncClient(**client_kwargs)
|
||||
|
||||
request = client.build_request(
|
||||
"GET",
|
||||
url,
|
||||
)
|
||||
try:
|
||||
response = await client.send(request, stream=True)
|
||||
except (asyncio.CancelledError, httpx.HTTPError) as e:
|
||||
await client.aclose()
|
||||
raise NotSuccess(str(e))
|
||||
|
||||
try:
|
||||
if response.status_code != 200:
|
||||
raise NotSuccess(f"Status code is {response.status_code}!")
|
||||
|
||||
content_type = response.headers.get("Content-Type")
|
||||
content_disposition = response.headers.get("Content-Disposition", "")
|
||||
|
||||
if (
|
||||
"text/html" in content_type
|
||||
and self.file_type.lower() != "html"
|
||||
and "html" not in content_disposition.lower()
|
||||
):
|
||||
raise ReceivedHTML()
|
||||
|
||||
return client, response, "application/zip" in content_type
|
||||
except (asyncio.CancelledError, httpx.HTTPError, NotSuccess, ReceivedHTML) as e:
|
||||
await response.aclose()
|
||||
await client.aclose()
|
||||
|
||||
if isinstance(e, httpx.HTTPError):
|
||||
raise NotSuccess(str(e))
|
||||
else:
|
||||
raise e
|
||||
|
||||
@classmethod
|
||||
async def _close_other_done(
|
||||
cls,
|
||||
done_tasks: set[asyncio.Task[tuple[httpx.AsyncClient, httpx.Response, bool]]],
|
||||
):
|
||||
for task in done_tasks:
|
||||
try:
|
||||
data = await task
|
||||
|
||||
await data[0].aclose()
|
||||
await data[1].aclose()
|
||||
except (
|
||||
NotSuccess,
|
||||
ReceivedHTML,
|
||||
ConvertationError,
|
||||
FileNotFoundError,
|
||||
ValueError,
|
||||
asyncio.InvalidStateError,
|
||||
asyncio.CancelledError,
|
||||
):
|
||||
continue
|
||||
|
||||
async def _wait_until_some_done(
|
||||
self, tasks: set[asyncio.Task[tuple[httpx.AsyncClient, httpx.Response, bool]]]
|
||||
) -> Optional[tuple[httpx.AsyncClient, httpx.Response, bool]]:
|
||||
tasks_ = tasks
|
||||
|
||||
while tasks_:
|
||||
done, pending = await asyncio.wait(
|
||||
tasks_, return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
|
||||
for task in done:
|
||||
try:
|
||||
data = task.result()
|
||||
|
||||
for t_task in pending:
|
||||
t_task.cancel()
|
||||
|
||||
await self._close_other_done(pending)
|
||||
await self._close_other_done(
|
||||
{ttask for ttask in done if ttask != task}
|
||||
)
|
||||
|
||||
return data
|
||||
except:
|
||||
continue
|
||||
|
||||
tasks_ = pending
|
||||
|
||||
return None
|
||||
|
||||
async def _write_response_content_to_ntf(self, temp_file, response: httpx.Response):
|
||||
async for chunk in response.aiter_bytes(2048):
|
||||
await temp_file.write(chunk)
|
||||
|
||||
await temp_file.flush()
|
||||
await temp_file.seek(0)
|
||||
|
||||
async def _unzip(self, response: httpx.Response, file_type: str) -> Optional[str]:
|
||||
async with asynctempfile.NamedTemporaryFile(delete=True) as temp_file:
|
||||
try:
|
||||
await self._write_response_content_to_ntf(temp_file, response)
|
||||
except httpx.HTTPError:
|
||||
return None
|
||||
|
||||
await temp_file.flush()
|
||||
|
||||
try:
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
process_pool_executor, unzip, temp_file.name, file_type
|
||||
)
|
||||
except (FileNotFoundError, zipfile.BadZipFile):
|
||||
return None
|
||||
|
||||
async def _download_with_converting(
|
||||
self,
|
||||
) -> tuple[httpx.AsyncClient, httpx.Response, bool]:
|
||||
tasks = set()
|
||||
|
||||
for source in env_config.FL_SOURCES:
|
||||
tasks.add(
|
||||
asyncio.create_task(self._download_from_source(source, file_type="fb2"))
|
||||
)
|
||||
|
||||
data = await self._wait_until_some_done(tasks)
|
||||
|
||||
if data is None:
|
||||
raise ValueError
|
||||
|
||||
client, response, is_zip = data
|
||||
|
||||
try:
|
||||
if is_zip:
|
||||
filename_to_convert = await self._unzip(response, "fb2")
|
||||
else:
|
||||
async with asynctempfile.NamedTemporaryFile(delete=False) as temp_file:
|
||||
await self._write_response_content_to_ntf(temp_file, response)
|
||||
filename_to_convert = temp_file.name
|
||||
finally:
|
||||
await response.aclose()
|
||||
await client.aclose()
|
||||
|
||||
if filename_to_convert is None:
|
||||
raise ValueError
|
||||
|
||||
form = {"format": self.file_type}
|
||||
files = {"file": open(filename_to_convert, "rb")}
|
||||
|
||||
converter_client = httpx.AsyncClient(timeout=5 * 60)
|
||||
converter_request = converter_client.build_request(
|
||||
"POST", env_config.CONVERTER_URL, data=form, files=files
|
||||
)
|
||||
|
||||
try:
|
||||
converter_response = await converter_client.send(
|
||||
converter_request, stream=True
|
||||
)
|
||||
except (httpx.ConnectError, httpx.ReadTimeout, asyncio.CancelledError):
|
||||
await converter_client.aclose()
|
||||
raise ConvertationError
|
||||
finally:
|
||||
await aiofiles.os.remove(filename_to_convert)
|
||||
|
||||
try:
|
||||
if response.status_code != 200:
|
||||
raise ConvertationError
|
||||
|
||||
return converter_client, converter_response, False
|
||||
except (asyncio.CancelledError, ConvertationError):
|
||||
await converter_response.aclose()
|
||||
await converter_client.aclose()
|
||||
await aiofiles.os.remove(filename_to_convert)
|
||||
raise
|
||||
|
||||
async def _get_content(self) -> Optional[tuple[AsyncIterator[bytes], str]]:
|
||||
tasks = set()
|
||||
|
||||
for source in env_config.FL_SOURCES:
|
||||
tasks.add(asyncio.create_task(self._download_from_source(source)))
|
||||
|
||||
if self.file_type.lower() in ["epub", "mobi"]:
|
||||
tasks.add(asyncio.create_task(self._download_with_converting()))
|
||||
|
||||
data = await self._wait_until_some_done(tasks)
|
||||
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
client, response, is_zip = data
|
||||
|
||||
try:
|
||||
if is_zip and self.file_type.lower() not in self.EXCLUDE_UNZIP:
|
||||
temp_filename = await self._unzip(response, self.file_type)
|
||||
else:
|
||||
async with asynctempfile.NamedTemporaryFile(delete=False) as temp_file:
|
||||
temp_filename = temp_file.name
|
||||
await self._write_response_content_to_ntf(temp_file, response)
|
||||
finally:
|
||||
await response.aclose()
|
||||
await client.aclose()
|
||||
|
||||
if temp_filename is None:
|
||||
return None
|
||||
|
||||
if self.need_zip:
|
||||
content_filename = await asyncio.get_event_loop().run_in_executor(
|
||||
process_pool_executor, zip, await self.get_filename(), temp_filename
|
||||
)
|
||||
await aiofiles.os.remove(temp_filename)
|
||||
else:
|
||||
content_filename = temp_filename
|
||||
|
||||
force_zip = is_zip and self.file_type.lower() in self.EXCLUDE_UNZIP
|
||||
|
||||
async def _content_iterator() -> AsyncIterator[bytes]:
|
||||
try:
|
||||
async with aiofiles.open(content_filename, "rb") as temp_file:
|
||||
while chunk := await temp_file.read(2048):
|
||||
yield cast(bytes, chunk)
|
||||
finally:
|
||||
await aiofiles.os.remove(content_filename)
|
||||
|
||||
return _content_iterator(), await self.get_final_filename(force_zip)
|
||||
|
||||
async def _get_book_data(self):
|
||||
return await BookLibraryClient.get_remote_book(self.source_id, self.book_id)
|
||||
|
||||
async def _download(self) -> Optional[tuple[AsyncIterator[bytes], str]]:
|
||||
await asyncio.wait([self.get_book_data_task, self.get_content_task])
|
||||
|
||||
return self.get_content_task.result()
|
||||
|
||||
@classmethod
|
||||
async def download(
|
||||
cls, remote_id: int, file_type: str, source_id: int
|
||||
) -> Optional[tuple[AsyncIterator[bytes], str]]:
|
||||
downloader = cls(remote_id, file_type, source_id)
|
||||
return await downloader._download()
|
||||
@@ -1,154 +0,0 @@
|
||||
import asyncio
|
||||
from concurrent.futures.process import ProcessPoolExecutor
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Optional
|
||||
import zipfile
|
||||
|
||||
import transliterate
|
||||
import transliterate.exceptions
|
||||
|
||||
from app.services.book_library import Book, BookAuthor
|
||||
|
||||
|
||||
process_pool_executor = ProcessPoolExecutor(2)
|
||||
|
||||
|
||||
def remove_temp_file(filename: str) -> bool:
|
||||
try:
|
||||
os.remove(filename)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def unzip(temp_zipfile: str, file_type: str) -> Optional[str]:
|
||||
zip_file = zipfile.ZipFile(temp_zipfile)
|
||||
|
||||
result = tempfile.NamedTemporaryFile(delete=False)
|
||||
|
||||
for name in zip_file.namelist():
|
||||
if file_type.lower() in name.lower() or name.lower() == "elector":
|
||||
with zip_file.open(name, "r") as internal_file:
|
||||
while chunk := internal_file.read(2048):
|
||||
result.write(chunk)
|
||||
|
||||
result.seek(0)
|
||||
return result.name
|
||||
|
||||
result.close()
|
||||
remove_temp_file(result.name)
|
||||
|
||||
raise FileNotFoundError
|
||||
|
||||
|
||||
def zip(
|
||||
filename: str,
|
||||
content_filename: str,
|
||||
) -> str:
|
||||
result = tempfile.NamedTemporaryFile(delete=False)
|
||||
|
||||
zip_file = zipfile.ZipFile(
|
||||
file=result,
|
||||
mode="w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
allowZip64=False,
|
||||
compresslevel=9,
|
||||
)
|
||||
|
||||
with open(content_filename, "rb") as content:
|
||||
with zip_file.open(filename, "w") as internal_file:
|
||||
while chunk := content.read(2048):
|
||||
internal_file.write(chunk)
|
||||
|
||||
for zfile in zip_file.filelist:
|
||||
zfile.create_system = 0
|
||||
|
||||
zip_file.close()
|
||||
result.close()
|
||||
|
||||
return result.name
|
||||
|
||||
|
||||
def get_short_name(author: BookAuthor) -> str:
|
||||
name_parts = []
|
||||
|
||||
if author.last_name:
|
||||
name_parts.append(author.last_name)
|
||||
|
||||
if author.first_name:
|
||||
name_parts.append(author.first_name[:1])
|
||||
|
||||
if author.middle_name:
|
||||
name_parts.append(author.middle_name[:1])
|
||||
|
||||
return " ".join(name_parts)
|
||||
|
||||
|
||||
def get_filename(book_id: int, book: Book, file_type: str) -> str:
|
||||
filename_parts = []
|
||||
|
||||
file_type_ = "fb2.zip" if file_type == "fb2zip" else file_type
|
||||
|
||||
if book.authors:
|
||||
filename_parts.append(
|
||||
"_".join([get_short_name(a) for a in book.authors]) + "_-_"
|
||||
)
|
||||
|
||||
if book.title.startswith(" "):
|
||||
filename_parts.append(book.title[1:])
|
||||
else:
|
||||
filename_parts.append(book.title)
|
||||
|
||||
filename = "".join(filename_parts)
|
||||
|
||||
try:
|
||||
filename = transliterate.translit(filename, reversed=True)
|
||||
except transliterate.exceptions.LanguageDetectionError:
|
||||
pass
|
||||
|
||||
for c in "(),….’!\"?»«':":
|
||||
filename = filename.replace(c, "")
|
||||
|
||||
for c, r in (
|
||||
("—", "-"),
|
||||
("/", "_"),
|
||||
("№", "N"),
|
||||
(" ", "_"),
|
||||
("–", "-"),
|
||||
("á", "a"),
|
||||
(" ", "_"),
|
||||
("'", ""),
|
||||
):
|
||||
filename = filename.replace(c, r)
|
||||
|
||||
filename = re.sub(r"[^\x00-\x7f]", r"", filename)
|
||||
|
||||
right_part = f".{book_id}.{file_type_}"
|
||||
|
||||
return filename[: 64 - len(right_part) - 1] + right_part
|
||||
|
||||
|
||||
def async_retry(*exceptions: type[Exception], times: int = 1, delay: float = 1.0):
|
||||
"""
|
||||
:param times: retry count
|
||||
:param delay: delay time
|
||||
:param default_content: set default content
|
||||
:return
|
||||
"""
|
||||
|
||||
def func_wrapper(f):
|
||||
async def wrapper(*args, **kwargs):
|
||||
for retry in range(times):
|
||||
try:
|
||||
return await f(*args, **kwargs)
|
||||
except exceptions as e:
|
||||
if retry + 1 == times:
|
||||
raise e
|
||||
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
return wrapper
|
||||
|
||||
return func_wrapper
|
||||
@@ -1,44 +0,0 @@
|
||||
from fastapi import APIRouter, Depends, Response, status
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.depends import check_token
|
||||
from app.services.book_library import BookLibraryClient
|
||||
from app.services.dowloaders_manager import DownloadersManager
|
||||
from app.services.utils import get_filename as _get_filename
|
||||
|
||||
|
||||
router = APIRouter(
|
||||
tags=["downloader"],
|
||||
dependencies=[Depends(check_token)],
|
||||
)
|
||||
|
||||
|
||||
@router.get("/download/{source_id}/{remote_id}/{file_type}")
|
||||
async def download(source_id: int, remote_id: int, file_type: str):
|
||||
downloader = await DownloadersManager.get_downloader(source_id)
|
||||
|
||||
result = await downloader.download(remote_id, file_type, source_id)
|
||||
|
||||
if result is None:
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
content, filename = result
|
||||
|
||||
return StreamingResponse(
|
||||
content, headers={"Content-Disposition": f"attachment; filename={filename}"}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/filename/{book_id}/{file_type}", response_model=str)
|
||||
async def get_filename(book_id: int, file_type: str):
|
||||
book = await BookLibraryClient.get_book(book_id)
|
||||
|
||||
return _get_filename(book.remote_id, book, file_type)
|
||||
|
||||
|
||||
healthcheck_router = APIRouter(tags=["healthcheck"])
|
||||
|
||||
|
||||
@healthcheck_router.get("/healthcheck")
|
||||
async def healthcheck():
|
||||
return "Ok!"
|
||||
45
src/config.rs
Normal file
45
src/config.rs
Normal file
@@ -0,0 +1,45 @@
|
||||
use serde::Deserialize;
|
||||
|
||||
fn get_env(env: &'static str) -> String {
|
||||
std::env::var(env).unwrap_or_else(|_| panic!("Cannot get the {} env variable", env))
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
pub struct SourceConfig {
|
||||
pub url: String,
|
||||
pub proxy: Option<String>
|
||||
}
|
||||
|
||||
pub struct Config {
|
||||
pub api_key: String,
|
||||
|
||||
pub fl_sources: Vec<SourceConfig>,
|
||||
|
||||
pub book_library_api_key: String,
|
||||
pub book_library_url: String,
|
||||
|
||||
pub converter_url: String,
|
||||
|
||||
pub sentry_dsn: String
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn load() -> Config {
|
||||
Config {
|
||||
api_key: get_env("API_KEY"),
|
||||
|
||||
fl_sources: serde_json::from_str(&get_env("FL_SOURCES")).unwrap(),
|
||||
|
||||
book_library_api_key: get_env("BOOK_LIBRARY_API_KEY"),
|
||||
book_library_url: get_env("BOOK_LIBRARY_URL"),
|
||||
|
||||
converter_url: get_env("CONVERTER_URL"),
|
||||
|
||||
sentry_dsn: get_env("SENTRY_DSN")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
pub static ref CONFIG: Config = Config::load();
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
from fastapi import FastAPI
|
||||
|
||||
from prometheus_fastapi_instrumentator import Instrumentator
|
||||
import sentry_sdk
|
||||
|
||||
from app.views import router, healthcheck_router
|
||||
from core.config import env_config
|
||||
|
||||
|
||||
sentry_sdk.init(
|
||||
env_config.SENTRY_DSN,
|
||||
)
|
||||
|
||||
|
||||
def start_app() -> FastAPI:
|
||||
app = FastAPI()
|
||||
|
||||
app.include_router(router)
|
||||
app.include_router(healthcheck_router)
|
||||
|
||||
Instrumentator().instrument(app).expose(app, include_in_schema=True)
|
||||
|
||||
return app
|
||||
@@ -1,4 +0,0 @@
|
||||
from fastapi.security import APIKeyHeader
|
||||
|
||||
|
||||
default_security = APIKeyHeader(name="Authorization")
|
||||
@@ -1,24 +0,0 @@
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseSettings, BaseModel
|
||||
|
||||
|
||||
class SourceConfig(BaseModel):
|
||||
URL: str
|
||||
PROXY: Optional[str]
|
||||
|
||||
|
||||
class EnvConfig(BaseSettings):
|
||||
API_KEY: str
|
||||
|
||||
FL_SOURCES: list[SourceConfig]
|
||||
|
||||
BOOK_LIBRARY_API_KEY: str
|
||||
BOOK_LIBRARY_URL: str
|
||||
|
||||
CONVERTER_URL: str
|
||||
|
||||
SENTRY_DSN: str
|
||||
|
||||
|
||||
env_config = EnvConfig()
|
||||
@@ -1,4 +0,0 @@
|
||||
from core.app import start_app
|
||||
|
||||
|
||||
app = start_app()
|
||||
29
src/main.rs
Normal file
29
src/main.rs
Normal file
@@ -0,0 +1,29 @@
|
||||
#[macro_use]
|
||||
extern crate lazy_static;
|
||||
|
||||
pub mod config;
|
||||
pub mod views;
|
||||
pub mod services;
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use axum::{Router, routing::get};
|
||||
use views::{download, get_filename};
|
||||
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
env_logger::init();
|
||||
|
||||
let app = Router::new()
|
||||
.route("/download/:source_id/:remote_id/:file_type", get(download))
|
||||
.route("/filename/:book_id/:file_type", get(get_filename));
|
||||
|
||||
let addr = SocketAddr::from(([0, 0, 0, 0], 8080));
|
||||
|
||||
log::info!("Start webserver...");
|
||||
axum::Server::bind(&addr)
|
||||
.serve(app.into_make_service())
|
||||
.await
|
||||
.unwrap();
|
||||
log::info!("Webserver shutdown...")
|
||||
}
|
||||
58
src/services/book_library/mod.rs
Normal file
58
src/services/book_library/mod.rs
Normal file
@@ -0,0 +1,58 @@
|
||||
pub mod types;
|
||||
|
||||
use serde::de::DeserializeOwned;
|
||||
|
||||
use crate::config;
|
||||
|
||||
async fn _make_request<T>(
|
||||
url: &str,
|
||||
params: Vec<(&str, String)>,
|
||||
) -> Result<T, Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let formated_url = format!("{}{}", &config::CONFIG.book_library_url, url);
|
||||
|
||||
log::debug!("{}", formated_url);
|
||||
|
||||
let response = client
|
||||
.get(formated_url)
|
||||
.query(¶ms)
|
||||
.header("Authorization", &config::CONFIG.book_library_api_key)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let response = match response {
|
||||
Ok(v) => v,
|
||||
Err(err) => return Err(Box::new(err)),
|
||||
};
|
||||
|
||||
let response = match response.error_for_status() {
|
||||
Ok(v) => v,
|
||||
Err(err) => return Err(Box::new(err)),
|
||||
};
|
||||
|
||||
match response.json::<T>().await {
|
||||
Ok(v) => Ok(v),
|
||||
Err(err) => Err(Box::new(err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_sources() -> Result<types::Source, Box<dyn std::error::Error + Send + Sync>> {
|
||||
_make_request("/api/v1/sources", vec![]).await
|
||||
}
|
||||
|
||||
pub async fn get_book(
|
||||
book_id: u32,
|
||||
) -> Result<types::Book, Box<dyn std::error::Error + Send + Sync>> {
|
||||
_make_request(format!("/api/v1/books/{book_id}").as_str(), vec![]).await
|
||||
}
|
||||
|
||||
pub async fn get_remote_book(
|
||||
source_id: u32,
|
||||
book_id: u32,
|
||||
) -> Result<types::Book, Box<dyn std::error::Error + Send + Sync>> {
|
||||
_make_request(format!("/api/v1/books/remote/{source_id}/{book_id}").as_ref(), vec![]).await
|
||||
}
|
||||
27
src/services/book_library/types.rs
Normal file
27
src/services/book_library/types.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
use serde::Deserialize;
|
||||
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Source {
|
||||
// id: u32,
|
||||
// name: String
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct BookAuthor {
|
||||
pub id: u32,
|
||||
pub first_name: String,
|
||||
pub last_name: String,
|
||||
pub middle_name: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Book {
|
||||
pub id: u32,
|
||||
pub remote_id: u32,
|
||||
pub title: String,
|
||||
pub lang: String,
|
||||
pub file_type: String,
|
||||
pub uploaded: String,
|
||||
pub authors: Vec<BookAuthor>,
|
||||
}
|
||||
38
src/services/covert.rs
Normal file
38
src/services/covert.rs
Normal file
@@ -0,0 +1,38 @@
|
||||
use reqwest::{Response, multipart::{Form, Part}, Body};
|
||||
use tempfile::SpooledTempFile;
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use crate::config;
|
||||
|
||||
use super::downloader::types::SpooledTempAsyncRead;
|
||||
|
||||
pub async fn convert_file(file: SpooledTempFile, file_type: String) -> Option<Response> {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let async_file = Body::wrap_stream(ReaderStream::new(SpooledTempAsyncRead::new(file)));
|
||||
let file_part = Part::stream(async_file).file_name("file");
|
||||
let form = Form::new()
|
||||
.text("format", file_type.clone())
|
||||
.part("file", file_part);
|
||||
|
||||
let response = client
|
||||
.post(&config::CONFIG.converter_url)
|
||||
.multipart(form)
|
||||
.send().await;
|
||||
|
||||
let response = match response {
|
||||
Ok(v) => v,
|
||||
Err(_) => {
|
||||
return None
|
||||
},
|
||||
};
|
||||
|
||||
let response = match response.error_for_status() {
|
||||
Ok(v) => v,
|
||||
Err(_) => {
|
||||
return None
|
||||
},
|
||||
};
|
||||
|
||||
Some(response)
|
||||
}
|
||||
202
src/services/downloader/mod.rs
Normal file
202
src/services/downloader/mod.rs
Normal file
@@ -0,0 +1,202 @@
|
||||
pub mod types;
|
||||
pub mod utils;
|
||||
pub mod zip;
|
||||
|
||||
use reqwest::Response;
|
||||
|
||||
use crate::config;
|
||||
|
||||
use self::types::{DownloadResult, Data, SpooledTempAsyncRead};
|
||||
use self::utils::response_to_tempfile;
|
||||
use self::zip::{unzip, zip};
|
||||
|
||||
use super::book_library::types::Book;
|
||||
use super::covert::convert_file;
|
||||
use super::{book_library::get_remote_book, filename_getter::get_filename_by_book};
|
||||
|
||||
use futures::stream::FuturesUnordered;
|
||||
use futures::StreamExt;
|
||||
|
||||
pub async fn download<'a>(
|
||||
book_id: &'a u32,
|
||||
book_file_type: &'a str,
|
||||
source_config: &'a config::SourceConfig,
|
||||
) -> Option<(Response, bool)> {
|
||||
let basic_url = &source_config.url;
|
||||
let proxy = &source_config.proxy;
|
||||
|
||||
let url = if book_file_type == "fb2" || book_file_type == "epub" || book_file_type == "mobi" {
|
||||
format!("{basic_url}/b/{book_id}/{book_file_type}")
|
||||
} else {
|
||||
format!("{basic_url}/b/{book_id}/download")
|
||||
};
|
||||
|
||||
let client = match proxy {
|
||||
Some(v) => {
|
||||
let proxy_data = reqwest::Proxy::http(v);
|
||||
reqwest::Client::builder()
|
||||
.proxy(proxy_data.unwrap())
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
None => reqwest::Client::new(),
|
||||
};
|
||||
|
||||
let response = client.get(url).send().await;
|
||||
|
||||
let response = match response {
|
||||
Ok(v) => v,
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let response = match response.error_for_status() {
|
||||
Ok(v) => v,
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let headers = response.headers();
|
||||
let content_type = match headers.get("Content-Type") {
|
||||
Some(v) => v.to_str().unwrap(),
|
||||
None => "",
|
||||
};
|
||||
|
||||
if book_file_type.to_lowercase() == "html" && content_type.contains("text/html") {
|
||||
return Some((response, false));
|
||||
}
|
||||
|
||||
if content_type.contains("text/html")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let is_zip = content_type.contains("application/zip");
|
||||
|
||||
Some((response, is_zip))
|
||||
}
|
||||
|
||||
pub async fn download_chain<'a>(
|
||||
book: &'a Book,
|
||||
file_type: &'a str,
|
||||
source_config: &'a config::SourceConfig,
|
||||
converting: bool
|
||||
) -> Option<DownloadResult> {
|
||||
let final_need_zip = file_type == "fb2zip";
|
||||
|
||||
let file_type_ = if converting {
|
||||
&book.file_type
|
||||
} else {
|
||||
file_type
|
||||
};
|
||||
|
||||
let (mut response, is_zip) = match download(&book.remote_id, file_type_, source_config).await {
|
||||
Some(v) => v,
|
||||
None => return None,
|
||||
};
|
||||
|
||||
if is_zip && book.file_type.to_lowercase() == "html" {
|
||||
let filename = get_filename_by_book(book, file_type, true);
|
||||
return Some(DownloadResult::new(Data::Response(response), filename));
|
||||
}
|
||||
|
||||
if !is_zip && !final_need_zip && !converting {
|
||||
let filename = get_filename_by_book(book, &book.file_type, false);
|
||||
return Some(DownloadResult::new(Data::Response(response), filename));
|
||||
};
|
||||
|
||||
let unziped_temp_file = {
|
||||
let temp_file_to_unzip_result = response_to_tempfile(&mut response).await;
|
||||
let temp_file_to_unzip = match temp_file_to_unzip_result {
|
||||
Some(v) => v,
|
||||
None => return None,
|
||||
};
|
||||
|
||||
match unzip(temp_file_to_unzip, "fb2") {
|
||||
Some(v) => v,
|
||||
None => return None,
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
let mut clean_file = if converting {
|
||||
match convert_file(unziped_temp_file, file_type.to_string()).await {
|
||||
Some(mut response) => {
|
||||
match response_to_tempfile(&mut response).await {
|
||||
Some(v) => v,
|
||||
None => return None,
|
||||
}
|
||||
},
|
||||
None => return None,
|
||||
}
|
||||
} else {
|
||||
unziped_temp_file
|
||||
};
|
||||
|
||||
if !final_need_zip {
|
||||
let t = SpooledTempAsyncRead::new(clean_file);
|
||||
let filename = get_filename_by_book(book, file_type, false);
|
||||
return Some(DownloadResult::new(Data::SpooledTempAsyncRead(t), filename));
|
||||
};
|
||||
|
||||
let t_file_type = if file_type == "fb2zip" { "fb2" } else { file_type };
|
||||
let filename = get_filename_by_book(book, t_file_type, false);
|
||||
match zip(&mut clean_file, filename.as_str()) {
|
||||
Some(v) => {
|
||||
let t = SpooledTempAsyncRead::new(v);
|
||||
let filename = get_filename_by_book(book, file_type, true);
|
||||
Some(DownloadResult::new(Data::SpooledTempAsyncRead(t), filename))
|
||||
},
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn start_download_futures(
|
||||
book: &Book,
|
||||
file_type: &str,
|
||||
) -> Option<DownloadResult> {
|
||||
let mut futures = FuturesUnordered::new();
|
||||
|
||||
for source_config in &config::CONFIG.fl_sources {
|
||||
futures.push(download_chain(
|
||||
book,
|
||||
file_type,
|
||||
source_config,
|
||||
false
|
||||
));
|
||||
|
||||
if file_type == "epub" || file_type == "fb2" {
|
||||
futures.push(download_chain(
|
||||
book,
|
||||
file_type.clone(),
|
||||
source_config,
|
||||
true
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
while let Some(result) = futures.next().await {
|
||||
match result {
|
||||
Some(v) => return Some(v),
|
||||
None => (),
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn book_download(
|
||||
source_id: u32,
|
||||
remote_id: u32,
|
||||
file_type: &str,
|
||||
) -> Result<Option<(DownloadResult, String)>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let book = match get_remote_book(source_id, remote_id).await {
|
||||
Ok(v) => v,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let filename = get_filename_by_book(&book, file_type, false);
|
||||
|
||||
match start_download_futures(&book, file_type).await {
|
||||
Some(v) => Ok(Some((v, filename))),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
64
src/services/downloader/types.rs
Normal file
64
src/services/downloader/types.rs
Normal file
@@ -0,0 +1,64 @@
|
||||
use reqwest::Response;
|
||||
use std::pin::Pin;
|
||||
use tempfile::SpooledTempFile;
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
use futures::TryStreamExt;
|
||||
use tokio_util::compat::FuturesAsyncReadCompatExt;
|
||||
|
||||
pub enum Data {
|
||||
Response(Response),
|
||||
SpooledTempAsyncRead(SpooledTempAsyncRead),
|
||||
}
|
||||
|
||||
pub struct DownloadResult {
|
||||
pub data: Data,
|
||||
pub filename: String,
|
||||
}
|
||||
|
||||
pub fn get_response_async_read(it: Response) -> impl AsyncRead {
|
||||
it.bytes_stream()
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))
|
||||
.into_async_read()
|
||||
.compat()
|
||||
}
|
||||
|
||||
impl DownloadResult {
|
||||
pub fn new(data: Data, filename: String) -> Self {
|
||||
Self { data, filename }
|
||||
}
|
||||
|
||||
pub fn get_async_read(self) -> Pin<Box<dyn AsyncRead + Send>> {
|
||||
match self.data {
|
||||
Data::Response(v) => Box::pin(get_response_async_read(v)),
|
||||
Data::SpooledTempAsyncRead(v) => Box::pin(v),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SpooledTempAsyncRead {
|
||||
file: SpooledTempFile,
|
||||
}
|
||||
|
||||
impl SpooledTempAsyncRead {
|
||||
pub fn new(file: SpooledTempFile) -> Self {
|
||||
Self { file }
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for SpooledTempAsyncRead {
|
||||
fn poll_read(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
_cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
let result = match std::io::Read::read(&mut self.get_mut().file, buf.initialize_unfilled()) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return std::task::Poll::Ready(Err(err)),
|
||||
};
|
||||
|
||||
buf.set_filled(result);
|
||||
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
36
src/services/downloader/utils.rs
Normal file
36
src/services/downloader/utils.rs
Normal file
@@ -0,0 +1,36 @@
|
||||
use reqwest::Response;
|
||||
use tempfile::SpooledTempFile;
|
||||
use bytes::Buf;
|
||||
|
||||
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
|
||||
|
||||
pub async fn response_to_tempfile(res: &mut Response) -> Option<SpooledTempFile> {
|
||||
let mut tmp_file = tempfile::spooled_tempfile(5 * 1024 * 1024);
|
||||
|
||||
{
|
||||
loop {
|
||||
let chunk = res.chunk().await;
|
||||
|
||||
let result = match chunk {
|
||||
Ok(v) => v,
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let data = match result {
|
||||
Some(v) => v,
|
||||
None => break,
|
||||
};
|
||||
|
||||
match tmp_file.write(data.chunk()) {
|
||||
Ok(_) => (),
|
||||
Err(_) => return None,
|
||||
}
|
||||
}
|
||||
|
||||
tmp_file.seek(SeekFrom::Start(0)).unwrap();
|
||||
}
|
||||
|
||||
Some(tmp_file)
|
||||
}
|
||||
60
src/services/downloader/zip.rs
Normal file
60
src/services/downloader/zip.rs
Normal file
@@ -0,0 +1,60 @@
|
||||
use std::io::{Seek, SeekFrom};
|
||||
|
||||
use tempfile::SpooledTempFile;
|
||||
use zip::write::FileOptions;
|
||||
|
||||
|
||||
pub fn unzip(tmp_file: SpooledTempFile, file_type: &str) -> Option<SpooledTempFile> {
|
||||
let mut archive = zip::ZipArchive::new(tmp_file).unwrap();
|
||||
|
||||
let file_type_lower = file_type.to_lowercase();
|
||||
|
||||
for i in 0..archive.len() {
|
||||
let mut file = archive.by_index(i).unwrap();
|
||||
let filename = file.name();
|
||||
|
||||
if filename.contains(&file_type_lower) || file.name().to_lowercase() == "elector" {
|
||||
let mut output_file = tempfile::spooled_tempfile(5 * 1024 * 1024);
|
||||
|
||||
match std::io::copy(&mut file, &mut output_file) {
|
||||
Ok(_) => (),
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
output_file.seek(SeekFrom::Start(0)).unwrap();
|
||||
|
||||
return Some(output_file);
|
||||
}
|
||||
}
|
||||
|
||||
return None;
|
||||
}
|
||||
|
||||
pub fn zip(tmp_file: &mut SpooledTempFile, filename: &str) -> Option<SpooledTempFile> {
|
||||
let output_file = tempfile::spooled_tempfile(5 * 1024 * 1024);
|
||||
let mut archive = zip::ZipWriter::new(output_file);
|
||||
|
||||
let options = FileOptions::default()
|
||||
.compression_level(Some(9))
|
||||
.compression_method(zip::CompressionMethod::Deflated)
|
||||
.unix_permissions(0o755);
|
||||
|
||||
match archive.start_file(filename, options) {
|
||||
Ok(_) => (),
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
match std::io::copy(tmp_file, &mut archive) {
|
||||
Ok(_) => (),
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let mut archive_result = match archive.finish() {
|
||||
Ok(v) => v,
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
archive_result.seek(SeekFrom::Start(0)).unwrap();
|
||||
|
||||
Some(archive_result)
|
||||
}
|
||||
77
src/services/filename_getter.rs
Normal file
77
src/services/filename_getter.rs
Normal file
@@ -0,0 +1,77 @@
|
||||
use translit::{gost779b_ru, CharsMapping, Transliterator};
|
||||
|
||||
use super::book_library::types::{BookAuthor, Book};
|
||||
|
||||
pub fn get_author_short_name(author: BookAuthor) -> String {
|
||||
let mut parts: Vec<String> = vec![];
|
||||
|
||||
if author.last_name.len() != 0 {
|
||||
parts.push(author.last_name);
|
||||
}
|
||||
|
||||
if author.first_name.len() != 0 {
|
||||
let first_char = author.first_name.chars().next().unwrap();
|
||||
parts.push(first_char.to_string());
|
||||
}
|
||||
|
||||
if author.middle_name.len() != 0 {
|
||||
let first_char = author.middle_name.chars().next().unwrap();
|
||||
parts.push(first_char.to_string());
|
||||
}
|
||||
|
||||
parts.join(" ")
|
||||
}
|
||||
|
||||
pub fn get_filename_by_book(book: &Book, file_type: &str, force_zip: bool) -> String {
|
||||
let book_id = book.remote_id;
|
||||
let mut filename_parts: Vec<String> = vec![];
|
||||
|
||||
let file_type_: String = if let "fb2zip" = file_type {
|
||||
"fb2.zip".to_string()
|
||||
} else if force_zip {
|
||||
format!("{file_type}.zip")
|
||||
} else {
|
||||
file_type.to_string()
|
||||
};
|
||||
|
||||
filename_parts.push(
|
||||
book.authors
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|author| get_author_short_name(author))
|
||||
.collect::<Vec<String>>()
|
||||
.join("_-_"),
|
||||
);
|
||||
filename_parts.push(book.title.trim().to_string());
|
||||
|
||||
let transliterator = Transliterator::new(gost779b_ru());
|
||||
let mut filename_without_type = transliterator.convert(&filename_parts.join(""), false);
|
||||
|
||||
for char in "(),….’!\"?»«':".get(..) {
|
||||
filename_without_type = filename_without_type.replace(char, "");
|
||||
}
|
||||
|
||||
let replace_char_map: CharsMapping = [
|
||||
("—", "-"),
|
||||
("/", "_"),
|
||||
("№", "N"),
|
||||
(" ", "_"),
|
||||
("–", "-"),
|
||||
("á", "a"),
|
||||
(" ", "_"),
|
||||
("'", ""),
|
||||
("`", ""),
|
||||
]
|
||||
.iter()
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
let replace_transliterator = Transliterator::new(replace_char_map);
|
||||
let normal_filename = replace_transliterator.convert(&filename_without_type, false);
|
||||
|
||||
let right_part = format!(".{book_id}.{file_type_}");
|
||||
let normal_filename_slice = std::cmp::min(64 - right_part.len() - 1, normal_filename.len());
|
||||
let left_part = normal_filename.get(..normal_filename_slice).unwrap();
|
||||
|
||||
format!("{left_part}{right_part}")
|
||||
}
|
||||
4
src/services/mod.rs
Normal file
4
src/services/mod.rs
Normal file
@@ -0,0 +1,4 @@
|
||||
pub mod book_library;
|
||||
pub mod filename_getter;
|
||||
pub mod downloader;
|
||||
pub mod covert;
|
||||
68
src/views.rs
Normal file
68
src/views.rs
Normal file
@@ -0,0 +1,68 @@
|
||||
use axum::{
|
||||
body::StreamBody,
|
||||
extract::Path,
|
||||
http::{header, HeaderMap, StatusCode},
|
||||
response::{IntoResponse, AppendHeaders},
|
||||
};
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use crate::{config, services::{book_library::get_book, filename_getter::get_filename_by_book, downloader::book_download}};
|
||||
|
||||
pub async fn download(
|
||||
Path((source_id, remote_id, file_type)): Path<(u32, u32, String)>,
|
||||
headers: HeaderMap
|
||||
) -> impl IntoResponse {
|
||||
let config_api_key = config::CONFIG.api_key.clone();
|
||||
|
||||
let api_key = match headers.get("Authorization") {
|
||||
Some(v) => v,
|
||||
None => return Err((StatusCode::FORBIDDEN, "No api-key!".to_string())),
|
||||
};
|
||||
|
||||
if config_api_key != api_key.to_str().unwrap() {
|
||||
return Err((StatusCode::FORBIDDEN, "Wrong api-key!".to_string()))
|
||||
}
|
||||
|
||||
let download_result = match book_download(source_id, remote_id, file_type.as_str()).await {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Err((StatusCode::NO_CONTENT, "Can't download!".to_string())),
|
||||
};
|
||||
|
||||
let (data, filename) = match download_result {
|
||||
Some(v) => v,
|
||||
None => return Err((StatusCode::NO_CONTENT, "Can't download!".to_string())),
|
||||
};
|
||||
|
||||
let reader = data.get_async_read();
|
||||
let stream = ReaderStream::new(reader);
|
||||
let body = StreamBody::new(stream);
|
||||
|
||||
let headers = AppendHeaders([
|
||||
(header::CONTENT_DISPOSITION, format!("attachment; filename={filename}"))
|
||||
]);
|
||||
|
||||
Ok((headers, body))
|
||||
}
|
||||
|
||||
pub async fn get_filename(
|
||||
Path((book_id, file_type)): Path<(u32, String)>,
|
||||
headers: HeaderMap
|
||||
) -> (StatusCode, String){
|
||||
let config_api_key = config::CONFIG.api_key.clone();
|
||||
|
||||
let api_key = match headers.get("Authorization") {
|
||||
Some(v) => v,
|
||||
None => return (StatusCode::FORBIDDEN, "No api-key!".to_string()),
|
||||
};
|
||||
|
||||
if config_api_key != api_key.to_str().unwrap() {
|
||||
return (StatusCode::FORBIDDEN, "Wrong api-key!".to_string())
|
||||
}
|
||||
|
||||
let filename = match get_book(book_id).await {
|
||||
Ok(book) => get_filename_by_book(&book, file_type.as_str(), false),
|
||||
Err(_) => return (StatusCode::BAD_REQUEST, "Book not found!".to_string()),
|
||||
};
|
||||
|
||||
(StatusCode::OK, filename)
|
||||
}
|
||||
Reference in New Issue
Block a user