flibusta-apps/telegram_files_cache_server
1
0
Fork 0
mirror of https://github.com/flibusta-apps/telegram_files_cache_server.git synced 2025-12-06 14:45:36 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use taskiq

Browse Source
This commit is contained in:
Bulat Kurbanov
2023-05-20 20:16:00 +02:00
parent 67478536e2
commit 7f6837b8f5
11 changed files with 241 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
poetry.lock generated
View File

@@ -2,14 +2,14 @@
[[package]] [[package]]
name = "alembic" name = "alembic"
version = "1.10.4" version = "1.11.1"
description = "A database migration tool for SQLAlchemy." description = "A database migration tool for SQLAlchemy."
category = "main" category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
{file = "alembic-1.10.4-py3-none-any.whl", hash = "sha256:43942c3d4bf2620c466b91c0f4fca136fe51ae972394a0cc8b90810d664e4f5c"}, {file = "alembic-1.11.1-py3-none-any.whl", hash = "sha256:dc871798a601fab38332e38d6ddb38d5e734f60034baeb8e2db5b642fccd8ab8"},
{file = "alembic-1.10.4.tar.gz", hash = "sha256:295b54bbb92c4008ab6a7dcd1e227e668416d6f84b98b3c4446a2bc6214a556b"}, {file = "alembic-1.11.1.tar.gz", hash = "sha256:6a810a6b012c88b33458fceb869aef09ac75d6ace5291915ba7fae44de372c01"},
] ]
[package.dependencies] [package.dependencies]
@@ -41,26 +41,6 @@ doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"]
trio = ["trio (>=0.16,<0.22)"] trio = ["trio (>=0.16,<0.22)"]
[[package]]
name = "arq"
version = "0.25.0"
description = "Job queues in python with asyncio and redis"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "arq-0.25.0-py3-none-any.whl", hash = "sha256:db072d0f39c0bc06b436db67ae1f315c81abc1527563b828955670531815290b"},
{file = "arq-0.25.0.tar.gz", hash = "sha256:d176ebadfba920c039dc578814d19b7814d67fa15f82fdccccaedb4330d65dae"},
]
[package.dependencies]
click = ">=8.0"
redis = {version = ">=4.2.0", extras = ["hiredis"]}
typing-extensions = ">=4.1.0"
[package.extras]
watch = ["watchfiles (>=0.16)"]
[[package]] [[package]]
name = "async-timeout" name = "async-timeout"
version = "4.0.2" version = "4.0.2"
@@ -214,19 +194,19 @@ files = [
[[package]] [[package]]
name = "fastapi" name = "fastapi"
version = "0.95.1" version = "0.95.2"
description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
category = "main" category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
{file = "fastapi-0.95.1-py3-none-any.whl", hash = "sha256:a870d443e5405982e1667dfe372663abf10754f246866056336d7f01c21dab07"}, {file = "fastapi-0.95.2-py3-none-any.whl", hash = "sha256:d374dbc4ef2ad9b803899bd3360d34c534adc574546e25314ab72c0c4411749f"},
{file = "fastapi-0.95.1.tar.gz", hash = "sha256:9569f0a381f8a457ec479d90fa01005cfddaae07546eb1f3fa035bc4797ae7d5"}, {file = "fastapi-0.95.2.tar.gz", hash = "sha256:4d9d3e8c71c73f11874bcf5e33626258d143252e329a01002f767306c64fb982"},
] ]
[package.dependencies] [package.dependencies]
pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0" pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0"
starlette = ">=0.26.1,<0.27.0" starlette = ">=0.27.0,<0.28.0"
[package.extras] [package.extras]
all = ["email-validator (>=1.1.1)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] all = ["email-validator (>=1.1.1)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"]
@@ -513,14 +493,14 @@ test = ["Cython (>=0.29.24,<0.30.0)"]
[[package]] [[package]]
name = "httpx" name = "httpx"
version = "0.24.0" version = "0.24.1"
description = "The next generation HTTP client." description = "The next generation HTTP client."
category = "main" category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
{file = "httpx-0.24.0-py3-none-any.whl", hash = "sha256:447556b50c1921c351ea54b4fe79d91b724ed2b027462ab9a329465d147d5a4e"}, {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"},
{file = "httpx-0.24.0.tar.gz", hash = "sha256:507d676fc3e26110d41df7d35ebd8b3b8585052450f4097401c9be59d928c63e"}, {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"},
] ]
[package.dependencies] [package.dependencies]
@@ -562,6 +542,26 @@ files = [
{file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"},
] ]
[[package]]
name = "importlib-metadata"
version = "6.6.0"
description = "Read metadata from Python packages"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"},
{file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"},
]
[package.dependencies]
zipp = ">=0.5"
[package.extras]
docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
perf = ["ipython"]
testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"]
[[package]] [[package]]
name = "mako" name = "mako"
version = "1.2.4" version = "1.2.4"
@@ -953,6 +953,17 @@ files = [
{file = "psycopg2_binary-2.9.6-cp39-cp39-win_amd64.whl", hash = "sha256:f6a88f384335bb27812293fdb11ac6aee2ca3f51d3c7820fe03de0a304ab6249"}, {file = "psycopg2_binary-2.9.6-cp39-cp39-win_amd64.whl", hash = "sha256:f6a88f384335bb27812293fdb11ac6aee2ca3f51d3c7820fe03de0a304ab6249"},
] ]
[[package]]
name = "pycron"
version = "3.0.0"
description = "Simple cron-like parser, which determines if current datetime matches conditions."
category = "main"
optional = false
python-versions = ">=3.5"
files = [
{file = "pycron-3.0.0.tar.gz", hash = "sha256:b916044e3e8253d5409c68df3ac64a3472c4e608dab92f40e8f595e5d3acb3de"},
]
[[package]] [[package]]
name = "pydantic" name = "pydantic"
version = "1.10.4" version = "1.10.4"
@@ -1093,14 +1104,14 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"
[[package]] [[package]]
name = "sentry-sdk" name = "sentry-sdk"
version = "1.22.2" version = "1.23.1"
description = "Python client for Sentry (https://sentry.io)" description = "Python client for Sentry (https://sentry.io)"
category = "main" category = "main"
optional = false optional = false
python-versions = "*" python-versions = "*"
files = [ files = [
{file = "sentry-sdk-1.22.2.tar.gz", hash = "sha256:5932c092c6e6035584eb74d77064e4bce3b7935dfc4a331349719a40db265840"}, {file = "sentry-sdk-1.23.1.tar.gz", hash = "sha256:0300fbe7a07b3865b3885929fb863a68ff01f59e3bcfb4e7953d0bf7fd19c67f"},
{file = "sentry_sdk-1.22.2-py2.py3-none-any.whl", hash = "sha256:cf89a5063ef84278d186aceaed6fb595bfe67d099298e537634a323664265669"}, {file = "sentry_sdk-1.23.1-py2.py3-none-any.whl", hash = "sha256:a884e2478e0b055776ea2b9234d5de9339b4bae0b3a5e74ae43d131db8ded27e"},
] ]
[package.dependencies] [package.dependencies]
@@ -1117,10 +1128,11 @@ chalice = ["chalice (>=1.16.0)"]
django = ["django (>=1.8)"] django = ["django (>=1.8)"]
falcon = ["falcon (>=1.4)"] falcon = ["falcon (>=1.4)"]
fastapi = ["fastapi (>=0.79.0)"] fastapi = ["fastapi (>=0.79.0)"]
flask = ["blinker (>=1.1)", "flask (>=0.11)"] flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"]
grpcio = ["grpcio (>=1.21.1)"] grpcio = ["grpcio (>=1.21.1)"]
httpx = ["httpx (>=0.16.0)"] httpx = ["httpx (>=0.16.0)"]
huey = ["huey (>=2)"] huey = ["huey (>=2)"]
loguru = ["loguru (>=0.5)"]
opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"]
pure-eval = ["asttokens", "executing", "pure-eval"] pure-eval = ["asttokens", "executing", "pure-eval"]
pymongo = ["pymongo (>=3.1)"] pymongo = ["pymongo (>=3.1)"]
@@ -1239,14 +1251,14 @@ sqlcipher = ["sqlcipher3-binary"]
[[package]] [[package]]
name = "starlette" name = "starlette"
version = "0.26.1" version = "0.27.0"
description = "The little ASGI library that shines." description = "The little ASGI library that shines."
category = "main" category = "main"
optional = false optional = false
python-versions = ">=3.7" python-versions = ">=3.7"
files = [ files = [
{file = "starlette-0.26.1-py3-none-any.whl", hash = "sha256:e87fce5d7cbdde34b76f0ac69013fd9d190d581d80681493016666e6f96c6d5e"}, {file = "starlette-0.27.0-py3-none-any.whl", hash = "sha256:918416370e846586541235ccd38a474c08b80443ed31c578a418e2209b3eef91"},
{file = "starlette-0.26.1.tar.gz", hash = "sha256:41da799057ea8620e4667a3e69a5b1923ebd32b1819c8fa75634bbe8d8bea9bd"}, {file = "starlette-0.27.0.tar.gz", hash = "sha256:6a6b0d042acb8d469a01eba54e9cda6cbd24ac602c4cd016723117d6a7e73b75"},
] ]
[package.dependencies] [package.dependencies]
@@ -1255,6 +1267,75 @@ anyio = ">=3.4.0,<5"
[package.extras] [package.extras]
full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"]
[[package]]
name = "taskiq"
version = "0.4.3"
description = "Distributed task queue with full async support"
category = "main"
optional = false
python-versions = ">=3.7,<4.0"
files = [
{file = "taskiq-0.4.3-py3-none-any.whl", hash = "sha256:1a9305aae8df5ebd5d2388382da74ac96bb0a9434eeb97ad1dd6e9d9ab9d03aa"},
{file = "taskiq-0.4.3.tar.gz", hash = "sha256:525644179f587cdacbc356f391639a1d6718ac9625251b36f80d45ee5e64f107"},
]
[package.dependencies]
importlib-metadata = "*"
pycron = ">=3.0.0,<4.0.0"
pydantic = ">=1.6.2,<2.0.0"
taskiq_dependencies = ">=1,<2"
typing-extensions = ">=3.10.0.0"
[package.extras]
metrics = ["prometheus_client (>=0,<1)"]
reload = ["gitignore-parser (>=0,<1)", "watchdog (>=2.1.9,<3.0.0)"]
uv = ["uvloop (>=0.16.0,<1)"]
zmq = ["pyzmq (>=23.2.0,<24.0.0)"]
[[package]]
name = "taskiq-dependencies"
version = "1.2.3"
description = "FastAPI like dependency injection implementation"
category = "main"
optional = false
python-versions = ">=3.7,<4.0"
files = [
{file = "taskiq_dependencies-1.2.3-py3-none-any.whl", hash = "sha256:504792d7e505f0db7b4a0b57c20d5b6fd6f1598103eb2c5b8cbbb4d07e0c1f29"},
{file = "taskiq_dependencies-1.2.3.tar.gz", hash = "sha256:22157aa538c3650ebdc8bdbc4618047d9bf7c29ce8697d7151def79e834ac2d1"},
]
[[package]]
name = "taskiq-fastapi"
version = "0.1.2"
description = "FastAPI integration for taskiq"
category = "main"
optional = false
python-versions = ">=3.7,<4.0"
files = [
{file = "taskiq_fastapi-0.1.2-py3-none-any.whl", hash = "sha256:dbe4728ff6ea5675c76ed81e17bfe2da249ff1e8387e4915111108782b7a5b36"},
{file = "taskiq_fastapi-0.1.2.tar.gz", hash = "sha256:7edff4a7e20dcf92fa79703a5c474f63b76863a9eb2e436353acb4a5f45a742e"},
]
[package.dependencies]
fastapi = "*"
taskiq = ">=0.3.1,<1"
[[package]]
name = "taskiq-redis"
version = "0.3.1"
description = "Redis integration for taskiq"
category = "main"
optional = false
python-versions = ">=3.7,<4.0"
files = [
{file = "taskiq_redis-0.3.1-py3-none-any.whl", hash = "sha256:437174b671a992a469fd0866cab2ed6317b9672cbc2a1a804d524f857ade2e6b"},
{file = "taskiq_redis-0.3.1.tar.gz", hash = "sha256:4d33e9a8aa247cce194f38466d7f3dbe4f5530ca0bbb9a46d9a5bb015a85a3c4"},
]
[package.dependencies]
redis = ">=4.2.0,<5.0.0"
taskiq = ">=0,<1"
[[package]] [[package]]
name = "typing-extensions" name = "typing-extensions"
version = "4.5.0" version = "4.5.0"
@@ -1491,7 +1572,23 @@ files = [
{file = "websockets-11.0.2.tar.gz", hash = "sha256:b1a69701eb98ed83dd099de4a686dc892c413d974fa31602bc00aca7cb988ac9"}, {file = "websockets-11.0.2.tar.gz", hash = "sha256:b1a69701eb98ed83dd099de4a686dc892c413d974fa31602bc00aca7cb988ac9"},
] ]
[[package]]
name = "zipp"
version = "3.15.0"
description = "Backport of pathlib-compatible object wrapper for zip files"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"},
{file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"},
]
[package.extras]
docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"]
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = "^3.11" python-versions = "^3.11"
content-hash = "e63feebc3e80b7d7e6c5cdaaadb0d32b59d61a970dd7698d13fb53ee6a808936" content-hash = "279cb2082f3a3784c6bc0776b61b2cfba07fd2a3c89651871e45d1b0147e9329"

14
pyproject.toml
View File

@@ -6,19 +6,21 @@ authors = ["Kurbanov Bulat <kurbanovbul@gmail.com>"]
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.11" python = "^3.11"
fastapi = "^0.95.1" fastapi = "^0.95.2"
httpx = "^0.24.0" httpx = "^0.24.1"
alembic = "^1.10.4" alembic = "^1.11.1"
uvicorn = {extras = ["standard"], version = "^0.22.0"} uvicorn = {extras = ["standard"], version = "^0.22.0"}
arq = "^0.25.0"
prometheus-fastapi-instrumentator = "^6.0.0" prometheus-fastapi-instrumentator = "^6.0.0"
uvloop = "^0.17.0" uvloop = "^0.17.0"
orjson = "^3.8.12" orjson = "^3.8.12"
sentry-sdk = "^1.22.2" sentry-sdk = "^1.23.1"
ormar = {extras = ["postgresql"], version = "^0.12.1"} ormar = {extras = ["postgresql"], version = "^0.12.1"}
pydantic = "^1.10.4" pydantic = "^1.10.4"
redis = {extras = ["hiredis"], version = "^4.5.5"} redis = {extras = ["hiredis"], version = "^4.5.5"}
msgpack = "^1.0.5" msgpack = "^1.0.5"
taskiq = "^0.4.3"
taskiq-redis = "^0.3.1"
taskiq-fastapi = "^0.1.2"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
pre-commit = "^2.21.0" pre-commit = "^2.21.0"
@@ -55,7 +57,7 @@ exclude = [
] ]
[tool.ruff.flake8-bugbear] [tool.ruff.flake8-bugbear]
extend-immutable-calls = ["fastapi.File", "fastapi.Form", "fastapi.Security"] extend-immutable-calls = ["fastapi.File", "fastapi.Form", "fastapi.Security", "taskiq.TaskiqDepends"]
[tool.ruff.mccabe] [tool.ruff.mccabe]
max-complexity = 15 max-complexity = 15

9
src/app/depends.py
View File

@@ -1,4 +1,7 @@
from fastapi import HTTPException, Security, status from fastapi import HTTPException, Request, Security, status
from redis.asyncio import ConnectionPool
from taskiq import TaskiqDepends
from core.auth import default_security from core.auth import default_security
from core.config import env_config from core.config import env_config
@@ -9,3 +12,7 @@ async def check_token(api_key: str = Security(default_security)):
raise HTTPException( raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!" status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!"
) )
def get_redis_pool(request: Request = TaskiqDepends()) -> ConnectionPool:
return request.app.state.redis_pool

66
src/app/services/cache_updater.py
View File

@@ -1,24 +1,22 @@
import collections import collections
from datetime import timedelta
from io import BytesIO from io import BytesIO
import logging import logging
import random
from tempfile import SpooledTemporaryFile from tempfile import SpooledTemporaryFile
from typing import Optional, cast from typing import Optional, cast
from fastapi import UploadFile from fastapi import UploadFile
from arq.connections import ArqRedis
from arq.worker import Retry
import httpx import httpx
from redis import asyncio as aioredis from redis.asyncio import ConnectionPool, Redis
from redis.exceptions import LockError from taskiq import TaskiqDepends
from app.depends import get_redis_pool
from app.models import CachedFile from app.models import CachedFile
from app.services.caption_getter import get_caption from app.services.caption_getter import get_caption
from app.services.downloader import download from app.services.downloader import download
from app.services.files_client import upload_file from app.services.files_client import upload_file
from app.services.library_client import Book, get_book, get_books, get_last_book_id from app.services.library_client import Book, get_book, get_books, get_last_book_id
from core.taskiq_worker import broker
logger = logging.getLogger("telegram_channel_files_manager") logger = logging.getLogger("telegram_channel_files_manager")
@@ -27,14 +25,17 @@ logger = logging.getLogger("telegram_channel_files_manager")
PAGE_SIZE = 100 PAGE_SIZE = 100
class Retry(Exception):
pass
class FileTypeNotAllowed(Exception): class FileTypeNotAllowed(Exception):
def __init__(self, message: str) -> None: def __init__(self, message: str) -> None:
super().__init__(message) super().__init__(message)
async def check_books_page(ctx: dict, page_number: int) -> None: @broker.task
arq_pool: ArqRedis = ctx["arq_pool"] async def check_books_page(page_number: int) -> bool:
page = await get_books(page_number, PAGE_SIZE) page = await get_books(page_number, PAGE_SIZE)
object_ids = [book.id for book in page.items] object_ids = [book.id for book in page.items]
@@ -48,25 +49,21 @@ async def check_books_page(ctx: dict, page_number: int) -> None:
for book in page.items: for book in page.items:
for file_type in book.available_types: for file_type in book.available_types:
if file_type not in cached_files_map[book.id]: if file_type not in cached_files_map[book.id]:
await arq_pool.enqueue_job( await cache_file_by_book_id.kiq(
"cache_file_by_book_id", book_id=book.id,
book.id, file_type=file_type,
file_type,
by_request=False, by_request=False,
_job_id=f"cache_file_by_book_id_{book.id}_{file_type}",
) )
return True
async def check_books(ctx: dict, *args, **kwargs) -> bool: # NOSONAR
arq_pool: ArqRedis = ctx["arq_pool"]
@broker.task
async def check_books(*args, **kwargs) -> bool: # NOSONAR
last_book_id = await get_last_book_id() last_book_id = await get_last_book_id()
for page_number in range(0, last_book_id // 100 + 1): for page_number in range(0, last_book_id // 100 + 1):
await arq_pool.enqueue_job( await check_books_page.kiq(page_number)
"check_books_page",
page_number,
)
return True return True
@@ -76,16 +73,13 @@ async def cache_file(book: Book, file_type: str) -> Optional[CachedFile]:
object_id=book.id, object_type=file_type object_id=book.id, object_type=file_type
).exists(): ).exists():
return return
retry_exc = Retry(defer=timedelta(minutes=15).seconds * random.random())
try: try:
data = await download(book.source.id, book.remote_id, file_type) data = await download(book.source.id, book.remote_id, file_type)
except httpx.HTTPError: except httpx.HTTPError:
raise retry_exc data = None
if data is None: if data is None:
raise retry_exc raise Retry
response, client, filename = data response, client, filename = data
caption = get_caption(book) caption = get_caption(book)
@@ -113,41 +107,31 @@ async def cache_file(book: Book, file_type: str) -> Optional[CachedFile]:
) )
@broker.task
async def cache_file_by_book_id( async def cache_file_by_book_id(
ctx: dict, # NOSONAR
book_id: int, book_id: int,
file_type: str, file_type: str,
by_request: bool = True, by_request: bool = True,
redis_pool: ConnectionPool = TaskiqDepends(get_redis_pool),
) -> Optional[CachedFile]: ) -> Optional[CachedFile]:
r_client: aioredis.Redis = ctx["redis"] book = await get_book(book_id, 3)
get_book_retry = 3 if by_request else 1
book = await get_book(book_id, get_book_retry)
if book is None: if book is None:
if by_request: if by_request:
return None return None
raise Retry(defer=15) raise Retry
if file_type not in book.available_types: if file_type not in book.available_types:
return None return None
lock = r_client.lock( async with Redis(connection_pool=redis_pool) as redis_client:
lock = redis_client.lock(
f"{book_id}_{file_type}", blocking_timeout=5, thread_local=False f"{book_id}_{file_type}", blocking_timeout=5, thread_local=False
) )
try:
try:
async with lock: async with lock:
result = await cache_file(book, file_type) result = await cache_file(book, file_type)
if by_request: if by_request:
return result return result
except LockError:
raise Retry( # noqa: B904
defer=timedelta(minutes=15).seconds * random.random()
)
except Retry as e:
if by_request:
return None return None
raise e

46
src/core/app.py
View File

@@ -1,38 +1,46 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI from fastapi import FastAPI
from fastapi.responses import ORJSONResponse from fastapi.responses import ORJSONResponse
from prometheus_fastapi_instrumentator import Instrumentator from prometheus_fastapi_instrumentator import Instrumentator
from redis.asyncio import ConnectionPool
from app.views import healthcheck_router, router from app.views import healthcheck_router, router
from core.arq_pool import get_arq_pool from core.config import REDIS_URL
from core.db import database from core.db import database
from core.redis_client import get_client from core.taskiq_worker import broker
import core.sentry # noqa: F401
@asynccontextmanager
async def lifespan(app: FastAPI):
database = app.state.database
if not database.is_connected:
await database.connect()
if not broker.is_worker_process:
await broker.startup()
yield
if database.is_connected:
await database.disconnect()
if not broker.is_worker_process:
await broker.shutdown()
await app.state.redis_pool.disconnect()
def start_app() -> FastAPI: def start_app() -> FastAPI:
app = FastAPI(default_response_class=ORJSONResponse) app = FastAPI(default_response_class=ORJSONResponse, lifespan=lifespan)
app.state.database = database app.state.database = database
app.state.redis_pool = ConnectionPool.from_url(REDIS_URL)
app.include_router(router) app.include_router(router)
app.include_router(healthcheck_router) app.include_router(healthcheck_router)
@app.on_event("startup")
async def startup() -> None:
database_ = app.state.database
if not database_.is_connected:
await database_.connect()
app.state.arq_pool = await get_arq_pool()
app.state.redis_client = get_client()
@app.on_event("shutdown")
async def shutdown() -> None:
database_ = app.state.database
if database_.is_connected:
await database_.disconnect()
Instrumentator( Instrumentator(
should_ignore_untemplated=True, should_ignore_untemplated=True,
excluded_handlers=["/docs", "/metrics", "/healthcheck"], excluded_handlers=["/docs", "/metrics", "/healthcheck"],

48
src/core/arq_pool.py
View File

@@ -1,48 +0,0 @@
import asyncio
from typing import Any
from arq.connections import ArqRedis, RedisSettings, create_pool
from arq.worker import JobExecutionFailed
import msgpack
from core.config import env_config
def default(obj: Any):
if isinstance(obj, asyncio.TimeoutError):
return msgpack.ExtType(0, "")
elif isinstance(obj, JobExecutionFailed):
return msgpack.ExtType(1, obj.args[0].encode())
raise TypeError("Unknown type: %r" % (obj,))
def ext_hook(code: int, data: bytes):
if code == 0:
return asyncio.TimeoutError()
elif code == 1:
return JobExecutionFailed((data.decode()))
return msgpack.ExtType(code, data)
def job_serializer(d):
return msgpack.packb(d, default=default, use_bin_type=True) # noqa: E731
def job_deserializer(b):
return msgpack.unpackb(b, ext_hook=ext_hook, raw=False) # noqa: E731
def get_redis_settings() -> RedisSettings:
return RedisSettings(
host=env_config.REDIS_HOST,
port=env_config.REDIS_PORT,
database=env_config.REDIS_DB,
)
async def get_arq_pool() -> ArqRedis:
return await create_pool(
get_redis_settings(),
job_serializer=job_serializer, # type: ignore
job_deserializer=job_deserializer, # noqa: E731
)

6
src/core/config.py
View File

@@ -26,4 +26,8 @@ class EnvConfig(BaseSettings):
SENTRY_DSN: str SENTRY_DSN: str
env_config = EnvConfig() env_config = EnvConfig() # type: ignore
REDIS_URL = (
f"redis://{env_config.REDIS_HOST}:{env_config.REDIS_PORT}/{env_config.REDIS_DB}"
)

9
src/core/redis_client.py
View File

@@ -1,9 +0,0 @@
from redis import asyncio as aioredis
from core.config import env_config
def get_client() -> aioredis.Redis:
return aioredis.Redis(
host=env_config.REDIS_HOST, port=env_config.REDIS_PORT, db=env_config.REDIS_DB
)

40
src/core/setup_arq.py
View File

@@ -1,40 +0,0 @@
from app.services.cache_updater import (
cache_file_by_book_id,
check_books,
check_books_page,
)
from core.arq_pool import (
get_arq_pool,
get_redis_settings,
job_deserializer,
job_serializer,
)
from core.db import database
from core.redis_client import get_client
import core.sentry # noqa: F401
async def startup(ctx):
if not database.is_connected:
await database.connect()
ctx["arq_pool"] = await get_arq_pool()
ctx["redis"] = get_client()
async def shutdown(ctx):
if database.is_connected:
await database.disconnect()
class WorkerSettings:
functions = [check_books, check_books_page, cache_file_by_book_id]
on_startup = startup
on_shutdown = shutdown
redis_settings = get_redis_settings()
max_jobs = 2
max_tries = 2
job_timeout = 10 * 60
expires_extra_ms = 7 * 24 * 60 * 1000
job_serializer = job_serializer
job_deserializer = job_deserializer

17
src/core/taskiq_worker.py Normal file
View File

@@ -0,0 +1,17 @@
from taskiq import SimpleRetryMiddleware
import taskiq_fastapi
from taskiq_redis import ListQueueBroker, RedisAsyncResultBackend
from core.config import REDIS_URL
broker = (
ListQueueBroker(url=REDIS_URL)
.with_result_backend(
RedisAsyncResultBackend(redis_url=REDIS_URL, result_ex_time=5 * 60)
)
.with_middlewares(SimpleRetryMiddleware())
)
taskiq_fastapi.init(broker, "main:app")

10
src/main.py
View File

@@ -1,4 +1,12 @@
from core.app import start_app import sentry_sdk
from core.app import start_app
from core.config import env_config
if env_config.SENTRY_DSN:
sentry_sdk.init(
dsn=env_config.SENTRY_DSN,
)
app = start_app() app = start_app()