Add arq

.github/workflows/build_docker_image.yml

@@ -47,3 +47,9 @@ jobs:
         uses: joelwmale/webhook-action@master
         with:
           url: ${{ secrets.WEBHOOK_URL }}
+
+      - 
+        name: Invoke deployment hook
+        uses: joelwmale/webhook-action@master
+        with:
+          url: ${{ secrets.WEBHOOK_URL2 }}

poetry.lock

@@ -23,6 +23,18 @@ PyMySQL = ">=0.9,<=0.9.3"
 [package.extras]
 sa = ["sqlalchemy (>=1.0)"]
 
+[[package]]
+name = "aioredis"
+version = "1.3.1"
+description = "asyncio (PEP 3156) Redis support"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+async-timeout = "*"
+hiredis = "*"
+
 [[package]]
 name = "anyio"
 version = "3.4.0"
@@ -40,6 +52,22 @@ doc = ["sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"]
 test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=6.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"]
 trio = ["trio (>=0.16)"]
 
+[[package]]
+name = "arq"
+version = "0.22"
+description = "Job queues in python with asyncio and redis"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+aioredis = ">=1.1.0,<2.0.0"
+click = ">=6.7"
+pydantic = ">=1"
+
+[package.extras]
+watch = ["watchgod (>=0.4)"]
+
 [[package]]
 name = "asgiref"
 version = "3.4.1"
@@ -51,6 +79,14 @@ python-versions = ">=3.6"
 [package.extras]
 tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"]
 
+[[package]]
+name = "async-timeout"
+version = "4.0.2"
+description = "Timeout context manager for asyncio programs"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
 [[package]]
 name = "asyncpg"
 version = "0.25.0"
@@ -128,6 +164,14 @@ category = "main"
 optional = false
 python-versions = ">=3.6"
 
+[[package]]
+name = "hiredis"
+version = "2.0.0"
+description = "Python wrapper for hiredis"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
 [[package]]
 name = "httpcore"
 version = "0.14.3"
@@ -274,7 +318,7 @@ standard = ["httptools (>=0.2.0,<0.4.0)", "watchgod (>=0.6)", "python-dotenv (>=
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "e82ee51aaa18bb146a524a296ada00fa905a92f8634c61503684091684a5d348"
+content-hash = "521f70378dce3b416c0117f29e614d544b9f078407e734d97d1ac4ea247a59db"
 
 [metadata.files]
 aiologger = [
@@ -284,14 +328,26 @@ aiomysql = [
     {file = "aiomysql-0.0.22-py3-none-any.whl", hash = "sha256:4e4a65914daacc40e70f992ddbeef32457561efbad8de41393e8ac5a84126a5a"},
     {file = "aiomysql-0.0.22.tar.gz", hash = "sha256:9bcf8f26d22e550f75cabd635fa19a55c45f835eea008275960cb37acadd622a"},
 ]
+aioredis = [
+    {file = "aioredis-1.3.1-py3-none-any.whl", hash = "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3"},
+    {file = "aioredis-1.3.1.tar.gz", hash = "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a"},
+]
 anyio = [
     {file = "anyio-3.4.0-py3-none-any.whl", hash = "sha256:2855a9423524abcdd652d942f8932fda1735210f77a6b392eafd9ff34d3fe020"},
     {file = "anyio-3.4.0.tar.gz", hash = "sha256:24adc69309fb5779bc1e06158e143e0b6d2c56b302a3ac3de3083c705a6ed39d"},
 ]
+arq = [
+    {file = "arq-0.22-py3-none-any.whl", hash = "sha256:55a0f933636c804b82c366a0e3710e9e5ed26a716251fa6742777d0b039f7f30"},
+    {file = "arq-0.22.tar.gz", hash = "sha256:c7bd98151cc83cec941ce5f660ede4bee888effd9a4692258ec8a9a0aff2f9f9"},
+]
 asgiref = [
     {file = "asgiref-3.4.1-py3-none-any.whl", hash = "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"},
     {file = "asgiref-3.4.1.tar.gz", hash = "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9"},
 ]
+async-timeout = [
+    {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"},
+    {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"},
+]
 asyncpg = [
     {file = "asyncpg-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf5e3408a14a17d480f36ebaf0401a12ff6ae5457fdf45e4e2775c51cc9517d3"},
     {file = "asyncpg-0.25.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2bc197fc4aca2fd24f60241057998124012469d2e414aed3f992579db0c88e3a"},
@@ -344,6 +400,49 @@ h11 = [
     {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"},
     {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"},
 ]
+hiredis = [
+    {file = "hiredis-2.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b4c8b0bc5841e578d5fb32a16e0c305359b987b850a06964bd5a62739d688048"},
+    {file = "hiredis-2.0.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0adea425b764a08270820531ec2218d0508f8ae15a448568109ffcae050fee26"},
+    {file = "hiredis-2.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:3d55e36715ff06cdc0ab62f9591607c4324297b6b6ce5b58cb9928b3defe30ea"},
+    {file = "hiredis-2.0.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:5d2a48c80cf5a338d58aae3c16872f4d452345e18350143b3bf7216d33ba7b99"},
+    {file = "hiredis-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:240ce6dc19835971f38caf94b5738092cb1e641f8150a9ef9251b7825506cb05"},
+    {file = "hiredis-2.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:5dc7a94bb11096bc4bffd41a3c4f2b958257085c01522aa81140c68b8bf1630a"},
+    {file = "hiredis-2.0.0-cp36-cp36m-win32.whl", hash = "sha256:139705ce59d94eef2ceae9fd2ad58710b02aee91e7fa0ccb485665ca0ecbec63"},
+    {file = "hiredis-2.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c39c46d9e44447181cd502a35aad2bb178dbf1b1f86cf4db639d7b9614f837c6"},
+    {file = "hiredis-2.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:adf4dd19d8875ac147bf926c727215a0faf21490b22c053db464e0bf0deb0485"},
+    {file = "hiredis-2.0.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0f41827028901814c709e744060843c77e78a3aca1e0d6875d2562372fcb405a"},
+    {file = "hiredis-2.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:508999bec4422e646b05c95c598b64bdbef1edf0d2b715450a078ba21b385bcc"},
+    {file = "hiredis-2.0.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:0d5109337e1db373a892fdcf78eb145ffb6bbd66bb51989ec36117b9f7f9b579"},
+    {file = "hiredis-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:04026461eae67fdefa1949b7332e488224eac9e8f2b5c58c98b54d29af22093e"},
+    {file = "hiredis-2.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a00514362df15af041cc06e97aebabf2895e0a7c42c83c21894be12b84402d79"},
+    {file = "hiredis-2.0.0-cp37-cp37m-win32.whl", hash = "sha256:09004096e953d7ebd508cded79f6b21e05dff5d7361771f59269425108e703bc"},
+    {file = "hiredis-2.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f8196f739092a78e4f6b1b2172679ed3343c39c61a3e9d722ce6fcf1dac2824a"},
+    {file = "hiredis-2.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:294a6697dfa41a8cba4c365dd3715abc54d29a86a40ec6405d677ca853307cfb"},
+    {file = "hiredis-2.0.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3dddf681284fe16d047d3ad37415b2e9ccdc6c8986c8062dbe51ab9a358b50a5"},
+    {file = "hiredis-2.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:dcef843f8de4e2ff5e35e96ec2a4abbdf403bd0f732ead127bd27e51f38ac298"},
+    {file = "hiredis-2.0.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:87c7c10d186f1743a8fd6a971ab6525d60abd5d5d200f31e073cd5e94d7e7a9d"},
+    {file = "hiredis-2.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:7f0055f1809b911ab347a25d786deff5e10e9cf083c3c3fd2dd04e8612e8d9db"},
+    {file = "hiredis-2.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:11d119507bb54e81f375e638225a2c057dda748f2b1deef05c2b1a5d42686048"},
+    {file = "hiredis-2.0.0-cp38-cp38-win32.whl", hash = "sha256:7492af15f71f75ee93d2a618ca53fea8be85e7b625e323315169977fae752426"},
+    {file = "hiredis-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:65d653df249a2f95673976e4e9dd7ce10de61cfc6e64fa7eeaa6891a9559c581"},
+    {file = "hiredis-2.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ae8427a5e9062ba66fc2c62fb19a72276cf12c780e8db2b0956ea909c48acff5"},
+    {file = "hiredis-2.0.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:3f5f7e3a4ab824e3de1e1700f05ad76ee465f5f11f5db61c4b297ec29e692b2e"},
+    {file = "hiredis-2.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:e3447d9e074abf0e3cd85aef8131e01ab93f9f0e86654db7ac8a3f73c63706ce"},
+    {file = "hiredis-2.0.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:8b42c0dc927b8d7c0eb59f97e6e34408e53bc489f9f90e66e568f329bff3e443"},
+    {file = "hiredis-2.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:b84f29971f0ad4adaee391c6364e6f780d5aae7e9226d41964b26b49376071d0"},
+    {file = "hiredis-2.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0b39ec237459922c6544d071cdcf92cbb5bc6685a30e7c6d985d8a3e3a75326e"},
+    {file = "hiredis-2.0.0-cp39-cp39-win32.whl", hash = "sha256:a7928283143a401e72a4fad43ecc85b35c27ae699cf5d54d39e1e72d97460e1d"},
+    {file = "hiredis-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:a4ee8000454ad4486fb9f28b0cab7fa1cd796fc36d639882d0b34109b5b3aec9"},
+    {file = "hiredis-2.0.0-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1f03d4dadd595f7a69a75709bc81902673fa31964c75f93af74feac2f134cc54"},
+    {file = "hiredis-2.0.0-pp36-pypy36_pp73-manylinux1_x86_64.whl", hash = "sha256:04927a4c651a0e9ec11c68e4427d917e44ff101f761cd3b5bc76f86aaa431d27"},
+    {file = "hiredis-2.0.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:a39efc3ade8c1fb27c097fd112baf09d7fd70b8cb10ef1de4da6efbe066d381d"},
+    {file = "hiredis-2.0.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:07bbf9bdcb82239f319b1f09e8ef4bdfaec50ed7d7ea51a56438f39193271163"},
+    {file = "hiredis-2.0.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:807b3096205c7cec861c8803a6738e33ed86c9aae76cac0e19454245a6bbbc0a"},
+    {file = "hiredis-2.0.0-pp37-pypy37_pp73-manylinux1_x86_64.whl", hash = "sha256:1233e303645f468e399ec906b6b48ab7cd8391aae2d08daadbb5cad6ace4bd87"},
+    {file = "hiredis-2.0.0-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:cb2126603091902767d96bcb74093bd8b14982f41809f85c9b96e519c7e1dc41"},
+    {file = "hiredis-2.0.0-pp37-pypy37_pp73-win32.whl", hash = "sha256:f52010e0a44e3d8530437e7da38d11fb822acfb0d5b12e9cd5ba655509937ca0"},
+    {file = "hiredis-2.0.0.tar.gz", hash = "sha256:81d6d8e39695f2c37954d1011c0480ef7cf444d4e3ae24bc5e89ee5de360139a"},
+]
 httpcore = [
     {file = "httpcore-0.14.3-py3-none-any.whl", hash = "sha256:9a98d2416b78976fc5396ff1f6b26ae9885efbb3105d24eed490f20ab4c95ec1"},
     {file = "httpcore-0.14.3.tar.gz", hash = "sha256:d10162a63265a0228d5807964bd964478cbdb5178f9a2eedfebb2faba27eef5d"},

pyproject.toml

@@ -13,6 +13,7 @@ aiologger = "^0.6.1"
 asyncpg = "^0.25.0"
 aiomysql = "^0.0.22"
 uvicorn = {extras = ["standart"], version = "^0.16.0"}
+arq = "^0.22"
 
 [tool.poetry.dev-dependencies]
 

src/app/services/updaters/__init__.py

@@ -1,11 +1,8 @@
 from enum import Enum
 
-from app.services.updaters.base import BaseUpdater
-from app.services.updaters.fl_updater import FlUpdater
-
 
 class UpdaterTypes(Enum):
     FL = "fl"
 
 
-UPDATERS: dict[UpdaterTypes, BaseUpdater] = {UpdaterTypes.FL: FlUpdater}
+UPDATERS: dict[UpdaterTypes, str] = {UpdaterTypes.FL: "run_fl_update"}

src/app/services/updaters/base.py

@@ -1,7 +0,0 @@
-from typing import Protocol
-
-
-class BaseUpdater(Protocol):
-    @classmethod
-    async def update(cls) -> bool:
-        ...

src/app/services/updaters/fl_updater.py

@@ -1,14 +1,17 @@
 import asyncio
+from logging import Logger
 from typing import Optional
 
-from aiologger import Logger
 import aiomysql
+from arq.connections import ArqRedis
 import asyncpg
 
-from app.services.updaters.base import BaseUpdater
 from core.config import env_config
 
 
+logger = Logger("fl_updater")
+
+
 async def run(cmd) -> tuple[bytes, bytes, Optional[int]]:
     proc = await asyncio.create_subprocess_shell(
         cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
@@ -26,7 +29,7 @@ def remove_dots(s: str):
     return s.replace(".", "")
 
 
-class FlUpdater(BaseUpdater):
+class FlUpdater:
     SOURCE: int
 
     FILES = [
@@ -52,15 +55,13 @@ class FlUpdater(BaseUpdater):
     sequences_updated_event: asyncio.Event
     genres_updated_event: asyncio.Event
 
-    logger: Logger
-
     async def _import_dump(self, filename: str):
         await run(
             f"wget -O - {env_config.FL_BASE_URL}/sql/{filename}.gz | gunzip | "
             f"mysql -h {env_config.MYSQL_HOST} -u {env_config.MYSQL_USER} "
             f'-p"{env_config.MYSQL_PASSWORD}" {env_config.MYSQL_DB_NAME}'
         )
-        await self.logger.info(f"Imported {filename}")
+        logger.info(f"Imported {filename}")
 
     async def _prepare(self):
         posgres_pool = await asyncpg.create_pool(
@@ -85,7 +86,7 @@ class FlUpdater(BaseUpdater):
         self.postgres_pool = posgres_pool
 
     async def _set_source(self):
-        await self.logger.info("Set source...")
+        logger.info("Set source...")
 
         source_row = await self.postgres_pool.fetchrow(
             "SELECT id FROM sources WHERE name = 'flibusta';"
@@ -102,7 +103,7 @@ class FlUpdater(BaseUpdater):
 
         self.SOURCE = source_row["id"]
 
-        await self.logger.info("Source has set!")
+        logger.info("Source has set!")
 
     async def _update_authors(self):
         def prepare_author(row: list):
@@ -114,7 +115,7 @@ class FlUpdater(BaseUpdater):
                 remove_wrong_ch(row[3]),
             ]
 
-        await self.logger.info("Update authors...")
+        logger.info("Update authors...")
 
         await self.postgres_pool.execute(
             """
@@ -157,7 +158,7 @@ class FlUpdater(BaseUpdater):
 
         self.authors_updated_event.set()
 
-        await self.logger.info("Authors updated!")
+        logger.info("Authors updated!")
 
     async def _update_books(self):
         replace_dict = {"ru-": "ru", "ru~": "ru"}
@@ -178,7 +179,7 @@ class FlUpdater(BaseUpdater):
                 row[5] == "1",
             ]
 
-        await self.logger.info("Update books...")
+        logger.info("Update books...")
 
         await self.postgres_pool.execute(
             """
@@ -223,13 +224,13 @@ class FlUpdater(BaseUpdater):
 
         self.books_updated_event.set()
 
-        await self.logger.info("Books updated!")
+        logger.info("Books updated!")
 
     async def _update_books_authors(self):
         await self.books_updated_event.wait()
         await self.authors_updated_event.wait()
 
-        await self.logger.info("Update books_authors...")
+        logger.info("Update books_authors...")
 
         await self.postgres_pool.execute(
             """
@@ -271,13 +272,13 @@ class FlUpdater(BaseUpdater):
                         [(self.SOURCE, *row) for row in rows],
                     )
 
-        await self.logger.info("Books_authors updated!")
+        logger.info("Books_authors updated!")
 
     async def _update_translations(self):
         await self.books_updated_event.wait()
         await self.authors_updated_event.wait()
 
-        await self.logger.info("Update translations...")
+        logger.info("Update translations...")
 
         await self.postgres_pool.execute(
             """
@@ -324,7 +325,7 @@ class FlUpdater(BaseUpdater):
                         [(self.SOURCE, *row) for row in rows],
                     )
 
-        await self.logger.info("Translations updated!")
+        logger.info("Translations updated!")
 
     async def _update_sequences(self):
         def prepare_sequence(row: list):
@@ -334,7 +335,7 @@ class FlUpdater(BaseUpdater):
                 remove_wrong_ch(row[1]),
             ]
 
-        await self.logger.info("Update sequences...")
+        logger.info("Update sequences...")
 
         await self.postgres_pool.execute(
             """
@@ -373,13 +374,13 @@ class FlUpdater(BaseUpdater):
 
         self.sequences_updated_event.set()
 
-        await self.logger.info("Sequences updated!")
+        logger.info("Sequences updated!")
 
     async def _update_sequences_info(self):
         await self.sequences_updated_event.wait()
         await self.books_updated_event.wait()
 
-        await self.logger.info("Update book_sequences...")
+        logger.info("Update book_sequences...")
 
         await self.postgres_pool.execute(
             """
@@ -429,12 +430,12 @@ class FlUpdater(BaseUpdater):
                         [[self.SOURCE, *row] for row in rows],
                     )
 
-        await self.logger.info("Book_sequences updated!")
+        logger.info("Book_sequences updated!")
 
     async def _update_book_annotations(self):
         await self.books_updated_event.wait()
 
-        await self.logger.info("Update book_annotations...")
+        logger.info("Update book_annotations...")
 
         await self.postgres_pool.execute(
             """
@@ -478,12 +479,12 @@ class FlUpdater(BaseUpdater):
                         [[self.SOURCE, *row] for row in rows],
                     )
 
-        await self.logger.info("Book_annotations updated!")
+        logger.info("Book_annotations updated!")
 
         await self._update_book_annotations_pic()
 
     async def _update_book_annotations_pic(self):
-        await self.logger.info("Update book_annotations_pic...")
+        logger.info("Update book_annotations_pic...")
 
         async with self.mysql_pool.acquire() as conn:
             async with conn.cursor() as cursor:
@@ -508,12 +509,12 @@ class FlUpdater(BaseUpdater):
                         [[self.SOURCE, *row] for row in rows],
                     )
 
-        await self.logger.info("Book_annotation_pics updated!")
+        logger.info("Book_annotation_pics updated!")
 
     async def _update_author_annotations(self):
         await self.authors_updated_event.wait()
 
-        await self.logger.info("Update author_annotations...")
+        logger.info("Update author_annotations...")
 
         await self.postgres_pool.execute(
             """
@@ -554,12 +555,12 @@ class FlUpdater(BaseUpdater):
                         [[self.SOURCE, *row] for row in rows],
                     )
 
-        await self.logger.info("Author_annotation_updated!")
+        logger.info("Author_annotation_updated!")
 
         await self._update_author_annotations_pics()
 
     async def _update_author_annotations_pics(self):
-        await self.logger.info("Update author_annotations_pic...")
+        logger.info("Update author_annotations_pic...")
 
         async with self.mysql_pool.acquire() as conn:
             async with conn.cursor() as cursor:
@@ -584,10 +585,10 @@ class FlUpdater(BaseUpdater):
                         [[self.SOURCE, *row] for row in rows],
                     )
 
-        await self.logger.info("Author_annotatioins_pic updated!")
+        logger.info("Author_annotatioins_pic updated!")
 
     async def _update_genres(self):
-        await self.logger.info("Update genres...")
+        logger.info("Update genres...")
 
         await self.postgres_pool.execute(
             """
@@ -627,13 +628,13 @@ class FlUpdater(BaseUpdater):
                         [[self.SOURCE, *row] for row in rows],
                     )
 
-        await self.logger.info("Genres updated!")
+        logger.info("Genres updated!")
 
     async def _update_books_genres(self):
         await self.books_updated_event.wait()
         await self.genres_updated_event.wait()
 
-        await self.logger.info("Update book_genres...")
+        logger.info("Update book_genres...")
 
         await self.postgres_pool.execute(
             """
@@ -675,15 +676,19 @@ class FlUpdater(BaseUpdater):
                         [(self.SOURCE, *row) for row in rows],
                     )
 
-        await self.logger.info("Book_genres updated!")
+        logger.info("Book_genres updated!")
-
-    async def _update(self) -> bool:
-        self.logger = Logger.with_default_handlers()
-
-        await self._prepare()
 
+    async def _import(self, ctx) -> bool:
         await asyncio.gather(*[self._import_dump(filename) for filename in self.FILES])
 
+        arq_pool: ArqRedis = ctx["arq_pool"]
+        await arq_pool.enqueue_job("run_fl_update2")
+
+        return True
+
+    async def _update(self, ctx) -> bool:
+        await self._prepare()
+
         await self._set_source()
 
         self.authors_updated_event = asyncio.Event()
@@ -706,7 +711,10 @@ class FlUpdater(BaseUpdater):
 
         return True
 
-    @classmethod
+
-    async def update(cls) -> bool:
+async def run_fl_update(ctx) -> bool:
-        updater = cls()
+    return await FlUpdater()._import(ctx)
-        return await updater._update()
+
+
+async def run_fl_update2(ctx) -> bool:
+    return await FlUpdater()._update(ctx)

src/app/views.py

@@ -1,4 +1,6 @@
-from fastapi import APIRouter, BackgroundTasks, Depends
+from fastapi import APIRouter, Depends, Request
+
+from arq.connections import ArqRedis
 
 from app.depends import check_token
 from app.services.updaters import UpdaterTypes, UPDATERS
@@ -8,9 +10,8 @@ router = APIRouter(tags=["updater"], dependencies=[Depends(check_token)])
 
 
 @router.post("/update/{updater}")
-async def update(updater: UpdaterTypes, background_tasks: BackgroundTasks):
+async def update(request: Request, updater: UpdaterTypes):
-    updater_ = UPDATERS[updater]
+    arq_pool: ArqRedis = request.app.state.arq_pool
-
+    await arq_pool.enqueue_job(UPDATERS[updater])
-    background_tasks.add_task(updater_.update)
 
     return "Ok!"

src/core/app.py

@@ -1,6 +1,7 @@
 from fastapi import FastAPI
 
 from app.views import router
+from core.arq_pool import get_arq_pool
 
 
 def start_app() -> FastAPI:
@@ -8,4 +9,8 @@ def start_app() -> FastAPI:
 
     app.include_router(router)
 
+    @app.on_event("startup")
+    async def startup() -> None:
+        app.state.arq_pool = await get_arq_pool()
+
     return app

src/core/arq_pool.py

@@ -0,0 +1,15 @@
+from arq.connections import create_pool, RedisSettings, ArqRedis
+
+from core.config import env_config
+
+
+def get_redis_settings() -> RedisSettings:
+    return RedisSettings(
+        host=env_config.REDIS_HOST,
+        port=env_config.REDIS_PORT,
+        database=env_config.REDIS_DB,
+    )
+
+
+async def get_arq_pool() -> ArqRedis:
+    return await create_pool(get_redis_settings())

src/core/config.py

@@ -24,6 +24,10 @@ class EnvConfig(BaseSettings):
     MYSQL_USER: str
     MYSQL_PASSWORD: str
 
+    REDIS_HOST: str
+    REDIS_PORT: int
+    REDIS_DB: int
+
     FL_BASE_URL: str
 
 

src/core/setup_arq.py

@@ -0,0 +1,14 @@
+from app.services.updaters.fl_updater import run_fl_update, run_fl_update2
+from core.arq_pool import get_redis_settings, get_arq_pool
+
+
+async def startup(ctx):
+    ctx["arq_pool"] = await get_arq_pool()
+
+
+class WorkerSettings:
+    functions = [run_fl_update, run_fl_update2]
+    on_startup = startup
+    redis_settings = get_redis_settings()
+    max_jobs = 1
+    job_timeout = 60 * 60