diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml index 41be1da..dfd6584 100644 --- a/.github/workflows/build_docker_image.yml +++ b/.github/workflows/build_docker_image.yml @@ -47,3 +47,9 @@ jobs: uses: joelwmale/webhook-action@master with: url: ${{ secrets.WEBHOOK_URL }} + + - + name: Invoke deployment hook + uses: joelwmale/webhook-action@master + with: + url: ${{ secrets.WEBHOOK_URL2 }} diff --git a/poetry.lock b/poetry.lock index 8bed6e0..273f867 100644 --- a/poetry.lock +++ b/poetry.lock @@ -23,6 +23,18 @@ PyMySQL = ">=0.9,<=0.9.3" [package.extras] sa = ["sqlalchemy (>=1.0)"] +[[package]] +name = "aioredis" +version = "1.3.1" +description = "asyncio (PEP 3156) Redis support" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +async-timeout = "*" +hiredis = "*" + [[package]] name = "anyio" version = "3.4.0" @@ -40,6 +52,22 @@ doc = ["sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=6.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"] trio = ["trio (>=0.16)"] +[[package]] +name = "arq" +version = "0.22" +description = "Job queues in python with asyncio and redis" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +aioredis = ">=1.1.0,<2.0.0" +click = ">=6.7" +pydantic = ">=1" + +[package.extras] +watch = ["watchgod (>=0.4)"] + [[package]] name = "asgiref" version = "3.4.1" @@ -51,6 +79,14 @@ python-versions = ">=3.6" [package.extras] tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"] +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "asyncpg" version = "0.25.0" @@ -128,6 +164,14 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "hiredis" +version = "2.0.0" +description = "Python wrapper for hiredis" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "httpcore" version = "0.14.3" @@ -274,7 +318,7 @@ standard = ["httptools (>=0.2.0,<0.4.0)", "watchgod (>=0.6)", "python-dotenv (>= [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "e82ee51aaa18bb146a524a296ada00fa905a92f8634c61503684091684a5d348" +content-hash = "521f70378dce3b416c0117f29e614d544b9f078407e734d97d1ac4ea247a59db" [metadata.files] aiologger = [ @@ -284,14 +328,26 @@ aiomysql = [ {file = "aiomysql-0.0.22-py3-none-any.whl", hash = "sha256:4e4a65914daacc40e70f992ddbeef32457561efbad8de41393e8ac5a84126a5a"}, {file = "aiomysql-0.0.22.tar.gz", hash = "sha256:9bcf8f26d22e550f75cabd635fa19a55c45f835eea008275960cb37acadd622a"}, ] +aioredis = [ + {file = "aioredis-1.3.1-py3-none-any.whl", hash = "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3"}, + {file = "aioredis-1.3.1.tar.gz", hash = "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a"}, +] anyio = [ {file = "anyio-3.4.0-py3-none-any.whl", hash = "sha256:2855a9423524abcdd652d942f8932fda1735210f77a6b392eafd9ff34d3fe020"}, {file = "anyio-3.4.0.tar.gz", hash = "sha256:24adc69309fb5779bc1e06158e143e0b6d2c56b302a3ac3de3083c705a6ed39d"}, ] +arq = [ + {file = "arq-0.22-py3-none-any.whl", hash = "sha256:55a0f933636c804b82c366a0e3710e9e5ed26a716251fa6742777d0b039f7f30"}, + {file = "arq-0.22.tar.gz", hash = "sha256:c7bd98151cc83cec941ce5f660ede4bee888effd9a4692258ec8a9a0aff2f9f9"}, +] asgiref = [ {file = "asgiref-3.4.1-py3-none-any.whl", hash = "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"}, {file = "asgiref-3.4.1.tar.gz", hash = "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9"}, ] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] asyncpg = [ {file = "asyncpg-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf5e3408a14a17d480f36ebaf0401a12ff6ae5457fdf45e4e2775c51cc9517d3"}, {file = "asyncpg-0.25.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2bc197fc4aca2fd24f60241057998124012469d2e414aed3f992579db0c88e3a"}, @@ -344,6 +400,49 @@ h11 = [ {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, ] +hiredis = [ + {file = "hiredis-2.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b4c8b0bc5841e578d5fb32a16e0c305359b987b850a06964bd5a62739d688048"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0adea425b764a08270820531ec2218d0508f8ae15a448568109ffcae050fee26"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:3d55e36715ff06cdc0ab62f9591607c4324297b6b6ce5b58cb9928b3defe30ea"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:5d2a48c80cf5a338d58aae3c16872f4d452345e18350143b3bf7216d33ba7b99"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:240ce6dc19835971f38caf94b5738092cb1e641f8150a9ef9251b7825506cb05"}, + {file = "hiredis-2.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:5dc7a94bb11096bc4bffd41a3c4f2b958257085c01522aa81140c68b8bf1630a"}, + {file = "hiredis-2.0.0-cp36-cp36m-win32.whl", hash = "sha256:139705ce59d94eef2ceae9fd2ad58710b02aee91e7fa0ccb485665ca0ecbec63"}, + {file = "hiredis-2.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c39c46d9e44447181cd502a35aad2bb178dbf1b1f86cf4db639d7b9614f837c6"}, + {file = "hiredis-2.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:adf4dd19d8875ac147bf926c727215a0faf21490b22c053db464e0bf0deb0485"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0f41827028901814c709e744060843c77e78a3aca1e0d6875d2562372fcb405a"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:508999bec4422e646b05c95c598b64bdbef1edf0d2b715450a078ba21b385bcc"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:0d5109337e1db373a892fdcf78eb145ffb6bbd66bb51989ec36117b9f7f9b579"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:04026461eae67fdefa1949b7332e488224eac9e8f2b5c58c98b54d29af22093e"}, + {file = "hiredis-2.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a00514362df15af041cc06e97aebabf2895e0a7c42c83c21894be12b84402d79"}, + {file = "hiredis-2.0.0-cp37-cp37m-win32.whl", hash = "sha256:09004096e953d7ebd508cded79f6b21e05dff5d7361771f59269425108e703bc"}, + {file = "hiredis-2.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f8196f739092a78e4f6b1b2172679ed3343c39c61a3e9d722ce6fcf1dac2824a"}, + {file = "hiredis-2.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:294a6697dfa41a8cba4c365dd3715abc54d29a86a40ec6405d677ca853307cfb"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3dddf681284fe16d047d3ad37415b2e9ccdc6c8986c8062dbe51ab9a358b50a5"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:dcef843f8de4e2ff5e35e96ec2a4abbdf403bd0f732ead127bd27e51f38ac298"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:87c7c10d186f1743a8fd6a971ab6525d60abd5d5d200f31e073cd5e94d7e7a9d"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:7f0055f1809b911ab347a25d786deff5e10e9cf083c3c3fd2dd04e8612e8d9db"}, + {file = "hiredis-2.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:11d119507bb54e81f375e638225a2c057dda748f2b1deef05c2b1a5d42686048"}, + {file = "hiredis-2.0.0-cp38-cp38-win32.whl", hash = "sha256:7492af15f71f75ee93d2a618ca53fea8be85e7b625e323315169977fae752426"}, + {file = "hiredis-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:65d653df249a2f95673976e4e9dd7ce10de61cfc6e64fa7eeaa6891a9559c581"}, + {file = "hiredis-2.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ae8427a5e9062ba66fc2c62fb19a72276cf12c780e8db2b0956ea909c48acff5"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:3f5f7e3a4ab824e3de1e1700f05ad76ee465f5f11f5db61c4b297ec29e692b2e"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:e3447d9e074abf0e3cd85aef8131e01ab93f9f0e86654db7ac8a3f73c63706ce"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:8b42c0dc927b8d7c0eb59f97e6e34408e53bc489f9f90e66e568f329bff3e443"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:b84f29971f0ad4adaee391c6364e6f780d5aae7e9226d41964b26b49376071d0"}, + {file = "hiredis-2.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0b39ec237459922c6544d071cdcf92cbb5bc6685a30e7c6d985d8a3e3a75326e"}, + {file = "hiredis-2.0.0-cp39-cp39-win32.whl", hash = "sha256:a7928283143a401e72a4fad43ecc85b35c27ae699cf5d54d39e1e72d97460e1d"}, + {file = "hiredis-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:a4ee8000454ad4486fb9f28b0cab7fa1cd796fc36d639882d0b34109b5b3aec9"}, + {file = "hiredis-2.0.0-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1f03d4dadd595f7a69a75709bc81902673fa31964c75f93af74feac2f134cc54"}, + {file = "hiredis-2.0.0-pp36-pypy36_pp73-manylinux1_x86_64.whl", hash = "sha256:04927a4c651a0e9ec11c68e4427d917e44ff101f761cd3b5bc76f86aaa431d27"}, + {file = "hiredis-2.0.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:a39efc3ade8c1fb27c097fd112baf09d7fd70b8cb10ef1de4da6efbe066d381d"}, + {file = "hiredis-2.0.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:07bbf9bdcb82239f319b1f09e8ef4bdfaec50ed7d7ea51a56438f39193271163"}, + {file = "hiredis-2.0.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:807b3096205c7cec861c8803a6738e33ed86c9aae76cac0e19454245a6bbbc0a"}, + {file = "hiredis-2.0.0-pp37-pypy37_pp73-manylinux1_x86_64.whl", hash = "sha256:1233e303645f468e399ec906b6b48ab7cd8391aae2d08daadbb5cad6ace4bd87"}, + {file = "hiredis-2.0.0-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:cb2126603091902767d96bcb74093bd8b14982f41809f85c9b96e519c7e1dc41"}, + {file = "hiredis-2.0.0-pp37-pypy37_pp73-win32.whl", hash = "sha256:f52010e0a44e3d8530437e7da38d11fb822acfb0d5b12e9cd5ba655509937ca0"}, + {file = "hiredis-2.0.0.tar.gz", hash = "sha256:81d6d8e39695f2c37954d1011c0480ef7cf444d4e3ae24bc5e89ee5de360139a"}, +] httpcore = [ {file = "httpcore-0.14.3-py3-none-any.whl", hash = "sha256:9a98d2416b78976fc5396ff1f6b26ae9885efbb3105d24eed490f20ab4c95ec1"}, {file = "httpcore-0.14.3.tar.gz", hash = "sha256:d10162a63265a0228d5807964bd964478cbdb5178f9a2eedfebb2faba27eef5d"}, diff --git a/pyproject.toml b/pyproject.toml index 4f73ef3..4c78580 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ aiologger = "^0.6.1" asyncpg = "^0.25.0" aiomysql = "^0.0.22" uvicorn = {extras = ["standart"], version = "^0.16.0"} +arq = "^0.22" [tool.poetry.dev-dependencies] diff --git a/src/app/services/updaters/__init__.py b/src/app/services/updaters/__init__.py index 7aaefba..45f782f 100644 --- a/src/app/services/updaters/__init__.py +++ b/src/app/services/updaters/__init__.py @@ -1,11 +1,8 @@ from enum import Enum -from app.services.updaters.base import BaseUpdater -from app.services.updaters.fl_updater import FlUpdater - class UpdaterTypes(Enum): FL = "fl" -UPDATERS: dict[UpdaterTypes, BaseUpdater] = {UpdaterTypes.FL: FlUpdater} +UPDATERS: dict[UpdaterTypes, str] = {UpdaterTypes.FL: "run_fl_update"} diff --git a/src/app/services/updaters/base.py b/src/app/services/updaters/base.py deleted file mode 100644 index 57eca09..0000000 --- a/src/app/services/updaters/base.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import Protocol - - -class BaseUpdater(Protocol): - @classmethod - async def update(cls) -> bool: - ... diff --git a/src/app/services/updaters/fl_updater.py b/src/app/services/updaters/fl_updater.py index ae1ac45..4d1ae54 100644 --- a/src/app/services/updaters/fl_updater.py +++ b/src/app/services/updaters/fl_updater.py @@ -1,14 +1,17 @@ import asyncio +from logging import Logger from typing import Optional -from aiologger import Logger import aiomysql +from arq.connections import ArqRedis import asyncpg -from app.services.updaters.base import BaseUpdater from core.config import env_config +logger = Logger("fl_updater") + + async def run(cmd) -> tuple[bytes, bytes, Optional[int]]: proc = await asyncio.create_subprocess_shell( cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE @@ -26,7 +29,7 @@ def remove_dots(s: str): return s.replace(".", "") -class FlUpdater(BaseUpdater): +class FlUpdater: SOURCE: int FILES = [ @@ -52,15 +55,13 @@ class FlUpdater(BaseUpdater): sequences_updated_event: asyncio.Event genres_updated_event: asyncio.Event - logger: Logger - async def _import_dump(self, filename: str): await run( f"wget -O - {env_config.FL_BASE_URL}/sql/{filename}.gz | gunzip | " f"mysql -h {env_config.MYSQL_HOST} -u {env_config.MYSQL_USER} " f'-p"{env_config.MYSQL_PASSWORD}" {env_config.MYSQL_DB_NAME}' ) - await self.logger.info(f"Imported {filename}") + logger.info(f"Imported {filename}") async def _prepare(self): posgres_pool = await asyncpg.create_pool( @@ -85,7 +86,7 @@ class FlUpdater(BaseUpdater): self.postgres_pool = posgres_pool async def _set_source(self): - await self.logger.info("Set source...") + logger.info("Set source...") source_row = await self.postgres_pool.fetchrow( "SELECT id FROM sources WHERE name = 'flibusta';" @@ -102,7 +103,7 @@ class FlUpdater(BaseUpdater): self.SOURCE = source_row["id"] - await self.logger.info("Source has set!") + logger.info("Source has set!") async def _update_authors(self): def prepare_author(row: list): @@ -114,7 +115,7 @@ class FlUpdater(BaseUpdater): remove_wrong_ch(row[3]), ] - await self.logger.info("Update authors...") + logger.info("Update authors...") await self.postgres_pool.execute( """ @@ -157,7 +158,7 @@ class FlUpdater(BaseUpdater): self.authors_updated_event.set() - await self.logger.info("Authors updated!") + logger.info("Authors updated!") async def _update_books(self): replace_dict = {"ru-": "ru", "ru~": "ru"} @@ -178,7 +179,7 @@ class FlUpdater(BaseUpdater): row[5] == "1", ] - await self.logger.info("Update books...") + logger.info("Update books...") await self.postgres_pool.execute( """ @@ -223,13 +224,13 @@ class FlUpdater(BaseUpdater): self.books_updated_event.set() - await self.logger.info("Books updated!") + logger.info("Books updated!") async def _update_books_authors(self): await self.books_updated_event.wait() await self.authors_updated_event.wait() - await self.logger.info("Update books_authors...") + logger.info("Update books_authors...") await self.postgres_pool.execute( """ @@ -271,13 +272,13 @@ class FlUpdater(BaseUpdater): [(self.SOURCE, *row) for row in rows], ) - await self.logger.info("Books_authors updated!") + logger.info("Books_authors updated!") async def _update_translations(self): await self.books_updated_event.wait() await self.authors_updated_event.wait() - await self.logger.info("Update translations...") + logger.info("Update translations...") await self.postgres_pool.execute( """ @@ -324,7 +325,7 @@ class FlUpdater(BaseUpdater): [(self.SOURCE, *row) for row in rows], ) - await self.logger.info("Translations updated!") + logger.info("Translations updated!") async def _update_sequences(self): def prepare_sequence(row: list): @@ -334,7 +335,7 @@ class FlUpdater(BaseUpdater): remove_wrong_ch(row[1]), ] - await self.logger.info("Update sequences...") + logger.info("Update sequences...") await self.postgres_pool.execute( """ @@ -373,13 +374,13 @@ class FlUpdater(BaseUpdater): self.sequences_updated_event.set() - await self.logger.info("Sequences updated!") + logger.info("Sequences updated!") async def _update_sequences_info(self): await self.sequences_updated_event.wait() await self.books_updated_event.wait() - await self.logger.info("Update book_sequences...") + logger.info("Update book_sequences...") await self.postgres_pool.execute( """ @@ -429,12 +430,12 @@ class FlUpdater(BaseUpdater): [[self.SOURCE, *row] for row in rows], ) - await self.logger.info("Book_sequences updated!") + logger.info("Book_sequences updated!") async def _update_book_annotations(self): await self.books_updated_event.wait() - await self.logger.info("Update book_annotations...") + logger.info("Update book_annotations...") await self.postgres_pool.execute( """ @@ -478,12 +479,12 @@ class FlUpdater(BaseUpdater): [[self.SOURCE, *row] for row in rows], ) - await self.logger.info("Book_annotations updated!") + logger.info("Book_annotations updated!") await self._update_book_annotations_pic() async def _update_book_annotations_pic(self): - await self.logger.info("Update book_annotations_pic...") + logger.info("Update book_annotations_pic...") async with self.mysql_pool.acquire() as conn: async with conn.cursor() as cursor: @@ -508,12 +509,12 @@ class FlUpdater(BaseUpdater): [[self.SOURCE, *row] for row in rows], ) - await self.logger.info("Book_annotation_pics updated!") + logger.info("Book_annotation_pics updated!") async def _update_author_annotations(self): await self.authors_updated_event.wait() - await self.logger.info("Update author_annotations...") + logger.info("Update author_annotations...") await self.postgres_pool.execute( """ @@ -554,12 +555,12 @@ class FlUpdater(BaseUpdater): [[self.SOURCE, *row] for row in rows], ) - await self.logger.info("Author_annotation_updated!") + logger.info("Author_annotation_updated!") await self._update_author_annotations_pics() async def _update_author_annotations_pics(self): - await self.logger.info("Update author_annotations_pic...") + logger.info("Update author_annotations_pic...") async with self.mysql_pool.acquire() as conn: async with conn.cursor() as cursor: @@ -584,10 +585,10 @@ class FlUpdater(BaseUpdater): [[self.SOURCE, *row] for row in rows], ) - await self.logger.info("Author_annotatioins_pic updated!") + logger.info("Author_annotatioins_pic updated!") async def _update_genres(self): - await self.logger.info("Update genres...") + logger.info("Update genres...") await self.postgres_pool.execute( """ @@ -627,13 +628,13 @@ class FlUpdater(BaseUpdater): [[self.SOURCE, *row] for row in rows], ) - await self.logger.info("Genres updated!") + logger.info("Genres updated!") async def _update_books_genres(self): await self.books_updated_event.wait() await self.genres_updated_event.wait() - await self.logger.info("Update book_genres...") + logger.info("Update book_genres...") await self.postgres_pool.execute( """ @@ -675,15 +676,19 @@ class FlUpdater(BaseUpdater): [(self.SOURCE, *row) for row in rows], ) - await self.logger.info("Book_genres updated!") - - async def _update(self) -> bool: - self.logger = Logger.with_default_handlers() - - await self._prepare() + logger.info("Book_genres updated!") + async def _import(self, ctx) -> bool: await asyncio.gather(*[self._import_dump(filename) for filename in self.FILES]) + arq_pool: ArqRedis = ctx["arq_pool"] + await arq_pool.enqueue_job("run_fl_update2") + + return True + + async def _update(self, ctx) -> bool: + await self._prepare() + await self._set_source() self.authors_updated_event = asyncio.Event() @@ -706,7 +711,10 @@ class FlUpdater(BaseUpdater): return True - @classmethod - async def update(cls) -> bool: - updater = cls() - return await updater._update() + +async def run_fl_update(ctx) -> bool: + return await FlUpdater()._import(ctx) + + +async def run_fl_update2(ctx) -> bool: + return await FlUpdater()._update(ctx) diff --git a/src/app/views.py b/src/app/views.py index 484b8a1..d2a680b 100644 --- a/src/app/views.py +++ b/src/app/views.py @@ -1,4 +1,6 @@ -from fastapi import APIRouter, BackgroundTasks, Depends +from fastapi import APIRouter, Depends, Request + +from arq.connections import ArqRedis from app.depends import check_token from app.services.updaters import UpdaterTypes, UPDATERS @@ -8,9 +10,8 @@ router = APIRouter(tags=["updater"], dependencies=[Depends(check_token)]) @router.post("/update/{updater}") -async def update(updater: UpdaterTypes, background_tasks: BackgroundTasks): - updater_ = UPDATERS[updater] - - background_tasks.add_task(updater_.update) +async def update(request: Request, updater: UpdaterTypes): + arq_pool: ArqRedis = request.app.state.arq_pool + await arq_pool.enqueue_job(UPDATERS[updater]) return "Ok!" diff --git a/src/core/app.py b/src/core/app.py index 4f37812..44656bf 100644 --- a/src/core/app.py +++ b/src/core/app.py @@ -1,6 +1,7 @@ from fastapi import FastAPI from app.views import router +from core.arq_pool import get_arq_pool def start_app() -> FastAPI: @@ -8,4 +9,8 @@ def start_app() -> FastAPI: app.include_router(router) + @app.on_event("startup") + async def startup() -> None: + app.state.arq_pool = await get_arq_pool() + return app diff --git a/src/core/arq_pool.py b/src/core/arq_pool.py new file mode 100644 index 0000000..e5de84d --- /dev/null +++ b/src/core/arq_pool.py @@ -0,0 +1,15 @@ +from arq.connections import create_pool, RedisSettings, ArqRedis + +from core.config import env_config + + +def get_redis_settings() -> RedisSettings: + return RedisSettings( + host=env_config.REDIS_HOST, + port=env_config.REDIS_PORT, + database=env_config.REDIS_DB, + ) + + +async def get_arq_pool() -> ArqRedis: + return await create_pool(get_redis_settings()) diff --git a/src/core/config.py b/src/core/config.py index 9dcc632..91958cd 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -24,6 +24,10 @@ class EnvConfig(BaseSettings): MYSQL_USER: str MYSQL_PASSWORD: str + REDIS_HOST: str + REDIS_PORT: int + REDIS_DB: int + FL_BASE_URL: str diff --git a/src/core/setup_arq.py b/src/core/setup_arq.py new file mode 100644 index 0000000..ecf432f --- /dev/null +++ b/src/core/setup_arq.py @@ -0,0 +1,14 @@ +from app.services.updaters.fl_updater import run_fl_update, run_fl_update2 +from core.arq_pool import get_redis_settings, get_arq_pool + + +async def startup(ctx): + ctx["arq_pool"] = await get_arq_pool() + + +class WorkerSettings: + functions = [run_fl_update, run_fl_update2] + on_startup = startup + redis_settings = get_redis_settings() + max_jobs = 1 + job_timeout = 60 * 60