flibusta-apps/library_updater
1
0
Fork 0
mirror of https://github.com/flibusta-apps/library_updater.git synced 2025-12-06 15:45:36 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add linters configs

Browse Source
This commit is contained in:
Kurbanov Bulat
2022-01-01 20:37:37 +03:00
parent 5def2d8ab3
commit 6ec5fede0f
13 changed files with 619 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
.github/workflows/linters.yaml vendored Normal file
View File

@@ -0,0 +1,35 @@
name: Linters
on:
push:
branches:
- main
pull_request:
types: [opened, synchronize, reopened]
jobs:
Run-Pre-Commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 32
- uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install pre-commit
run: pip3 install pre-commit
- name: Pre-commit (Push)
env:
SETUPTOOLS_USE_DISTUTILS: stdlib
if: ${{ github.event_name == 'push' }}
run: pre-commit run --source ${{ github.event.before }} --origin ${{ github.event.after }} --show-diff-on-failure
- name: Pre-commit (Pull-Request)
env:
SETUPTOOLS_USE_DISTUTILS: stdlib
if: ${{ github.event_name == 'pull_request' }}
run: pre-commit run --source ${{ github.event.pull_request.base.sha }} --origin ${{ github.event.pull_request.head.sha }} --show-diff-on-failure

20
.pre-commit-config.yaml Normal file
View File

@@ -0,0 +1,20 @@
exclude: 'docs|node_modules|migrations|.git|.tox'
repos:
- repo: https://github.com/ambv/black
rev: 21.12b0
hooks:
- id: black
language_version: python3.9
- repo: https://github.com/pycqa/isort
rev: 5.10.1
hooks:
- id: isort
- repo: https://github.com/csachs/pyproject-flake8
rev: v0.0.1a2.post1
hooks:
- id: pyproject-flake8
additional_dependencies: [
'-e', 'git+https://github.com/pycqa/pyflakes@1911c20#egg=pyflakes',
'-e', 'git+https://github.com/pycqa/pycodestyle@d219c68#egg=pycodestyle',
]

7
docker/build.dockerfile
View File

@@ -4,8 +4,11 @@ RUN apt-get update \
&& apt-get install --no-install-recommends -y gcc build-essential python3-dev libpq-dev libffi-dev \ && apt-get install --no-install-recommends -y gcc build-essential python3-dev libpq-dev libffi-dev \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
WORKDIR / WORKDIR /root/poetry
COPY ./requirements.txt ./ COPY pyproject.toml poetry.lock /root/poetry/
RUN pip install poetry --no-cache-dir \
&& poetry export --without-hashes > requirements.txt
ENV VENV_PATH=/opt/venv ENV VENV_PATH=/opt/venv
RUN python -m venv $VENV_PATH \ RUN python -m venv $VENV_PATH \

423
poetry.lock generated Normal file
View File

@@ -0,0 +1,423 @@
[[package]]
name = "aiologger"
version = "0.6.1"
description = "Asynchronous logging for python and asyncio"
category = "main"
optional = false
python-versions = ">=3.6"
[package.extras]
aiofiles = ["aiofiles (==0.4.0)"]
[[package]]
name = "aiomysql"
version = "0.0.22"
description = "MySQL driver for asyncio."
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
PyMySQL = ">=0.9,<=0.9.3"
[package.extras]
sa = ["sqlalchemy (>=1.0)"]
[[package]]
name = "anyio"
version = "3.4.0"
description = "High level compatibility layer for multiple asynchronous event loop implementations"
category = "main"
optional = false
python-versions = ">=3.6.2"
[package.dependencies]
idna = ">=2.8"
sniffio = ">=1.1"
[package.extras]
doc = ["sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"]
test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=6.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"]
trio = ["trio (>=0.16)"]
[[package]]
name = "asgiref"
version = "3.4.1"
description = "ASGI specs, helper code, and adapters"
category = "main"
optional = false
python-versions = ">=3.6"
[package.extras]
tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"]
[[package]]
name = "asyncpg"
version = "0.25.0"
description = "An asyncio PostgreSQL driver"
category = "main"
optional = false
python-versions = ">=3.6.0"
[package.extras]
dev = ["Cython (>=0.29.24,<0.30.0)", "pytest (>=6.0)", "Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0.15.3)"]
docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)"]
test = ["pycodestyle (>=2.7.0,<2.8.0)", "flake8 (>=3.9.2,<3.10.0)", "uvloop (>=0.15.3)"]
[[package]]
name = "certifi"
version = "2021.10.8"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "charset-normalizer"
version = "2.0.9"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
optional = false
python-versions = ">=3.5.0"
[package.extras]
unicode_backport = ["unicodedata2"]
[[package]]
name = "click"
version = "8.0.3"
description = "Composable command line interface toolkit"
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
colorama = {version = "*", markers = "platform_system == \"Windows\""}
[[package]]
name = "colorama"
version = "0.4.4"
description = "Cross-platform colored terminal text."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[[package]]
name = "fastapi"
version = "0.70.1"
description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
category = "main"
optional = false
python-versions = ">=3.6.1"
[package.dependencies]
pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1.7.3,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0"
starlette = "0.16.0"
[package.extras]
all = ["requests (>=2.24.0,<3.0.0)", "jinja2 (>=2.11.2,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "itsdangerous (>=1.1.0,<3.0.0)", "pyyaml (>=5.3.1,<6.0.0)", "ujson (>=4.0.1,<5.0.0)", "orjson (>=3.2.1,<4.0.0)", "email_validator (>=1.1.1,<2.0.0)", "uvicorn[standard] (>=0.12.0,<0.16.0)"]
dev = ["python-jose[cryptography] (>=3.3.0,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "autoflake (>=1.4.0,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "uvicorn[standard] (>=0.12.0,<0.16.0)"]
doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=7.1.9,<8.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "typer-cli (>=0.0.12,<0.0.13)", "pyyaml (>=5.3.1,<6.0.0)"]
test = ["pytest (>=6.2.4,<7.0.0)", "pytest-cov (>=2.12.0,<4.0.0)", "mypy (==0.910)", "flake8 (>=3.8.3,<4.0.0)", "black (==21.9b0)", "isort (>=5.0.6,<6.0.0)", "requests (>=2.24.0,<3.0.0)", "httpx (>=0.14.0,<0.19.0)", "email_validator (>=1.1.1,<2.0.0)", "sqlalchemy (>=1.3.18,<1.5.0)", "peewee (>=3.13.3,<4.0.0)", "databases[sqlite] (>=0.3.2,<0.6.0)", "orjson (>=3.2.1,<4.0.0)", "ujson (>=4.0.1,<5.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "flask (>=1.1.2,<3.0.0)", "anyio[trio] (>=3.2.1,<4.0.0)", "types-ujson (==0.1.1)", "types-orjson (==3.6.0)", "types-dataclasses (==0.1.7)"]
[[package]]
name = "h11"
version = "0.12.0"
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "httpcore"
version = "0.14.3"
description = "A minimal low-level HTTP client."
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
anyio = ">=3.0.0,<4.0.0"
certifi = "*"
h11 = ">=0.11,<0.13"
sniffio = ">=1.0.0,<2.0.0"
[package.extras]
http2 = ["h2 (>=3,<5)"]
[[package]]
name = "httpx"
version = "0.21.1"
description = "The next generation HTTP client."
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
certifi = "*"
charset-normalizer = "*"
httpcore = ">=0.14.0,<0.15.0"
rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]}
sniffio = "*"
[package.extras]
brotli = ["brotlicffi", "brotli"]
cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10.0.0,<11.0.0)", "pygments (>=2.0.0,<3.0.0)"]
http2 = ["h2 (>=3,<5)"]
[[package]]
name = "idna"
version = "3.3"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=3.5"
[[package]]
name = "pydantic"
version = "1.9.0"
description = "Data validation and settings management using python 3.6 type hinting"
category = "main"
optional = false
python-versions = ">=3.6.1"
[package.dependencies]
python-dotenv = {version = ">=0.10.4", optional = true, markers = "extra == \"dotenv\""}
typing-extensions = ">=3.7.4.3"
[package.extras]
dotenv = ["python-dotenv (>=0.10.4)"]
email = ["email-validator (>=1.0.3)"]
[[package]]
name = "pymysql"
version = "0.9.3"
description = "Pure Python MySQL Driver"
category = "main"
optional = false
python-versions = "*"
[package.extras]
rsa = ["cryptography"]
[[package]]
name = "python-dotenv"
version = "0.19.2"
description = "Read key-value pairs from a .env file and set them as environment variables"
category = "main"
optional = false
python-versions = ">=3.5"
[package.extras]
cli = ["click (>=5.0)"]
[[package]]
name = "rfc3986"
version = "1.5.0"
description = "Validating URI References per RFC 3986"
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
idna = {version = "*", optional = true, markers = "extra == \"idna2008\""}
[package.extras]
idna2008 = ["idna"]
[[package]]
name = "sniffio"
version = "1.2.0"
description = "Sniff out which async library your code is running under"
category = "main"
optional = false
python-versions = ">=3.5"
[[package]]
name = "starlette"
version = "0.16.0"
description = "The little ASGI library that shines."
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
anyio = ">=3.0.0,<4"
[package.extras]
full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests", "graphene"]
[[package]]
name = "typing-extensions"
version = "4.0.1"
description = "Backported and Experimental Type Hints for Python 3.6+"
category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "uvicorn"
version = "0.16.0"
description = "The lightning-fast ASGI server."
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
asgiref = ">=3.4.0"
click = ">=7.0"
h11 = ">=0.8"
[package.extras]
standard = ["httptools (>=0.2.0,<0.4.0)", "watchgod (>=0.6)", "python-dotenv (>=0.13)", "PyYAML (>=5.1)", "websockets (>=9.1)", "websockets (>=10.0)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "colorama (>=0.4)"]
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "e82ee51aaa18bb146a524a296ada00fa905a92f8634c61503684091684a5d348"
[metadata.files]
aiologger = [
{file = "aiologger-0.6.1.tar.gz", hash = "sha256:1b6b8f00d74a588339b657ff60ffa9f64c53873887a008934c66e1a673ea68cd"},
]
aiomysql = [
{file = "aiomysql-0.0.22-py3-none-any.whl", hash = "sha256:4e4a65914daacc40e70f992ddbeef32457561efbad8de41393e8ac5a84126a5a"},
{file = "aiomysql-0.0.22.tar.gz", hash = "sha256:9bcf8f26d22e550f75cabd635fa19a55c45f835eea008275960cb37acadd622a"},
]
anyio = [
{file = "anyio-3.4.0-py3-none-any.whl", hash = "sha256:2855a9423524abcdd652d942f8932fda1735210f77a6b392eafd9ff34d3fe020"},
{file = "anyio-3.4.0.tar.gz", hash = "sha256:24adc69309fb5779bc1e06158e143e0b6d2c56b302a3ac3de3083c705a6ed39d"},
]
asgiref = [
{file = "asgiref-3.4.1-py3-none-any.whl", hash = "sha256:ffc141aa908e6f175673e7b1b3b7af4fdb0ecb738fc5c8b88f69f055c2415214"},
{file = "asgiref-3.4.1.tar.gz", hash = "sha256:4ef1ab46b484e3c706329cedeff284a5d40824200638503f5768edb6de7d58e9"},
]
asyncpg = [
{file = "asyncpg-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf5e3408a14a17d480f36ebaf0401a12ff6ae5457fdf45e4e2775c51cc9517d3"},
{file = "asyncpg-0.25.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2bc197fc4aca2fd24f60241057998124012469d2e414aed3f992579db0c88e3a"},
{file = "asyncpg-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a70783f6ffa34cc7dd2de20a873181414a34fd35a4a208a1f1a7f9f695e4ec4"},
{file = "asyncpg-0.25.0-cp310-cp310-win32.whl", hash = "sha256:43cde84e996a3afe75f325a68300093425c2f47d340c0fc8912765cf24a1c095"},
{file = "asyncpg-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:56d88d7ef4341412cd9c68efba323a4519c916979ba91b95d4c08799d2ff0c09"},
{file = "asyncpg-0.25.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a84d30e6f850bac0876990bcd207362778e2208df0bee8be8da9f1558255e634"},
{file = "asyncpg-0.25.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:beaecc52ad39614f6ca2e48c3ca15d56e24a2c15cbfdcb764a4320cc45f02fd5"},
{file = "asyncpg-0.25.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:6f8f5fc975246eda83da8031a14004b9197f510c41511018e7b1bedde6968e92"},
{file = "asyncpg-0.25.0-cp36-cp36m-win32.whl", hash = "sha256:ddb4c3263a8d63dcde3d2c4ac1c25206bfeb31fa83bd70fd539e10f87739dee4"},
{file = "asyncpg-0.25.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bf6dc9b55b9113f39eaa2057337ce3f9ef7de99a053b8a16360395ce588925cd"},
{file = "asyncpg-0.25.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:acb311722352152936e58a8ee3c5b8e791b24e84cd7d777c414ff05b3530ca68"},
{file = "asyncpg-0.25.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a61fb196ce4dae2f2fa26eb20a778db21bbee484d2e798cb3cc988de13bdd1b"},
{file = "asyncpg-0.25.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2633331cbc8429030b4f20f712f8d0fbba57fa8555ee9b2f45f981b81328b256"},
{file = "asyncpg-0.25.0-cp37-cp37m-win32.whl", hash = "sha256:863d36eba4a7caa853fd7d83fad5fd5306f050cc2fe6e54fbe10cdb30420e5e9"},
{file = "asyncpg-0.25.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fe471ccd915b739ca65e2e4dbd92a11b44a5b37f2e38f70827a1c147dafe0fa8"},
{file = "asyncpg-0.25.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:72a1e12ea0cf7c1e02794b697e3ca967b2360eaa2ce5d4bfdd8604ec2d6b774b"},
{file = "asyncpg-0.25.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4327f691b1bdb222df27841938b3e04c14068166b3a97491bec2cb982f49f03e"},
{file = "asyncpg-0.25.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:739bbd7f89a2b2f6bc44cb8bf967dab12c5bc714fcbe96e68d512be45ecdf962"},
{file = "asyncpg-0.25.0-cp38-cp38-win32.whl", hash = "sha256:18d49e2d93a7139a2fdbd113e320cc47075049997268a61bfbe0dde680c55471"},
{file = "asyncpg-0.25.0-cp38-cp38-win_amd64.whl", hash = "sha256:191fe6341385b7fdea7dbdcf47fd6db3fd198827dcc1f2b228476d13c05a03c6"},
{file = "asyncpg-0.25.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:52fab7f1b2c29e187dd8781fce896249500cf055b63471ad66332e537e9b5f7e"},
{file = "asyncpg-0.25.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a738f1b2876f30d710d3dc1e7858160a0afe1603ba16bf5f391f5316eb0ed855"},
{file = "asyncpg-0.25.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4105f57ad1e8fbc8b1e535d8fcefa6ce6c71081228f08680c6dea24384ff0e"},
{file = "asyncpg-0.25.0-cp39-cp39-win32.whl", hash = "sha256:f55918ded7b85723a5eaeb34e86e7b9280d4474be67df853ab5a7fa0cc7c6bf2"},
{file = "asyncpg-0.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:649e2966d98cc48d0646d9a4e29abecd8b59d38d55c256d5c857f6b27b7407ac"},
{file = "asyncpg-0.25.0.tar.gz", hash = "sha256:63f8e6a69733b285497c2855464a34de657f2cccd25aeaeeb5071872e9382540"},
]
certifi = [
{file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
{file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
]
charset-normalizer = [
{file = "charset-normalizer-2.0.9.tar.gz", hash = "sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c"},
{file = "charset_normalizer-2.0.9-py3-none-any.whl", hash = "sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721"},
]
click = [
{file = "click-8.0.3-py3-none-any.whl", hash = "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3"},
{file = "click-8.0.3.tar.gz", hash = "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"},
]
colorama = [
{file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"},
{file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"},
]
fastapi = [
{file = "fastapi-0.70.1-py3-none-any.whl", hash = "sha256:5367226c7bcd7bfb2e17edaf225fd9a983095b1372281e9a3eb661336fb93748"},
{file = "fastapi-0.70.1.tar.gz", hash = "sha256:21d03979b5336375c66fa5d1f3126c6beca650d5d2166fbb78345a30d33c8d06"},
]
h11 = [
{file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"},
{file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"},
]
httpcore = [
{file = "httpcore-0.14.3-py3-none-any.whl", hash = "sha256:9a98d2416b78976fc5396ff1f6b26ae9885efbb3105d24eed490f20ab4c95ec1"},
{file = "httpcore-0.14.3.tar.gz", hash = "sha256:d10162a63265a0228d5807964bd964478cbdb5178f9a2eedfebb2faba27eef5d"},
]
httpx = [
{file = "httpx-0.21.1-py3-none-any.whl", hash = "sha256:208e5ef2ad4d105213463cfd541898ed9d11851b346473539a8425e644bb7c66"},
{file = "httpx-0.21.1.tar.gz", hash = "sha256:02af20df486b78892a614a7ccd4e4e86a5409ec4981ab0e422c579a887acad83"},
]
idna = [
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
{file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
]
pydantic = [
{file = "pydantic-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cb23bcc093697cdea2708baae4f9ba0e972960a835af22560f6ae4e7e47d33f5"},
{file = "pydantic-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d5278bd9f0eee04a44c712982343103bba63507480bfd2fc2790fa70cd64cf4"},
{file = "pydantic-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab624700dc145aa809e6f3ec93fb8e7d0f99d9023b713f6a953637429b437d37"},
{file = "pydantic-1.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8d7da6f1c1049eefb718d43d99ad73100c958a5367d30b9321b092771e96c25"},
{file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3c3b035103bd4e2e4a28da9da7ef2fa47b00ee4a9cf4f1a735214c1bcd05e0f6"},
{file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3011b975c973819883842c5ab925a4e4298dffccf7782c55ec3580ed17dc464c"},
{file = "pydantic-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:086254884d10d3ba16da0588604ffdc5aab3f7f09557b998373e885c690dd398"},
{file = "pydantic-1.9.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0fe476769acaa7fcddd17cadd172b156b53546ec3614a4d880e5d29ea5fbce65"},
{file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8e9dcf1ac499679aceedac7e7ca6d8641f0193c591a2d090282aaf8e9445a46"},
{file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1e4c28f30e767fd07f2ddc6f74f41f034d1dd6bc526cd59e63a82fe8bb9ef4c"},
{file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c86229333cabaaa8c51cf971496f10318c4734cf7b641f08af0a6fbf17ca3054"},
{file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:c0727bda6e38144d464daec31dff936a82917f431d9c39c39c60a26567eae3ed"},
{file = "pydantic-1.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:dee5ef83a76ac31ab0c78c10bd7d5437bfdb6358c95b91f1ba7ff7b76f9996a1"},
{file = "pydantic-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9c9bdb3af48e242838f9f6e6127de9be7063aad17b32215ccc36a09c5cf1070"},
{file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee7e3209db1e468341ef41fe263eb655f67f5c5a76c924044314e139a1103a2"},
{file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b6037175234850ffd094ca77bf60fb54b08b5b22bc85865331dd3bda7a02fa1"},
{file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b2571db88c636d862b35090ccf92bf24004393f85c8870a37f42d9f23d13e032"},
{file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8b5ac0f1c83d31b324e57a273da59197c83d1bb18171e512908fe5dc7278a1d6"},
{file = "pydantic-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bbbc94d0c94dd80b3340fc4f04fd4d701f4b038ebad72c39693c794fd3bc2d9d"},
{file = "pydantic-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e0896200b6a40197405af18828da49f067c2fa1f821491bc8f5bde241ef3f7d7"},
{file = "pydantic-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bdfdadb5994b44bd5579cfa7c9b0e1b0e540c952d56f627eb227851cda9db77"},
{file = "pydantic-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:574936363cd4b9eed8acdd6b80d0143162f2eb654d96cb3a8ee91d3e64bf4cf9"},
{file = "pydantic-1.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c556695b699f648c58373b542534308922c46a1cda06ea47bc9ca45ef5b39ae6"},
{file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f947352c3434e8b937e3aa8f96f47bdfe6d92779e44bb3f41e4c213ba6a32145"},
{file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5e48ef4a8b8c066c4a31409d91d7ca372a774d0212da2787c0d32f8045b1e034"},
{file = "pydantic-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:96f240bce182ca7fe045c76bcebfa0b0534a1bf402ed05914a6f1dadff91877f"},
{file = "pydantic-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:815ddebb2792efd4bba5488bc8fde09c29e8ca3227d27cf1c6990fc830fd292b"},
{file = "pydantic-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c5b77947b9e85a54848343928b597b4f74fc364b70926b3c4441ff52620640c"},
{file = "pydantic-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c68c3bc88dbda2a6805e9a142ce84782d3930f8fdd9655430d8576315ad97ce"},
{file = "pydantic-1.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a79330f8571faf71bf93667d3ee054609816f10a259a109a0738dac983b23c3"},
{file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f5a64b64ddf4c99fe201ac2724daada8595ada0d102ab96d019c1555c2d6441d"},
{file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a733965f1a2b4090a5238d40d983dcd78f3ecea221c7af1497b845a9709c1721"},
{file = "pydantic-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cc6a4cb8a118ffec2ca5fcb47afbacb4f16d0ab8b7350ddea5e8ef7bcc53a16"},
{file = "pydantic-1.9.0-py3-none-any.whl", hash = "sha256:085ca1de245782e9b46cefcf99deecc67d418737a1fd3f6a4f511344b613a5b3"},
{file = "pydantic-1.9.0.tar.gz", hash = "sha256:742645059757a56ecd886faf4ed2441b9c0cd406079c2b4bee51bcc3fbcd510a"},
]
pymysql = [
{file = "PyMySQL-0.9.3-py2.py3-none-any.whl", hash = "sha256:3943fbbbc1e902f41daf7f9165519f140c4451c179380677e6a848587042561a"},
{file = "PyMySQL-0.9.3.tar.gz", hash = "sha256:d8c059dcd81dedb85a9f034d5e22dcb4442c0b201908bede99e306d65ea7c8e7"},
]
python-dotenv = [
{file = "python-dotenv-0.19.2.tar.gz", hash = "sha256:a5de49a31e953b45ff2d2fd434bbc2670e8db5273606c1e737cc6b93eff3655f"},
{file = "python_dotenv-0.19.2-py2.py3-none-any.whl", hash = "sha256:32b2bdc1873fd3a3c346da1c6db83d0053c3c62f28f1f38516070c4c8971b1d3"},
]
rfc3986 = [
{file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"},
{file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"},
]
sniffio = [
{file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"},
{file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"},
]
starlette = [
{file = "starlette-0.16.0-py3-none-any.whl", hash = "sha256:38eb24bf705a2c317e15868e384c1b8a12ca396e5a3c3a003db7e667c43f939f"},
{file = "starlette-0.16.0.tar.gz", hash = "sha256:e1904b5d0007aee24bdd3c43994be9b3b729f4f58e740200de1d623f8c3a8870"},
]
typing-extensions = [
{file = "typing_extensions-4.0.1-py3-none-any.whl", hash = "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b"},
{file = "typing_extensions-4.0.1.tar.gz", hash = "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e"},
]
uvicorn = [
{file = "uvicorn-0.16.0-py3-none-any.whl", hash = "sha256:d8c839231f270adaa6d338d525e2652a0b4a5f4c2430b5c4ef6ae4d11776b0d2"},
{file = "uvicorn-0.16.0.tar.gz", hash = "sha256:eacb66afa65e0648fcbce5e746b135d09722231ffffc61883d4fac2b62fbea8d"},
]

64
pyproject.toml Normal file
View File

@@ -0,0 +1,64 @@
[tool.poetry]
name = "library_updater"
version = "0.1.0"
description = ""
authors = ["Kurbanov Bulat <kurbanovbul@gmail.com>"]
[tool.poetry.dependencies]
python = "^3.9"
fastapi = "^0.70.1"
pydantic = {extras = ["dotenv"], version = "^1.9.0"}
httpx = "^0.21.1"
aiologger = "^0.6.1"
asyncpg = "^0.25.0"
aiomysql = "^0.0.22"
uvicorn = {extras = ["standart"], version = "^0.16.0"}
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.black]
include = '\.pyi?$'
exclude = '''
/(
\.git
| \.vscode
| \venv
| alembic
)/
'''
[tool.flake8]
ignore = [
# Whitespace before ':' ( https://www.flake8rules.com/rules/E203.html )
"E203"
]
max-line-length=88
max-complexity = 15
select = "B,C,E,F,W,T4,B9"
exclude = [
# No need to traverse our git directory
".git",
# There's no value in checking cache directories
"__pycache__",
# The conf file is mostly autogenerated, ignore it
"src/app/alembic/*",
# The old directory contains Flake8 2.0
]
per-file-ignores = [
"src/app/services/updaters/fl_updater.py:E501",
]
[tool.isort]
profile = "black"
only_sections = true
force_sort_within_sections = true
lines_after_imports = 2
lexicographical = true
sections = ["FUTURE", "STDLIB", "BASEFRAMEWORK", "FRAMEWORKEXT", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
known_baseframework = ["fastapi",]
known_frameworkext = ["starlette",]
src_paths = ["src"]

7
requirements.txt
View File

@@ -1,7 +0,0 @@
fastapi
pydantic[dotenv]
asyncpg
aiomysql
uvicorn[standart]
aiologger
httpx

4
src/app/depends.py
View File

@@ -6,4 +6,6 @@ from core.config import env_config
async def check_token(api_key: str = Security(default_security)): async def check_token(api_key: str = Security(default_security)):
if api_key != env_config.API_KEY: if api_key != env_config.API_KEY:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!") raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail="Wrong api key!"
)

6
src/app/services/updaters/__init__.py
View File

@@ -5,9 +5,7 @@ from app.services.updaters.fl_updater import FlUpdater
class UpdaterTypes(Enum): class UpdaterTypes(Enum):
FL = 'fl' FL = "fl"
UPDATERS: dict[UpdaterTypes, BaseUpdater] = { UPDATERS: dict[UpdaterTypes, BaseUpdater] = {UpdaterTypes.FL: FlUpdater}
UpdaterTypes.FL: FlUpdater
}

132
src/app/services/updaters/fl_updater.py
View File

@@ -1,50 +1,49 @@
from typing import Optional
import asyncio import asyncio
import platform import platform
from typing import Optional
import asyncpg
import aiomysql
from aiologger import Logger from aiologger import Logger
import aiomysql
import asyncpg
from core.config import env_config
from app.services.updaters.base import BaseUpdater from app.services.updaters.base import BaseUpdater
from app.services.webhook import WebhookSender from app.services.webhook import WebhookSender
from core.config import env_config
async def run(cmd) -> tuple[bytes, bytes, Optional[int]]: async def run(cmd) -> tuple[bytes, bytes, Optional[int]]:
proc = await asyncio.create_subprocess_shell( proc = await asyncio.create_subprocess_shell(
cmd, cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
stdout=asyncio.subprocess.PIPE, )
stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate() stdout, stderr = await proc.communicate()
return stdout, stderr, proc.returncode return stdout, stderr, proc.returncode
def remove_wrong_ch(s: str): def remove_wrong_ch(s: str):
return s.replace(";", "").replace("\n", " ").replace('ё', 'е') return s.replace(";", "").replace("\n", " ").replace("ё", "е")
def remove_dots(s: str): def remove_dots(s: str):
return s.replace('.', '') return s.replace(".", "")
class FlUpdater(BaseUpdater): class FlUpdater(BaseUpdater):
SOURCE: int SOURCE: int
FILES = [ FILES = [
'lib.libavtor.sql', "lib.libavtor.sql",
'lib.libbook.sql', "lib.libbook.sql",
'lib.libavtorname.sql', "lib.libavtorname.sql",
'lib.libtranslator.sql', "lib.libtranslator.sql",
'lib.libseqname.sql', "lib.libseqname.sql",
'lib.libseq.sql', "lib.libseq.sql",
'lib.libgenre.sql', "lib.libgenre.sql",
'lib.libgenrelist.sql', "lib.libgenrelist.sql",
'lib.b.annotations.sql', "lib.b.annotations.sql",
'lib.b.annotations_pics.sql', "lib.b.annotations_pics.sql",
'lib.a.annotations.sql', "lib.a.annotations.sql",
'lib.a.annotations_pics.sql', "lib.a.annotations_pics.sql",
] ]
postgres_pool: asyncpg.Pool postgres_pool: asyncpg.Pool
@@ -59,7 +58,7 @@ class FlUpdater(BaseUpdater):
logger: Logger logger: Logger
async def log(self, message): async def log(self, message):
if 'windows' in self.platform.lower(): if "windows" in self.platform.lower():
print(message) print(message)
else: else:
await self.logger.info(message) await self.logger.info(message)
@@ -86,7 +85,7 @@ class FlUpdater(BaseUpdater):
result = await run( result = await run(
f"wget -O - {env_config.FL_BASE_URL}/sql/{filename}.gz | gunzip | " f"wget -O - {env_config.FL_BASE_URL}/sql/{filename}.gz | gunzip | "
f"mysql -h {env_config.MYSQL_HOST} -u {env_config.MYSQL_USER} " f"mysql -h {env_config.MYSQL_HOST} -u {env_config.MYSQL_USER} "
f"-p\"{env_config.MYSQL_PASSWORD}\" {env_config.MYSQL_DB_NAME}" f'-p"{env_config.MYSQL_PASSWORD}" {env_config.MYSQL_DB_NAME}'
) )
await self.log(f"Imported {filename}: {result}.") await self.log(f"Imported {filename}: {result}.")
@@ -113,7 +112,7 @@ class FlUpdater(BaseUpdater):
self.postgres_pool = posgres_pool self.postgres_pool = posgres_pool
async def _set_source(self): async def _set_source(self):
await self.log('Set source...') await self.log("Set source...")
source_row = await self.postgres_pool.fetchrow( source_row = await self.postgres_pool.fetchrow(
"SELECT id FROM sources WHERE name = 'flibusta';" "SELECT id FROM sources WHERE name = 'flibusta';"
@@ -128,7 +127,7 @@ class FlUpdater(BaseUpdater):
"SELECT id FROM sources WHERE name = 'flibusta';" "SELECT id FROM sources WHERE name = 'flibusta';"
) )
self.SOURCE = source_row['id'] self.SOURCE = source_row["id"]
await self.log("Source has set!") await self.log("Source has set!")
@@ -139,7 +138,7 @@ class FlUpdater(BaseUpdater):
row[0], row[0],
remove_wrong_ch(row[1]), remove_wrong_ch(row[1]),
remove_wrong_ch(row[2]), remove_wrong_ch(row[2]),
remove_wrong_ch(row[3]) remove_wrong_ch(row[3]),
] ]
await self.log("Update authors...") await self.log("Update authors...")
@@ -150,13 +149,13 @@ class FlUpdater(BaseUpdater):
"SELECT AvtorId, FirstName, LastName, MiddleName FROM libavtorname;" "SELECT AvtorId, FirstName, LastName, MiddleName FROM libavtorname;"
) )
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO authors (source, remote_id, first_name, last_name, middle_name) " "INSERT INTO authors (source, remote_id, first_name, last_name, middle_name) "
"VALUES ($1, $2, cast($3 as varchar), cast($4 as varchar), cast($5 as varchar)) " "VALUES ($1, $2, cast($3 as varchar), cast($4 as varchar), cast($5 as varchar)) "
"ON CONFLICT (source, remote_id) " "ON CONFLICT (source, remote_id) "
"DO UPDATE SET first_name = EXCLUDED.first_name, last_name = EXCLUDED.last_name, middle_name = EXCLUDED.middle_name;", "DO UPDATE SET first_name = EXCLUDED.first_name, last_name = EXCLUDED.last_name, middle_name = EXCLUDED.middle_name;",
[prepare_author(row) for row in rows] [prepare_author(row) for row in rows],
) )
self.authors_updated_event.set() self.authors_updated_event.set()
@@ -164,10 +163,7 @@ class FlUpdater(BaseUpdater):
await self.log("Authors updated!") await self.log("Authors updated!")
async def _update_books(self): async def _update_books(self):
replace_dict = { replace_dict = {"ru-": "ru", "ru~": "ru"}
"ru-": "ru",
"ru~": "ru"
}
def fix_lang(lang: str) -> str: def fix_lang(lang: str) -> str:
lower_lang = lang.lower() lower_lang = lang.lower()
@@ -182,7 +178,7 @@ class FlUpdater(BaseUpdater):
fix_lang(row[2]), fix_lang(row[2]),
row[3], row[3],
row[4], row[4],
row[5] == '1' row[5] == "1",
] ]
await self.log("Update books...") await self.log("Update books...")
@@ -193,14 +189,14 @@ class FlUpdater(BaseUpdater):
"SELECT BookId, Title, Lang, FileType, Time, Deleted FROM libbook;" "SELECT BookId, Title, Lang, FileType, Time, Deleted FROM libbook;"
) )
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO books (source, remote_id, title, lang, file_type, uploaded, is_deleted) " "INSERT INTO books (source, remote_id, title, lang, file_type, uploaded, is_deleted) "
"VALUES ($1, $2, cast($3 as varchar), cast($4 as varchar), cast($5 as varchar), $6, $7) " "VALUES ($1, $2, cast($3 as varchar), cast($4 as varchar), cast($5 as varchar), $6, $7) "
"ON CONFLICT (source, remote_id) " "ON CONFLICT (source, remote_id) "
"DO UPDATE SET title = EXCLUDED.title, lang = EXCLUDED.lang, file_type = EXCLUDED.file_type, " "DO UPDATE SET title = EXCLUDED.title, lang = EXCLUDED.lang, file_type = EXCLUDED.file_type, "
"uploaded = EXCLUDED.uploaded, is_deleted = EXCLUDED.is_deleted;", "uploaded = EXCLUDED.uploaded, is_deleted = EXCLUDED.is_deleted;",
[prepare_book(row) for row in rows] [prepare_book(row) for row in rows],
) )
self.books_updated_event.set() self.books_updated_event.set()
@@ -215,18 +211,16 @@ class FlUpdater(BaseUpdater):
async with self.mysql_pool.acquire() as conn: async with self.mysql_pool.acquire() as conn:
async with conn.cursor() as cursor: async with conn.cursor() as cursor:
await cursor.execute( await cursor.execute("SELECT BookId, AvtorId FROM libavtor;")
"SELECT BookId, AvtorId FROM libavtor;"
)
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO book_authors (book, author) " "INSERT INTO book_authors (book, author) "
"SELECT " "SELECT "
"(SELECT id FROM books WHERE source = $1 AND remote_id = $2), " "(SELECT id FROM books WHERE source = $1 AND remote_id = $2), "
"(SELECT id FROM authors WHERE source = $1 AND remote_id = $3) " "(SELECT id FROM authors WHERE source = $1 AND remote_id = $3) "
"ON CONFLICT (book, author) DO NOTHING;", "ON CONFLICT (book, author) DO NOTHING;",
[(self.SOURCE, *row) for row in rows] [(self.SOURCE, *row) for row in rows],
) )
await self.log("Books_authors updated!") await self.log("Books_authors updated!")
@@ -244,7 +238,7 @@ class FlUpdater(BaseUpdater):
"WHERE BookId IN (SELECT BookId FROM libbook);" "WHERE BookId IN (SELECT BookId FROM libbook);"
) )
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO translations (book, author, position) " "INSERT INTO translations (book, author, position) "
"SELECT " "SELECT "
@@ -253,7 +247,7 @@ class FlUpdater(BaseUpdater):
"$4 " "$4 "
"ON CONFLICT (book, author) " "ON CONFLICT (book, author) "
"DO UPDATE SET position = EXCLUDED.position;", "DO UPDATE SET position = EXCLUDED.position;",
[(self.SOURCE, *row) for row in rows] [(self.SOURCE, *row) for row in rows],
) )
await self.log("Translations updated!") await self.log("Translations updated!")
@@ -270,17 +264,15 @@ class FlUpdater(BaseUpdater):
async with self.mysql_pool.acquire() as conn: async with self.mysql_pool.acquire() as conn:
async with conn.cursor() as cursor: async with conn.cursor() as cursor:
await cursor.execute( await cursor.execute("SELECT SeqId, SeqName FROM libseqname;")
"SELECT SeqId, SeqName FROM libseqname;"
)
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO sequences (source, remote_id, name) " "INSERT INTO sequences (source, remote_id, name) "
"VALUES ($1, $2, cast($3 as varchar)) " "VALUES ($1, $2, cast($3 as varchar)) "
"ON CONFLICT (source, remote_id) " "ON CONFLICT (source, remote_id) "
"DO UPDATE SET name = EXCLUDED.name;", "DO UPDATE SET name = EXCLUDED.name;",
[prepare_sequence(row) for row in rows] [prepare_sequence(row) for row in rows],
) )
self.sequences_updated_event.set() self.sequences_updated_event.set()
@@ -302,7 +294,7 @@ class FlUpdater(BaseUpdater):
"SeqId IN (SELECT SeqId FROM libseqname);" "SeqId IN (SELECT SeqId FROM libseqname);"
) )
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO book_sequences (book, sequence, position) " "INSERT INTO book_sequences (book, sequence, position) "
"SELECT " "SELECT "
@@ -311,7 +303,7 @@ class FlUpdater(BaseUpdater):
"$4 " "$4 "
"ON CONFLICT (book, sequence) " "ON CONFLICT (book, sequence) "
"DO UPDATE SET position = EXCLUDED.position;", "DO UPDATE SET position = EXCLUDED.position;",
[[self.SOURCE, *row] for row in rows] [[self.SOURCE, *row] for row in rows],
) )
await self.log("Book_sequences updated!") await self.log("Book_sequences updated!")
@@ -328,7 +320,7 @@ class FlUpdater(BaseUpdater):
"WHERE BookId IN (SELECT BookId FROM libbook);" "WHERE BookId IN (SELECT BookId FROM libbook);"
) )
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO book_annotations (book, title, text) " "INSERT INTO book_annotations (book, title, text) "
"SELECT " "SELECT "
@@ -336,7 +328,7 @@ class FlUpdater(BaseUpdater):
"$3, $4 " "$3, $4 "
"ON CONFLICT (book) " "ON CONFLICT (book) "
"DO UPDATE SET title = EXCLUDED.title, text = EXCLUDED.text;", "DO UPDATE SET title = EXCLUDED.title, text = EXCLUDED.text;",
[[self.SOURCE, *row] for row in rows] [[self.SOURCE, *row] for row in rows],
) )
await self.log("Book_annotations updated!") await self.log("Book_annotations updated!")
@@ -348,17 +340,15 @@ class FlUpdater(BaseUpdater):
async with self.mysql_pool.acquire() as conn: async with self.mysql_pool.acquire() as conn:
async with conn.cursor() as cursor: async with conn.cursor() as cursor:
await cursor.execute( await cursor.execute("SELECT BookId, File FROM libbpics;")
"SELECT BookId, File FROM libbpics;"
)
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"UPDATE book_annotations " "UPDATE book_annotations "
"SET file = cast($3 as varchar) " "SET file = cast($3 as varchar) "
"FROM (SELECT id FROM books WHERE source = $1 AND remote_id = $2) as books " "FROM (SELECT id FROM books WHERE source = $1 AND remote_id = $2) as books "
"WHERE book = books.id;", "WHERE book = books.id;",
[[self.SOURCE, *row] for row in rows] [[self.SOURCE, *row] for row in rows],
) )
await self.log("Book_annotation_pics updated!") await self.log("Book_annotation_pics updated!")
@@ -374,7 +364,7 @@ class FlUpdater(BaseUpdater):
"SELECT AvtorId, Title, Body FROM libaannotations;" "SELECT AvtorId, Title, Body FROM libaannotations;"
) )
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO author_annotations (author, title, text) " "INSERT INTO author_annotations (author, title, text) "
"SELECT " "SELECT "
@@ -382,7 +372,7 @@ class FlUpdater(BaseUpdater):
"$3, $4 " "$3, $4 "
"ON CONFLICT (author) " "ON CONFLICT (author) "
"DO UPDATE SET title = EXCLUDED.title, text = EXCLUDED.text;", "DO UPDATE SET title = EXCLUDED.title, text = EXCLUDED.text;",
[[self.SOURCE, *row] for row in rows] [[self.SOURCE, *row] for row in rows],
) )
await self.log("Author_annotation_updated!") await self.log("Author_annotation_updated!")
@@ -394,17 +384,15 @@ class FlUpdater(BaseUpdater):
async with self.mysql_pool.acquire() as conn: async with self.mysql_pool.acquire() as conn:
async with conn.cursor() as cursor: async with conn.cursor() as cursor:
await cursor.execute( await cursor.execute("SELECT AvtorId, File FROM libapics;")
"SELECT AvtorId, File FROM libapics;"
)
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"UPDATE author_annotations " "UPDATE author_annotations "
"SET file = cast($3 as varchar) " "SET file = cast($3 as varchar) "
"FROM (SELECT id FROM authors WHERE source = $1 AND remote_id = $2) as authors " "FROM (SELECT id FROM authors WHERE source = $1 AND remote_id = $2) as authors "
"WHERE author = authors.id;", "WHERE author = authors.id;",
[[self.SOURCE, *row] for row in rows] [[self.SOURCE, *row] for row in rows],
) )
await self.log("Author_annotatioins_pic updated!") await self.log("Author_annotatioins_pic updated!")
@@ -418,13 +406,13 @@ class FlUpdater(BaseUpdater):
"SELECT GenreId, GenreCode, GenreDesc, GenreMeta FROM libgenrelist;" "SELECT GenreId, GenreCode, GenreDesc, GenreMeta FROM libgenrelist;"
) )
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO genres (source, remote_id, code, description, meta) " "INSERT INTO genres (source, remote_id, code, description, meta) "
"VALUES ($1, $2, cast($3 as varchar), cast($4 as varchar), cast($5 as varchar)) " "VALUES ($1, $2, cast($3 as varchar), cast($4 as varchar), cast($5 as varchar)) "
"ON CONFLICT (source, remote_id) " "ON CONFLICT (source, remote_id) "
"DO UPDATE SET code = EXCLUDED.code, description = EXCLUDED.description, meta = EXCLUDED.meta;", "DO UPDATE SET code = EXCLUDED.code, description = EXCLUDED.description, meta = EXCLUDED.meta;",
[[self.SOURCE, *row] for row in rows] [[self.SOURCE, *row] for row in rows],
) )
await self.log("Genres updated!") await self.log("Genres updated!")
@@ -437,18 +425,16 @@ class FlUpdater(BaseUpdater):
async with self.mysql_pool.acquire() as conn: async with self.mysql_pool.acquire() as conn:
async with conn.cursor() as cursor: async with conn.cursor() as cursor:
await cursor.execute( await cursor.execute("SELECT BookId, GenreId FROM libgenre;")
"SELECT BookId, GenreId FROM libgenre;"
)
while (rows := await cursor.fetchmany(32)): while rows := await cursor.fetchmany(32):
await self.postgres_pool.executemany( await self.postgres_pool.executemany(
"INSERT INTO book_genres (book, genre) " "INSERT INTO book_genres (book, genre) "
"SELECT " "SELECT "
"(SELECT id FROM books WHERE source = $1 AND remote_id = $2), " "(SELECT id FROM books WHERE source = $1 AND remote_id = $2), "
"(SELECT id FROM genres WHERE source = $1 AND remote_id = $3) " "(SELECT id FROM genres WHERE source = $1 AND remote_id = $3) "
"ON CONFLICT (book, author) DO NOTHING;", "ON CONFLICT (book, author) DO NOTHING;",
[(self.SOURCE, *row) for row in rows] [(self.SOURCE, *row) for row in rows],
) )
await self.log("Book_genres updated!") await self.log("Book_genres updated!")
@@ -461,9 +447,7 @@ class FlUpdater(BaseUpdater):
await self._drop_tables() await self._drop_tables()
await asyncio.gather( await asyncio.gather(*[self._import_dump(filename) for filename in self.FILES])
*[self._import_dump(filename) for filename in self.FILES]
)
await self._set_source() await self._set_source()
@@ -482,7 +466,7 @@ class FlUpdater(BaseUpdater):
self._update_book_annotations(), self._update_book_annotations(),
self._update_author_annotations(), self._update_author_annotations(),
self._update_genres(), self._update_genres(),
self._update_books_genres() self._update_books_genres(),
) )
await WebhookSender.send() await WebhookSender.send()

2
src/app/services/webhook.py
View File

@@ -7,7 +7,7 @@ class WebhookSender:
@classmethod @classmethod
async def _make_request(cls, webhook: WebhookConfig): async def _make_request(cls, webhook: WebhookConfig):
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
request_maker= getattr(client, webhook.method) request_maker = getattr(client, webhook.method)
await request_maker(webhook.url, headers=webhook.headers) await request_maker(webhook.url, headers=webhook.headers)
@classmethod @classmethod

11
src/app/views.py
View File

@@ -1,21 +1,16 @@
from fastapi import APIRouter, BackgroundTasks, Depends from fastapi import APIRouter, BackgroundTasks, Depends
from app.services.updaters import UpdaterTypes, UPDATERS
from app.depends import check_token from app.depends import check_token
from app.services.updaters import UpdaterTypes, UPDATERS
router = APIRouter( router = APIRouter(tags=["updater"], dependencies=[Depends(check_token)])
tags=["updater"],
dependencies=[Depends(check_token)]
)
@router.post("/update/{updater}") @router.post("/update/{updater}")
async def update(updater: UpdaterTypes, background_tasks: BackgroundTasks): async def update(updater: UpdaterTypes, background_tasks: BackgroundTasks):
updater_ = UPDATERS[updater] updater_ = UPDATERS[updater]
background_tasks.add_task( background_tasks.add_task(updater_.update)
updater_.update
)
return True return True

3
src/core/config.py
View File

@@ -1,9 +1,10 @@
from typing import Optional, Union, Literal from typing import Optional, Union, Literal
from pydantic import BaseModel, BaseSettings from pydantic import BaseModel, BaseSettings
class WebhookConfig(BaseModel): class WebhookConfig(BaseModel):
method: Union[Literal['get'], Literal['post']] method: Union[Literal["get"], Literal["post"]]
url: str url: str
headers: dict[str, str] headers: dict[str, str]

1
src/main.py
View File

@@ -1,3 +1,4 @@
from core.app import start_app from core.app import start_app
app = start_app() app = start_app()