diff --git a/src/app/services/updaters/fl_updater.py b/src/app/services/updaters/fl_updater.py index 8d6966c..7cdce88 100644 --- a/src/app/services/updaters/fl_updater.py +++ b/src/app/services/updaters/fl_updater.py @@ -1,5 +1,5 @@ import asyncio -from logging import Logger +import logging from typing import Optional import aiomysql @@ -9,7 +9,13 @@ import asyncpg from core.config import env_config -logger = Logger("fl_updater") +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) + +logger.addHandler(ch) async def run(cmd) -> tuple[bytes, bytes, Optional[int]]: @@ -29,6 +35,26 @@ def remove_dots(s: str): return s.replace(".", "") +def fix_annotation_text(text: str) -> str: + replace_map = { + "

": "", + '

': "", + "

": "", + "

": "", + " ": "", + "[b]": "", + "[/b]": "", + "[hr]": "", + } + + t = text + + for key, value in replace_map: + t = t.replace(key, value) + + return t + + class FlUpdater: SOURCE: int @@ -461,7 +487,7 @@ class FlUpdater: self.SOURCE, row[0], row[1], - row[2].lstrip("

").rstrip("

"), + fix_annotation_text(row[2]), ] async with self.mysql_pool.acquire() as conn: @@ -543,6 +569,14 @@ class FlUpdater: """ ) + def fix_annotation(row) -> list: + return [ + self.SOURCE, + row[0], + row[1], + fix_annotation_text(row[2]), + ] + async with self.mysql_pool.acquire() as conn: async with conn.cursor() as cursor: await cursor.execute("SELECT COUNT(*) FROM libaannotations;") @@ -560,7 +594,7 @@ class FlUpdater: await self.postgres_pool.executemany( "SELECT update_author_annotation($1, $2, cast($3 as varchar), cast($4 as text));", - [[self.SOURCE, *row] for row in rows], + [fix_annotation(row) for row in rows], ) logger.info("Author_annotation_updated!") diff --git a/src/core/setup_arq.py b/src/core/setup_arq.py index ecf432f..62b1dbe 100644 --- a/src/core/setup_arq.py +++ b/src/core/setup_arq.py @@ -11,4 +11,4 @@ class WorkerSettings: on_startup = startup redis_settings = get_redis_settings() max_jobs = 1 - job_timeout = 60 * 60 + job_timeout = 15 * 60