Add redis analytics

package.json

@@ -18,13 +18,14 @@
     "chunk-text": "^2.0.1",
     "docker-ip-get": "^1.1.5",
     "envalid": "^7.2.2",
+    "esbuild": "^0.14.2",
     "express": "^4.17.1",
     "got": "^11.8.3",
     "js-base64": "^3.7.2",
     "moment": "^2.29.1",
+    "redis": "^4.0.6",
     "safe-compare": "^1.1.4",
     "telegraf": "^4.4.2",
-    "esbuild": "^0.14.2",
     "typescript": "^4.5.2"
   },
   "devDependencies": {

src/analytics/users_counter.ts

@@ -1,39 +1,121 @@
+import * as Sentry from '@sentry/node';
+import { createClient, RedisClientType } from 'redis';
+import moment from 'moment';
+import BotsManager from '@/bots/manager';
+
+import env from '@/config';
+
+
+Sentry.init({
+    dsn: env.SENTRY_DSN,
+});
+
+
+enum RedisKeys {
+    UsersActivity = "users_activity",
+    RequestsCount = "requests_count",
+}
+
+
 export default class UsersCounter {
-    static bots: {[key: string]: Set<number>} = {};
+    static _redisClient: RedisClientType | null = null;
-    static allUsers: Set<number> = new Set();
+ 
-    static requests = 0;
+    static async _getClient() {
+        if (this._redisClient === null) {
+            this._redisClient = createClient({
+                url: `redis://${env.REDIS_HOST}:${env.REDIS_PORT}/${env.REDIS_DB}`
+            });
 
-    static take(userId: number, bot: string) {
+            this._redisClient.on('error', (err) => {
-        const isExists = this.bots[bot];
+                console.log(err);
+                Sentry.captureException(err);
+            });
 
-        if (!isExists) {
+            await this._redisClient.connect();
-            this.bots[bot] = new Set();
         }
 
-        this.bots[bot].add(userId);
+        return this._redisClient;
-        this.allUsers.add(userId);
-        this.requests++;
     }
 
-    static getAllUsersCount(): number {
+    static async _getBotsUsernames(): Promise<string[]> {        
-        return this.allUsers.size;
+        const promises = Object.values(BotsManager.bots).map(async (bot) => {
+            const botInfo = await bot.telegram.getMe();
+            return botInfo.username;
+        });
+
+        return Promise.all(promises); 
     }
 
-    static getUsersByBots(): {[bot: string]: number} {
+    static async _getUsersByBot(bot: string): Promise<number[]> {
+        const client = await this._getClient();
+
+        return (await client.hKeys(`${RedisKeys.UsersActivity}_${bot}`)).map((userId) => parseInt(userId));
+    }
+
+    static async _getAllUsersCount(): Promise<number> {
+        const botsUsernames = await this._getBotsUsernames();
+
+        const users = new Set<number>();
+
+        await Promise.all(
+            botsUsernames.map(async (bot) => {
+                (await this._getUsersByBot(bot)).forEach((user) => users.add(user));
+            })
+        );
+
+        return users.size;
+    }
+
+    static async _getUsersByBots(): Promise<{[bot: string]: number}> {
+        const botsUsernames = await this._getBotsUsernames();
+
         const result: {[bot: string]: number} = {};
 
-        Object.keys(this.bots).forEach((bot: string) => result[bot] = this.bots[bot].size);
+        await Promise.all(
+            botsUsernames.map(async (bot) => {
+                result[bot] = (await this._getUsersByBot(bot)).length;
+            })
+        );
 
         return result;
     }
 
-    static getMetrics(): string {
+    static async _incrementRequests() {
+        const client = await this._getClient();
+
+        const exists = await client.exists(RedisKeys.RequestsCount);
+
+        if (!exists) {
+            await client.set(RedisKeys.RequestsCount, 0);
+        }
+
+        await client.incr(RedisKeys.RequestsCount);
+    }
+
+    static async _getRequestsCount(): Promise<number> {
+        const client = await this._getClient();
+
+        const result = await client.get(RedisKeys.RequestsCount);
+
+        if (result === null) return 0;
+
+        return parseInt(result);
+    }
+
+    static async take(userId: number, bot: string) {
+        const client = await this._getClient();
+
+        await client.hSet(`${RedisKeys.UsersActivity}_${bot}`, userId, moment().format());
+        await this._incrementRequests();
+    }
+
+    static async getMetrics(): Promise<string> {
         const lines = [];
 
-        lines.push(`all_users_count ${this.getAllUsersCount()}`);
+        lines.push(`all_users_count ${await this._getAllUsersCount()}`);
-        lines.push(`requests_count ${this.requests}`);
+        lines.push(`requests_count ${await this._getRequestsCount()}`);
 
-        const usersByBots = this.getUsersByBots();
+        const usersByBots = await this._getUsersByBots();
 
         Object.keys(usersByBots).forEach((bot: string) => {
             lines.push(`users_count{bot="${bot}"} ${usersByBots[bot]}`)

src/bots/manager/index.ts

@@ -140,7 +140,9 @@ export default class BotsManager {
         });
 
         application.get("/metrics", (req, res) => {
-            res.send(UsersCounter.getMetrics());
+            UsersCounter.getMetrics().then((response) => {
+                res.send(response);
+            });
         });
 
         application.use((req: Request, res: Response, next: NextFunction) => this.handleUpdate(req, res, next));

src/config.ts

@@ -19,4 +19,7 @@ export default cleanEnv(process.env, {
     USER_SETTINGS_URL: str(),
     USER_SETTINGS_API_KEY: str(),
     NETWORK_IP_PREFIX: str(),
+    REDIS_HOST: str(),
+    REDIS_PORT: num(),
+    REDIS_DB: num(),
 });

yarn.lock

@@ -2,6 +2,41 @@
 # yarn lockfile v1
 
 
+"@node-redis/bloom@1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@node-redis/bloom/-/bloom-1.0.1.tgz#144474a0b7dc4a4b91badea2cfa9538ce0a1854e"
+  integrity sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==
+
+"@node-redis/client@1.0.5":
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/@node-redis/client/-/client-1.0.5.tgz#ebac5e2bbf12214042a37621604973a954ede755"
+  integrity sha512-ESZ3bd1f+od62h4MaBLKum+klVJfA4wAeLHcVQBkoXa1l0viFesOWnakLQqKg+UyrlJhZmXJWtu0Y9v7iTMrig==
+  dependencies:
+    cluster-key-slot "1.1.0"
+    generic-pool "3.8.2"
+    redis-parser "3.0.0"
+    yallist "4.0.0"
+
+"@node-redis/graph@1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@node-redis/graph/-/graph-1.0.0.tgz#baf8eaac4a400f86ea04d65ec3d65715fd7951ab"
+  integrity sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==
+
+"@node-redis/json@1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@node-redis/json/-/json-1.0.2.tgz#8ad2d0f026698dc1a4238cc3d1eb099a3bee5ab8"
+  integrity sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==
+
+"@node-redis/search@1.0.5":
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/@node-redis/search/-/search-1.0.5.tgz#96050007eb7c50a7e47080320b4f12aca8cf94c4"
+  integrity sha512-MCOL8iCKq4v+3HgEQv8zGlSkZyXSXtERgrAJ4TSryIG/eLFy84b57KmNNa/V7M1Q2Wd2hgn2nPCGNcQtk1R1OQ==
+
+"@node-redis/time-series@1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@node-redis/time-series/-/time-series-1.0.2.tgz#5dd3638374edd85ebe0aa6b0e87addc88fb9df69"
+  integrity sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==
+
 "@sentry/core@6.18.2":
   version "6.18.2"
   resolved "https://registry.yarnpkg.com/@sentry/core/-/core-6.18.2.tgz#d27619b7b4a4b90e2cfdc254d40ee9d630b251b9"
@@ -408,6 +443,11 @@ clone-response@^1.0.2:
   dependencies:
     mimic-response "^1.0.0"
 
+cluster-key-slot@1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
+  integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
+
 color-convert@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3"
@@ -787,6 +827,11 @@ fsevents@~2.3.2:
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
   integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
 
+generic-pool@3.8.2:
+  version "3.8.2"
+  resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9"
+  integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==
+
 get-stream@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
@@ -1335,6 +1380,30 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
+redis-errors@^1.0.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
+  integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=
+
+redis-parser@3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
+  integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=
+  dependencies:
+    redis-errors "^1.0.0"
+
+redis@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/redis/-/redis-4.0.6.tgz#a2ded4d9f4f4bad148e54781051618fc684cd858"
+  integrity sha512-IaPAxgF5dV0jx+A9l6yd6R9/PAChZIoAskDVRzUODeLDNhsMlq7OLLTmu0AwAr0xjrJ1bibW5xdpRwqIQ8Q0Xg==
+  dependencies:
+    "@node-redis/bloom" "1.0.1"
+    "@node-redis/client" "1.0.5"
+    "@node-redis/graph" "1.0.0"
+    "@node-redis/json" "1.0.2"
+    "@node-redis/search" "1.0.5"
+    "@node-redis/time-series" "1.0.2"
+
 registry-auth-token@^4.0.0:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-4.2.1.tgz#6d7b4006441918972ccd5fedcd41dc322c79b250"
@@ -1685,7 +1754,7 @@ xdg-basedir@^4.0.0:
   resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"
   integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==
 
-yallist@^4.0.0:
+yallist@4.0.0, yallist@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
   integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==