From 4d08aeb2fe89f559729399a84075bfa92ef2faac Mon Sep 17 00:00:00 2001 From: Alan Hamlett Date: Tue, 27 Aug 2024 20:36:44 +0200 Subject: [PATCH] finish core --- package-lock.json | 54 +++++-- package.json | 2 + src/components/MainList.tsx | 2 +- src/config/config.test.ts | 8 +- src/config/config.ts | 12 +- src/core/WakaTimeCore.ts | 247 +++++++++++++---------------- src/types/heartbeats.ts | 8 +- src/types/sites.ts | 1 + src/utils/apiKey.ts | 11 -- src/utils/changeExtensionStatus.ts | 8 +- src/utils/user.ts | 4 +- 11 files changed, 178 insertions(+), 179 deletions(-) diff --git a/package-lock.json b/package-lock.json index df738d3..d26469c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "react-redux": "^8.0.5", "react-transition-group": "^4.4.5", "redux-logger": "^4.0.0", + "uuid": "^10.0.0", "webextension-polyfill": "^0.10.0" }, "devDependencies": { @@ -49,6 +50,7 @@ "@types/redux-logger": "^3.0.9", "@types/shelljs": "^0.8.8", "@types/sinon-chrome": "^2.2.11", + "@types/uuid": "^10.0.0", "@types/wait-on": "^5.2.0", "@types/webextension-polyfill": "^0.10.0", "@typescript-eslint/eslint-plugin": "^4.33.0", @@ -4096,6 +4098,13 @@ "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz", "integrity": "sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==" }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/wait-on": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/@types/wait-on/-/wait-on-5.3.1.tgz", @@ -18779,6 +18788,17 @@ "node": ">=0.8" } }, + "node_modules/request/node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "dev": true, + "license": "MIT", + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/require-at": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/require-at/-/require-at-1.0.6.tgz", @@ -21388,13 +21408,16 @@ "dev": true }, "node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "dev": true, + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", "bin": { - "uuid": "bin/uuid" + "uuid": "dist/bin/uuid" } }, "node_modules/v8-compile-cache": { @@ -25578,6 +25601,12 @@ "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz", "integrity": "sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==" }, + "@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true + }, "@types/wait-on": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/@types/wait-on/-/wait-on-5.3.1.tgz", @@ -36628,6 +36657,12 @@ "psl": "^1.1.28", "punycode": "^2.1.1" } + }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "dev": true } } }, @@ -38625,10 +38660,9 @@ "dev": true }, "uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "dev": true + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==" }, "v8-compile-cache": { "version": "2.3.0", diff --git a/package.json b/package.json index ac86170..251cb8a 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "react-redux": "^8.0.5", "react-transition-group": "^4.4.5", "redux-logger": "^4.0.0", + "uuid": "^10.0.0", "webextension-polyfill": "^0.10.0" }, "devDependencies": { @@ -70,6 +71,7 @@ "@types/redux-logger": "^3.0.9", "@types/shelljs": "^0.8.8", "@types/sinon-chrome": "^2.2.11", + "@types/uuid": "^10.0.0", "@types/wait-on": "^5.2.0", "@types/webextension-polyfill": "^0.10.0", "@typescript-eslint/eslint-plugin": "^4.33.0", diff --git a/src/components/MainList.tsx b/src/components/MainList.tsx index 92ba5e4..26b09a9 100644 --- a/src/components/MainList.tsx +++ b/src/components/MainList.tsx @@ -40,7 +40,7 @@ export default function MainList({ const disableLogging = async (): Promise => { dispatch(setLoggingEnabled(false)); await browser.storage.sync.set({ loggingEnabled: false }); - await changeExtensionState('notLogging'); + await changeExtensionState('trackingDisabled'); }; return ( diff --git a/src/config/config.test.ts b/src/config/config.test.ts index 6fc1980..871be13 100644 --- a/src/config/config.test.ts +++ b/src/config/config.test.ts @@ -29,7 +29,7 @@ describe('wakatime config', () => { "colors": { "allGood": "", "lightTheme": "white", - "notLogging": "gray", + "trackingDisabled": "gray", "notSignedIn": "red", }, "currentUserApiEndPoint": "/users/current", @@ -46,7 +46,7 @@ describe('wakatime config', () => { "udemy.com", "w3schools.com", ], - "heartbeatApiEndPoint": "/users/current/heartbeats", + "heartbeatApiEndPoint": "/users/current/heartbeats.bulk", "hostname": "", "loggingEnabled": true, "loggingStyle": "deny", @@ -71,7 +71,7 @@ describe('wakatime config', () => { ], "states": [ "allGood", - "notLogging", + "trackingDisabled", "notSignedIn", "ignored", ], @@ -80,7 +80,7 @@ describe('wakatime config', () => { "tooltips": { "allGood": "", "ignored": "This URL is ignored", - "notLogging": "Not logging", + "trackingDisabled": "Not logging", "notSignedIn": "Not signed In", }, "trackSocialMedia": true, diff --git a/src/config/config.ts b/src/config/config.ts index 2ffd421..45443fa 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -3,7 +3,7 @@ import browser from 'webextension-polyfill'; /** * Logging */ -export type ExtensionStatus = 'allGood' | 'notLogging' | 'notSignedIn' | 'ignored'; +export type ExtensionStatus = 'allGood' | 'trackingDisabled' | 'notSignedIn' | 'ignored'; /** * Supported logging style */ @@ -35,7 +35,7 @@ interface SuccessOrFailAlert { interface Colors { allGood: ''; lightTheme: 'white'; - notLogging: 'gray'; + trackingDisabled: 'gray'; notSignedIn: 'red'; } /** @@ -44,7 +44,7 @@ interface Colors { interface Tooltips { allGood: string; ignored: string; - notLogging: string; + trackingDisabled: string; notSignedIn: string; } @@ -126,7 +126,7 @@ const config: Config = { colors: { allGood: '', lightTheme: 'white', - notLogging: 'gray', + trackingDisabled: 'gray', notSignedIn: 'red', }, @@ -176,7 +176,7 @@ const config: Config = { 'youtube.com', ], - states: ['allGood', 'notLogging', 'notSignedIn', 'ignored'], + states: ['allGood', 'trackingDisabled', 'notSignedIn', 'ignored'], summariesApiEndPoint: process.env.SUMMARIES_API_URL ?? '/users/current/summaries', @@ -185,7 +185,7 @@ const config: Config = { tooltips: { allGood: '', ignored: 'This URL is ignored', - notLogging: 'Not logging', + trackingDisabled: 'Not logging', notSignedIn: 'Not signed In', }, trackSocialMedia: true, diff --git a/src/core/WakaTimeCore.ts b/src/core/WakaTimeCore.ts index 3949829..0b39caf 100644 --- a/src/core/WakaTimeCore.ts +++ b/src/core/WakaTimeCore.ts @@ -3,16 +3,15 @@ import browser, { Tabs } from 'webextension-polyfill'; /* eslint-disable no-fallthrough */ /* eslint-disable default-case */ import moment from 'moment'; -import { OptionalHeartbeat } from 'src/types/sites'; -import { getOperatingSystem } from '../utils'; +import { v4 as uuid4 } from 'uuid'; +import { OptionalHeartbeat } from '../types/sites'; +import { getOperatingSystem, IS_EDGE, IS_FIREFOX } from '../utils'; import { changeExtensionStatus } from '../utils/changeExtensionStatus'; import getDomainFromUrl, { getDomain } from '../utils/getDomainFromUrl'; import { getSettings, Settings } from '../utils/settings'; import config, { ExtensionStatus } from '../config/config'; -import { EntityType, Heartbeat } from '../types/heartbeats'; -import { getApiKey } from '../utils/apiKey'; -import { getLoggingType } from '../utils/logging'; +import { EntityType, Heartbeat, HeartbeatsBulkResponse } from '../types/heartbeats'; class WakaTimeCore { tabsWithDevtoolsOpen: Tabs.Tab[]; @@ -32,17 +31,15 @@ class WakaTimeCore { const dbConnection = await openDB('wakatime', 1, { upgrade(db, oldVersion) { // Create a store of objects - const store = db.createObjectStore('cacheHeartbeats', { - // The `time` property of the object will be the key, and be incremented automatically - keyPath: 'time', + const store = db.createObjectStore('heartbeatQueue', { + keyPath: 'id', }); // Switch over the oldVersion, *without breaks*, to allow the database to be incrementally upgraded. switch (oldVersion) { case 0: // Placeholder to execute when database is created (oldVersion is 0) case 1: - // Create an index called `type` based on the `type` property of objects in the store - store.createIndex('time', 'time'); + store.createIndex('id', 'id'); } }, }); @@ -95,7 +92,7 @@ class WakaTimeCore { async handleActivity(tabId: number) { const settings = await getSettings(); if (!settings.loggingEnabled) { - await changeExtensionStatus('notLogging'); + await changeExtensionStatus('trackingDisabled'); return; } @@ -118,7 +115,9 @@ class WakaTimeCore { if (!this.shouldSendHeartbeat(heartbeat)) return; // append heartbeat to queue - await this.db?.add('cacheHeartbeats', heartbeat); + await this.db?.add('heartbeatQueue', heartbeat); + + await this.sendHeartbeats(); } async getCurrentTab(tabId: number): Promise { @@ -145,63 +144,112 @@ class WakaTimeCore { const entity = settings.loggingType === 'domain' ? getDomainFromUrl(url) : url; return { - branch: heartbeat?.branch, + branch: heartbeat?.branch ?? '<>', category: heartbeat?.category, entity: heartbeat?.entity ?? entity, - entityType: heartbeat?.entityType ?? (settings.loggingType as EntityType), + id: uuid4(), language: heartbeat?.language, - project: heartbeat?.project, + project: heartbeat?.project ?? '<>', + time: this.getCurrentTime(), + type: heartbeat?.entityType ?? (settings.loggingType as EntityType), }; } - /** - * Given the heartbeat and logging type it creates a payload and - * sends an ajax post request to the API. - * - * @param heartbeat - * @param debug - */ - async sendHeartbeat( - heartbeat: Heartbeat, - apiKey: string, - navigationPayload: Record, - ): Promise { - console.log('Sending Heartbeat', heartbeat); - let payload; + getCurrentTime(): string { + const m = moment(); + return `${m.format('x').slice(0, -3)}.${m.format('x').slice(-3)}`; + } - const loggingType = await getLoggingType(); - // Get only the domain from the entity. - // And send that in heartbeat - if (loggingType == 'domain') { - heartbeat.url = getDomainFromUrl(heartbeat.url); - payload = await this.preparePayload(heartbeat, 'domain'); - await this.sendPostRequestToApi( - { ...payload, ...navigationPayload }, - apiKey, - heartbeat.hostname, - ); + async sendHeartbeats(): Promise { + const settings = await browser.storage.sync.get({ + apiKey: config.apiKey, + apiUrl: config.apiUrl, + heartbeatApiEndPoint: config.heartbeatApiEndPoint, + hostname: '', + }); + if (!settings.apiKey) { + await changeExtensionStatus('notSignedIn'); + return; } - // Send entity in heartbeat - else if (loggingType == 'url') { - payload = await this.preparePayload(heartbeat, 'url'); - await this.sendPostRequestToApi( - { ...payload, ...navigationPayload }, - apiKey, - heartbeat.hostname, - ); + + const heartbeats = (await this.db?.getAll('heartbeatQueue', undefined, 50)) as + | Heartbeat[] + | undefined; + if (!heartbeats || heartbeats.length === 0) return; + + await this.db?.delete( + 'heartbeatQueue', + heartbeats.map((heartbeat) => heartbeat.id), + ); + + const userAgent = await this.getUserAgent(); + + try { + const request: RequestInit = { + body: JSON.stringify( + heartbeats.map((heartbeat) => { + return { ...heartbeat, userAgent }; + }), + ), + credentials: 'omit', + method: 'POST', + }; + if (typeof settings.hostname === 'string' && settings.hostname) { + request.headers = { + 'X-Machine-Name': settings.hostname, + }; + } + + const url = `${settings.apiUrl}${settings.heartbeatApiEndPoint}?api_key=${settings.apiKey}`; + const response = await fetch(url, request); + if (response.status === 401) { + await this.putHeartbeatsBackInQueue(heartbeats); + await changeExtensionStatus('notSignedIn'); + return; + } + const data = (await response.json()) as HeartbeatsBulkResponse; + if (data.error) { + await this.putHeartbeatsBackInQueue(heartbeats); + console.error(data.error); + return; + } + if (response.status === 201) { + await Promise.all( + (data.responses ?? []).map(async (resp, respNumber) => { + if (resp[0].error) { + await this.putHeartbeatsBackInQueue(heartbeats.filter((h, i) => i === respNumber)); + console.error(resp[0].error); + } else if (resp[1] === 201 && resp[0].data?.id) { + await changeExtensionStatus('allGood'); + // await this.db?.delete('heartbeatQueue', resp[0].data.id); + } else { + if (resp[1] !== 400) { + await this.putHeartbeatsBackInQueue(heartbeats.filter((h, i) => i === respNumber)); + } + console.error( + `Heartbeat ${resp[0].data?.id ?? respNumber} returned status: ${resp[1]}`, + ); + } + return resp; + }), + ); + } else { + await this.putHeartbeatsBackInQueue(heartbeats); + console.error(`Heartbeat response status: ${response.status}`); + } + } catch (err: unknown) { + console.error(err); + await this.putHeartbeatsBackInQueue(heartbeats); } } - /** - * Creates payload for the heartbeat and returns it as JSON. - * - * @param heartbeat - * @param type - * @param debug - * @returns {*} - * @private - */ - async preparePayload(heartbeat: Heartbeat, type: string): Promise> { + async putHeartbeatsBackInQueue(heartbeats: Heartbeat[]): Promise { + await Promise.all( + heartbeats.map(async (heartbeat) => this.db?.add('heartbeatQueue', heartbeat)), + ); + } + + async getUserAgent(): Promise { const os = await getOperatingSystem(); let browserName = 'chrome'; let userAgent; @@ -214,88 +262,7 @@ class WakaTimeCore { } else { userAgent = navigator.userAgent.match(/Chrome\/\S+/g)?.[0]; } - const payload: Record = { - entity: heartbeat.url, - time: moment().format('X'), - type: type, - user_agent: `${userAgent} ${os} ${browserName}-wakatime/${config.version}`, - }; - - payload.project = heartbeat.project ?? '<>'; - payload.branch = heartbeat.branch ?? '<>'; - - return payload; - } - - /** - * Sends AJAX request with payload to the heartbeat API as JSON. - * - * @param payload - * @param method - * @returns {*} - */ - async sendPostRequestToApi( - payload: Record, - apiKey = '', - hostname = '', - ): Promise { - try { - const items = await browser.storage.sync.get({ - apiUrl: config.apiUrl, - heartbeatApiEndPoint: config.heartbeatApiEndPoint, - }); - - const request: RequestInit = { - body: JSON.stringify(payload), - credentials: 'omit', - method: 'POST', - }; - if (hostname) { - request.headers = { - 'X-Machine-Name': hostname, - }; - } - const response = await fetch( - `${items.apiUrl}${items.heartbeatApiEndPoint}?api_key=${apiKey}`, - request, - ); - await response.json(); - } catch (err: unknown) { - if (this.db) { - await this.db.add('cacheHeartbeats', payload); - } - - await changeExtensionStatus('notSignedIn'); - } - } - - /** - * Sends cached heartbeats request to wakatime api - * @param requests - */ - async sendCachedHeartbeatsRequest(): Promise { - const apiKey = await getApiKey(); - if (!apiKey) { - return changeExtensionStatus('notLogging'); - } - - if (this.db) { - const requests = await this.db.getAll('cacheHeartbeats'); - await this.db.clear('cacheHeartbeats'); - const chunkSize = 50; // Create batches of max 50 request - for (let i = 0; i < requests.length; i += chunkSize) { - const chunk = requests.slice(i, i + chunkSize); - const requestsPromises: Promise[] = []; - chunk.forEach((request: Record) => - requestsPromises.push(this.sendPostRequestToApi(request, apiKey)), - ); - try { - await Promise.all(requestsPromises); - } catch (error: unknown) { - console.log('Error sending heartbeats'); - } - } - } + return `${userAgent} ${os} ${browserName}-wakatime/${config.version}`; } } diff --git a/src/types/heartbeats.ts b/src/types/heartbeats.ts index e39e6c9..065bdf5 100644 --- a/src/types/heartbeats.ts +++ b/src/types/heartbeats.ts @@ -2,9 +2,11 @@ export interface Heartbeat { branch?: string | null; category?: Category | null; entity: string; - entityType: EntityType; + id: string; language?: string | null; project?: string | null; + time: string; + type: EntityType; } export enum Category { @@ -30,3 +32,7 @@ export interface ProjectDetails { language: string; project: string; } + +export type HeartbeatsBulkResponse = { error?: string; responses?: HeartbeatResponse[] }; + +export type HeartbeatResponse = [{ data?: { id: string }; error?: string }, number]; diff --git a/src/types/sites.ts b/src/types/sites.ts index 0ae61b7..1a7e7d9 100644 --- a/src/types/sites.ts +++ b/src/types/sites.ts @@ -8,6 +8,7 @@ export enum KnownSite { github = 'github', gitlab = 'gitlab', googlemeet = 'googlemeet', + slack = 'slack', stackoverflow = 'stackoverflow', travisci = 'travisci', vercel = 'vercel', diff --git a/src/utils/apiKey.ts b/src/utils/apiKey.ts index 8414635..56c86e4 100644 --- a/src/utils/apiKey.ts +++ b/src/utils/apiKey.ts @@ -1,6 +1,3 @@ -import browser from 'webextension-polyfill'; -import config from '../config/config'; - export default function apiKeyInvalid(key?: string): string { const err = 'Invalid api key... check https://wakatime.com/settings for your key'; if (!key) return err; @@ -11,11 +8,3 @@ export default function apiKeyInvalid(key?: string): string { if (!re.test(key)) return err; return ''; } - -export async function getApiKey(): Promise { - const storage = await browser.storage.sync.get({ - apiKey: config.apiKey, - }); - const apiKey = storage.apiKey as string; - return apiKey; -} diff --git a/src/utils/changeExtensionStatus.ts b/src/utils/changeExtensionStatus.ts index 8ffd9e1..62f6763 100644 --- a/src/utils/changeExtensionStatus.ts +++ b/src/utils/changeExtensionStatus.ts @@ -13,16 +13,16 @@ export async function changeExtensionStatus(status: ExtensionStatus): Promise => { if (items.loggingEnabled === true) { await changeExtensionState('allGood'); } else { - await changeExtensionState('notLogging'); + await changeExtensionState('trackingDisabled'); } } catch (err: unknown) { await changeExtensionState('notSignedIn'); @@ -121,7 +121,7 @@ export const fetchUserData = async ( if (items.loggingEnabled === true) { await changeExtensionState('allGood'); } else { - await changeExtensionState('notLogging'); + await changeExtensionState('trackingDisabled'); } dispatch(setLoggingEnabled(items.loggingEnabled as boolean));