From 1fad81f3ab445bf276c4db234383c1d18c74d2a7 Mon Sep 17 00:00:00 2001 From: moonrailgun Date: Fri, 5 Jan 2024 23:05:19 +0800 Subject: [PATCH] feat: add daily cron job to calc workspace usage --- .../migration.sql | 21 +++ prisma/schema.prisma | 19 ++- prisma/zod/index.ts | 1 + prisma/zod/workspace.ts | 4 +- prisma/zod/workspacedailyusage.ts | 26 ++++ src/server/cronjob/index.ts | 127 ++++++++++++++++++ src/server/main.ts | 3 + 7 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 prisma/migrations/20240105145715_add_workspace_daily_usage/migration.sql create mode 100644 prisma/zod/workspacedailyusage.ts create mode 100644 src/server/cronjob/index.ts diff --git a/prisma/migrations/20240105145715_add_workspace_daily_usage/migration.sql b/prisma/migrations/20240105145715_add_workspace_daily_usage/migration.sql new file mode 100644 index 0000000..c0253a9 --- /dev/null +++ b/prisma/migrations/20240105145715_add_workspace_daily_usage/migration.sql @@ -0,0 +1,21 @@ +-- CreateTable +CREATE TABLE "WorkspaceDailyUsage" ( + "id" VARCHAR(30) NOT NULL, + "workspaceId" VARCHAR(30) NOT NULL, + "date" DATE NOT NULL, + "websiteAcceptedCount" INTEGER NOT NULL, + "websiteEventCount" INTEGER NOT NULL, + "monitorExecutionCount" INTEGER NOT NULL, + "createdAt" TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "WorkspaceDailyUsage_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkspaceDailyUsage_id_key" ON "WorkspaceDailyUsage"("id"); + +-- CreateIndex +CREATE UNIQUE INDEX "WorkspaceDailyUsage_workspaceId_date_key" ON "WorkspaceDailyUsage"("workspaceId", "date"); + +-- AddForeignKey +ALTER TABLE "WorkspaceDailyUsage" ADD CONSTRAINT "WorkspaceDailyUsage_workspaceId_fkey" FOREIGN KEY ("workspaceId") REFERENCES "Workspace"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index ce3854d..9d4ec63 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -48,7 +48,8 @@ model Workspace { monitorStatusPages MonitorStatusPage[] // for user currentWorkspace - selectedUsers User[] + selectedUsers User[] + workspaceDailyUsage WorkspaceDailyUsage[] } model WorkspacesOnUsers { @@ -135,7 +136,7 @@ model WebsiteEvent { referrerQuery String? @db.VarChar(500) referrerDomain String? @db.VarChar(500) pageTitle String? @db.VarChar(500) - eventType Int @default(1) @db.Integer + eventType Int @default(1) @db.Integer // 1 is view, 2 is click event eventName String? @db.VarChar(50) createdAt DateTime @default(now()) @db.Timestamptz(6) @@ -323,3 +324,17 @@ model MonitorStatusPage { @@index([slug]) } + +model WorkspaceDailyUsage { + id String @id @unique @default(cuid()) @db.VarChar(30) + workspaceId String @db.VarChar(30) + date DateTime @db.Date // calc every day + websiteAcceptedCount Int // website accept any request count + websiteEventCount Int // website accept event request count + monitorExecutionCount Int // monitor exec number count + createdAt DateTime @default(now()) @db.Timestamptz(6) + + workspace Workspace @relation(fields: [workspaceId], references: [id], onUpdate: Cascade, onDelete: Cascade) + + @@unique([workspaceId, date]) +} diff --git a/prisma/zod/index.ts b/prisma/zod/index.ts index 298109b..e33a8e9 100644 --- a/prisma/zod/index.ts +++ b/prisma/zod/index.ts @@ -14,3 +14,4 @@ export * from "./monitorevent" export * from "./monitordata" export * from "./monitorstatus" export * from "./monitorstatuspage" +export * from "./workspacedailyusage" diff --git a/prisma/zod/workspace.ts b/prisma/zod/workspace.ts index b74df8a..5a27bc0 100644 --- a/prisma/zod/workspace.ts +++ b/prisma/zod/workspace.ts @@ -1,6 +1,6 @@ import * as z from "zod" import * as imports from "./schemas" -import { CompleteWorkspacesOnUsers, RelatedWorkspacesOnUsersModelSchema, CompleteWebsite, RelatedWebsiteModelSchema, CompleteNotification, RelatedNotificationModelSchema, CompleteMonitor, RelatedMonitorModelSchema, CompleteMonitorStatusPage, RelatedMonitorStatusPageModelSchema, CompleteUser, RelatedUserModelSchema } from "./index" +import { CompleteWorkspacesOnUsers, RelatedWorkspacesOnUsersModelSchema, CompleteWebsite, RelatedWebsiteModelSchema, CompleteNotification, RelatedNotificationModelSchema, CompleteMonitor, RelatedMonitorModelSchema, CompleteMonitorStatusPage, RelatedMonitorStatusPageModelSchema, CompleteUser, RelatedUserModelSchema, CompleteWorkspaceDailyUsage, RelatedWorkspaceDailyUsageModelSchema } from "./index" // Helper schema for JSON fields type Literal = boolean | number | string @@ -27,6 +27,7 @@ export interface CompleteWorkspace extends z.infer monitors: CompleteMonitor[] monitorStatusPages: CompleteMonitorStatusPage[] selectedUsers: CompleteUser[] + workspaceDailyUsage: CompleteWorkspaceDailyUsage[] } /** @@ -41,4 +42,5 @@ export const RelatedWorkspaceModelSchema: z.ZodSchema = z.laz monitors: RelatedMonitorModelSchema.array(), monitorStatusPages: RelatedMonitorStatusPageModelSchema.array(), selectedUsers: RelatedUserModelSchema.array(), + workspaceDailyUsage: RelatedWorkspaceDailyUsageModelSchema.array(), })) diff --git a/prisma/zod/workspacedailyusage.ts b/prisma/zod/workspacedailyusage.ts new file mode 100644 index 0000000..ef22a74 --- /dev/null +++ b/prisma/zod/workspacedailyusage.ts @@ -0,0 +1,26 @@ +import * as z from "zod" +import * as imports from "./schemas" +import { CompleteWorkspace, RelatedWorkspaceModelSchema } from "./index" + +export const WorkspaceDailyUsageModelSchema = z.object({ + id: z.string(), + workspaceId: z.string(), + date: z.date(), + websiteAcceptedCount: z.number().int(), + websiteEventCount: z.number().int(), + monitorExecutionCount: z.number().int(), + createdAt: z.date(), +}) + +export interface CompleteWorkspaceDailyUsage extends z.infer { + workspace: CompleteWorkspace +} + +/** + * RelatedWorkspaceDailyUsageModelSchema contains all relations on your model in addition to the scalars + * + * NOTE: Lazy required in case of potential circular dependencies within schema + */ +export const RelatedWorkspaceDailyUsageModelSchema: z.ZodSchema = z.lazy(() => WorkspaceDailyUsageModelSchema.extend({ + workspace: RelatedWorkspaceModelSchema, +})) diff --git a/src/server/cronjob/index.ts b/src/server/cronjob/index.ts new file mode 100644 index 0000000..b630d22 --- /dev/null +++ b/src/server/cronjob/index.ts @@ -0,0 +1,127 @@ +import { Cron } from 'croner'; +import { logger } from '../utils/logger'; +import { prisma } from '../model/_client'; +import dayjs from 'dayjs'; +import { Prisma } from '@prisma/client'; + +type WebsiteEventCountSqlReturn = { + workspace_id: string; + count: bigint; +}[]; + +export function initCronjob() { + const dailyJob = Cron('0 2 * * *', async () => { + logger.info('Start statistics usage'); + + await statdailyUsage(); + }); + + logger.info('Daily job will start at:', dailyJob.nextRun()?.toISOString()); +} + +async function statdailyUsage() { + logger.info('Statistics Workspace Daily Usage Start'); + const start = dayjs().subtract(1, 'day').startOf('day').toDate(); + const end = dayjs().startOf('day').toDate(); + const date = dayjs().subtract(1, 'day').toDate(); + + const [websiteAcceptCountRes, websiteEventCountRes, monitorExecutionCount] = + await Promise.all([ + await prisma.$queryRaw` + SELECT + w.id AS workspace_id, + COALESCE(COUNT(we.id), 0) AS count + FROM + "Workspace" w + LEFT JOIN + "Website" ws ON w.id = ws."workspaceId" + LEFT JOIN + "WebsiteEvent" we ON ws.id = we."websiteId" + AND we."createdAt" >= ${start} + AND we."createdAt" < ${end} + GROUP BY + w.id, w.name + ORDER BY + w.id`, + await prisma.$queryRaw` + SELECT + w.id AS workspace_id, + COALESCE(COUNT(we.id), 0) AS count + FROM + "Workspace" w + LEFT JOIN + "Website" ws ON w.id = ws."workspaceId" + LEFT JOIN + "WebsiteEvent" we ON ws.id = we."websiteId" + AND we."eventType" = 2 + AND we."createdAt" >= ${start} + AND we."createdAt" < ${end} + GROUP BY + w.id, w.name + ORDER BY + w.id`, + await prisma.$queryRaw` + SELECT + w.id AS workspace_id, + COALESCE(COUNT(md.id), 0) AS count + FROM + "Workspace" w + LEFT JOIN + "Monitor" m ON w.id = m."workspaceId" + LEFT JOIN + "MonitorData" md ON m.id = md."monitorId" + AND md."createdAt" >= ${start} + AND md."createdAt" < ${end} + GROUP BY + w.id + ORDER BY + w.id`, + ]); + + const map: Map = new Map(); + + const blank: Omit = + { + websiteAcceptedCount: 0, + websiteEventCount: 0, + monitorExecutionCount: 0, + date, + }; + + websiteAcceptCountRes.forEach((item) => { + const workspaceId = item.workspace_id; + map.set(workspaceId, { + ...blank, + ...map.get(workspaceId), + workspaceId, + websiteAcceptedCount: Number(item.count), + }); + }); + + websiteEventCountRes.forEach((item) => { + const workspaceId = item.workspace_id; + map.set(workspaceId, { + ...blank, + ...map.get(workspaceId), + workspaceId, + websiteEventCount: Number(item.count), + }); + }); + + monitorExecutionCount.forEach((item) => { + const workspaceId = item.workspace_id; + map.set(workspaceId, { + ...blank, + ...map.get(workspaceId), + workspaceId, + monitorExecutionCount: Number(item.count), + }); + }); + + await prisma.workspaceDailyUsage.createMany({ + data: Array.from(map.values()), + skipDuplicates: true, + }); + + logger.info('Statistics Workspace Daily Usage Completed'); +} diff --git a/src/server/main.ts b/src/server/main.ts index 40de856..6933abf 100644 --- a/src/server/main.ts +++ b/src/server/main.ts @@ -23,6 +23,7 @@ import { settings } from './utils/settings'; import { env } from './utils/env'; import cors from 'cors'; import { serverStatusRouter } from './router/serverStatus'; +import { initCronjob } from './cronjob'; const port = settings.port; @@ -33,6 +34,8 @@ initUdpServer(port); initSocketio(httpServer); +initCronjob(); + monitorManager.startAll(); app.use(compression());