From 08ef1fea912d918f8bfbd32bc6158b9969662f25 Mon Sep 17 00:00:00 2001 From: moonrailgun Date: Sun, 3 Sep 2023 19:28:53 +0800 Subject: [PATCH] feat: website/send api is completed --- .gitignore | 2 + index.html | 6 + package.json | 16 +- pnpm-lock.yaml | 160 +++++++++++++++++- prisma/schema.prisma | 25 ++- scripts/build-geo.ts | 55 +++++++ scripts/build-tracker.ts | 1 + src/client/components/HealthBar.tsx | 3 +- src/server/main.ts | 4 +- src/server/model/website.ts | 246 ++++++++++++++++++++++++++++ src/server/router/website.ts | 106 ++++++++++++ src/server/utils/common.ts | 131 +++++++++++++++ src/server/utils/const.ts | 59 +++++++ src/server/utils/detect.ts | 157 ++++++++++++++++++ src/server/utils/types.ts | 14 ++ src/tracker/index.js | 2 +- 16 files changed, 968 insertions(+), 19 deletions(-) create mode 100644 scripts/build-geo.ts create mode 100644 src/server/model/website.ts create mode 100644 src/server/router/website.ts create mode 100644 src/server/utils/common.ts create mode 100644 src/server/utils/detect.ts create mode 100644 src/server/utils/types.ts diff --git a/.gitignore b/.gitignore index 2bd1803..436c399 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .env +public/tracker.js +geo # Logs logs diff --git a/index.html b/index.html index 7acaff1..eb998a5 100644 --- a/index.html +++ b/index.html @@ -9,5 +9,11 @@
+ diff --git a/package.json b/package.json index 94a0a80..bb2b01e 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,9 @@ "scripts": { "dev": "nodemon src/server/main.ts -w src/server", "start": "NODE_ENV=production ts-node src/server/main.ts", - "build": "vite build && pnpm build:tracker", + "build": "vite build && pnpm build:tracker && pnpm build:geo", "build:tracker": "ts-node scripts/build-tracker.ts", + "build:geo": "ts-node scripts/build-geo.ts", "db:push": "prisma db push", "db:generate": "prisma generate", "db:studio": "prisma studio" @@ -15,18 +16,23 @@ "@ant-design/charts": "^1.4.2", "@ant-design/icons": "^5.2.5", "@prisma/client": "^5.2.0", + "@types/uuid": "^9.0.3", "antd": "^5.8.5", "axios": "^1.5.0", "bcryptjs": "^2.4.3", "clsx": "^2.0.0", "compose-middleware": "^5.0.1", "compression": "^1.7.4", + "dayjs": "^1.11.9", + "detect-browser": "^5.3.0", "dotenv": "^16.3.1", "express": "^4.18.2", "express-async-errors": "^3.1.1", "express-validator": "^7.0.1", + "is-localhost-ip": "^2.0.0", "jsonwebtoken": "^9.0.2", "lodash-es": "^4.17.21", + "maxmind": "^4.3.11", "nanoid": "^3.3.6", "passport": "^0.6.0", "passport-jwt": "^4.0.1", @@ -34,11 +40,14 @@ "react-dom": "^18.2.0", "react-router": "^6.15.0", "react-router-dom": "^6.15.0", + "request-ip": "^3.3.0", "socket.io": "^4.7.2", "socket.io-client": "^4.7.2", "ts-node": "^10.9.1", "typescript": "^4.9.5", - "vite-express": "^0.10.0" + "uuid": "^9.0.0", + "vite-express": "^0.10.0", + "yup": "^1.2.0" }, "devDependencies": { "@types/bcryptjs": "^2.4.3", @@ -51,12 +60,15 @@ "@types/passport-jwt": "^3.0.9", "@types/react": "^18.2.21", "@types/react-dom": "^18.2.7", + "@types/request-ip": "^0.0.38", + "@types/tar": "^6.1.5", "@vitejs/plugin-react": "^4.0.4", "autoprefixer": "^10.4.15", "nodemon": "^2.0.22", "postcss": "^8.4.29", "prisma": "^5.2.0", "tailwindcss": "^3.3.3", + "tar": "^6.1.15", "vite": "^4.4.9" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1e0c641..19cc90d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,6 +10,9 @@ dependencies: '@prisma/client': specifier: ^5.2.0 version: 5.2.0(prisma@5.2.0) + '@types/uuid': + specifier: ^9.0.3 + version: 9.0.3 antd: specifier: ^5.8.5 version: 5.8.5(react-dom@18.2.0)(react@18.2.0) @@ -28,6 +31,12 @@ dependencies: compression: specifier: ^1.7.4 version: 1.7.4 + dayjs: + specifier: ^1.11.9 + version: 1.11.9 + detect-browser: + specifier: ^5.3.0 + version: 5.3.0 dotenv: specifier: ^16.3.1 version: 16.3.1 @@ -40,12 +49,18 @@ dependencies: express-validator: specifier: ^7.0.1 version: 7.0.1 + is-localhost-ip: + specifier: ^2.0.0 + version: 2.0.0 jsonwebtoken: specifier: ^9.0.2 version: 9.0.2 lodash-es: specifier: ^4.17.21 version: 4.17.21 + maxmind: + specifier: ^4.3.11 + version: 4.3.11 nanoid: specifier: ^3.3.6 version: 3.3.6 @@ -67,6 +82,9 @@ dependencies: react-router-dom: specifier: ^6.15.0 version: 6.15.0(react-dom@18.2.0)(react@18.2.0) + request-ip: + specifier: ^3.3.0 + version: 3.3.0 socket.io: specifier: ^4.7.2 version: 4.7.2 @@ -79,9 +97,15 @@ dependencies: typescript: specifier: ^4.9.5 version: 4.9.5 + uuid: + specifier: ^9.0.0 + version: 9.0.0 vite-express: specifier: ^0.10.0 version: 0.10.0 + yup: + specifier: ^1.2.0 + version: 1.2.0 devDependencies: '@types/bcryptjs': @@ -114,6 +138,12 @@ devDependencies: '@types/react-dom': specifier: ^18.2.7 version: 18.2.7 + '@types/request-ip': + specifier: ^0.0.38 + version: 0.0.38 + '@types/tar': + specifier: ^6.1.5 + version: 6.1.5 '@vitejs/plugin-react': specifier: ^4.0.4 version: 4.0.4(vite@4.4.9) @@ -132,6 +162,9 @@ devDependencies: tailwindcss: specifier: ^3.3.3 version: 3.3.3(ts-node@10.9.1) + tar: + specifier: ^6.1.15 + version: 6.1.15 vite: specifier: ^4.4.9 version: 4.4.9(@types/node@18.17.12) @@ -1951,6 +1984,12 @@ packages: csstype: 3.1.2 dev: true + /@types/request-ip@0.0.38: + resolution: {integrity: sha512-1yeq8UuK/tUBqLXRY24gjeFvrSNaGNcOcZLQjHlnuw8iu+qE/vTQ64TUcLWorr607NKLfFakdoYEXXHXrLLKCw==} + dependencies: + '@types/node': 18.17.12 + dev: true + /@types/scheduler@0.16.3: resolution: {integrity: sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==} dev: true @@ -1970,6 +2009,17 @@ packages: '@types/node': 18.17.12 dev: true + /@types/tar@6.1.5: + resolution: {integrity: sha512-qm2I/RlZij5RofuY7vohTpYNaYcrSQlN2MyjucQc7ZweDwaEWkdN/EeNh6e9zjK6uEm6PwjdMXkcj05BxZdX1Q==} + dependencies: + '@types/node': 18.17.12 + minipass: 4.2.8 + dev: true + + /@types/uuid@9.0.3: + resolution: {integrity: sha512-taHQQH/3ZyI3zP8M/puluDEIEvtQHVYcC6y3N8ijFtAd28+Ey/G4sg1u2gB01S8MwybLOKAp9/yCMu/uR5l3Ug==} + dev: false + /@vitejs/plugin-react@4.0.4(vite@4.4.9): resolution: {integrity: sha512-7wU921ABnNYkETiMaZy7XqpueMnpu5VxvVps13MjmCo+utBdD79sZzrApHawHtVX66cCJQQTXFcjH0y9dSUK8g==} engines: {node: ^14.18.0 || >=16.0.0} @@ -2368,6 +2418,11 @@ packages: fsevents: 2.3.3 dev: true + /chownr@2.0.0: + resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} + engines: {node: '>=10'} + dev: true + /clamp@1.0.1: resolution: {integrity: sha512-kgMuFyE78OC6Dyu3Dy7vcx4uy97EIbVxJB/B0eJ3bUNAkwdNcxYzgKltnyADiYwsR7SEqkkUPsEUT//OVS6XMA==} dev: false @@ -3147,6 +3202,13 @@ packages: engines: {node: '>= 0.6'} dev: false + /fs-minipass@2.1.0: + resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} + engines: {node: '>= 8'} + dependencies: + minipass: 3.3.6 + dev: true + /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} @@ -3482,6 +3544,11 @@ packages: is-extglob: 2.1.1 dev: true + /is-localhost-ip@2.0.0: + resolution: {integrity: sha512-vlgs2cSgMOfnKU8c1ewgKPyum9rVrjjLLW2HBdL5i0iAJjOs8NY55ZBd/hqUTaYR0EO9CKZd3hVSC2HlIbygTQ==} + engines: {node: '>=12'} + dev: false + /is-negative-zero@2.0.2: resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==} engines: {node: '>= 0.4'} @@ -3784,6 +3851,14 @@ packages: resolution: {integrity: sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg==} dev: false + /maxmind@4.3.11: + resolution: {integrity: sha512-tJDrKbUzN6PSA88tWgg0L2R4Ln00XwecYQJPFI+RvlF2k1sx6VQYtuQ1SVxm8+bw5tF7GWV4xyb+3/KyzEpPUw==} + engines: {node: '>=12', npm: '>=6'} + dependencies: + mmdb-lib: 2.0.2 + tiny-lru: 11.0.1 + dev: false + /mdn-data@2.0.14: resolution: {integrity: sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==} dev: false @@ -3842,6 +3917,37 @@ packages: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} dev: false + /minipass@3.3.6: + resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} + engines: {node: '>=8'} + dependencies: + yallist: 4.0.0 + dev: true + + /minipass@4.2.8: + resolution: {integrity: sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==} + engines: {node: '>=8'} + dev: true + + /minipass@5.0.0: + resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} + engines: {node: '>=8'} + dev: true + + /minizlib@2.1.2: + resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} + engines: {node: '>= 8'} + dependencies: + minipass: 3.3.6 + yallist: 4.0.0 + dev: true + + /mkdirp@1.0.4: + resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} + engines: {node: '>=10'} + hasBin: true + dev: true + /ml-array-max@1.2.4: resolution: {integrity: sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==} dependencies: @@ -3869,6 +3975,11 @@ packages: ml-array-rescale: 1.3.7 dev: false + /mmdb-lib@2.0.2: + resolution: {integrity: sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA==} + engines: {node: '>=10', npm: '>=6'} + dev: false + /moment@2.29.4: resolution: {integrity: sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==} dev: false @@ -4207,6 +4318,10 @@ packages: react-is: 16.13.1 dev: false + /property-expr@2.0.5: + resolution: {integrity: sha512-IJUkICM5dP5znhCckHSv30Q4b5/JA5enCtkRHYaOVOAocnH/1BQEYTC5NMfT3AVl/iXKdr3aqQbQn9DxyWknwA==} + dev: false + /protocol-buffers-schema@3.6.0: resolution: {integrity: sha512-TdDRD+/QNdrCGCE7v8340QyuXd4kIWIgapsE2+n/SaGiSSbomYl4TjHlvIoCWRpE7wFt02EpB35VVA2ImcBVqw==} dev: false @@ -5017,6 +5132,10 @@ packages: engines: {node: '>=0.10'} dev: false + /request-ip@3.3.0: + resolution: {integrity: sha512-cA6Xh6e0fDBBBwH77SLJaJPBmD3nWVAcF9/XAcsrIHdjhFzFiB5aNQFytdjCGPezU3ROwrR11IddKAM08vohxA==} + dev: false + /resize-observer-polyfill@1.5.1: resolution: {integrity: sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==} dev: false @@ -5484,6 +5603,18 @@ packages: through: 2.3.8 dev: false + /tar@6.1.15: + resolution: {integrity: sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A==} + engines: {node: '>=10'} + dependencies: + chownr: 2.0.0 + fs-minipass: 2.1.0 + minipass: 5.0.0 + minizlib: 2.1.2 + mkdirp: 1.0.4 + yallist: 4.0.0 + dev: true + /thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -5511,6 +5642,15 @@ packages: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} dev: false + /tiny-case@1.0.3: + resolution: {integrity: sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q==} + dev: false + + /tiny-lru@11.0.1: + resolution: {integrity: sha512-iNgFugVuQgBKrqeO/mpiTTgmBsTP0WL6yeuLfLs/Ctf0pI/ixGqIRm8sDCwMcXGe9WWvt2sGXI5mNqZbValmJg==} + engines: {node: '>=12'} + dev: false + /tinycolor2@1.6.0: resolution: {integrity: sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==} dev: false @@ -5604,6 +5744,11 @@ packages: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} dev: false + /type-fest@2.19.0: + resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} + engines: {node: '>=12.20'} + dev: false + /type-is@1.6.18: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} @@ -5725,6 +5870,11 @@ packages: engines: {node: '>= 0.4.0'} dev: false + /uuid@9.0.0: + resolution: {integrity: sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==} + hasBin: true + dev: false + /v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} @@ -5857,7 +6007,6 @@ packages: /yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: false /yaml@2.3.2: resolution: {integrity: sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==} @@ -5876,3 +6025,12 @@ packages: /yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} + + /yup@1.2.0: + resolution: {integrity: sha512-PPqYKSAXjpRCgLgLKVGPA33v5c/WgEx3wi6NFjIiegz90zSwyMpvTFp/uGcVnnbx6to28pgnzp/q8ih3QRjLMQ==} + dependencies: + property-expr: 2.0.5 + tiny-case: 1.0.3 + toposort: 2.0.2 + type-fest: 2.19.0 + dev: false diff --git a/prisma/schema.prisma b/prisma/schema.prisma index a34eb24..3efd51b 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -8,16 +8,16 @@ generator client { } model User { - id String @id @unique @default(uuid()) @db.Uuid - username String @unique @db.VarChar(255) - password String @db.VarChar(60) - role String @db.VarChar(50) - createdAt DateTime? @default(now()) @db.Timestamptz(6) - updatedAt DateTime? @updatedAt @db.Timestamptz(6) - deletedAt DateTime? @db.Timestamptz(6) - currentWorkspaceId String? @db.Uuid + id String @id @unique @default(uuid()) @db.Uuid + username String @unique @db.VarChar(255) + password String @db.VarChar(60) + role String @db.VarChar(50) + createdAt DateTime? @default(now()) @db.Timestamptz(6) + updatedAt DateTime? @updatedAt @db.Timestamptz(6) + deletedAt DateTime? @db.Timestamptz(6) + currentWorkspaceId String? @db.Uuid - currentWorkspace Workspace? @relation(fields: [currentWorkspaceId], references: [id]) + currentWorkspace Workspace? @relation(fields: [currentWorkspaceId], references: [id]) workspaces WorkspacesOnUsers[] } @@ -101,7 +101,7 @@ model WebsiteSession { } model WebsiteEvent { - id String @id() @db.Uuid + id String @id() @default(uuid()) @db.Uuid websiteId String @db.Uuid sessionId String @db.Uuid createdAt DateTime? @default(now()) @db.Timestamptz(6) @@ -130,7 +130,7 @@ model WebsiteEvent { } model WebsiteEventData { - id String @id() @db.Uuid + id String @id() @default(uuid()) @db.Uuid websiteId String @db.Uuid websiteEventId String @db.Uuid eventKey String @db.VarChar(500) @@ -151,10 +151,9 @@ model WebsiteEventData { } model WebsiteSessionData { - id String @id() @db.Uuid + id String @id() @default(uuid()) @db.Uuid websiteId String @db.Uuid sessionId String @db.Uuid - sessionKey String @db.VarChar(500) stringValue String? @db.VarChar(500) numberValue Decimal? @db.Decimal(19, 4) dateValue DateTime? @db.Timestamptz(6) diff --git a/scripts/build-geo.ts b/scripts/build-geo.ts new file mode 100644 index 0000000..077e282 --- /dev/null +++ b/scripts/build-geo.ts @@ -0,0 +1,55 @@ +require('dotenv').config(); +import fs from 'fs'; +import path from 'path'; +import https from 'https'; +import zlib from 'zlib'; +import tar from 'tar'; + +if (process.env.VERCEL) { + console.log('Vercel environment detected. Skipping geo setup.'); + process.exit(0); +} + +const db = 'GeoLite2-City'; + +let url = `https://raw.githubusercontent.com/GitSquared/node-geolite2-redist/master/redist/${db}.tar.gz`; + +if (process.env.MAXMIND_LICENSE_KEY) { + url = + `https://download.maxmind.com/app/geoip_download` + + `?edition_id=${db}&license_key=${process.env.MAXMIND_LICENSE_KEY}&suffix=tar.gz`; +} + +const dest = path.resolve(__dirname, '../geo'); + +if (!fs.existsSync(dest)) { + fs.mkdirSync(dest); +} + +const download = (url: string): Promise => + new Promise((resolve) => { + https.get(url, (res) => { + resolve(res.pipe(zlib.createGunzip({})).pipe(tar.t())); + }); + }); + +download(url).then( + (res) => + new Promise((resolve, reject) => { + res.on('entry', (entry) => { + if (entry.path.endsWith('.mmdb')) { + const filename = path.join(dest, path.basename(entry.path)); + entry.pipe(fs.createWriteStream(filename)); + + console.log('Saved geo database:', filename); + } + }); + + res.on('error', (e) => { + reject(e); + }); + res.on('finish', () => { + resolve(); + }); + }) +); diff --git a/scripts/build-tracker.ts b/scripts/build-tracker.ts index 7ff5f27..2bf73fb 100644 --- a/scripts/build-tracker.ts +++ b/scripts/build-tracker.ts @@ -13,6 +13,7 @@ vite formats: ['iife'], }, emptyOutDir: false, + outDir: resolve(__dirname, '../public'), }, }) .then((res) => { diff --git a/src/client/components/HealthBar.tsx b/src/client/components/HealthBar.tsx index 5f79b62..91ab7d5 100644 --- a/src/client/components/HealthBar.tsx +++ b/src/client/components/HealthBar.tsx @@ -9,8 +9,9 @@ interface HealthBarProps { export const HealthBar: React.FC = React.memo((props) => { return (
- {props.beats.map((beat) => ( + {props.beats.map((beat, i) => (
{ res.status(500); diff --git a/src/server/model/website.ts b/src/server/model/website.ts new file mode 100644 index 0000000..0d09bed --- /dev/null +++ b/src/server/model/website.ts @@ -0,0 +1,246 @@ +import { Website, WebsiteSession } from '@prisma/client'; +import { flattenJSON, hashUuid, isUuid } from '../utils/common'; +import { prisma } from './_client'; +import { Request } from 'express'; +import { getClientInfo } from '../utils/detect'; +import { + DATA_TYPE, + EVENT_NAME_LENGTH, + EVENT_TYPE, + URL_LENGTH, +} from '../utils/const'; +import type { DynamicData } from '../utils/types'; + +export interface WebsiteEventPayload { + data?: object; + hostname: string; + language?: string; + referrer?: string; + screen?: string; + title?: string; + url?: string; + website: string; + name?: string; +} + +export async function findSession(req: Request): Promise<{ + id: any; + websiteId: string; + hostname: string; + browser: string; + os: any; + device: string; + screen: string; + language: string; + country: any; + subdivision1: any; + subdivision2: any; + city: any; + workspaceId: string; +}> { + // Verify payload + const { payload } = req.body; + + const { + website: websiteId, + hostname, + screen, + language, + } = payload as WebsiteEventPayload; + + // Check the hostname value for legality to eliminate dirty data + const validHostnameRegex = /^[\w-.]+$/; + if (typeof hostname === 'string' && !validHostnameRegex.test(hostname)) { + throw new Error('Invalid hostname.'); + } + + if (!isUuid(websiteId)) { + throw new Error('Invalid website ID.'); + } + + // Find website + const website = await loadWebsite(websiteId); + + if (!website) { + throw new Error(`Website not found: ${websiteId}.`); + } + + const { + userAgent, + browser, + os, + ip, + country, + subdivision1, + subdivision2, + city, + device, + } = await getClientInfo(req, payload); + + const sessionId = hashUuid(websiteId, hostname!, ip, userAgent!); + + // Find session + let session = await loadSession(sessionId); + + // Create a session if not found + if (!session) { + try { + session = await prisma.websiteSession.create({ + data: { + id: sessionId, + websiteId, + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1, + subdivision2, + city, + }, + }); + } catch (e: any) { + if (!e.message.toLowerCase().includes('unique constraint')) { + throw e; + } + } + } + + const res: any = { ...session!, workspaceId: website.workspaceId }; + + return res; +} + +async function loadWebsite(websiteId: string): Promise { + const website = await prisma.website.findUnique({ + where: { + id: websiteId, + }, + }); + + if (!website || website.deletedAt) { + return null; + } + + return website; +} + +async function loadSession(sessionId: string): Promise { + const session = await prisma.websiteSession.findUnique({ + where: { + id: sessionId, + }, + }); + + if (!session) { + return null; + } + + return session; +} + +export async function saveWebsiteEvent(data: { + sessionId: string; + websiteId: string; + urlPath: string; + urlQuery?: string; + referrerPath?: string; + referrerQuery?: string; + referrerDomain?: string; + pageTitle?: string; + eventName?: string; + eventData?: any; +}) { + const { + websiteId, + sessionId, + urlPath, + urlQuery, + referrerPath, + referrerQuery, + referrerDomain, + eventName, + eventData, + pageTitle, + } = data; + + const websiteEvent = await prisma.websiteEvent.create({ + data: { + websiteId, + sessionId, + urlPath: urlPath?.substring(0, URL_LENGTH), + urlQuery: urlQuery?.substring(0, URL_LENGTH), + referrerPath: referrerPath?.substring(0, URL_LENGTH), + referrerQuery: referrerQuery?.substring(0, URL_LENGTH), + referrerDomain: referrerDomain?.substring(0, URL_LENGTH), + pageTitle, + eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, + }, + }); + + if (eventData) { + const jsonKeys = flattenJSON(eventData); + + // id, websiteEventId, eventStringValue + const flattendData = jsonKeys.map((a) => ({ + websiteEventId: websiteEvent.id, + websiteId, + eventKey: a.key, + stringValue: + a.dynamicDataType === DATA_TYPE.number + ? parseFloat(a.value).toFixed(4) + : a.dynamicDataType === DATA_TYPE.date + ? a.value.split('.')[0] + 'Z' + : a.value.toString(), + numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, + dateValue: + a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null, + dataType: a.dynamicDataType, + })); + + await prisma.websiteEventData.createMany({ + data: flattendData, + }); + } + + return websiteEvent; +} + +export async function saveWebsiteSessionData(data: { + websiteId: string; + sessionId: string; + sessionData: DynamicData; +}) { + const { websiteId, sessionId, sessionData } = data; + + const jsonKeys = flattenJSON(sessionData); + + const flattendData = jsonKeys.map((a) => ({ + websiteId, + sessionId, + key: a.key, + stringValue: + a.dynamicDataType === DATA_TYPE.number + ? parseFloat(a.value).toFixed(4) + : a.dynamicDataType === DATA_TYPE.date + ? a.value.split('.')[0] + 'Z' + : a.value.toString(), + numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, + dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null, + dataType: a.dynamicDataType, + })); + + return prisma.$transaction([ + prisma.websiteSessionData.deleteMany({ + where: { + sessionId, + }, + }), + prisma.websiteSessionData.createMany({ + data: flattendData, + }), + ]); +} diff --git a/src/server/router/website.ts b/src/server/router/website.ts new file mode 100644 index 0000000..6e45040 --- /dev/null +++ b/src/server/router/website.ts @@ -0,0 +1,106 @@ +import { Router } from 'express'; +import { body, validate } from '../middleware/validate'; +import * as yup from 'yup'; +import { COLLECTION_TYPE, HOSTNAME_REGEX } from '../utils/const'; +import { + findSession, + saveWebsiteEvent, + saveWebsiteSessionData, +} from '../model/website'; + +export const websiteRouter = Router(); + +websiteRouter.post( + '/send', + validate( + body('payload') + .exists() + .withMessage('payload should be existed') + .isObject() + .custom(async (input) => { + return yup + .object() + .shape({ + data: yup.object(), + hostname: yup.string().matches(HOSTNAME_REGEX).max(100), + language: yup.string().max(35), + referrer: yup.string().max(500), + screen: yup.string().max(11), + title: yup.string().max(500), + url: yup.string().max(500), + website: yup.string().uuid().required(), + name: yup.string().max(50), + }) + .required() + .validate(input) + .catch((err) => {}); + }), + body('type') + .exists() + .withMessage('type should be existed') + .isString() + .matches(/event|identify/i) + ), + async (req, res) => { + // https://github1s.com/umami-software/umami/blob/master/src/pages/api/send.ts + + const { type, payload } = req.body; + const { + url, + referrer, + name: eventName, + data: eventData, + title: pageTitle, + } = payload; + + const session = await findSession(req); + + if (type === COLLECTION_TYPE.event) { + let [urlPath, urlQuery] = url?.split('?') || []; + let [referrerPath, referrerQuery] = referrer?.split('?') || []; + let referrerDomain; + + if (!urlPath) { + urlPath = '/'; + } + + if (referrerPath?.startsWith('http')) { + const refUrl = new URL(referrer); + referrerPath = refUrl.pathname; + referrerQuery = refUrl.search.substring(1); + referrerDomain = refUrl.hostname.replace(/www\./, ''); + } + + if (process.env.REMOVE_TRAILING_SLASH) { + urlPath = urlPath.replace(/.+\/$/, ''); + } + + await saveWebsiteEvent({ + urlPath, + urlQuery, + referrerPath, + referrerQuery, + referrerDomain, + pageTitle, + eventName, + eventData, + ...session, + sessionId: session.id, + }); + } + + if (type === COLLECTION_TYPE.identify) { + if (!eventData) { + throw new Error('Data required'); + } + + await saveWebsiteSessionData({ + ...session, + sessionData: eventData, + sessionId: session.id, + }); + } + + res.send(); + } +); diff --git a/src/server/utils/common.ts b/src/server/utils/common.ts new file mode 100644 index 0000000..61d352d --- /dev/null +++ b/src/server/utils/common.ts @@ -0,0 +1,131 @@ +import { v4, v5, validate } from 'uuid'; +import crypto from 'crypto'; +import { DATA_TYPE } from './const'; +import { DynamicDataType } from './types'; +import dayjs from 'dayjs'; + +export function isUuid(value: string) { + return validate(value); +} + +export function safeDecodeURIComponent( + s: string | string[] | undefined | null +): string | undefined | null { + if (s === undefined || s === null) { + return s; + } + + if (Array.isArray(s)) { + s = String(s); + } + + try { + return decodeURIComponent(s); + } catch (e) { + return s; + } +} + +function hash(...args: string[]) { + return crypto.createHash('sha512').update(args.join('')).digest('hex'); +} + +export function hashUuid(...args: string[]) { + if (!args.length) { + return v4(); + } + + return v5(hash(...args), v5.DNS); +} + +export function isValidDate(input: any) { + return dayjs(input).isValid(); +} + +export function flattenJSON( + eventData: { [key: string]: any }, + keyValues: { + key: string; + value: any; + dynamicDataType: DynamicDataType; + }[] = [], + parentKey = '' +): { key: string; value: any; dynamicDataType: DynamicDataType }[] { + return Object.keys(eventData).reduce( + (acc, key) => { + const value = eventData[key]; + const type = typeof eventData[key]; + + // nested object + if ( + value && + type === 'object' && + !Array.isArray(value) && + !isValidDate(value) + ) { + flattenJSON(value, acc.keyValues, getKeyName(key, parentKey)); + } else { + createKey(getKeyName(key, parentKey), value, acc); + } + + return acc; + }, + { keyValues, parentKey } + ).keyValues; +} + +function getKeyName(key: string, parentKey?: string) { + if (!parentKey) { + return key; + } + + return `${parentKey}.${key}`; +} + +function createKey( + key: string, + value: any, + acc: { keyValues: any[]; parentKey: string } +) { + const type = getDataType(value); + + let dynamicDataType = null; + + switch (type) { + case 'number': + dynamicDataType = DATA_TYPE.number; + break; + case 'string': + dynamicDataType = DATA_TYPE.string; + break; + case 'boolean': + dynamicDataType = DATA_TYPE.boolean; + value = value ? 'true' : 'false'; + break; + case 'date': + dynamicDataType = DATA_TYPE.date; + break; + case 'object': + dynamicDataType = DATA_TYPE.array; + value = JSON.stringify(value); + break; + default: + dynamicDataType = DATA_TYPE.string; + break; + } + + acc.keyValues.push({ key, value, dynamicDataType }); +} + +function getDataType(value: any): string { + let type: string = typeof value; + + if ( + (type === 'string' && isValidDate(value)) || + isValidDate(dayjs(value).toISOString()) + ) { + type = 'date'; + } + + return type; +} diff --git a/src/server/utils/const.ts b/src/server/utils/const.ts index 4be83c2..1f75368 100644 --- a/src/server/utils/const.ts +++ b/src/server/utils/const.ts @@ -7,3 +7,62 @@ export const ROLES = { owner: 'owner', readOnly: 'readOnly', } as const; + +export const HOSTNAME_REGEX = + /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])$/; + +export const COLLECTION_TYPE = { + event: 'event', + identify: 'identify', +}; + +export const DESKTOP_OS = [ + 'BeOS', + 'Chrome OS', + 'Linux', + 'Mac OS', + 'Open BSD', + 'OS/2', + 'QNX', + 'Sun OS', + 'Windows 10', + 'Windows 2000', + 'Windows 3.11', + 'Windows 7', + 'Windows 8', + 'Windows 8.1', + 'Windows 95', + 'Windows 98', + 'Windows ME', + 'Windows Server 2003', + 'Windows Vista', + 'Windows XP', +]; + +export const MOBILE_OS = [ + 'Amazon OS', + 'Android OS', + 'BlackBerry OS', + 'iOS', + 'Windows Mobile', +]; + +export const DESKTOP_SCREEN_WIDTH = 1920; +export const LAPTOP_SCREEN_WIDTH = 1024; +export const MOBILE_SCREEN_WIDTH = 479; + +export const URL_LENGTH = 500; +export const EVENT_NAME_LENGTH = 50; + +export const EVENT_TYPE = { + pageView: 1, + customEvent: 2, +} as const; + +export const DATA_TYPE = { + string: 1, + number: 2, + boolean: 3, + date: 4, + array: 5, +} as const; diff --git a/src/server/utils/detect.ts b/src/server/utils/detect.ts new file mode 100644 index 0000000..4ef23b3 --- /dev/null +++ b/src/server/utils/detect.ts @@ -0,0 +1,157 @@ +import { Request } from 'express'; +import type { WebsiteEventPayload } from '../model/website'; +import { getClientIp } from 'request-ip'; +import isLocalhost from 'is-localhost-ip'; +import { safeDecodeURIComponent } from './common'; +import { browserName, detectOS, OperatingSystem } from 'detect-browser'; +import { + DESKTOP_OS, + DESKTOP_SCREEN_WIDTH, + LAPTOP_SCREEN_WIDTH, + MOBILE_OS, + MOBILE_SCREEN_WIDTH, +} from './const'; +import path from 'path'; +import maxmind, { Reader, CityResponse } from 'maxmind'; + +let lookup: Reader; + +export async function getClientInfo( + req: Request, + payload: WebsiteEventPayload +) { + const userAgent = req.headers['user-agent']; + const ip = getIpAddress(req); + const location = await getLocation(ip, req); + const country = location?.country; + const subdivision1 = location?.subdivision1; + const subdivision2 = location?.subdivision2; + const city = location?.city; + const browser = browserName(userAgent ?? ''); + const os = detectOS(userAgent ?? ''); + const device = getDevice(payload.screen, os); + + return { + userAgent, + browser, + os, + ip, + country, + subdivision1, + subdivision2, + city, + device, + }; +} + +export function getIpAddress(req: Request): string { + // Custom header + if ( + process.env.CLIENT_IP_HEADER && + req.headers[process.env.CLIENT_IP_HEADER] + ) { + return String(req.headers[process.env.CLIENT_IP_HEADER]); + } + // Cloudflare + else if (req.headers['cf-connecting-ip']) { + return String(req.headers['cf-connecting-ip']); + } + + return getClientIp(req)!; +} + +export async function getLocation(ip: string, req: Request) { + // Ignore local ips + if (await isLocalhost(ip)) { + return; + } + + // Cloudflare headers + if (req.headers['cf-ipcountry']) { + const country = safeDecodeURIComponent(req.headers['cf-ipcountry']); + const subdivision1 = safeDecodeURIComponent(req.headers['cf-region-code']); + const city = safeDecodeURIComponent(req.headers['cf-ipcity']); + + return { + country, + subdivision1: getRegionCode(country, subdivision1), + city, + }; + } + + // Vercel headers + if (req.headers['x-vercel-ip-country']) { + const country = safeDecodeURIComponent(req.headers['x-vercel-ip-country']); + const subdivision1 = safeDecodeURIComponent( + req.headers['x-vercel-ip-country-region'] + ); + const city = safeDecodeURIComponent(req.headers['x-vercel-ip-city']); + + return { + country, + subdivision1: getRegionCode(country, subdivision1), + city, + }; + } + + // Database lookup + if (!lookup) { + const dir = path.join(process.cwd(), 'geo'); + + lookup = await maxmind.open(path.resolve(dir, 'GeoLite2-City.mmdb')); + } + + const result = lookup.get(ip); + + if (result) { + return { + country: result.country?.iso_code ?? result?.registered_country?.iso_code, + subdivision1: result.subdivisions?.[0]?.iso_code, + subdivision2: result.subdivisions?.[1]?.names?.en, + city: result.city?.names?.en, + }; + } +} + +function getRegionCode( + country: string | null | undefined, + region: string | null | undefined +) { + if (!country || !region) { + return undefined; + } + + return region.includes('-') ? region : `${country}-${region}`; +} + +export function getDevice( + screen: string | undefined, + os: OperatingSystem | null +) { + if (!screen) return; + if (!os) return; + + const [width] = screen.split('x').map((n) => Number(n)); + + if (DESKTOP_OS.includes(os)) { + if (os === 'Chrome OS' || width < DESKTOP_SCREEN_WIDTH) { + return 'laptop'; + } + return 'desktop'; + } else if (MOBILE_OS.includes(os)) { + if (os === 'Amazon OS' || width > MOBILE_SCREEN_WIDTH) { + return 'tablet'; + } + return 'mobile'; + } + + if (width >= DESKTOP_SCREEN_WIDTH) { + return 'desktop'; + } else if (width >= LAPTOP_SCREEN_WIDTH) { + return 'laptop'; + } else if (width >= MOBILE_SCREEN_WIDTH) { + return 'tablet'; + } else { + return 'mobile'; + } +} diff --git a/src/server/utils/types.ts b/src/server/utils/types.ts new file mode 100644 index 0000000..8fe0b41 --- /dev/null +++ b/src/server/utils/types.ts @@ -0,0 +1,14 @@ +import { DATA_TYPE } from './const'; + +type ObjectValues = T[keyof T]; +export type DynamicDataType = ObjectValues; + +export interface DynamicData { + [key: string]: + | number + | string + | DynamicData + | number[] + | string[] + | DynamicData[]; +} diff --git a/src/tracker/index.js b/src/tracker/index.js index 211220e..4ff6d9d 100644 --- a/src/tracker/index.js +++ b/src/tracker/index.js @@ -28,7 +28,7 @@ const root = hostUrl ? hostUrl.replace(/\/$/, '') : currentScript.src.split('/').slice(0, -1).join('/'); - const endpoint = `${root}/api/send`; + const endpoint = `${root}/api/website/send`; const screen = `${width}x${height}`; const eventRegex = /data-tianji-event-([\w-_]+)/; const eventNameAttribute = _data + 'tianji-event';