From 99a6c91b1b63eb0b07ed8f05b42175f8ea6d9f51 Mon Sep 17 00:00:00 2001 From: moonrailgun Date: Mon, 29 Jan 2024 18:54:44 +0800 Subject: [PATCH] feat: parse ip location and storage in db --- src/server/model/telemetry.ts | 6 ++ src/server/model/website.ts | 26 +++---- src/server/package.json | 3 +- .../migration.sql | 9 +++ src/server/prisma/schema.prisma | 62 ++++++++-------- src/server/prisma/zod/telemetrysession.ts | 3 + src/server/prisma/zod/websitesession.ts | 3 + src/server/utils/__tests__/detect.test.ts | 24 +++++++ src/server/utils/detect.ts | 71 +++++++------------ 9 files changed, 119 insertions(+), 88 deletions(-) create mode 100644 src/server/prisma/migrations/20240129091132_add_ip_location_info/migration.sql create mode 100644 src/server/utils/__tests__/detect.test.ts diff --git a/src/server/model/telemetry.ts b/src/server/model/telemetry.ts index e65fdad..3e151b5 100644 --- a/src/server/model/telemetry.ts +++ b/src/server/model/telemetry.ts @@ -80,6 +80,9 @@ async function findSession(req: Request, url: string) { subdivision1, subdivision2, city, + longitude, + latitude, + accuracyRadius, } = await getRequestInfo(req); const sessionId = hashUuid(workspaceId, hostname, ip, userAgent!); @@ -99,6 +102,9 @@ async function findSession(req: Request, url: string) { subdivision1, subdivision2, city, + longitude, + latitude, + accuracyRadius, }, }); } catch (e: any) { diff --git a/src/server/model/website.ts b/src/server/model/website.ts index 4dfdf88..7900deb 100644 --- a/src/server/model/website.ts +++ b/src/server/model/website.ts @@ -26,21 +26,11 @@ export interface WebsiteEventPayload { name?: string; } -export async function findSession(req: Request): Promise<{ - id: any; - websiteId: string; - hostname: string; - browser: string; - os: any; - device: string; - screen: string; - language: string; - country: any; - subdivision1: any; - subdivision2: any; - city: any; - workspaceId: string; -}> { +export async function findSession(req: Request): Promise< + WebsiteSession & { + workspaceId: string; + } +> { // Verify payload const { payload } = req.body; @@ -88,6 +78,9 @@ export async function findSession(req: Request): Promise<{ subdivision1, subdivision2, city, + longitude, + latitude, + accuracyRadius, device, } = await getClientInfo(req, payload); @@ -114,6 +107,9 @@ export async function findSession(req: Request): Promise<{ subdivision1, subdivision2, city, + longitude, + latitude, + accuracyRadius, }, }); } catch (e: any) { diff --git a/src/server/package.json b/src/server/package.json index cf9afa3..55dbb69 100644 --- a/src/server/package.json +++ b/src/server/package.json @@ -12,7 +12,8 @@ "db:migrate:dev": "prisma migrate dev", "db:migrate:apply": "prisma migrate deploy", "db:studio": "prisma studio", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "vitest", + "test:ci": "vitest --run" }, "keywords": [], "author": "moonrailgun ", diff --git a/src/server/prisma/migrations/20240129091132_add_ip_location_info/migration.sql b/src/server/prisma/migrations/20240129091132_add_ip_location_info/migration.sql new file mode 100644 index 0000000..9454f18 --- /dev/null +++ b/src/server/prisma/migrations/20240129091132_add_ip_location_info/migration.sql @@ -0,0 +1,9 @@ +-- AlterTable +ALTER TABLE "TelemetrySession" ADD COLUMN "accuracyRadius" INTEGER, +ADD COLUMN "latitude" DOUBLE PRECISION, +ADD COLUMN "longitude" DOUBLE PRECISION; + +-- AlterTable +ALTER TABLE "WebsiteSession" ADD COLUMN "accuracyRadius" INTEGER, +ADD COLUMN "latitude" DOUBLE PRECISION, +ADD COLUMN "longitude" DOUBLE PRECISION; diff --git a/src/server/prisma/schema.prisma b/src/server/prisma/schema.prisma index cec56c4..39b9d47 100644 --- a/src/server/prisma/schema.prisma +++ b/src/server/prisma/schema.prisma @@ -95,20 +95,23 @@ model Website { } model WebsiteSession { - id String @id @unique @db.Uuid - websiteId String @db.VarChar(30) - hostname String? @db.VarChar(100) - browser String? @db.VarChar(20) - os String? @db.VarChar(20) - device String? @db.VarChar(20) - screen String? @db.VarChar(11) - language String? @db.VarChar(35) - ip String? @db.VarChar(45) // The max length of ipv6 which adapter with ipv4 is 45(for example: [::ffff:192.168.100.228] => 0000:0000:0000:0000:0000:ffff:192.168.100.228) - country String? @db.Char(2) - subdivision1 String? @db.VarChar(20) - subdivision2 String? @db.VarChar(50) - city String? @db.VarChar(50) - createdAt DateTime @default(now()) @db.Timestamptz(6) + id String @id @unique @db.Uuid + websiteId String @db.VarChar(30) + hostname String? @db.VarChar(100) + browser String? @db.VarChar(20) + os String? @db.VarChar(20) + device String? @db.VarChar(20) + screen String? @db.VarChar(11) + language String? @db.VarChar(35) + ip String? @db.VarChar(45) // The max length of ipv6 which adapter with ipv4 is 45(for example: [::ffff:192.168.100.228] => 0000:0000:0000:0000:0000:ffff:192.168.100.228) + country String? @db.Char(2) + subdivision1 String? @db.VarChar(20) + subdivision2 String? @db.VarChar(50) + city String? @db.VarChar(50) + longitude Float? + latitude Float? + accuracyRadius Int? + createdAt DateTime @default(now()) @db.Timestamptz(6) website Website @relation(fields: [websiteId], references: [id], onUpdate: Cascade, onDelete: Cascade) @@ -198,17 +201,20 @@ model WebsiteSessionData { } model TelemetrySession { - id String @id @unique @db.Uuid - workspaceId String @db.VarChar(30) - hostname String? @db.VarChar(100) - browser String? @db.VarChar(20) - os String? @db.VarChar(20) - ip String? @db.VarChar(45) - country String? @db.Char(2) - subdivision1 String? @db.VarChar(20) - subdivision2 String? @db.VarChar(50) - city String? @db.VarChar(50) - createdAt DateTime @default(now()) @db.Timestamptz(6) + id String @id @unique @db.Uuid + workspaceId String @db.VarChar(30) + hostname String? @db.VarChar(100) + browser String? @db.VarChar(20) + os String? @db.VarChar(20) + ip String? @db.VarChar(45) + country String? @db.Char(2) + subdivision1 String? @db.VarChar(20) + subdivision2 String? @db.VarChar(50) + city String? @db.VarChar(50) + longitude Float? + latitude Float? + accuracyRadius Int? + createdAt DateTime @default(now()) @db.Timestamptz(6) telemetryEvent TelemetryEvent[] @@ -345,12 +351,12 @@ model WorkspaceDailyUsage { } model WorkspaceAuditLog { - id String @id @default(cuid()) @db.VarChar(30) - workspaceId String @db.VarChar(30) + id String @id @default(cuid()) @db.VarChar(30) + workspaceId String @db.VarChar(30) content String relatedId String? relatedType WorkspaceAuditLogType? - createdAt DateTime @default(now()) @db.Timestamptz(6) + createdAt DateTime @default(now()) @db.Timestamptz(6) workspace Workspace @relation(fields: [workspaceId], references: [id], onUpdate: Cascade, onDelete: Cascade) diff --git a/src/server/prisma/zod/telemetrysession.ts b/src/server/prisma/zod/telemetrysession.ts index 48ea5fd..f6ef3d0 100644 --- a/src/server/prisma/zod/telemetrysession.ts +++ b/src/server/prisma/zod/telemetrysession.ts @@ -13,6 +13,9 @@ export const TelemetrySessionModelSchema = z.object({ subdivision1: z.string().nullish(), subdivision2: z.string().nullish(), city: z.string().nullish(), + longitude: z.number().nullish(), + latitude: z.number().nullish(), + accuracyRadius: z.number().int().nullish(), createdAt: z.date(), }) diff --git a/src/server/prisma/zod/websitesession.ts b/src/server/prisma/zod/websitesession.ts index 1214ed0..6c3e700 100644 --- a/src/server/prisma/zod/websitesession.ts +++ b/src/server/prisma/zod/websitesession.ts @@ -16,6 +16,9 @@ export const WebsiteSessionModelSchema = z.object({ subdivision1: z.string().nullish(), subdivision2: z.string().nullish(), city: z.string().nullish(), + longitude: z.number().nullish(), + latitude: z.number().nullish(), + accuracyRadius: z.number().int().nullish(), createdAt: z.date(), }) diff --git a/src/server/utils/__tests__/detect.test.ts b/src/server/utils/__tests__/detect.test.ts new file mode 100644 index 0000000..a723115 --- /dev/null +++ b/src/server/utils/__tests__/detect.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, test } from 'vitest'; +import { getLocation } from '../detect'; + +describe('detect', () => { + describe('getLocation', () => { + test('should detect local ip', async () => { + const location = await getLocation('127.0.0.1'); + + expect(location).toBeUndefined(); + }); + + test('should detect public ip', async () => { + const location = await getLocation('76.76.21.123'); + + expect(location).toHaveProperty('country', 'US'); + expect(location).toHaveProperty('subdivision1', 'CA'); + expect(location).toHaveProperty('subdivision2', undefined); + expect(location).toHaveProperty('city', 'Walnut'); + expect(location).toHaveProperty('longitude', -117.8512); + expect(location).toHaveProperty('latitude', 34.0233); + expect(location).toHaveProperty('accuracy_radius', 20); + }); + }); +}); diff --git a/src/server/utils/detect.ts b/src/server/utils/detect.ts index 89aa040..9d9e7e6 100644 --- a/src/server/utils/detect.ts +++ b/src/server/utils/detect.ts @@ -2,7 +2,6 @@ import { Request } from 'express'; import type { WebsiteEventPayload } from '../model/website'; import { getClientIp } from 'request-ip'; import isLocalhost from 'is-localhost-ip'; -import { safeDecodeURIComponent } from './common'; import { browserName, detectOS, OperatingSystem } from 'detect-browser'; import { DESKTOP_OS, @@ -14,16 +13,19 @@ import { import maxmind, { Reader, CityResponse } from 'maxmind'; import { libraryPath } from './lib'; -let lookup: Reader; - export async function getRequestInfo(req: Request) { const userAgent = req.headers['user-agent']; const ip = getIpAddress(req); - const location = await getLocation(ip, req); - const country = location?.country; - const subdivision1 = location?.subdivision1; - const subdivision2 = location?.subdivision2; - const city = location?.city; + const location = await getLocation(ip); + const { + country, + subdivision1, + subdivision2, + city, + longitude, + latitude, + accuracyRadius, + } = location ?? {}; const browser = browserName(userAgent ?? ''); const os = detectOS(userAgent ?? ''); @@ -36,6 +38,9 @@ export async function getRequestInfo(req: Request) { subdivision1, subdivision2, city, + longitude, + latitude, + accuracyRadius, }; } @@ -68,40 +73,13 @@ export function getIpAddress(req: Request): string { return getClientIp(req)!; } -export async function getLocation(ip: string, req: Request) { +let lookup: Reader; +export async function getLocation(ip: string) { // Ignore local ips if (await isLocalhost(ip)) { return; } - // Cloudflare headers - if (req.headers['cf-ipcountry']) { - const country = safeDecodeURIComponent(req.headers['cf-ipcountry']); - const subdivision1 = safeDecodeURIComponent(req.headers['cf-region-code']); - const city = safeDecodeURIComponent(req.headers['cf-ipcity']); - - return { - country, - subdivision1: getRegionCode(country, subdivision1), - city, - }; - } - - // Vercel headers - if (req.headers['x-vercel-ip-country']) { - const country = safeDecodeURIComponent(req.headers['x-vercel-ip-country']); - const subdivision1 = safeDecodeURIComponent( - req.headers['x-vercel-ip-country-region'] - ); - const city = safeDecodeURIComponent(req.headers['x-vercel-ip-city']); - - return { - country, - subdivision1: getRegionCode(country, subdivision1), - city, - }; - } - // Database lookup if (!lookup) { lookup = await maxmind.open(libraryPath.geoPath); @@ -109,14 +87,19 @@ export async function getLocation(ip: string, req: Request) { const result = lookup.get(ip); - if (result) { - return { - country: result.country?.iso_code ?? result?.registered_country?.iso_code, - subdivision1: result.subdivisions?.[0]?.iso_code, - subdivision2: result.subdivisions?.[1]?.names?.en, - city: result.city?.names?.en, - }; + if (!result) { + return; } + + return { + country: result.country?.iso_code ?? result?.registered_country?.iso_code, + subdivision1: result.subdivisions?.[0]?.iso_code, + subdivision2: result.subdivisions?.[1]?.names?.en, + city: result.city?.names?.en, + longitude: result.location?.longitude, + latitude: result.location?.latitude, + accuracyRadius: result.location?.accuracy_radius, + }; } function getRegionCode(