diff --git a/package.json b/package.json index 8efe981..269988b 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "dev:server": "cd src/server && pnpm dev", "start": "cd src/server && cross-env NODE_ENV=production node ./dist/src/server/main.js", "start:docker": "pnpm start:docker:db && pnpm start", - "start:docker:db": "cd src/server && pnpm db:migrate:apply", + "start:docker:db": "cd src/server && pnpm db:migrate:apply && pnpm db:migrate:script", "test": "vitest", "build": "pnpm build:tracker && pnpm build:app && pnpm build:geo", "build:app": "pnpm build:server && pnpm build:client", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95a9e5d..72d8215 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,5 +1,9 @@ lockfileVersion: '6.0' +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + overrides: dayjs: 1.11.10 @@ -400,6 +404,9 @@ importers: nodemon: specifier: ^3.0.3 version: 3.0.3 + p-map: + specifier: 4.0.0 + version: 4.0.0 prisma: specifier: 5.4.2 version: 5.4.2 @@ -16630,6 +16637,7 @@ packages: /pify@4.0.1: resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==} engines: {node: '>=6'} + requiresBuild: true dev: true optional: true @@ -17439,6 +17447,7 @@ packages: /prr@1.0.1: resolution: {integrity: sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==} + requiresBuild: true dev: true optional: true @@ -18103,7 +18112,7 @@ packages: engines: {node: '>=8.x'} peerDependencies: date-fns: '>= 2.x' - dayjs: '>= 1.x' + dayjs: 1.11.10 luxon: '>= 3.x' moment: '>= 2.x' react: '>=16.9.0' @@ -18132,7 +18141,7 @@ packages: engines: {node: '>=8.x'} peerDependencies: date-fns: '>= 2.x' - dayjs: '>= 1.x' + dayjs: 1.11.10 luxon: '>= 3.x' moment: '>= 2.x' react: '>=16.9.0' @@ -22737,7 +22746,3 @@ packages: /zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} dev: true - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false diff --git a/src/server/package.json b/src/server/package.json index 36875a1..c37f331 100644 --- a/src/server/package.json +++ b/src/server/package.json @@ -11,6 +11,7 @@ "db:generate": "prisma generate", "db:migrate:dev": "prisma migrate dev", "db:migrate:apply": "prisma migrate deploy", + "db:migrate:script": "ts-node ./prisma/scripts/populate-ip-location.ts", "db:studio": "prisma studio", "db:seed:website": "ts-node ./tests/seeds/website.ts", "test": "vitest", @@ -83,6 +84,7 @@ "@types/uuid": "^9.0.7", "execa": "^5.1.1", "nodemon": "^3.0.3", + "p-map": "4.0.0", "prisma": "5.4.2", "prisma-json-types-generator": "3.0.3", "prisma-zod-generator": "0.8.13", diff --git a/src/server/prisma/scripts/populate-ip-location.ts b/src/server/prisma/scripts/populate-ip-location.ts new file mode 100644 index 0000000..f31de9d --- /dev/null +++ b/src/server/prisma/scripts/populate-ip-location.ts @@ -0,0 +1,104 @@ +import { PrismaClient } from '@prisma/client'; +import { getLocation } from '../../utils/detect'; +import pMap from 'p-map'; + +const prisma = new PrismaClient(); + +/** + * Update ip latitude and longitude + */ +async function main() { + console.log(`[populate-ip-location] start scan`); + const start = Date.now(); + const noParseWebsiteSessions = await prisma.websiteSession.findMany({ + where: { + ip: { + not: null, + }, + latitude: null, + }, + }); + + console.log( + `[populate-ip-location] find ${noParseWebsiteSessions.length} records wait to parse location` + ); + + let count = 0; + const queue: { + id: string; + latitude: number | undefined; + longitude: number | undefined; + accuracyRadius: number | undefined; + }[] = []; + + for (const session of noParseWebsiteSessions) { + if (!session.ip) { + continue; + } + + const res = await getLocation(session.ip); + if (!res) { + continue; + } + + const { latitude, longitude, accuracyRadius } = res; + queue.push({ + id: session.id, + latitude, + longitude, + accuracyRadius, + }); + count++; + } + + console.log(`[populate-ip-location] find ${count} records wait to update`); + + if (count === 0) { + return; + } + + let current = 0; + + await pMap( + queue, + async (item) => { + await prisma.websiteSession.update({ + where: { + id: item.id, + }, + data: { + latitude: item.latitude, + longitude: item.longitude, + accuracyRadius: item.accuracyRadius, + }, + }); + + current++; + + if (current % 100 === 0) { + console.log( + `[populate-ip-location] updated ${current}/${count} records` + ); + } + }, + { + concurrency: 20, + } + ); + + console.log( + `[populate-ip-location] update ${count} records location, usage: ${ + Date.now() - start + }ms` + ); +} + +main() + .catch(async (e) => { + console.error(e); + process.exit(1); + }) + .finally(async () => { + console.log(`[populate-ip-location] end`); + await prisma.$disconnect(); + });