chore: add populate ip location script

package.json

@@ -8,7 +8,7 @@
     "dev:server": "cd src/server && pnpm dev",
     "start": "cd src/server && cross-env NODE_ENV=production node ./dist/src/server/main.js",
     "start:docker": "pnpm start:docker:db && pnpm start",
-    "start:docker:db": "cd src/server && pnpm db:migrate:apply",
+    "start:docker:db": "cd src/server && pnpm db:migrate:apply && pnpm db:migrate:script",
     "test": "vitest",
     "build": "pnpm build:tracker && pnpm build:app && pnpm build:geo",
     "build:app": "pnpm build:server && pnpm build:client",

pnpm-lock.yaml

@@ -1,5 +1,9 @@
 lockfileVersion: '6.0'
 
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
 overrides:
   dayjs: 1.11.10
 
@@ -400,6 +404,9 @@ importers:
       nodemon:
         specifier: ^3.0.3
         version: 3.0.3
+      p-map:
+        specifier: 4.0.0
+        version: 4.0.0
       prisma:
         specifier: 5.4.2
         version: 5.4.2
@@ -16630,6 +16637,7 @@ packages:
   /pify@4.0.1:
     resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==}
     engines: {node: '>=6'}
+    requiresBuild: true
     dev: true
     optional: true
 
@@ -17439,6 +17447,7 @@ packages:
 
   /prr@1.0.1:
     resolution: {integrity: sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==}
+    requiresBuild: true
     dev: true
     optional: true
 
@@ -18103,7 +18112,7 @@ packages:
     engines: {node: '>=8.x'}
     peerDependencies:
       date-fns: '>= 2.x'
-      dayjs: '>= 1.x'
+      dayjs: 1.11.10
       luxon: '>= 3.x'
       moment: '>= 2.x'
       react: '>=16.9.0'
@@ -18132,7 +18141,7 @@ packages:
     engines: {node: '>=8.x'}
     peerDependencies:
       date-fns: '>= 2.x'
-      dayjs: '>= 1.x'
+      dayjs: 1.11.10
       luxon: '>= 3.x'
       moment: '>= 2.x'
       react: '>=16.9.0'
@@ -22737,7 +22746,3 @@ packages:
   /zwitch@2.0.4:
     resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
     dev: true
-
-settings:
-  autoInstallPeers: true
-  excludeLinksFromLockfile: false

src/server/package.json

@@ -11,6 +11,7 @@
     "db:generate": "prisma generate",
     "db:migrate:dev": "prisma migrate dev",
     "db:migrate:apply": "prisma migrate deploy",
+    "db:migrate:script": "ts-node ./prisma/scripts/populate-ip-location.ts",
     "db:studio": "prisma studio",
     "db:seed:website": "ts-node ./tests/seeds/website.ts",
     "test": "vitest",
@@ -83,6 +84,7 @@
     "@types/uuid": "^9.0.7",
     "execa": "^5.1.1",
     "nodemon": "^3.0.3",
+    "p-map": "4.0.0",
     "prisma": "5.4.2",
     "prisma-json-types-generator": "3.0.3",
     "prisma-zod-generator": "0.8.13",

src/server/prisma/scripts/populate-ip-location.ts

@@ -0,0 +1,104 @@
+import { PrismaClient } from '@prisma/client';
+import { getLocation } from '../../utils/detect';
+import pMap from 'p-map';
+
+const prisma = new PrismaClient();
+
+/**
+ * Update ip latitude and longitude
+ */
+async function main() {
+  console.log(`[populate-ip-location] start scan`);
+  const start = Date.now();
+  const noParseWebsiteSessions = await prisma.websiteSession.findMany({
+    where: {
+      ip: {
+        not: null,
+      },
+      latitude: null,
+    },
+  });
+
+  console.log(
+    `[populate-ip-location] find ${noParseWebsiteSessions.length} records wait to parse location`
+  );
+
+  let count = 0;
+  const queue: {
+    id: string;
+    latitude: number | undefined;
+    longitude: number | undefined;
+    accuracyRadius: number | undefined;
+  }[] = [];
+
+  for (const session of noParseWebsiteSessions) {
+    if (!session.ip) {
+      continue;
+    }
+
+    const res = await getLocation(session.ip);
+    if (!res) {
+      continue;
+    }
+
+    const { latitude, longitude, accuracyRadius } = res;
+    queue.push({
+      id: session.id,
+      latitude,
+      longitude,
+      accuracyRadius,
+    });
+    count++;
+  }
+
+  console.log(`[populate-ip-location] find ${count} records wait to update`);
+
+  if (count === 0) {
+    return;
+  }
+
+  let current = 0;
+
+  await pMap(
+    queue,
+    async (item) => {
+      await prisma.websiteSession.update({
+        where: {
+          id: item.id,
+        },
+        data: {
+          latitude: item.latitude,
+          longitude: item.longitude,
+          accuracyRadius: item.accuracyRadius,
+        },
+      });
+
+      current++;
+
+      if (current % 100 === 0) {
+        console.log(
+          `[populate-ip-location] updated ${current}/${count} records`
+        );
+      }
+    },
+    {
+      concurrency: 20,
+    }
+  );
+
+  console.log(
+    `[populate-ip-location] update ${count} records location, usage: ${
+      Date.now() - start
+    }ms`
+  );
+}
+
+main()
+  .catch(async (e) => {
+    console.error(e);
+    process.exit(1);
+  })
+  .finally(async () => {
+    console.log(`[populate-ip-location] end`);
+    await prisma.$disconnect();
+  });