Integration updates

Author: jmgasper

.gitignore

@@ -63,4 +63,6 @@ typings/
 # next.js build output
 .next
 
-migrator/data
+migrator/data
+
+src/scripts/seed/*.json

prisma/migrations/20250830001238_indices_and_stats/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "resources"."Resource" ADD COLUMN     "legacyId" INTEGER;

prisma/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
 # It should be added in your version-control system (e.g., Git)
-provider = "postgresql"
+provider = "postgresql"

src/scripts/seed-tables.js

@@ -2,6 +2,9 @@
  * Seed table data in database
  */
 const _ = require('lodash')
+const fs = require('fs')
+const path = require('path')
+const { v4: uuid } = require('uuid')
 const logger = require('../common/logger')
 const prisma = require('../common/prisma').getClient()
 
@@ -40,36 +43,97 @@ async function importResourceRolePhaseDependency () {
 }
 
 async function importResource () {
-  let data = require('./seed/Resource.json')
-  let memberData = {}
-  data = _.map(data, d => {
-    const r = _.omit(d, ['created', 'rating'])
-    r.createdAt = d.created
-    // collect member data
-    const memberId = _.parseInt(d.memberId)
-    if (!_.keys(memberData).includes(memberId)) {
-      memberData[memberId] = {
-        userId: memberId,
-        handle: d.memberHandle,
-        handleLower: d.memberHandle.toLowerCase(),
-        maxRating: _.get(d, 'rating', null),
-        createdBy
+  const logPath = path.join(__dirname, 'seed', 'resource-import-skipped.log')
+  // truncate existing log
+  try { fs.writeFileSync(logPath, '') } catch (e) { /* ignore */ }
+
+  const writeSkip = (reason, rec) => {
+    const line = JSON.stringify({ reason, record: rec }) + '\n'
+    fs.appendFileSync(logPath, line)
+  }
+
+  const data = require('./seed/Resource.json')
+  const total = data.length
+
+  const memberDataMap = {}
+  let success = 0
+  let skipped = 0
+  const start = Date.now()
+  const progressTimer = setInterval(() => {
+    const elapsed = (Date.now() - start) / 1000
+    const processed = success + skipped
+    const rate = processed > 0 ? (processed / elapsed) : 0
+    const pct = total > 0 ? ((processed / total) * 100) : 0
+    const remaining = Math.max(total - processed, 0)
+    const eta = rate > 0 ? (remaining / rate) : Infinity
+    logger.info(`Resource import progress: ${processed}/${total} (${pct.toFixed(1)}%), success=${success}, skipped=${skipped}, rate=${rate.toFixed(2)} rec/s, elapsed=${elapsed.toFixed(1)}s, ETA=${Number.isFinite(eta) ? eta.toFixed(1) : '∞'}s`)
+  }, 5000)
+
+  for (const d of data) {
+    // Validate required fields
+    const missing = []
+    for (const f of ['challengeId', 'memberId', 'memberHandle', 'roleId']) {
+      if (_.isNil(d[f]) || _.toString(d[f]).trim() === '') missing.push(f)
+    }
+    if (missing.length) {
+      skipped += 1
+      writeSkip(`Missing required field(s): ${missing.join(', ')}`, d)
+      continue
+    }
+
+    // Build record for Prisma. Remove fields not in schema or that conflict.
+    const r = _.omit(d, ['created', 'updated', 'rating', 'legacyId'])
+    // Ensure id exists; generate if absent
+    if (!r.id) r.id = uuid()
+    // Normalize timestamps
+    if (d.created) r.createdAt = new Date(d.created)
+    if (d.updated) r.updatedAt = new Date(d.updated)
+    // Ensure strings for string columns
+    r.challengeId = _.toString(r.challengeId)
+    r.memberId = _.toString(r.memberId)
+    r.memberHandle = _.toString(r.memberHandle)
+    r.roleId = _.toString(r.roleId)
+    if (!r.createdBy) r.createdBy = createdBy
+
+    try {
+      await prisma.resource.create({ data: r })
+      success += 1
+      // collect member data only for successfully inserted resources
+      const mid = _.parseInt(d.memberId)
+      if (!_.has(memberDataMap, mid)) {
+        memberDataMap[mid] = {
+          userId: mid,
+          handle: d.memberHandle,
+          handleLower: _.toString(d.memberHandle).toLowerCase(),
+          maxRating: _.get(d, 'rating', null),
+          createdBy
+        }
       }
+    } catch (err) {
+      skipped += 1
+      writeSkip(`Insert failed: ${err.message}`, d)
     }
-    return r
-  })
-  await prisma.resource.createMany({ data })
-  logger.info('Imported ResourceRole data')
-  // import memberProfile and memberStats
-  memberData = _.values(memberData)
-  await prisma.memberStats.createMany({ data: memberData })
-  await prisma.memberProfile.createMany({ data: _.map(memberData, d => _.omit(d, 'maxRating')) })
-  logger.info('Imported Member data')
+  }
+
+  clearInterval(progressTimer)
+  const elapsed = (Date.now() - start) / 1000
+  const rate = (success + skipped) > 0 ? ((success + skipped) / elapsed).toFixed(2) : '0.00'
+  logger.info(`Imported Resource data: success=${success}, skipped=${skipped}, total=${total}, elapsed=${elapsed.toFixed(1)}s, rate=${rate} rec/s`)
+
+  // import memberProfile and memberStats for members associated with successfully inserted resources
+  const memberData = _.values(memberDataMap)
+  if (memberData.length) {
+    await prisma.memberStats.createMany({ data: memberData })
+    await prisma.memberProfile.createMany({ data: _.map(memberData, (m) => _.omit(m, 'maxRating')) })
+    logger.info(`Imported Member data: count=${memberData.length}`)
+  } else {
+    logger.info('No member data to import')
+  }
 }
 
 async function main () {
-  await importResourceRole()
-  await importResourceRolePhaseDependency()
+  //await importResourceRole()
+  //await importResourceRolePhaseDependency()
   await importResource()
 }