Fixes for migrator to allow for incremental updates and update the link in the email to the new review app

Author: jmgasper

migrator/README.md

@@ -70,15 +70,59 @@
   CREATED_BY=my-migrator node src/index.js member-profiles
   ```
 
+  ## 🕒 Incremental Migration with Start Date
+
+  The `--start-date` parameter unlocks incremental migrations by filtering records on their creation and update timestamps. Run a full import in advance, then rerun with a start date to pick up only the recent changes—keeping the final cut-over fast and predictable.
+
+  ### 🛠️ Usage Examples
+
+  ```bash
+  node src/index.js resources ./data/challenge-api.resources.json --start-date=2024-01-15
+  node src/index.js resource-roles --start-date=2024-03-01
+  node src/migrateAll.js --start-date=2024-02-20
+  ```
+
+  ### 🔍 Filtering Behavior
+
+  - Records are imported when `created`/`createdAt` **or** `updatedAt` is on or after the provided start date
+  - Records missing **both** date fields are skipped when a start date is supplied
+  - Records with both date fields present but older than the start date are skipped
+  - Dates must use the ISO `YYYY-MM-DD` format
+  - Without `--start-date`, all records are imported (default behavior)
+
+  ### 🔄 Incremental Migration Workflow
+
+  1. **Bulk load ahead of time (Jan 1)** — seed the database without a start date:
+
+     ```bash
+     node src/migrateAll.js
+     ```
+
+  2. **Wait until the cut-over window (Mar 15)** — allow upstream systems to keep producing data as usual.
+
+  3. **Refresh with recent updates** — bring the target in sync by importing only records touched since Jan 1:
+
+     ```bash
+     node src/migrateAll.js --start-date=2024-01-01
+     ```
+
+     This staggered approach turns a multi-hour bulk migration into a quick incremental catch-up that typically completes in minutes.
+
+  ### 📌 Model Support
+
+  - `resources`: filters on `created` and `updatedAt`
+  - `resource-roles`: filters on `createdAt` and `updatedAt`
+  - `resource-role-phase-dependencies`: filters on `createdAt` and `updatedAt`
+
   ## 🧩 Available Migration Steps
 
-  | Step                                | Auto Strategy | Description                                                                                       |
-  |-------------------------------------|---------------|---------------------------------------------------------------------------------------------------|
-  | `member-profiles`                  | ✅            | Auto strategy: uses `stream-json` (batch) for files larger than 3MB, and `loadJSON` (simple) otherwise |
-  | `member-stats`                     | ✅            | Auto strategy: uses `stream-json` (batch) for files larger than 3MB, and `loadJSON` (simple) otherwise |
-  | `resource-roles`                   | ❌            | Simple in-memory migration using `loadJSON`, not expected to be large                             |
-  | `resource-role-phase-dependencies` | ❌            | Simple in-memory migration using `loadJSON`, not expected to be large                             |
-  | `resources`                        | ✅            | Auto strategy for NDJSON files: uses `readline` + batch for files > 3 MB, otherwise simple line-by-line       |
+  | Step                                | Auto Strategy | Start Date Support | Description                                                                                       |
+  |-------------------------------------|---------------|--------------------|---------------------------------------------------------------------------------------------------|
+  | `member-profiles`                  | ✅            | ❌                 | Auto strategy: uses `stream-json` (batch) for files larger than 3MB, and `loadJSON` (simple) otherwise |
+  | `member-stats`                     | ✅            | ❌                 | Auto strategy: uses `stream-json` (batch) for files larger than 3MB, and `loadJSON` (simple) otherwise |
+  | `resource-roles`                   | ❌            | ✅                 | Simple in-memory migration using `loadJSON`, not expected to be large                             |
+  | `resource-role-phase-dependencies` | ❌            | ✅                 | Simple in-memory migration using `loadJSON`, not expected to be large                             |
+  | `resources`                        | ✅            | ✅                 | Auto strategy for NDJSON files: uses `readline` + batch for files > 3 MB, otherwise simple line-by-line       |
 
   > ⚙️ **Why Auto Strategy?**
 >
@@ -110,9 +154,10 @@ You can now run all migration steps sequentially using the following script:
 
 ```bash
 node src/migrateAll.js
+node src/migrateAll.js --start-date=2024-01-15
 ```
 
-This script will automatically execute each step in order (`resource-roles`, `resource-role-phase-dependencies`, `resources`), logging progress and duration for each. Ideal for full dataset migration in one command.
+This script will automatically execute each step in order (`resource-roles`, `resource-role-phase-dependencies`, `resources`), logging progress and duration for each. When provided, the start date is passed to every step, so only records touched on or after the target date are processed—ideal for full imports followed by a quick incremental catch-up in one command.
 
   ## 📒 Error Logs
   All failed migrations are logged under the `logs/` folder by model:
@@ -224,5 +269,3 @@ node src/validation/validateMemberProfiles.js > logs/memberprofile_validation.lo
 
 ---
 
-
-

migrator/src/index.js

@@ -14,28 +14,49 @@ const { migrateResourceAuto } = require('./migrators/migrateResourceAuto');   //
 // Map each step name to its corresponding function.
 // Each function logs execution time and calls its migrator.
 const steps = {
-  'resource-roles': async (filePath) => {
+  'resource-roles': async (filePath, startDate) => {
     const start = Date.now();
-    await migrateResourceRoleAuto(filePath);
+    await migrateResourceRoleAuto(filePath, startDate);
     console.log(`⏱️  Duration: ${((Date.now() - start) / 1000).toFixed(2)}s.`);
   },
-  'resource-role-phase-dependencies': async (filePath) => {
+  'resource-role-phase-dependencies': async (filePath, startDate) => {
     const start = Date.now();
-    await migrateResourceRolePhaseDependencyAuto(filePath);
+    await migrateResourceRolePhaseDependencyAuto(filePath, startDate);
     console.log(`⏱️  Duration: ${((Date.now() - start) / 1000).toFixed(2)}s.`);
   },
-  'resources': async (filePath) => {
+  'resources': async (filePath, startDate) => {
     const start = Date.now();
-    await migrateResourceAuto(filePath);
+    await migrateResourceAuto(filePath, startDate);
     console.log(`⏱️  Duration: ${((Date.now() - start) / 1000).toFixed(2)}s.`);
   }
 };
 
 // === EXECUTION ENTRYPOINT ===
 // Determines which migration step to execute and handles file path input
 (async () => {
-  const step = process.argv[2];           // First argument: migration step name
-  const customPath = process.argv[3];     // Second argument (optional): custom file path
+  const args = process.argv.slice(2);
+  const step = args[0];
+  let customPath = null;
+  let startDate = null;
+  const startDatePattern = /^\d{4}-\d{2}-\d{2}$/;
+
+  for (const arg of args.slice(1)) {
+    if (arg.startsWith('--start-date=')) {
+      const value = arg.split('=')[1];
+      if (!value || !startDatePattern.test(value)) {
+        console.error('❌ Invalid --start-date format. Expected YYYY-MM-DD.');
+        process.exit(1);
+      }
+      const parsedDate = new Date(value);
+      if (isNaN(parsedDate.getTime())) {
+        console.error('❌ Invalid --start-date value.');
+        process.exit(1);
+      }
+      startDate = value;
+    } else if (!arg.startsWith('-') && !customPath) {
+      customPath = arg;
+    }
+  }
 
   // Default file paths for each step
   const defaultPaths = {
@@ -47,7 +68,7 @@ const steps = {
   // Show help if step is invalid
   if (!steps[step]) {
     console.log('❌ Invalid migration step.\nUsage:');
-    console.log('  node src/index.js <step-name> [custom-path]');
+    console.log('  node src/index.js <step-name> [custom-path] [--start-date=YYYY-MM-DD]');
     console.log('\nAvailable steps:');
     console.log(Object.keys(steps).map(s => `  - ${s}`).join('\n'));
     process.exit(1);
@@ -64,7 +85,7 @@ const steps = {
   // Run selected migration step
   try {
     console.log(`🚀 Starting ${step} migration from ${filePath}`);
-    await steps[step](filePath);
+    await steps[step](filePath, startDate);
     console.log(`Step '${step}' completed successfully.`);
   } catch (error) {
     console.error(`❌ Error during '${step}':`, error.message);

migrator/src/migrateAll.js

@@ -22,7 +22,21 @@ const steps = [
 ];
 
 (async () => {
+  const startDateArg = process.argv.find(arg => arg.startsWith('--start-date='));
+  let startDate = null;
+  if (startDateArg) {
+    startDate = startDateArg.split('=')[1];
+    const startDatePattern = /^\d{4}-\d{2}-\d{2}$/;
+    if (!startDate || !startDatePattern.test(startDate)) {
+      console.error('❌ Invalid --start-date format. Expected YYYY-MM-DD.');
+      process.exit(1);
+    }
+  }
+
   console.log('🚀 Starting full migration (all steps)...\n');
+  if (startDate) {
+    console.log(`🔁 Incremental mode enabled. Filtering records from ${startDate}.\n`);
+  }
 
   for (const step of steps) {
     const filePath = path.resolve(defaultPaths[step.name]);
@@ -36,7 +50,7 @@ const steps = [
     const start = Date.now();
 
     try {
-      await step.fn(filePath);
+      await step.fn(filePath, startDate);
       const duration = ((Date.now() - start) / 1000).toFixed(2);
       console.log(`✅ '${step.name}' completed in ${duration}s\n`);
     } catch (error) {

migrator/src/migrators/migrateResource.js

@@ -10,7 +10,7 @@ const fs = require('fs');
 const readline = require('readline');
 const prisma = require('../clients/prismaClient');
 
-async function migrateResource(filePath) {
+async function migrateResource(filePath, startDate) {
   const fileStream = fs.createReadStream(filePath);
   const rl = readline.createInterface({
     input: fileStream,
@@ -19,13 +19,37 @@ async function migrateResource(filePath) {
 
   let successCount = 0;
   let failCount = 0;
+  let skippedCount = 0;
+  const startDateObj = startDate ? new Date(startDate) : null;
 
   for await (const line of rl) {
     if (!line.trim()) continue;
 
     try {
       const jsonLine = JSON.parse(line);
       const data = jsonLine._source;
+
+      const createdRaw = data.created;
+      const updatedRaw = data.updatedAt;
+
+      let createdDate = createdRaw ? new Date(createdRaw) : null;
+      if (createdDate && isNaN(createdDate.getTime())) {
+        createdDate = null;
+      }
+
+      let updatedDate = updatedRaw ? new Date(updatedRaw) : null;
+      if (updatedDate && isNaN(updatedDate.getTime())) {
+        updatedDate = null;
+      }
+
+      const createdBeforeOrMissing = !createdDate || (startDateObj ? createdDate < startDateObj : false);
+      const updatedBeforeOrMissing = !updatedDate || (startDateObj ? updatedDate < startDateObj : false);
+
+      if (startDateObj && createdBeforeOrMissing && updatedBeforeOrMissing) {
+        skippedCount++;
+        continue;
+      }
+
       const createdBy = data.createdBy || process.env.CREATED_BY;
       const phaseChangeNotifications = Object.prototype.hasOwnProperty.call(data, 'phaseChangeNotifications')
         ? data.phaseChangeNotifications
@@ -72,7 +96,7 @@ async function migrateResource(filePath) {
     }
   }
 
-  console.log(`✅ Resource migration finished: ${successCount} success, ${failCount} failed`);
+  console.log(`✅ Resource migration finished: ${successCount} success, ${failCount} failed, ${skippedCount} skipped`);
 }
 
 module.exports = { migrateResource };

migrator/src/migrators/migrateResourceAuto.js

@@ -16,16 +16,16 @@ const { migrateResource: migrateBatch } = require('./migrateResourceBatch');
 
 const FILE_SIZE_THRESHOLD = 3 * 1024 * 1024; // 3 MB
 
-async function migrateResourceAuto(filePath) {
+async function migrateResourceAuto(filePath, startDate) {
   const stats = fs.statSync(filePath);
   const fileSize = stats.size;
 
   if (fileSize < FILE_SIZE_THRESHOLD) {
     // Using normal migration (in-memory)
-    await migrateResource(filePath);
+    await migrateResource(filePath, startDate);
   } else {
     // Using batch migration (streaming)
-    await migrateBatch(filePath);
+    await migrateBatch(filePath, startDate);
   }
 }
 

migrator/src/migrators/migrateResourceBatch.js

@@ -13,7 +13,12 @@ const prisma = require('../clients/prismaClient');
 const { countFileLines } = require('../utils/countFileLines');
 const { createSimpleProgressBar } = require('../utils/progressLogger');
 
-async function migrateResource(filePath) {
+const parseTimestamp = (value) => {
+  const t = value ? new Date(value) : null;
+  return t && !isNaN(t.getTime()) ? t : null;
+};
+
+async function migrateResource(filePath, startDate) {
   // Estimar el total de líneas del archivo NDJSON
   const totalRecords = await countFileLines(filePath);
   const progress = createSimpleProgressBar(Math.ceil(totalRecords / 100));
@@ -28,6 +33,14 @@ async function migrateResource(filePath) {
   let batch = [];
   let successCount = 0;
   let failCount = 0;
+  let skippedCount = 0;
+  const startDateObj = startDate ? new Date(startDate) : null;
+  const isInvalidStartDate = startDate && isNaN(startDateObj.getTime());
+  const filterStartDate = !isInvalidStartDate ? startDateObj : null;
+
+  if (isInvalidStartDate) {
+    console.warn('migrateResource: invalid startDate provided; disabling date filter.', startDate);
+  }
 
   async function processBatch(batch) {
     const results = await Promise.allSettled(
@@ -83,6 +96,20 @@ async function migrateResource(filePath) {
     try {
       const jsonLine = JSON.parse(line);
       const data = jsonLine._source;
+      if (filterStartDate) {
+        const createdDate = parseTimestamp(data.created);
+        const updatedAtDate = parseTimestamp(data.updatedAt);
+
+        const shouldSkip =
+          (!createdDate && !updatedAtDate) ||
+          (((createdDate && createdDate < filterStartDate) || !createdDate) &&
+            ((updatedAtDate && updatedAtDate < filterStartDate) || !updatedAtDate));
+
+        if (shouldSkip) {
+          skippedCount++;
+          continue;
+        }
+      }
       batch.push(data);
 
       if (batch.length >= batchSize) {
@@ -102,6 +129,9 @@ async function migrateResource(filePath) {
 
   progress.done();
   console.log(`✅ Resource migration finished: ${successCount} success, ${failCount} failed`);
+  if (skippedCount > 0) {
+    console.log(`ℹ️ Resource migration skipped ${skippedCount} record(s) before the start date`);
+  }
 }
 
 module.exports = { migrateResource };