feat: add more details to upload-flow (#169)

* fix: add verbose cli decorator

* fix: use onProgress fn in filecoin-pin

* fix: add ipni-announcement-check

* feat: more details during add/import

* fix: better UI during upload-flow

* Adjusted wording

* Updated gitignore to exclude .env

* fix: check ipni advertisement during upload

* fix: executeUpload only awaits ipniValidationPromise if it exists

* test: validateIPNIadvertisement returns true in import.test

* fix: better onProgressTypes

* chore: enable strictFucntionTypes tsc option

* Update tsconfig.json

* Update src/test/unit/import.test.ts

* chore: fix lint

* Update src/core/upload/index.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fix: remove unused var and if block

* Update src/utils/cli-logger.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/core/upload/index.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Steve Loeppky <stvn@loeppky.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

.gitignore

@@ -1,3 +1,4 @@
+.env
 node_modules/
 package-lock.json
 boxo/

src/add/add.ts

@@ -19,6 +19,7 @@ import {
 import { cleanupTempCar, createCarFromPath } from '../core/unixfs/index.js'
 import { parseCLIAuth, parseProviderOptions } from '../utils/cli-auth.js'
 import { cancel, createSpinner, formatFileSize, intro, outro } from '../utils/cli-helpers.js'
+import { log } from '../utils/cli-logger.js'
 import type { AddOptions, AddResult } from './types.js'
 
 /**
@@ -175,6 +176,10 @@ export async function runAdd(options: AddOptions): Promise<AddResult> {
     })
 
     spinner.stop(`${pc.green('✓')} Storage context ready`)
+    log.spinnerSection('Storage Context', [
+      pc.gray(`Data Set ID: ${storage.dataSetId}`),
+      pc.gray(`Provider: ${providerInfo.name || providerInfo.serviceProvider}`),
+    ])
 
     // Create service object for upload function
     const synapseService: SynapseService = { synapse, storage, providerInfo }

src/common/upload-flow.ts

@@ -5,13 +5,19 @@
  * including payment validation, storage context creation, and result display.
  */
 
-import type { Synapse } from '@filoz/synapse-sdk'
+import type { PieceCID, Synapse } from '@filoz/synapse-sdk'
 import type { CID } from 'multiformats/cid'
 import pc from 'picocolors'
 import type { Logger } from 'pino'
 import { DEFAULT_LOCKUP_DAYS, type PaymentCapacityCheck } from '../core/payments/index.js'
 import { cleanupSynapseService, type SynapseService } from '../core/synapse/index.js'
-import { checkUploadReadiness, executeUpload, getDownloadURL, type SynapseUploadResult } from '../core/upload/index.js'
+import {
+  checkUploadReadiness,
+  executeUpload,
+  getDownloadURL,
+  getServiceURL,
+  type SynapseUploadResult,
+} from '../core/upload/index.js'
 import { formatUSDFC } from '../core/utils/format.js'
 import { autoFund } from '../payments/fund.js'
 import type { AutoFundOptions } from '../payments/types.js'
@@ -245,19 +251,122 @@ export async function performUpload(
 
   spinner?.start('Uploading to Filecoin...')
 
+  // Track parallel operations with their messages
+  const pendingOps = new Map<string, string>()
+  let transactionHash: string | undefined
+
+  function getSpinnerMessage() {
+    return Array.from(pendingOps.values())
+      .map((op) => op)
+      .join(' & ')
+  }
+
+  function completeOperation(
+    operationKey: string,
+    completionMessage: string,
+    type: 'success' | 'warning' | 'info' | 'none' = 'success'
+  ) {
+    pendingOps.delete(operationKey)
+
+    switch (type) {
+      case 'success':
+        spinner?.stop(`${pc.green('✓')} ${completionMessage}`)
+        break
+      case 'warning':
+        spinner?.stop(`${pc.yellow('⚠')} ${completionMessage}`)
+        break
+      default:
+        spinner?.stop(completionMessage)
+        break
+    }
+
+    // Restart spinner with remaining operations if any
+    if (pendingOps.size > 0) {
+      spinner?.start(getSpinnerMessage())
+    }
+  }
+
+  let pieceCid: PieceCID | undefined
+  function getIpniAdvertisementMsg(attemptCount: number): string {
+    return `Checking for IPNI advertisement (check #${attemptCount})`
+  }
+
   const uploadResult = await executeUpload(synapseService, carData, rootCid, {
     logger,
     contextId: `${contextType}-${Date.now()}`,
-    ipniValidation: { enabled: false },
-    callbacks: {
-      onUploadComplete: () => {
-        spinner?.message('Upload complete, adding to data set...')
-      },
-      onPieceAdded: (transaction) => {
-        if (transaction) {
-          spinner?.message('Piece added to data set, confirming on-chain...')
+    onProgress(event) {
+      switch (event.type) {
+        case 'onUploadComplete': {
+          pieceCid = event.data.pieceCid
+          spinner?.stop(`${pc.green('✓')} Upload complete`)
+          const serviceURL = getServiceURL(synapseService.providerInfo)
+          if (serviceURL != null && serviceURL !== '') {
+            log.spinnerSection('Download IPFS CAR from SP', [
+              pc.gray(`${serviceURL.replace(/\/$/, '')}/ipfs/${rootCid}`),
+            ])
+          }
+          spinner?.start('Adding piece to DataSet...')
+          break
+        }
+        case 'onPieceAdded': {
+          spinner?.stop(`${pc.green('✓')} Piece added to DataSet (unconfirmed on-chain)`)
+          if (event.data.txHash) {
+            transactionHash = event.data.txHash
+          }
+          log.spinnerSection('Explorer URLs', [
+            pc.gray(`Piece: https://pdp.vxb.ai/calibration/piece/${pieceCid}`),
+            pc.gray(
+              `Transaction: https://${synapseService.synapse.getNetwork()}.filfox.info/en/message/${transactionHash}`
+            ),
+          ])
+
+          pendingOps.set('chain', 'Confirming piece added to DataSet on-chain')
+
+          spinner?.start(getSpinnerMessage())
+          break
+        }
+        case 'onPieceConfirmed': {
+          completeOperation('chain', `Piece added to DataSet (confirmed on-chain)`, 'success')
+          break
+        }
+
+        case 'ipniAdvertisement.retryUpdate': {
+          if (event.data.retryCount === 0) {
+            pendingOps.set('ipni', getIpniAdvertisementMsg(1))
+          }
+          pendingOps.set('ipni', getIpniAdvertisementMsg(event.data.retryCount + 1))
+          spinner?.message(getSpinnerMessage())
+          break
         }
-      },
+        case 'ipniAdvertisement.complete': {
+          const isIpniAdvertisementSuccessful = event.data.result
+          const message = isIpniAdvertisementSuccessful
+            ? `IPNI advertisement successful. IPFS retrieval possible.`
+            : `IPNI advertisement pending`
+
+          completeOperation('ipni', message, isIpniAdvertisementSuccessful ? 'success' : 'warning')
+
+          if (isIpniAdvertisementSuccessful) {
+            log.spinnerSection('IPFS Retrieval URLs', [
+              pc.gray(`ipfs://${rootCid}`),
+              pc.gray(`https://inbrowser.link/ipfs/${rootCid}`),
+              pc.gray(`https://dweb.link/ipfs/${rootCid}`),
+            ])
+          }
+          break
+        }
+        case 'ipniAdvertisement.failed': {
+          logger.error({ error: event.data.error }, 'Error checking IPNI advertisement')
+          completeOperation('ipni', `IPNI advertisement check failed`, 'warning')
+          log.spinnerSection('IPNI advertisement check failed', [
+            pc.gray(`IPNI advertisement does not exist at http://filecoinpin.contact/cid/${rootCid}`),
+          ])
+          break
+        }
+        default: {
+          break
+        }
+      }
     },
   })
 

src/core/upload/index.ts

@@ -1,4 +1,4 @@
-import type { Synapse, UploadCallbacks } from '@filoz/synapse-sdk'
+import type { Synapse } from '@filoz/synapse-sdk'
 import type { CID } from 'multiformats/cid'
 import type { Logger } from 'pino'
 import {
@@ -11,24 +11,26 @@ import {
   validatePaymentRequirements,
 } from '../payments/index.js'
 import { isSessionKeyMode, type SynapseService } from '../synapse/index.js'
+import type { ProgressEvent, ProgressEventHandler } from '../utils/types.js'
 import {
   type ValidateIPNIAdvertisementOptions,
+  type ValidateIPNIProgressEvents,
   validateIPNIAdvertisement,
 } from '../utils/validate-ipni-advertisement.js'
-import { type SynapseUploadResult, uploadToSynapse } from './synapse.js'
+import { type SynapseUploadResult, type UploadProgressEvents, uploadToSynapse } from './synapse.js'
 
-export type { SynapseUploadOptions, SynapseUploadResult } from './synapse.js'
+export type { SynapseUploadOptions, SynapseUploadResult, UploadProgressEvents } from './synapse.js'
 export { getDownloadURL, getServiceURL, uploadToSynapse } from './synapse.js'
 
 /**
  * Options for evaluating whether an upload can proceed.
  */
-export type UploadReadinessProgressEvent =
-  | { type: 'checking-balances' }
-  | { type: 'checking-allowances' }
-  | { type: 'configuring-allowances' }
-  | { type: 'allowances-configured'; transactionHash?: string }
-  | { type: 'validating-capacity' }
+export type UploadReadinessProgressEvents =
+  | ProgressEvent<'checking-balances'>
+  | ProgressEvent<'checking-allowances'>
+  | ProgressEvent<'configuring-allowances'>
+  | ProgressEvent<'allowances-configured', { transactionHash?: string }>
+  | ProgressEvent<'validating-capacity'>
 
 export interface UploadReadinessOptions {
   /** Initialized Synapse instance. */
@@ -41,7 +43,7 @@ export interface UploadReadinessOptions {
    */
   autoConfigureAllowances?: boolean
   /** Optional callback for progress updates. */
-  onProgress?: (event: UploadReadinessProgressEvent) => void
+  onProgress?: ProgressEventHandler<UploadReadinessProgressEvents>
 }
 
 /**
@@ -129,7 +131,7 @@ export async function checkUploadReadiness(options: UploadReadinessOptions): Pro
     const setResult = await setMaxAllowances(synapse)
     allowancesUpdated = true
     allowanceTxHash = setResult.transactionHash
-    onProgress?.({ type: 'allowances-configured', transactionHash: allowanceTxHash })
+    onProgress?.({ type: 'allowances-configured', data: { transactionHash: allowanceTxHash } })
   }
 
   onProgress?.({ type: 'validating-capacity' })
@@ -179,8 +181,8 @@ export interface UploadExecutionOptions {
   logger: Logger
   /** Optional identifier to help correlate logs. */
   contextId?: string
-  /** Optional callbacks mirroring Synapse SDK upload callbacks. */
-  callbacks?: UploadCallbacks
+  /** Optional umbrella onProgress receiving child progress events. */
+  onProgress?: ProgressEventHandler<(UploadProgressEvents | ValidateIPNIProgressEvents) & {}>
   /** Optional metadata to associate with the upload. */
   metadata?: Record<string, string>
   /**
@@ -193,7 +195,7 @@ export interface UploadExecutionOptions {
      * @default: true
      */
     enabled?: boolean
-  } & ValidateIPNIAdvertisementOptions
+  } & Omit<ValidateIPNIAdvertisementOptions, 'onProgress'>
 }
 
 export interface UploadExecutionResult extends SynapseUploadResult {
@@ -219,40 +221,40 @@ export async function executeUpload(
   rootCid: CID,
   options: UploadExecutionOptions
 ): Promise<UploadExecutionResult> {
-  const { logger, contextId, callbacks } = options
+  const { logger, contextId } = options
   let transactionHash: string | undefined
   let ipniValidationPromise: Promise<boolean> | undefined
 
-  const mergedCallbacks: UploadCallbacks = {
-    onUploadComplete: (pieceCid) => {
-      callbacks?.onUploadComplete?.(pieceCid)
-      // Begin IPNI validation as soon as the upload completes
-      if (options.ipniValidation?.enabled !== false && ipniValidationPromise == null) {
-        try {
-          const { enabled: _enabled, ...rest } = options.ipniValidation ?? {}
-          ipniValidationPromise = validateIPNIAdvertisement(rootCid, {
-            ...rest,
-            logger,
-          })
-        } catch (error) {
-          logger.error({ error }, 'Could not begin IPNI advertisement validation')
-          ipniValidationPromise = Promise.resolve(false)
+  const onProgress: ProgressEventHandler<UploadProgressEvents | ValidateIPNIProgressEvents> = (event) => {
+    switch (event.type) {
+      case 'onPieceAdded': {
+        // Begin IPNI validation as soon as the piece is added and parked in the data set
+        if (options.ipniValidation?.enabled !== false && ipniValidationPromise == null) {
+          try {
+            const { enabled: _enabled, ...rest } = options.ipniValidation ?? {}
+            ipniValidationPromise = validateIPNIAdvertisement(rootCid, {
+              ...rest,
+              logger,
+            })
+          } catch (error) {
+            logger.error({ error }, 'Could not begin IPNI advertisement validation')
+            ipniValidationPromise = Promise.resolve(false)
+          }
         }
+        if (event.data.txHash != null) {
+          transactionHash = event.data.txHash
+        }
+        break
       }
-    },
-    onPieceAdded: (txHash) => {
-      if (txHash) {
-        transactionHash = txHash
+      default: {
+        break
       }
-      callbacks?.onPieceAdded?.(txHash)
-    },
-    onPieceConfirmed: (pieceIds) => {
-      callbacks?.onPieceConfirmed?.(pieceIds)
-    },
+    }
+    options.onProgress?.(event)
   }
 
   const uploadOptions: Parameters<typeof uploadToSynapse>[4] = {
-    callbacks: mergedCallbacks,
+    onProgress,
   }
   if (contextId) {
     uploadOptions.contextId = contextId

src/core/upload/synapse.ts

@@ -4,17 +4,23 @@
  * This module provides a reusable upload pattern for CAR files to Filecoin
  * via Synapse SDK, used by both the import command and pinning server.
  */
-import type { UploadOptions } from '@filoz/synapse-sdk'
+import type { PieceCID, UploadOptions } from '@filoz/synapse-sdk'
 import { METADATA_KEYS, type ProviderInfo, type UploadCallbacks } from '@filoz/synapse-sdk'
 import type { CID } from 'multiformats/cid'
 import type { Logger } from 'pino'
 import type { SynapseService } from '../synapse/index.js'
+import type { ProgressEvent, ProgressEventHandler } from '../utils/types.js'
+
+export type UploadProgressEvents =
+  | ProgressEvent<'onUploadComplete', { pieceCid: PieceCID }>
+  | ProgressEvent<'onPieceAdded', { txHash: `0x${string}` | undefined }>
+  | ProgressEvent<'onPieceConfirmed', { pieceIds: number[] }>
 
 export interface SynapseUploadOptions {
   /**
    * Optional callbacks for monitoring upload progress
    */
-  callbacks?: UploadCallbacks
+  onProgress?: ProgressEventHandler<UploadProgressEvents>
 
   /**
    * Context identifier for logging (e.g., pinId, import job ID)
@@ -71,7 +77,7 @@ export async function uploadToSynapse(
   logger: Logger,
   options: SynapseUploadOptions = {}
 ): Promise<SynapseUploadResult> {
-  const { callbacks, contextId = 'upload' } = options
+  const { onProgress, contextId = 'upload' } = options
 
   // Merge provided callbacks with logging callbacks
   const uploadCallbacks: UploadCallbacks = {
@@ -84,7 +90,7 @@ export async function uploadToSynapse(
         },
         'Upload to PDP server complete'
       )
-      callbacks?.onUploadComplete?.(pieceCid)
+      onProgress?.({ type: 'onUploadComplete', data: { pieceCid } })
     },
 
     onPieceAdded: (txHash) => {
@@ -106,7 +112,7 @@ export async function uploadToSynapse(
           'Piece added to data set'
         )
       }
-      callbacks?.onPieceAdded?.(txHash)
+      onProgress?.({ type: 'onPieceAdded', data: { txHash } })
     },
 
     onPieceConfirmed: (pieceIds) => {
@@ -118,7 +124,7 @@ export async function uploadToSynapse(
         },
         'Piece addition confirmed on-chain'
       )
-      callbacks?.onPieceConfirmed?.(pieceIds)
+      onProgress?.({ type: 'onPieceConfirmed', data: { pieceIds } })
     },
   }
 

src/core/utils/index.ts

@@ -1,2 +1,3 @@
 export * from './format.js'
+export * from './types.js'
 export * from './validate-ipni-advertisement.js'

src/test/unit/import.test.ts

@@ -76,6 +76,9 @@ vi.mock('../../core/payments/index.js', async () => {
     }),
   }
 })
+vi.mock('../../core/utils/validate-ipni-advertisement.js', () => ({
+  validateIPNIAdvertisement: vi.fn().mockResolvedValue(true),
+}))
 
 vi.mock('../../payments/setup.js', () => ({
   formatUSDFC: vi.fn((amount) => `${amount} USDFC`),