feat: data-set CLI uses filecoin-pin/core/data-set (#211)

* feat: add core/data-set methods

* Update src/core/data-set/types.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/core/data-set/get-data-set-pieces.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update src/core/data-set/get-data-set-pieces.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* feat: data-set CLI uses filecoin-pin/core/data-set

* fix: remove batchSize from core/data-set

* fix: add createdWithFilecoinPin to DataSetSummary

also adds some jsdoc comments explaining that metadata does exist

* fix: call sp-registry for specific data-set providers

* test: cleanup core/data-set unit tests

* fix: cleanup pieceSize calculations and types

* chore: remove leaf count info from data-set

* fix: listDataSets only gets providerInfo if asked

* fix: major data-set overhaul

* refactor: use only one dataSet display function

* fix: re-arrange provider info

* Update display.ts

Co-authored-by: Rod Vagg <rod@vagg.org>

* Update display.ts

Co-authored-by: Rod Vagg <rod@vagg.org>

* Update display.ts

Co-authored-by: Rod Vagg <rod@vagg.org>

* Update src/data-set/display.ts

Co-authored-by: Rod Vagg <rod@vagg.org>

* fix: get data-set details with show <id>

* chore: fix lint

* chore: dont output unnecessary details

see https://github.com/filecoin-project/filecoin-pin/pull/211\#discussion_r2502272156

* chore: only display commission if not zero

* fix: metadata values wrapped in double-quotes

* fix: ls --all shows data-sets not created by filecoin-pin

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Rod Vagg <rod@vagg.org>

Author: 3 people

src/commands/data-set.ts

@@ -1,24 +1,47 @@
 import { Command } from 'commander'
-import { runDataSetCommand } from '../data-set/run.js'
-import type { DataSetCommandOptions } from '../data-set/types.js'
-import { addAuthOptions } from '../utils/cli-options.js'
+import { runDataSetDetailsCommand, runDataSetListCommand } from '../data-set/run.js'
+import type { DataSetCommandOptions, DataSetListCommandOptions } from '../data-set/types.js'
+import { addAuthOptions, addProviderOptions } from '../utils/cli-options.js'
 
-export const dataSetCommand = new Command('data-set')
-  .description('Inspect data sets managed through Filecoin Onchain Cloud')
-  .argument('[dataSetId]', 'Optional data set ID to inspect')
-  .option('--ls', 'List all data sets for the configured account')
-  .action(async (dataSetId: string | undefined, options) => {
+export const dataSetCommand = new Command('data-set').description(
+  'Inspect data sets managed through Filecoin Onchain Cloud'
+)
+
+export const dataSetShowCommand = new Command('show')
+  .description('Display detailed information about a data set')
+  .argument('<dataSetId>', 'Display detailed information about a data set')
+  .action(async (dataSetId: string, options) => {
     try {
       const commandOptions: DataSetCommandOptions = {
         ...options,
-        ls: options.ls,
+      }
+      const dataSetIdNumber = Number.parseInt(dataSetId, 10)
+      if (Number.isNaN(dataSetIdNumber)) {
+        throw new Error('Invalid data set ID')
       }
 
-      await runDataSetCommand(dataSetId, commandOptions)
+      await runDataSetDetailsCommand(dataSetIdNumber, commandOptions)
     } catch (error) {
       console.error('Data set command failed:', error instanceof Error ? error.message : error)
       process.exit(1)
     }
   })
+addAuthOptions(dataSetShowCommand)
+
+export const dataSetListCommand = new Command('list')
+  .alias('ls')
+  .description('List all data sets for the configured account')
+  .option('--all', 'Show all data sets, not just the ones created with filecoin-pin', false)
+  .action(async (options: DataSetListCommandOptions) => {
+    try {
+      await runDataSetListCommand(options)
+    } catch (error) {
+      console.error('Data set list command failed:', error instanceof Error ? error.message : error)
+      process.exit(1)
+    }
+  })
+addAuthOptions(dataSetListCommand)
+addProviderOptions(dataSetListCommand)
 
-addAuthOptions(dataSetCommand)
+dataSetCommand.addCommand(dataSetShowCommand)
+dataSetCommand.addCommand(dataSetListCommand)

src/core/data-set/get-data-set-pieces.ts

@@ -7,6 +7,7 @@
  */
 
 import { METADATA_KEYS, type StorageContext, type Synapse, WarmStorageService } from '@filoz/synapse-sdk'
+import { getSizeFromPieceCID } from '@filoz/synapse-sdk/piece'
 import { isStorageContextWithDataSetId } from './type-guards.js'
 import type {
   DataSetPiecesResult,
@@ -62,9 +63,18 @@ export async function getDataSetPieces(
   try {
     const getPiecesOptions = { ...(signal && { signal }) }
     for await (const piece of storageContext.getPieces(getPiecesOptions)) {
-      const pieceInfo: PieceInfo = {
-        pieceId: piece.pieceId,
-        pieceCid: piece.pieceCid.toString(),
+      const pieceId = piece.pieceId
+      const pieceCid = piece.pieceCid
+      const pieceInfo: PieceInfo = { pieceId, pieceCid: pieceCid.toString() }
+
+      // Calculate piece size from CID
+      try {
+        pieceInfo.size = getSizeFromPieceCID(pieceCid)
+      } catch (error) {
+        logger?.warn(
+          { pieceId: piece.pieceId, pieceCid: piece.pieceCid.toString(), error },
+          'Failed to calculate piece size from CID'
+        )
       }
 
       pieces.push(pieceInfo)
@@ -80,11 +90,20 @@ export async function getDataSetPieces(
     await enrichPiecesWithMetadata(synapse, storageContext, pieces, warnings, logger)
   }
 
-  return {
+  // Calculate total size from pieces that have sizes
+  const piecesWithSizes = pieces.filter((p): p is PieceInfo & { size: number } => p.size != null)
+
+  const result: DataSetPiecesResult = {
     pieces,
     dataSetId: storageContext.dataSetId,
     warnings,
   }
+
+  if (piecesWithSizes.length > 0) {
+    result.totalSizeBytes = piecesWithSizes.reduce((sum, piece) => sum + BigInt(piece.size), 0n)
+  }
+
+  return result
 }
 
 /**

src/core/data-set/get-detailed-data-set.ts

@@ -0,0 +1,43 @@
+import type { Synapse } from '@filoz/synapse-sdk'
+import { getDataSetPieces } from './get-data-set-pieces.js'
+import { listDataSets } from './list-data-sets.js'
+import type { DataSetSummary, ListDataSetsOptions } from './types.js'
+
+export async function getDetailedDataSet(
+  synapse: Synapse,
+  dataSetId: number,
+  options?: ListDataSetsOptions
+): Promise<DataSetSummary> {
+  const logger = options?.logger
+  const dataSets = await listDataSets(synapse, {
+    ...options,
+    withProviderDetails: true,
+    filter: (dataSet) => dataSet.pdpVerifierDataSetId === dataSetId,
+  })
+
+  const dataSet = dataSets[0]
+  if (dataSets.length === 0 || dataSet == null) {
+    logger?.error({ dataSetId }, `Data set ${dataSetId} not found`)
+    throw new Error(`Data set ${dataSetId} not found`)
+  }
+
+  const storageContext = await synapse.storage.createContext({
+    dataSetId: dataSet.dataSetId,
+  })
+
+  const piecesResult = await getDataSetPieces(synapse, storageContext, {
+    includeMetadata: true,
+    logger,
+  })
+
+  const result: DataSetSummary = {
+    ...dataSet,
+    pieces: piecesResult.pieces,
+  }
+
+  if (piecesResult.totalSizeBytes != null) {
+    result.totalSizeBytes = piecesResult.totalSizeBytes
+  }
+
+  return result
+}

src/core/data-set/index.ts

@@ -15,5 +15,6 @@
  */
 
 export * from './get-data-set-pieces.js'
+export * from './get-detailed-data-set.js'
 export * from './list-data-sets.js'
 export * from './types.js'

src/core/data-set/list-data-sets.ts

@@ -7,6 +7,7 @@
  */
 
 import type { ProviderInfo, Synapse } from '@filoz/synapse-sdk'
+import { WarmStorageService } from '@filoz/synapse-sdk'
 import { SPRegistryService } from '@filoz/synapse-sdk/sp-registry'
 import { DEFAULT_DATA_SET_METADATA } from '../synapse/constants.js'
 import type { DataSetSummary, ListDataSetsOptions } from './types.js'
@@ -34,27 +35,33 @@ import type { DataSetSummary, ListDataSetsOptions } from './types.js'
 export async function listDataSets(synapse: Synapse, options?: ListDataSetsOptions): Promise<DataSetSummary[]> {
   const logger = options?.logger
   const address = options?.address ?? (await synapse.getClient().getAddress())
+  const withProviderDetails = options?.withProviderDetails ?? false
+  const filter = options?.filter
 
   // Step 1: Find data sets
   const dataSets = await synapse.storage.findDataSets(address)
 
+  const filteredDataSets = filter ? dataSets.filter(filter) : dataSets
+
   // Step 2: Collect unique provider IDs from data sets
-  const uniqueProviderIds = Array.from(new Set(dataSets.map((ds) => ds.providerId)))
+  const uniqueProviderIds = withProviderDetails ? Array.from(new Set(filteredDataSets.map((ds) => ds.providerId))) : []
 
   // Step 3: Fetch provider info for the specific provider IDs using sp-registry
   let providerMap: Map<number, ProviderInfo> = new Map()
   if (uniqueProviderIds.length > 0) {
     try {
-      const spRegistry = new SPRegistryService(synapse.getProvider(), synapse.getNetwork())
+      const warmStorageService = await WarmStorageService.create(synapse.getProvider(), synapse.getWarmStorageAddress())
+      const serviceProviderRegistryAddress = await warmStorageService.getServiceProviderRegistryAddress()
+      const spRegistry = new SPRegistryService(synapse.getProvider(), serviceProviderRegistryAddress)
       const providers = await spRegistry.getProviders(uniqueProviderIds)
-      providerMap = new Map(providers.map((provider) => [provider.id, provider] as const))
+      providerMap = new Map(providers.map((provider) => [provider.id, provider]))
     } catch (error) {
       logger?.warn({ error }, 'Failed to fetch provider info from sp-registry for provider enrichment')
     }
   }
 
   // Map SDK datasets to our summary format (spread all fields, add dataSetId alias, provider, and filecoin-pin creation flag)
-  return dataSets.map((ds) => {
+  return filteredDataSets.map((ds) => {
     // Check if this dataset was created by filecoin-pin by looking for our DEFAULT_DATA_SET_METADATA fields
     const createdWithFilecoinPin = Object.entries(DEFAULT_DATA_SET_METADATA).every(
       ([key, value]) => ds.metadata[key] === value

src/core/data-set/types.ts

@@ -35,6 +35,8 @@ export interface DataSetPiecesResult {
   pieces: PieceInfo[]
   /** Dataset ID these pieces belong to */
   dataSetId: number
+  /** Total size of all pieces in bytes (sum of individual piece sizes) */
+  totalSizeBytes?: bigint
   /** Non-fatal warnings encountered during retrieval */
   warnings?: Warning[]
 }
@@ -62,6 +64,7 @@ export interface Warning {
  * - Provider enrichment (optional provider field)
  * - Dataset metadata (inherited from EnhancedDataSetInfo.metadata - key-value pairs from WarmStorage)
  * - Filecoin-pin creation flag (indicates if created by filecoin-pin)
+ * - Optional detailed information (pieces, metadata, size calculations, warnings)
  *
  * The dataSetId alias makes pdpVerifierDataSetId more discoverable.
  */
@@ -70,6 +73,10 @@ export interface DataSetSummary extends EnhancedDataSetInfo {
   dataSetId: number
   /** Provider information (enriched from getStorageInfo if available) */
   provider: ProviderInfo | undefined
+  /** Total size in bytes (optional, calculated from piece sizes) */
+  totalSizeBytes?: bigint
+  /** Pieces in the dataset (optional, populated when fetching detailed info) */
+  pieces?: PieceInfo[]
   /** Indicates if this dataset was created by filecoin-pin (has WITH_IPFS_INDEXING and source='filecoin-pin' metadata) */
   createdWithFilecoinPin: boolean
 }
@@ -81,7 +88,24 @@ export interface ListDataSetsOptions {
   /** Address to list datasets for (defaults to synapse client address) */
   address?: string
   /** Logger instance for debugging (optional) */
-  logger?: Logger
+  logger?: Logger | undefined
+  /**
+   * Whether to get the provider details from the SP registry
+   *
+   * @default false
+   */
+  withProviderDetails?: boolean
+
+  /**
+   * Filter function to apply to the data sets before additional processing
+   *
+   * Note: The filter receives raw EnhancedDataSetInfo objects from the SDK
+   * (with pdpVerifierDataSetId field) before transformation to DataSetSummary
+   *
+   * @param dataSet - Raw dataset from SDK storage.findDataSets()
+   * @returns true to include the dataset, false to exclude it
+   */
+  filter?: undefined | ((dataSet: EnhancedDataSetInfo) => boolean)
 }
 
 /**
@@ -93,7 +117,7 @@ export interface GetDataSetPiecesOptions {
   /** Abort signal for cancellation */
   signal?: AbortSignal
   /** Logger instance for debugging (optional) */
-  logger?: Logger
+  logger?: Logger | undefined
 }
 
 export type StorageContextWithDataSetId = StorageContext & { dataSetId: number }