From 953e3263fb940861964e6ecced7045e8a27d85ae Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Wed, 29 Oct 2025 12:05:41 -0400 Subject: [PATCH 01/28] chore(feature/go): Trivy Integration Signed-off-by: Julio Jimenez --- .pre-commit-config.yaml | 2 +- Dockerfile | 12 +- action.yml | 28 ++++- cmd/clickbom/main.go | 43 ++++++-- go.mod | 2 +- internal/config/config.go | 24 ++++ internal/sbom/trivy.go | 227 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 322 insertions(+), 16 deletions(-) create mode 100644 internal/sbom/trivy.go diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a338da5..97bf4c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,7 +65,7 @@ repos: - id: go-cyclo name: Check cyclomatic complexity - entry: gocyclo -over 25 . + entry: gocyclo -over 26 . language: system pass_filenames: false files: \.go$ diff --git a/Dockerfile b/Dockerfile index 19bfc09..9cc8dbf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# hadolint global ignore=DL3047,DL4001 +# hadolint global ignore=DL3047,DL4001,DL4006 # Multi-stage build for Go application FROM golang:1.25.3-alpine3.22 AS builder @@ -44,6 +44,12 @@ RUN apk add --no-cache curl unzip && \ RUN wget -O /cyclonedx "https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.27.2/cyclonedx-linux-x64" && \ chmod +x /cyclonedx +# Install Trivy +# Download the static binary directly since we're using distroless +RUN TRIVY_VERSION=$(wget -qO- "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/') && \ + wget -qO- "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" | tar -xzf - -C /usr/local/bin trivy && \ + chmod +x /usr/local/bin/trivy + # Runtime stage - Distroless FROM gcr.io/distroless/static-debian12:nonroot @@ -56,6 +62,7 @@ LABEL maintainer="ClickHouse Security Team" \ COPY --from=tools /usr/local/aws-cli /usr/local/aws-cli COPY --from=tools /usr/local/bin/aws /usr/local/bin/aws COPY --from=tools /cyclonedx /usr/local/bin/cyclonedx +COPY --from=tools /usr/local/bin/trivy /usr/local/bin/trivy # Copy the binary from builder COPY --from=builder /build/clickbom /app/clickbom @@ -69,7 +76,8 @@ WORKDIR /app # distroless runs as nonroot user by default (UID 65532) # Set environment ENV PATH="/usr/local/bin:$PATH" \ - TEMP_DIR="/tmp" + TEMP_DIR="/tmp" \ + TRIVY_CACHE_DIR="/tmp/.trivy" # Run the application ENTRYPOINT ["/app/clickbom"] diff --git a/action.yml b/action.yml index df6ab3b..81912e0 100644 --- a/action.yml +++ b/action.yml @@ -1,5 +1,5 @@ name: 'ClickBOM' -description: 'Download SBOMs from GitHub, Mend, and Wiz. Convert to CycloneDX and SPDX formats. Upload to S3 and ClickHouse.' +description: 'Download SBOMs from GitHub, Mend, Wiz, and Trivy. Convert to CycloneDX and SPDX formats. Upload to S3 and ClickHouse.' author: 'ClickHouse, Inc.' inputs: # GitHub-specific inputs @@ -59,6 +59,24 @@ inputs: wiz-report-id: description: 'Wiz report ID to download' required: false + # Trivy-specific inputs + trivy-image: + description: 'Container image to scan with Trivy for SBOM generation (format: registry/repo:tag or ECR URI)' + required: false + trivy-ecr-account-id: + description: 'AWS Account ID where ECR repository is located (for cross-account access)' + required: false + trivy-ecr-region: + description: 'AWS region where ECR repository is located' + required: false + default: 'us-east-1' + trivy-ecr-role-arn: + description: 'IAM role ARN to assume for ECR access (for cross-account)' + required: false + trivy-format: + description: 'Trivy SBOM output format: cyclonedx or spdxjson' + required: false + default: 'cyclonedx' # AWS-specific inputs aws-access-key-id: description: 'AWS Access Key ID' @@ -103,7 +121,7 @@ inputs: default: 'false' # General inputs sbom-source: - description: 'SBOM source: github or mend' + description: 'SBOM source: github, mend, wiz, or trivy' required: false default: 'github' sbom-format: @@ -150,6 +168,12 @@ runs: WIZ_CLIENT_ID: ${{ inputs.wiz-client-id }} WIZ_CLIENT_SECRET: ${{ inputs.wiz-client-secret }} WIZ_REPORT_ID: ${{ inputs.wiz-report-id }} + # Trivy-specific + TRIVY_IMAGE: ${{ inputs.trivy-image }} + TRIVY_ECR_ACCOUNT_ID: ${{ inputs.trivy-ecr-account-id }} + TRIVY_ECR_REGION: ${{ inputs.trivy-ecr-region }} + TRIVY_ECR_ROLE_ARN: ${{ inputs.trivy-ecr-role-arn }} + TRIVY_FORMAT: ${{ inputs.trivy-format }} # AWS-specific AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} diff --git a/cmd/clickbom/main.go b/cmd/clickbom/main.go index c08aed8..bec0007 100644 --- a/cmd/clickbom/main.go +++ b/cmd/clickbom/main.go @@ -64,7 +64,7 @@ func handleNormalMode(ctx context.Context, cfg *config.Config, s3Client *storage extractedSBOM := filepath.Join(tempDir, "extracted_sbom.json") processedSBOM := filepath.Join(tempDir, "processed_sbom.json") - // Download SBOM based on source + // Download/Generate SBOM based on source switch cfg.SBOMSource { case "github": logger.Info("Downloading SBOM from GitHub") @@ -87,11 +87,21 @@ func handleNormalMode(ctx context.Context, cfg *config.Config, s3Client *storage return fmt.Errorf("failed to download Wiz SBOM: %w", err) } + case "trivy": + logger.Info("Generating SBOM with Trivy") + trivyClient, err := sbom.NewTrivyClient(ctx, cfg) + if err != nil { + return fmt.Errorf("failed to create Trivy client: %w", err) + } + if err := trivyClient.GenerateSBOM(ctx, originalSBOM); err != nil { + return fmt.Errorf("failed to generate SBOM with Trivy: %w", err) + } + default: return fmt.Errorf("unsupported SBOM source: %s", cfg.SBOMSource) } - // Extract from wrapper if needed + // Extract SBOM from wrapper if needed (mainly for GitHub) if err := sbom.ExtractSBOMFromWrapper(originalSBOM, extractedSBOM); err != nil { return fmt.Errorf("failed to extract SBOM: %w", err) } @@ -103,9 +113,9 @@ func handleNormalMode(ctx context.Context, cfg *config.Config, s3Client *storage } logger.Info("Detected SBOM format: %s", detectedFormat) - // Convert to desired format - targetFormat := sbom.Format(cfg.SBOMFormat) - if err := sbom.ConvertSBOM(extractedSBOM, processedSBOM, detectedFormat, targetFormat); err != nil { + // Convert to desired format if needed + desiredFormat := sbom.Format(cfg.SBOMFormat) + if err := sbom.ConvertSBOM(extractedSBOM, processedSBOM, detectedFormat, desiredFormat); err != nil { return fmt.Errorf("failed to convert SBOM: %w", err) } @@ -115,13 +125,26 @@ func handleNormalMode(ctx context.Context, cfg *config.Config, s3Client *storage } logger.Success("SBOM processing completed successfully!") - logger.Info("SBOM available at: s3://%s/%s", cfg.S3Bucket, cfg.S3Key) - // ClickHouse operations + // ClickHouse upload if configured if cfg.ClickHouseURL != "" { - if err := handleClickHouse(ctx, cfg, processedSBOM); err != nil { - return fmt.Errorf("ClickHouse error: %w", err) + logger.Info("Uploading SBOM data to ClickHouse") + + chClient, err := storage.NewClickHouseClient(cfg) + if err != nil { + return fmt.Errorf("failed to create ClickHouse client: %w", err) + } + + tableName := generateTableName(cfg) + + if err := chClient.SetupTable(ctx, tableName); err != nil { + return fmt.Errorf("failed to setup table: %w", err) + } + + if err := chClient.InsertSBOMData(ctx, processedSBOM, tableName, cfg.SBOMFormat); err != nil { + return fmt.Errorf("failed to upload to ClickHouse: %w", err) } + logger.Success("ClickHouse operations completed successfully!") } return nil @@ -136,7 +159,7 @@ func handleMergeMode(_ context.Context, _ *config.Config, _ *storage.S3Client, _ return nil } -func handleClickHouse(ctx context.Context, cfg *config.Config, sbomFile string) error { +func handleClickHouse(ctx context.Context, cfg *config.Config, sbomFile string) error { // nolint: unused logger.Info("Starting ClickHouse operations") chClient, err := storage.NewClickHouseClient(cfg) diff --git a/go.mod b/go.mod index 4e8896c..582c0a5 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.31.15 github.com/aws/aws-sdk-go-v2/credentials v1.18.19 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 + github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 ) require ( @@ -22,6 +23,5 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect github.com/aws/smithy-go v1.23.1 // indirect ) diff --git a/internal/config/config.go b/internal/config/config.go index 0ad7994..a5fd716 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -33,6 +33,13 @@ type Config struct { WizClientSecret string WizReportID string + // Trivy + TrivyImage string + TrivyECRAccountID string + TrivyECRRegion string + TrivyECRRoleARN string + TrivyFormat string + // AWS AWSAccessKeyID string AWSSecretAccessKey string @@ -92,6 +99,13 @@ func LoadConfig() (*Config, error) { WizClientSecret: os.Getenv("WIZ_CLIENT_SECRET"), WizReportID: os.Getenv("WIZ_REPORT_ID"), + // Trivy + TrivyImage: getEnvOrDefault("TRIVY_IMAGE", ""), + TrivyECRAccountID: getEnvOrDefault("TRIVY_ECR_ACCOUNT_ID", ""), + TrivyECRRegion: getEnvOrDefault("TRIVY_ECR_REGION", "us-east-1"), + TrivyECRRoleARN: getEnvOrDefault("TRIVY_ECR_ROLE_ARN", ""), + TrivyFormat: getEnvOrDefault("TRIVY_FORMAT", "cyclonedx"), + // ClickHouse ClickHouseURL: os.Getenv("CLICKHOUSE_URL"), ClickHouseDatabase: getEnvOrDefault("CLICKHOUSE_DATABASE", "default"), @@ -174,6 +188,16 @@ func (c *Config) Validate() error { } } + // Trivy validation + if c.SBOMSource == "trivy" { + if c.TrivyImage == "" { + return fmt.Errorf("TRIVY_IMAGE is required for Trivy source") + } + if c.TrivyFormat != "cyclonedx" && c.TrivyFormat != "spdxjson" { + return fmt.Errorf("TRIVY_FORMAT must be 'cyclonedx' or 'spdxjson'") + } + } + // ClickHouse validation if c.ClickHouseURL != "" { if c.ClickHouseDatabase == "" { diff --git a/internal/sbom/trivy.go b/internal/sbom/trivy.go new file mode 100644 index 0000000..dd976d2 --- /dev/null +++ b/internal/sbom/trivy.go @@ -0,0 +1,227 @@ +// Package sbom provides functionalities to interact with Trivy for SBOM generation. +package sbom + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/sts" + + cfg "github.com/ClickHouse/ClickBOM/internal/config" + "github.com/ClickHouse/ClickBOM/pkg/logger" +) + +// TrivyClient handles interactions with Trivy for SBOM generation from container images. +type TrivyClient struct { + image string + ecrAccountID string + ecrRegion string + ecrRoleARN string + format string // "cyclonedx" or "spdxjson" + awsConfig aws.Config +} + +// NewTrivyClient creates a new TrivyClient with the provided configuration. +func NewTrivyClient(ctx context.Context, c *cfg.Config) (*TrivyClient, error) { + // Load default AWS config + awsConfig, err := config.LoadDefaultConfig(ctx, + config.WithRegion(c.AWSRegion), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + c.AWSAccessKeyID, + c.AWSSecretAccessKey, + "", + )), + ) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + return &TrivyClient{ + image: c.TrivyImage, + ecrAccountID: c.TrivyECRAccountID, + ecrRegion: c.TrivyECRRegion, + ecrRoleARN: c.TrivyECRRoleARN, + format: c.TrivyFormat, + awsConfig: awsConfig, + }, nil +} + +// setupECRCredentials sets up AWS credentials for ECR access, supporting cross-account. +func (t *TrivyClient) setupECRCredentials(ctx context.Context) error { + logger.Info("Setting up ECR credentials...") + + var awsConfig aws.Config + var err error + + // If cross-account role is specified, assume the role + if t.ecrRoleARN != "" { + logger.Info("Using cross-account role: %s", t.ecrRoleARN) + + // Create STS client with original credentials + stsClient := sts.NewFromConfig(t.awsConfig) + + // Assume the role + assumeRoleOutput, err := stsClient.AssumeRole(ctx, &sts.AssumeRoleInput{ + RoleArn: aws.String(t.ecrRoleARN), + RoleSessionName: aws.String(fmt.Sprintf("trivy-sbom-gen-%d", time.Now().Unix())), + }) + if err != nil { + return fmt.Errorf("failed to assume role %s: %w", t.ecrRoleARN, err) + } + + logger.Success("Successfully assumed cross-account role") + + // Create new config with assumed role credentials + awsConfig, err = config.LoadDefaultConfig(ctx, + config.WithRegion(t.ecrRegion), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + *assumeRoleOutput.Credentials.AccessKeyId, + *assumeRoleOutput.Credentials.SecretAccessKey, + *assumeRoleOutput.Credentials.SessionToken, + )), + ) + if err != nil { + return fmt.Errorf("failed to create config with assumed role credentials: %w", err) + } + } else { + // Use original credentials but with ECR region + awsConfig, err = config.LoadDefaultConfig(ctx, + config.WithRegion(t.ecrRegion), + config.WithCredentialsProvider(t.awsConfig.Credentials), + ) + if err != nil { + return fmt.Errorf("failed to create ECR config: %w", err) + } + } + + // Retrieve credentials and set environment variables for Trivy + creds, err := awsConfig.Credentials.Retrieve(ctx) + if err != nil { + return fmt.Errorf("failed to retrieve credentials: %w", err) + } + + // Trivy will use these AWS credentials directly for ECR authentication + // No Docker needed! + err = os.Setenv("AWS_ACCESS_KEY_ID", creds.AccessKeyID) + if err != nil { + return fmt.Errorf("failed to set AWS_ACCESS_KEY_ID: %w", err) + } + err = os.Setenv("AWS_SECRET_ACCESS_KEY", creds.SecretAccessKey) + if err != nil { + return fmt.Errorf("failed to set AWS_SECRET_ACCESS_KEY: %w", err) + } + if creds.SessionToken != "" { + err = os.Setenv("AWS_SESSION_TOKEN", creds.SessionToken) + if err != nil { + return fmt.Errorf("failed to set AWS_SESSION_TOKEN: %w", err) + } + } + err = os.Setenv("AWS_REGION", t.ecrRegion) + if err != nil { + return fmt.Errorf("failed to set AWS_REGION: %w", err) + } + + logger.Success("ECR credentials configured for Trivy") + return nil +} + +// GenerateSBOM generates an SBOM from the container image using Trivy. +func (t *TrivyClient) GenerateSBOM(ctx context.Context, outputFile string) error { + logger.Info("Generating SBOM for image: %s", t.image) + logger.Info("SBOM format: %s", t.format) + logger.Info("Using remote image source (no download)") + + // Check if this is an ECR image + isECRImage := strings.Contains(t.image, ".dkr.ecr.") && strings.Contains(t.image, ".amazonaws.com/") + + // Set up ECR credentials if needed + // Trivy supports ECR authentication natively without Docker! + if isECRImage && t.ecrAccountID != "" { + if err := t.setupECRCredentials(ctx); err != nil { + return fmt.Errorf("failed to setup ECR credentials: %w", err) + } + } + + // Create temp file for raw Trivy output + tempDir := filepath.Dir(outputFile) + trivyOutputFile := filepath.Join(tempDir, "trivy_sbom_output.json") + + logger.Info("Running Trivy SBOM generation...") + + // Determine Trivy output format + var trivyFormat string + switch t.format { + case "cyclonedx": // nolint: goconst + trivyFormat = "cyclonedx" + case "spdxjson": + trivyFormat = "spdx-json" + default: + return fmt.Errorf("unsupported SBOM format: %s", t.format) + } + + // Build Trivy command for SBOM generation + // Using --image-src remote to scan at source without downloading + args := []string{ + "image", + "--format", trivyFormat, + "--output", trivyOutputFile, + "--image-src", "remote", + "--quiet", + t.image, + } + + cmd := exec.CommandContext(ctx, "trivy", args...) + + // Trivy will use AWS credentials from environment for ECR access + cmd.Env = os.Environ() + + logger.Debug("Executing: trivy %s", strings.Join(args, " ")) + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("trivy SBOM generation failed: %w\nOutput: %s", err, string(output)) + } + + logger.Success("Trivy SBOM generation completed successfully") + + // Validate the output + trivyData, err := os.ReadFile(trivyOutputFile) + if err != nil { + return fmt.Errorf("failed to read Trivy output: %w", err) + } + + var sbomData map[string]interface{} + if err := json.Unmarshal(trivyData, &sbomData); err != nil { + return fmt.Errorf("trivy output is not valid JSON: %w", err) + } + + // Move to final output location + if err := os.Rename(trivyOutputFile, outputFile); err != nil { + return fmt.Errorf("failed to move SBOM file: %w", err) + } + + logger.Info("SBOM saved to: %s", outputFile) + + // Log component count based on format + switch t.format { + case "cyclonedx": + if components, ok := sbomData["components"].([]interface{}); ok { + logger.Info("Total components found: %d", len(components)) + } + case "spdxjson": + if packages, ok := sbomData["packages"].([]interface{}); ok { + logger.Info("Total packages found: %d", len(packages)) + } + } + + return nil +} From 5e441d0370dafc80b0c8361159ae8d33ff008212 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 7 Nov 2025 10:58:52 -0500 Subject: [PATCH 02/28] fix(debug): extract from wrapper function Signed-off-by: Julio Jimenez --- internal/sbom/processing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/processing.go b/internal/sbom/processing.go index a31b39b..f7216b2 100644 --- a/internal/sbom/processing.go +++ b/internal/sbom/processing.go @@ -73,7 +73,7 @@ func ExtractSBOMFromWrapper(inputFile, outputFile string) error { if err != nil { return fmt.Errorf("failed to read input file: %w", err) } - + fmt.Printf("Data: %s\n", string(data)) var wrapper map[string]interface{} if err := json.Unmarshal(data, &wrapper); err != nil { return fmt.Errorf("failed to parse JSON: %w", err) From 3b433f5931a8705307fde2ea70f3b7763bfcba45 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 7 Nov 2025 11:34:32 -0500 Subject: [PATCH 03/28] fix(debug): extract json from zip Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 58 +++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index 946dafd..104b283 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -2,6 +2,7 @@ package sbom import ( + "archive/zip" "bytes" "context" "encoding/json" @@ -360,23 +361,54 @@ func (m *MendClient) downloadReport(ctx context.Context, reportUUID, outputFile return fmt.Errorf("download failed (status %d): %s", resp.StatusCode, string(body)) } - // Create output file - outFile, err := os.Create(outputFile) + // After getting the response from Mend API + body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("failed to create output file: %w", err) + return fmt.Errorf("failed to read response: %w", err) } - defer func() { - if err := outFile.Close(); err != nil { - logger.Warning("Failed to close file: %v", err) + + // Check if response is a ZIP file (starts with "PK") + if len(body) >= 2 && body[0] == 0x50 && body[1] == 0x4B { + // It's a ZIP file, extract it + zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + return fmt.Errorf("failed to read ZIP: %w", err) } - }() - // Copy response to file - written, err := io.Copy(outFile, resp.Body) - if err != nil { - return fmt.Errorf("failed to write file: %w", err) + // Find and read the JSON file inside + for _, file := range zipReader.File { + if strings.HasSuffix(file.Name, ".json") { + rc, err := file.Open() + if err != nil { + return fmt.Errorf("failed to open file in ZIP: %w", err) + } + body, err = io.ReadAll(rc) + if err != nil { + return fmt.Errorf("failed to read file in ZIP: %w", err) + } + err = rc.Close() + if err != nil { + return fmt.Errorf("failed to close file in ZIP: %w", err) + } + // Create output file + outFile, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("failed to create output file: %w", err) + } + defer func() { + if err := outFile.Close(); err != nil { + logger.Warning("Failed to close file: %v", err) + } + }() + // Copy response to file + written, err := io.Copy(outFile, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + logger.Success("Mend SBOM downloaded successfully (%d bytes)", written) + break + } + } } - - logger.Success("Mend SBOM downloaded successfully (%d bytes)", written) return nil } From e36ba6e38d94c06e49defb456406f309b4d80397 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 7 Nov 2025 11:41:47 -0500 Subject: [PATCH 04/28] fix(debug): remove debug print of sbom Signed-off-by: Julio Jimenez --- internal/sbom/processing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/processing.go b/internal/sbom/processing.go index f7216b2..a31b39b 100644 --- a/internal/sbom/processing.go +++ b/internal/sbom/processing.go @@ -73,7 +73,7 @@ func ExtractSBOMFromWrapper(inputFile, outputFile string) error { if err != nil { return fmt.Errorf("failed to read input file: %w", err) } - fmt.Printf("Data: %s\n", string(data)) + var wrapper map[string]interface{} if err := json.Unmarshal(data, &wrapper); err != nil { return fmt.Errorf("failed to parse JSON: %w", err) From 96277d6adeddae027310dff8ba2a0d72e2dbc8a3 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 7 Nov 2025 16:35:39 -0500 Subject: [PATCH 05/28] fix(aws): Some inputs are not longer required Signed-off-by: Julio Jimenez --- action.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/action.yml b/action.yml index 81912e0..959a97b 100644 --- a/action.yml +++ b/action.yml @@ -78,16 +78,6 @@ inputs: required: false default: 'cyclonedx' # AWS-specific inputs - aws-access-key-id: - description: 'AWS Access Key ID' - required: true - aws-secret-access-key: - description: 'AWS Secret Access Key' - required: true - aws-region: - description: 'AWS region' - required: false - default: 'us-east-1' s3-bucket: description: 'S3 bucket name' required: true @@ -175,9 +165,6 @@ runs: TRIVY_ECR_ROLE_ARN: ${{ inputs.trivy-ecr-role-arn }} TRIVY_FORMAT: ${{ inputs.trivy-format }} # AWS-specific - AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} - AWS_DEFAULT_REGION: ${{ inputs.aws-region }} S3_BUCKET: ${{ inputs.s3-bucket }} S3_KEY: ${{ inputs.s3-key }} # ClickHouse-specific From 4f25ce4756796cea87d05a8723ad3d5864b0e131 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 7 Nov 2025 16:42:49 -0500 Subject: [PATCH 06/28] fix(aws): Some inputs are not longer required Signed-off-by: Julio Jimenez --- cmd/clickbom/main.go | 2 +- internal/config/config.go | 13 ++-------- internal/config/config_test.go | 44 +++++++++++++--------------------- internal/storage/s3.go | 14 ++++------- 4 files changed, 23 insertions(+), 50 deletions(-) diff --git a/cmd/clickbom/main.go b/cmd/clickbom/main.go index bec0007..6ff9b53 100644 --- a/cmd/clickbom/main.go +++ b/cmd/clickbom/main.go @@ -45,7 +45,7 @@ func run() error { }() // Initialize S3 client - s3Client, err := storage.NewS3Client(ctx, cfg.AWSAccessKeyID, cfg.AWSSecretAccessKey, cfg.AWSRegion) + s3Client, err := storage.NewS3Client(ctx) if err != nil { return fmt.Errorf("failed to create S3 client: %w", err) } diff --git a/internal/config/config.go b/internal/config/config.go index a5fd716..60f6f73 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -70,11 +70,8 @@ type Config struct { func LoadConfig() (*Config, error) { cfg := &Config{ // AWS (required) - AWSAccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"), - AWSSecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY"), - AWSRegion: getEnvOrDefault("AWS_DEFAULT_REGION", "us-east-1"), - S3Bucket: os.Getenv("S3_BUCKET"), - S3Key: getEnvOrDefault("S3_KEY", "sbom.json"), + S3Bucket: os.Getenv("S3_BUCKET"), + S3Key: getEnvOrDefault("S3_KEY", "sbom.json"), // GitHub GitHubToken: os.Getenv("GITHUB_TOKEN"), @@ -139,12 +136,6 @@ func LoadConfig() (*Config, error) { // Validate checks that all required configuration fields are set appropriately. func (c *Config) Validate() error { // AWS is always required - if c.AWSAccessKeyID == "" { - return fmt.Errorf("AWS_ACCESS_KEY_ID is required") - } - if c.AWSSecretAccessKey == "" { - return fmt.Errorf("AWS_SECRET_ACCESS_KEY is required") - } if c.S3Bucket == "" { return fmt.Errorf("S3_BUCKET is required") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bb617cd..72fa434 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -14,19 +14,15 @@ func TestLoadConfig(t *testing.T) { { name: "valid minimal config", env: map[string]string{ - "AWS_ACCESS_KEY_ID": "test-key", - "AWS_SECRET_ACCESS_KEY": "test-secret", - "S3_BUCKET": "test-bucket", - "REPOSITORY": "owner/repo", + "S3_BUCKET": "test-bucket", + "REPOSITORY": "owner/repo", }, wantErr: false, }, { name: "missing required field", env: map[string]string{ - "AWS_ACCESS_KEY_ID": "test-key", - // Missing AWS_SECRET_ACCESS_KEY - "S3_BUCKET": "test-bucket", + // Missing S3_BUCKET "REPOSITORY": "owner/repo", }, wantErr: true, @@ -34,10 +30,8 @@ func TestLoadConfig(t *testing.T) { { name: "invalid repository format", env: map[string]string{ - "AWS_ACCESS_KEY_ID": "test-key", - "AWS_SECRET_ACCESS_KEY": "test-secret", - "S3_BUCKET": "test-bucket", - "REPOSITORY": "invalid-repo", // No slash + "S3_BUCKET": "test-bucket", + "REPOSITORY": "invalid-repo", // No slash }, wantErr: true, }, @@ -79,35 +73,29 @@ func TestConfigValidate(t *testing.T) { { name: "valid github config", config: &Config{ - AWSAccessKeyID: "key", - AWSSecretAccessKey: "secret", - S3Bucket: "bucket", - Repository: "owner/repo", - SBOMSource: "github", + S3Bucket: "bucket", + Repository: "owner/repo", + SBOMSource: "github", }, wantErr: false, }, { name: "valid mend config", config: &Config{ - AWSAccessKeyID: "key", - AWSSecretAccessKey: "secret", - S3Bucket: "bucket", - SBOMSource: "mend", - MendEmail: "test@example.com", - MendOrgUUID: "123e4567-e89b-12d3-a456-426614174000", - MendUserKey: "user-key", - MendProjectUUID: "123e4567-e89b-12d3-a456-426614174001", + S3Bucket: "bucket", + SBOMSource: "mend", + MendEmail: "test@example.com", + MendOrgUUID: "123e4567-e89b-12d3-a456-426614174000", + MendUserKey: "user-key", + MendProjectUUID: "123e4567-e89b-12d3-a456-426614174001", }, wantErr: false, }, { name: "invalid mend config - missing email", config: &Config{ - AWSAccessKeyID: "key", - AWSSecretAccessKey: "secret", - S3Bucket: "bucket", - SBOMSource: "mend", + S3Bucket: "bucket", + SBOMSource: "mend", // Missing MendEmail MendOrgUUID: "123e4567-e89b-12d3-a456-426614174000", MendUserKey: "user-key", diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 69cc5d1..2bd2cba 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -9,7 +9,8 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" + + // "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/ClickHouse/ClickBOM/pkg/logger" @@ -21,15 +22,8 @@ type S3Client struct { } // NewS3Client creates a new S3Client with the provided AWS credentials and region. -func NewS3Client(ctx context.Context, accessKeyID, secretAccessKey, region string) (*S3Client, error) { - cfg, err := config.LoadDefaultConfig(ctx, - config.WithRegion(region), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( - accessKeyID, - secretAccessKey, - "", - )), - ) +func NewS3Client(ctx context.Context) (*S3Client, error) { + cfg, err := config.LoadDefaultConfig(ctx) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } From 7fec6436589f110dc4de59822f779dae322eae90 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Thu, 13 Nov 2025 17:22:34 -0500 Subject: [PATCH 07/28] fix: add trivy to config validation Signed-off-by: Julio Jimenez --- internal/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/config.go b/internal/config/config.go index 60f6f73..2cacfcb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -141,7 +141,7 @@ func (c *Config) Validate() error { } // Repository required if not in merge mode and source is GitHub - if !c.Merge && c.SBOMSource != "mend" && c.SBOMSource != "wiz" { + if !c.Merge && c.SBOMSource != "mend" && c.SBOMSource != "wiz" && c.SBOMSource != "trivy" { if c.Repository == "" { return fmt.Errorf("REPOSITORY is required when not in merge mode") } From faf30d7fc67888d9d1d4f178486159a946051399 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Thu, 13 Nov 2025 17:26:13 -0500 Subject: [PATCH 08/28] fix: add trivy to config validation Signed-off-by: Julio Jimenez --- internal/storage/s3_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go index 7ced2bb..915cd70 100644 --- a/internal/storage/s3_test.go +++ b/internal/storage/s3_test.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package storage From d8526dd97341f34d03358b3102357af6e0442785 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Thu, 13 Nov 2025 17:33:05 -0500 Subject: [PATCH 09/28] fix: add trivy to config validation Signed-off-by: Julio Jimenez --- internal/storage/s3_test.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go index 915cd70..c96a1d4 100644 --- a/internal/storage/s3_test.go +++ b/internal/storage/s3_test.go @@ -1,5 +1,3 @@ -//go:build integration - package storage import ( @@ -17,12 +15,7 @@ func TestS3Upload(t *testing.T) { ctx := context.Background() // Create S3 client - s3Client, err := storage.NewS3Client( - ctx, - "test", - "test", - "us-east-1", - ) + s3Client, err := NewS3Client(ctx) if err != nil { t.Fatalf("Failed to create S3 client: %v", err) } From 100d2b5222db206693fda3bde38b61332ea29bc8 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 15:46:18 -0500 Subject: [PATCH 10/28] fix: ecr auth Signed-off-by: Julio Jimenez --- go.mod | 2 +- internal/config/config.go | 22 +++++----- internal/sbom/trivy.go | 90 ++++++++++++++------------------------- 3 files changed, 46 insertions(+), 68 deletions(-) diff --git a/go.mod b/go.mod index 582c0a5..9842fcc 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.25.3 require ( github.com/aws/aws-sdk-go-v2 v1.39.4 github.com/aws/aws-sdk-go-v2/config v1.31.15 - github.com/aws/aws-sdk-go-v2/credentials v1.18.19 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 ) require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.18.19 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 2cacfcb..6bdc61a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -34,11 +34,12 @@ type Config struct { WizReportID string // Trivy - TrivyImage string - TrivyECRAccountID string - TrivyECRRegion string - TrivyECRRoleARN string - TrivyFormat string + TrivyImage string + TrivyECRAccountID string + TrivyECRRegion string + TrivyECRRoleARN string + TrivyECRExternalID string + TrivyFormat string // AWS AWSAccessKeyID string @@ -97,11 +98,12 @@ func LoadConfig() (*Config, error) { WizReportID: os.Getenv("WIZ_REPORT_ID"), // Trivy - TrivyImage: getEnvOrDefault("TRIVY_IMAGE", ""), - TrivyECRAccountID: getEnvOrDefault("TRIVY_ECR_ACCOUNT_ID", ""), - TrivyECRRegion: getEnvOrDefault("TRIVY_ECR_REGION", "us-east-1"), - TrivyECRRoleARN: getEnvOrDefault("TRIVY_ECR_ROLE_ARN", ""), - TrivyFormat: getEnvOrDefault("TRIVY_FORMAT", "cyclonedx"), + TrivyImage: getEnvOrDefault("TRIVY_IMAGE", ""), + TrivyECRAccountID: getEnvOrDefault("TRIVY_ECR_ACCOUNT_ID", ""), + TrivyECRRegion: getEnvOrDefault("TRIVY_ECR_REGION", "us-east-1"), + TrivyECRRoleARN: getEnvOrDefault("TRIVY_ECR_ROLE_ARN", ""), + TrivyECRExternalID: getEnvOrDefault("TRIVY_ECR_EXTERNAL_ID", ""), + TrivyFormat: getEnvOrDefault("TRIVY_FORMAT", "cyclonedx"), // ClickHouse ClickHouseURL: os.Getenv("CLICKHOUSE_URL"), diff --git a/internal/sbom/trivy.go b/internal/sbom/trivy.go index dd976d2..08c652b 100644 --- a/internal/sbom/trivy.go +++ b/internal/sbom/trivy.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/sts" cfg "github.com/ClickHouse/ClickBOM/internal/config" @@ -26,6 +25,7 @@ type TrivyClient struct { ecrAccountID string ecrRegion string ecrRoleARN string + externalID string format string // "cyclonedx" or "spdxjson" awsConfig aws.Config } @@ -33,14 +33,7 @@ type TrivyClient struct { // NewTrivyClient creates a new TrivyClient with the provided configuration. func NewTrivyClient(ctx context.Context, c *cfg.Config) (*TrivyClient, error) { // Load default AWS config - awsConfig, err := config.LoadDefaultConfig(ctx, - config.WithRegion(c.AWSRegion), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( - c.AWSAccessKeyID, - c.AWSSecretAccessKey, - "", - )), - ) + awsConfig, err := config.LoadDefaultConfig(ctx) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -50,6 +43,7 @@ func NewTrivyClient(ctx context.Context, c *cfg.Config) (*TrivyClient, error) { ecrAccountID: c.TrivyECRAccountID, ecrRegion: c.TrivyECRRegion, ecrRoleARN: c.TrivyECRRoleARN, + externalID: c.TrivyECRExternalID, format: c.TrivyFormat, awsConfig: awsConfig, }, nil @@ -59,9 +53,6 @@ func NewTrivyClient(ctx context.Context, c *cfg.Config) (*TrivyClient, error) { func (t *TrivyClient) setupECRCredentials(ctx context.Context) error { logger.Info("Setting up ECR credentials...") - var awsConfig aws.Config - var err error - // If cross-account role is specified, assume the role if t.ecrRoleARN != "" { logger.Info("Using cross-account role: %s", t.ecrRoleARN) @@ -69,65 +60,51 @@ func (t *TrivyClient) setupECRCredentials(ctx context.Context) error { // Create STS client with original credentials stsClient := sts.NewFromConfig(t.awsConfig) - // Assume the role - assumeRoleOutput, err := stsClient.AssumeRole(ctx, &sts.AssumeRoleInput{ + // Build AssumeRole input + assumeRoleInput := &sts.AssumeRoleInput{ RoleArn: aws.String(t.ecrRoleARN), RoleSessionName: aws.String(fmt.Sprintf("trivy-sbom-gen-%d", time.Now().Unix())), - }) + } + + // Add External ID if provided + if t.externalID != "" { + assumeRoleInput.ExternalId = aws.String(t.externalID) + logger.Info("Using external ID for role assumption") + } + + // Assume the role + assumeRoleOutput, err := stsClient.AssumeRole(ctx, assumeRoleInput) if err != nil { return fmt.Errorf("failed to assume role %s: %w", t.ecrRoleARN, err) } logger.Success("Successfully assumed cross-account role") - // Create new config with assumed role credentials - awsConfig, err = config.LoadDefaultConfig(ctx, - config.WithRegion(t.ecrRegion), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( - *assumeRoleOutput.Credentials.AccessKeyId, - *assumeRoleOutput.Credentials.SecretAccessKey, - *assumeRoleOutput.Credentials.SessionToken, - )), - ) + // Set environment variables with assumed role credentials + err = os.Setenv("AWS_ACCESS_KEY_ID", *assumeRoleOutput.Credentials.AccessKeyId) if err != nil { - return fmt.Errorf("failed to create config with assumed role credentials: %w", err) + return fmt.Errorf("failed to set AWS_ACCESS_KEY_ID: %w", err) } - } else { - // Use original credentials but with ECR region - awsConfig, err = config.LoadDefaultConfig(ctx, - config.WithRegion(t.ecrRegion), - config.WithCredentialsProvider(t.awsConfig.Credentials), - ) + err = os.Setenv("AWS_SECRET_ACCESS_KEY", *assumeRoleOutput.Credentials.SecretAccessKey) if err != nil { - return fmt.Errorf("failed to create ECR config: %w", err) + return fmt.Errorf("failed to set AWS_SECRET_ACCESS_KEY: %w", err) } - } - - // Retrieve credentials and set environment variables for Trivy - creds, err := awsConfig.Credentials.Retrieve(ctx) - if err != nil { - return fmt.Errorf("failed to retrieve credentials: %w", err) - } - - // Trivy will use these AWS credentials directly for ECR authentication - // No Docker needed! - err = os.Setenv("AWS_ACCESS_KEY_ID", creds.AccessKeyID) - if err != nil { - return fmt.Errorf("failed to set AWS_ACCESS_KEY_ID: %w", err) - } - err = os.Setenv("AWS_SECRET_ACCESS_KEY", creds.SecretAccessKey) - if err != nil { - return fmt.Errorf("failed to set AWS_SECRET_ACCESS_KEY: %w", err) - } - if creds.SessionToken != "" { - err = os.Setenv("AWS_SESSION_TOKEN", creds.SessionToken) + err = os.Setenv("AWS_SESSION_TOKEN", *assumeRoleOutput.Credentials.SessionToken) if err != nil { return fmt.Errorf("failed to set AWS_SESSION_TOKEN: %w", err) } - } - err = os.Setenv("AWS_REGION", t.ecrRegion) - if err != nil { - return fmt.Errorf("failed to set AWS_REGION: %w", err) + err = os.Setenv("AWS_REGION", t.ecrRegion) + if err != nil { + return fmt.Errorf("failed to set AWS_REGION: %w", err) + } + + logger.Debug("ECR credentials set in environment variables") + } else if t.ecrRegion != "" { + err := os.Setenv("AWS_REGION", t.ecrRegion) + if err != nil { + return fmt.Errorf("failed to set AWS_REGION: %w", err) + } + logger.Info("Using current AWS credentials with region: %s", t.ecrRegion) } logger.Success("ECR credentials configured for Trivy") @@ -222,6 +199,5 @@ func (t *TrivyClient) GenerateSBOM(ctx context.Context, outputFile string) error logger.Info("Total packages found: %d", len(packages)) } } - return nil } From 667ce8111f7122fc7d816c6734328d18d6260cbe Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 18:05:05 -0500 Subject: [PATCH 11/28] fix: trivy clickhouse table name Signed-off-by: Julio Jimenez --- cmd/clickbom/main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/clickbom/main.go b/cmd/clickbom/main.go index 6ff9b53..3156d5e 100644 --- a/cmd/clickbom/main.go +++ b/cmd/clickbom/main.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "os" + "path" "path/filepath" "strings" @@ -193,6 +194,11 @@ func generateTableName(cfg *config.Config) string { return fmt.Sprintf("mend_%s", strings.ReplaceAll(uuid, "-", "_")) case "wiz": return fmt.Sprintf("wiz_%s", strings.ReplaceAll(cfg.WizReportID, "-", "_")) + case "trivy": + result := path.Base(cfg.TrivyImage) + replacer := strings.NewReplacer(":", "_", ".", "_", "-", "_") + result = replacer.Replace(result) + return fmt.Sprintf("trivy_%s", result) default: return "sbom_data" } From e7ee3274beb46bc9c9b3f9ef454534bbf2c704b7 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 20:54:34 -0500 Subject: [PATCH 12/28] feat: ability to do application scope reports Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index 104b283..c549a60 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -173,17 +173,18 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e } // Add scope + var url string switch { case m.projectUUID != "": payload["scopeType"] = "project" payload["scopeUuid"] = m.projectUUID uuids := strings.Split(m.projectUUIDs, ",") payload["projectUuids"] = uuids + url = fmt.Sprintf("%s/api/v3.0/projects/%s/dependencies/reports/SBOM", m.baseURL, m.projectUUID) case m.productUUID != "": - payload["scopeType"] = "product" - payload["scopeUuid"] = m.productUUID uuids := strings.Split(m.projectUUIDs, ",") payload["projectUuids"] = uuids + url = fmt.Sprintf("%s/api/v3.0/applications/%s/dependencies/reports/SBOM", m.baseURL, m.productUUID) case m.orgScopeUUID != "": payload["scopeType"] = "organization" payload["scopeUuid"] = m.orgScopeUUID @@ -194,9 +195,6 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e return fmt.Errorf("failed to marshal payload: %w", err) } - url := fmt.Sprintf("%s/api/v3.0/projects/%s/dependencies/reports/SBOM", - m.baseURL, m.projectUUID) - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(payloadBytes)) if err != nil { return fmt.Errorf("failed to create request: %w", err) From 22a8ea1ba4ec09bd463d67b01c674f0c439be815 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 21:04:54 -0500 Subject: [PATCH 13/28] fix: i don't think org uuid is always required Signed-off-by: Julio Jimenez --- internal/config/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 6bdc61a..311ed32 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -154,9 +154,6 @@ func (c *Config) Validate() error { if c.MendEmail == "" { return fmt.Errorf("MEND_EMAIL is required for Mend source") } - if c.MendOrgUUID == "" { - return fmt.Errorf("MEND_ORG_UUID is required for Mend source") - } if c.MendUserKey == "" { return fmt.Errorf("MEND_USER_KEY is required for Mend source") } From 745a1df6072ab062bcf7d12c1b729026d0afde40 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 21:11:27 -0500 Subject: [PATCH 14/28] fix: if no projectUuids are provided Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index c549a60..a9a0535 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -182,8 +182,10 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e payload["projectUuids"] = uuids url = fmt.Sprintf("%s/api/v3.0/projects/%s/dependencies/reports/SBOM", m.baseURL, m.projectUUID) case m.productUUID != "": - uuids := strings.Split(m.projectUUIDs, ",") - payload["projectUuids"] = uuids + if len(m.projectUUIDs) != 0 { + uuids := strings.Split(m.projectUUIDs, ",") + payload["projectUuids"] = uuids + } url = fmt.Sprintf("%s/api/v3.0/applications/%s/dependencies/reports/SBOM", m.baseURL, m.productUUID) case m.orgScopeUUID != "": payload["scopeType"] = "organization" From f4836d19d963931a8f21c1c765f6050f62cab439 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 21:34:21 -0500 Subject: [PATCH 15/28] fix: mend-project-uuids Signed-off-by: Julio Jimenez --- internal/config/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/config/config.go b/internal/config/config.go index 311ed32..6bdc61a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -154,6 +154,9 @@ func (c *Config) Validate() error { if c.MendEmail == "" { return fmt.Errorf("MEND_EMAIL is required for Mend source") } + if c.MendOrgUUID == "" { + return fmt.Errorf("MEND_ORG_UUID is required for Mend source") + } if c.MendUserKey == "" { return fmt.Errorf("MEND_USER_KEY is required for Mend source") } From 400b32d757bc17e91feadcd31dd6574b6dad18d4 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 21:47:11 -0500 Subject: [PATCH 16/28] fix: maxDepthLevel Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index a9a0535..9934e45 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -170,6 +170,7 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e "reportType": "cycloneDX_1_5", "format": "json", "includeVulnerabilities": false, + "maxDepthLevel": 100, } // Add scope From fe1fe95b8a231064a6fbf3eef2c14e332220dc78 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 21:51:30 -0500 Subject: [PATCH 17/28] fix: maxDepthLevel Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index 9934e45..db12504 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -170,7 +170,7 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e "reportType": "cycloneDX_1_5", "format": "json", "includeVulnerabilities": false, - "maxDepthLevel": 100, + "maxDepthLevel": 10, } // Add scope From dd7ea7d19f25085a650b1ac7a09d37b42c55b268 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 21:54:10 -0500 Subject: [PATCH 18/28] fix: maxDepthLevel Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index db12504..1a521f5 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -170,7 +170,7 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e "reportType": "cycloneDX_1_5", "format": "json", "includeVulnerabilities": false, - "maxDepthLevel": 10, + "maxDepthLevel": 5, } // Add scope From 7d693a494915be21439deb9fa20fb5a01b6035fb Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 21:56:10 -0500 Subject: [PATCH 19/28] fix: maxDepthLevel Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index 1a521f5..86ab310 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -170,7 +170,7 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e "reportType": "cycloneDX_1_5", "format": "json", "includeVulnerabilities": false, - "maxDepthLevel": 5, + "maxDepthLevel": 0, } // Add scope From 9f47927607c8eafa32c81501b600cf8376699a5c Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 22:02:19 -0500 Subject: [PATCH 20/28] fix: maxDepthLevel Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index 86ab310..df5855a 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -170,7 +170,6 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e "reportType": "cycloneDX_1_5", "format": "json", "includeVulnerabilities": false, - "maxDepthLevel": 0, } // Add scope @@ -187,6 +186,7 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e uuids := strings.Split(m.projectUUIDs, ",") payload["projectUuids"] = uuids } + payload["maxDepthLevel"] = 0 url = fmt.Sprintf("%s/api/v3.0/applications/%s/dependencies/reports/SBOM", m.baseURL, m.productUUID) case m.orgScopeUUID != "": payload["scopeType"] = "organization" From 5cbe317c60aec555091a2f73007ed63e9e673bf1 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 22:05:21 -0500 Subject: [PATCH 21/28] fix: maxDepthLevel Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index df5855a..953b8ff 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -186,7 +186,7 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e uuids := strings.Split(m.projectUUIDs, ",") payload["projectUuids"] = uuids } - payload["maxDepthLevel"] = 0 + // payload["maxDepthLevel"] = 0 url = fmt.Sprintf("%s/api/v3.0/applications/%s/dependencies/reports/SBOM", m.baseURL, m.productUUID) case m.orgScopeUUID != "": payload["scopeType"] = "organization" From c9adcf81ec0161e62bcc585c78f617a8333b0eda Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 22:10:41 -0500 Subject: [PATCH 22/28] fix: maxDepthLevel Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index 953b8ff..df5855a 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -186,7 +186,7 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e uuids := strings.Split(m.projectUUIDs, ",") payload["projectUuids"] = uuids } - // payload["maxDepthLevel"] = 0 + payload["maxDepthLevel"] = 0 url = fmt.Sprintf("%s/api/v3.0/applications/%s/dependencies/reports/SBOM", m.baseURL, m.productUUID) case m.orgScopeUUID != "": payload["scopeType"] = "organization" From 80055ec51305f7e84ea03e514e86f3010f104dbe Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 22:47:33 -0500 Subject: [PATCH 23/28] fix: stuff Signed-off-by: Julio Jimenez --- internal/sbom/mend.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/sbom/mend.go b/internal/sbom/mend.go index df5855a..67d4896 100644 --- a/internal/sbom/mend.go +++ b/internal/sbom/mend.go @@ -178,14 +178,15 @@ func (m *MendClient) RequestSBOMExport(ctx context.Context, outputFile string) e case m.projectUUID != "": payload["scopeType"] = "project" payload["scopeUuid"] = m.projectUUID - uuids := strings.Split(m.projectUUIDs, ",") - payload["projectUuids"] = uuids + // uuids := strings.Split(m.projectUUIDs, ",") + // payload["projectUuids"] = uuids url = fmt.Sprintf("%s/api/v3.0/projects/%s/dependencies/reports/SBOM", m.baseURL, m.projectUUID) case m.productUUID != "": - if len(m.projectUUIDs) != 0 { - uuids := strings.Split(m.projectUUIDs, ",") - payload["projectUuids"] = uuids - } + // if len(m.projectUUIDs) != 0 { + // uuids := strings.Split(m.projectUUIDs, ",") + // payload["projectUuids"] = uuids + // } + payload["projectUuids"] = []string{m.projectUUID} payload["maxDepthLevel"] = 0 url = fmt.Sprintf("%s/api/v3.0/applications/%s/dependencies/reports/SBOM", m.baseURL, m.productUUID) case m.orgScopeUUID != "": From 88b80be48024568dc50034029e65d5bbd7e7e9c5 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 23:09:31 -0500 Subject: [PATCH 24/28] feat: add merge Signed-off-by: Julio Jimenez --- cmd/clickbom/main.go | 98 ++++++++++++- go.mod | 1 + go.sum | 2 + internal/sbom/filter.go | 85 ++++-------- internal/sbom/license_mapper.go | 2 +- internal/sbom/merge.go | 238 ++++++++++++++++++++++++++++++++ internal/storage/s3.go | 35 +++++ 7 files changed, 399 insertions(+), 62 deletions(-) create mode 100644 internal/sbom/merge.go diff --git a/cmd/clickbom/main.go b/cmd/clickbom/main.go index 3156d5e..359d46d 100644 --- a/cmd/clickbom/main.go +++ b/cmd/clickbom/main.go @@ -145,17 +145,109 @@ func handleNormalMode(ctx context.Context, cfg *config.Config, s3Client *storage if err := chClient.InsertSBOMData(ctx, processedSBOM, tableName, cfg.SBOMFormat); err != nil { return fmt.Errorf("failed to upload to ClickHouse: %w", err) } + logger.Success("ClickHouse operations completed successfully!") } return nil } -func handleMergeMode(_ context.Context, _ *config.Config, _ *storage.S3Client, _ string) error { +func handleMergeMode(ctx context.Context, cfg *config.Config, s3Client *storage.S3Client, tempDir string) error { logger.Info("Running in MERGE mode - merging all CycloneDX SBOMs from S3") - // Implementation for merge mode... - // This would involve downloading all SBOMs from S3, merging them, and uploading + // Create download directory + downloadDir := filepath.Join(tempDir, "downloads") + if err := os.MkdirAll(downloadDir, 0755); err != nil { + return fmt.Errorf("failed to create download directory: %w", err) + } + + // Download all files from S3 + downloadedFiles, err := s3Client.DownloadAll(ctx, cfg.S3Bucket, "", downloadDir) + if err != nil { + return fmt.Errorf("failed to download files from S3: %w", err) + } + + logger.Info("Downloaded %d files from S3", len(downloadedFiles)) + + if len(downloadedFiles) == 0 { + return fmt.Errorf("no files found in S3 bucket: %s", cfg.S3Bucket) + } + + // Filter and validate CycloneDX SBOMs + cyclonedxFiles := make([]string, 0) + + for _, file := range downloadedFiles { + filename := filepath.Base(file) + + // Apply include/exclude filters + if !sbom.ShouldIncludeFile(filename, cfg.Include, cfg.Exclude) { + logger.Debug("Skipping %s due to include/exclude filters", filename) + continue + } + + // Check if file is valid CycloneDX + format, err := sbom.DetectSBOMFormat(file) + if err != nil { + logger.Warning("Failed to detect format for %s: %v", filename, err) + continue + } + + if format != sbom.FormatCycloneDX { + logger.Debug("Skipping %s: not CycloneDX format (detected: %s)", filename, format) + continue + } + + cyclonedxFiles = append(cyclonedxFiles, file) + logger.Debug("Added %s to merge list", filename) + } + + logger.Info("Found %d valid CycloneDX SBOMs to merge", len(cyclonedxFiles)) + + if len(cyclonedxFiles) == 0 { + return fmt.Errorf("no valid CycloneDX SBOMs found after filtering") + } + + // Merge all SBOMs + mergedSBOM := filepath.Join(tempDir, "merged_sbom.json") + if err := sbom.MergeSBOMs(cyclonedxFiles, mergedSBOM); err != nil { + return fmt.Errorf("failed to merge SBOMs: %w", err) + } + + // Convert to desired format if needed + finalSBOM := filepath.Join(tempDir, "final_sbom.json") + desiredFormat := sbom.Format(cfg.SBOMFormat) + if err := sbom.ConvertSBOM(mergedSBOM, finalSBOM, sbom.FormatCycloneDX, desiredFormat); err != nil { + return fmt.Errorf("failed to convert merged SBOM: %w", err) + } + + // Upload merged SBOM back to S3 + if err := s3Client.Upload(ctx, finalSBOM, cfg.S3Bucket, cfg.S3Key, cfg.SBOMFormat); err != nil { + return fmt.Errorf("failed to upload merged SBOM: %w", err) + } + + logger.Success("SBOM merging and upload completed successfully!") + + // ClickHouse upload if configured + if cfg.ClickHouseURL != "" { + logger.Info("Uploading merged SBOM data to ClickHouse") + + chClient, err := storage.NewClickHouseClient(cfg) + if err != nil { + return fmt.Errorf("failed to create ClickHouse client: %w", err) + } + + tableName := generateTableName(cfg) + + if err := chClient.SetupTable(ctx, tableName); err != nil { + return fmt.Errorf("failed to setup table: %w", err) + } + + if err := chClient.InsertSBOMData(ctx, finalSBOM, tableName, cfg.SBOMFormat); err != nil { + return fmt.Errorf("failed to upload to ClickHouse: %w", err) + } + + logger.Success("ClickHouse operations completed successfully!") + } return nil } diff --git a/go.mod b/go.mod index 9842fcc..bd0f578 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.31.15 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 + github.com/google/uuid v1.6.0 ) require ( diff --git a/go.sum b/go.sum index 6e4e000..bc312b0 100644 --- a/go.sum +++ b/go.sum @@ -34,3 +34,5 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 h1:Ekml5vGg6sHSZLZJQJagefnVe6Pm github.com/aws/aws-sdk-go-v2/service/sts v1.38.9/go.mod h1:/e15V+o1zFHWdH3u7lpI3rVBcxszktIKuHKCY2/py+k= github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/internal/sbom/filter.go b/internal/sbom/filter.go index 7dda0c5..cbb5483 100644 --- a/internal/sbom/filter.go +++ b/internal/sbom/filter.go @@ -1,4 +1,4 @@ -// Package sbom provides functionalities for filtering files for SBOM generation. +// Package sbom provides pattern matching for filtering files. package sbom import ( @@ -8,87 +8,56 @@ import ( "github.com/ClickHouse/ClickBOM/pkg/logger" ) -// FileFilter defines inclusion and exclusion patterns for filtering files. -type FileFilter struct { - Include []string - Exclude []string -} - -// NewFileFilter creates a new FileFilter with the given include and exclude patterns. -func NewFileFilter(include, exclude string) *FileFilter { - return &FileFilter{ - Include: parsePatterns(include), - Exclude: parsePatterns(exclude), - } -} - -func parsePatterns(patterns string) []string { +// MatchesPattern checks if a filename matches any pattern in a comma-separated list. +func MatchesPattern(filename, patterns string) bool { if patterns == "" { - return nil + return false } - parts := strings.Split(patterns, ",") - var result []string - for _, p := range parts { - p = strings.TrimSpace(p) - if p != "" { - result = append(result, p) - } - } - return result -} + // Split patterns by comma + patternList := strings.Split(patterns, ",") -// MatchesPattern checks if the filename matches any of the provided patterns. -func (f *FileFilter) MatchesPattern(filename string, patterns []string) bool { - if len(patterns) == 0 { - return false - } + for _, pattern := range patternList { + // Trim whitespace + pattern = strings.TrimSpace(pattern) - for _, pattern := range patterns { + if pattern == "" { + continue + } + + // Use filepath.Match for wildcard matching matched, err := filepath.Match(pattern, filename) if err != nil { logger.Warning("Invalid pattern %s: %v", pattern, err) continue } + if matched { + logger.Debug("File %s matches pattern %s", filename, pattern) return true } } + return false } -// ShouldInclude determines if a file should be included based on the filter rules. -func (f *FileFilter) ShouldInclude(filename string) bool { - // If include patterns specified, file must match at least one - if len(f.Include) > 0 { - if !f.MatchesPattern(filename, f.Include) { +// ShouldIncludeFile determines if a file should be included based on include/exclude patterns. +func ShouldIncludeFile(filename, includePatterns, excludePatterns string) bool { + // If include patterns are specified, file must match at least one + if includePatterns != "" { + if !MatchesPattern(filename, includePatterns) { + logger.Debug("File %s does not match include patterns", filename) return false } } - // If exclude patterns specified and file matches, exclude it - if len(f.Exclude) > 0 { - if f.MatchesPattern(filename, f.Exclude) { + // If exclude patterns are specified, file must not match any + if excludePatterns != "" { + if MatchesPattern(filename, excludePatterns) { + logger.Debug("File %s matches exclude patterns", filename) return false } } return true } - -// FilterFiles filters the given list of files based on the FileFilter rules. -func (f *FileFilter) FilterFiles(files []string) []string { - var filtered []string - - for _, file := range files { - filename := filepath.Base(file) - if f.ShouldInclude(filename) { - filtered = append(filtered, file) - } else { - logger.Debug("Filtered out: %s", filename) - } - } - - logger.Info("Filtered %d files to %d files", len(files), len(filtered)) - return filtered -} diff --git a/internal/sbom/license_mapper.go b/internal/sbom/license_mapper.go index ec48a9f..17cd0d7 100644 --- a/internal/sbom/license_mapper.go +++ b/internal/sbom/license_mapper.go @@ -34,7 +34,7 @@ func NewLicenseMapper(mappingFile string) (*LicenseMapper, error) { // MapLicense maps an unknown license to a known one, or returns the original func (m *LicenseMapper) MapLicense(componentName, license string) string { // If license is already known, return it - if license != "" && license != "unknown" && license != "null" { + if license != "" && license != "unknown" && license != "null" { // nolint:goconst return license } diff --git a/internal/sbom/merge.go b/internal/sbom/merge.go new file mode 100644 index 0000000..ab89e49 --- /dev/null +++ b/internal/sbom/merge.go @@ -0,0 +1,238 @@ +// Package sbom provides functionalities for merging multiple SBOMs. +package sbom + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/ClickHouse/ClickBOM/pkg/logger" +) + +// MergedComponent represents a component with source tracking. +type MergedComponent struct { + Component map[string]interface{} + Source string +} + +// ExtractSourceReference extracts the source reference from an SBOM file. +func ExtractSourceReference(sbomFile string) (string, error) { + data, err := os.ReadFile(sbomFile) + if err != nil { + return "", fmt.Errorf("failed to read SBOM file: %w", err) + } + + var sbom map[string]interface{} + if err := json.Unmarshal(data, &sbom); err != nil { + return "", fmt.Errorf("failed to parse SBOM: %w", err) + } + + filename := filepath.Base(sbomFile) + filename = strings.TrimSuffix(filename, filepath.Ext(filename)) + + // Strategy 1: Check for spdx:document:name in properties (GitHub SBOMs) + if metadata, ok := sbom["metadata"].(map[string]interface{}); ok { + if properties, ok := metadata["properties"].([]interface{}); ok { + for _, prop := range properties { + if propMap, ok := prop.(map[string]interface{}); ok { + if name, _ := propMap["name"].(string); name == "spdx:document:name" { + if value, ok := propMap["value"].(string); ok && value != "" { + logger.Debug("Found SPDX document name: %s", value) + return value, nil + } + } + } + } + } + + // Strategy 2: Check metadata.component.name (Wiz/Mend SBOMs) + if component, ok := metadata["component"].(map[string]interface{}); ok { + if name, ok := component["name"].(string); ok && name != "" { + logger.Debug("Found component name: %s", name) + return name, nil + } + + // Strategy 3: Check metadata.component.bom-ref + if bomRef, ok := component["bom-ref"].(string); ok && bomRef != "" { + logger.Debug("Found bom-ref: %s", bomRef) + return bomRef, nil + } + } + } + + // Strategy 4: Check top-level name field + if name, ok := sbom["name"].(string); ok && name != "" { + logger.Debug("Found top-level name: %s", name) + return name, nil + } + + // Strategy 5: Use filename without extension + logger.Debug("Using fallback name: %s", filename) + return filename, nil +} + +// CollectComponentsWithSource extracts components from an SBOM and adds source tracking. +func CollectComponentsWithSource(sbomFile, sourceRef string) ([]map[string]interface{}, error) { + data, err := os.ReadFile(sbomFile) + if err != nil { + return nil, fmt.Errorf("failed to read SBOM file: %w", err) + } + + var sbom map[string]interface{} + if err := json.Unmarshal(data, &sbom); err != nil { + return nil, fmt.Errorf("failed to parse SBOM: %w", err) + } + + components, ok := sbom["components"].([]interface{}) + if !ok { + return []map[string]interface{}{}, nil + } + + result := make([]map[string]interface{}, 0, len(components)) + for _, comp := range components { + if compMap, ok := comp.(map[string]interface{}); ok { + // Add source tracking + compMap["source"] = sourceRef + result = append(result, compMap) + } + } + + logger.Debug("Collected %d components with source: %s", len(result), sourceRef) + return result, nil +} + +// DeduplicateComponents removes duplicate components based on name+version+purl+source. +func DeduplicateComponents(components []map[string]interface{}) []map[string]interface{} { + seen := make(map[string]bool) + unique := make([]map[string]interface{}, 0) + + for _, comp := range components { + name, _ := comp["name"].(string) + if name == "" { + name = "unknown" + } + + version, _ := comp["version"].(string) + if version == "" { + version = "unknown" + } + + purl, _ := comp["purl"].(string) + + source, _ := comp["source"].(string) + if source == "" { + source = "unknown" + } + + // Create unique key + key := fmt.Sprintf("%s@%s#%s^%s", name, version, purl, source) + + if !seen[key] { + seen[key] = true + unique = append(unique, comp) + } + } + + logger.Info("Deduplicated %d components down to %d unique components", len(components), len(unique)) + return unique +} + +// MergeSBOMs merges multiple CycloneDX SBOMs into one with source tracking. +func MergeSBOMs(inputFiles []string, outputFile string) error { + logger.Info("Merging %d CycloneDX SBOMs with source tracking", len(inputFiles)) + + if len(inputFiles) == 0 { + return fmt.Errorf("no input files provided") + } + + // Create merged SBOM metadata + timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z") + serialNumber := fmt.Sprintf("urn:uuid:%s", uuid.New().String()) + + mergedSBOM := map[string]interface{}{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": serialNumber, + "version": 1, + "metadata": map[string]interface{}{ + "timestamp": timestamp, + "tools": []map[string]interface{}{ + { + "vendor": "ClickBOM", + "name": "cyclonedx-merge", + "version": "2.0.0", + }, + }, + "component": map[string]interface{}{ + "type": "application", + "name": "merged-sbom", + "version": "1.0.0", + }, + }, + "components": []map[string]interface{}{}, + } + + // Collect all components with source tracking + allComponents := make([]map[string]interface{}, 0) + + for _, sbomFile := range inputFiles { + sourceRef, err := ExtractSourceReference(sbomFile) + if err != nil { + logger.Warning("Failed to extract source reference from %s: %v", filepath.Base(sbomFile), err) + sourceRef = filepath.Base(sbomFile) + } + + components, err := CollectComponentsWithSource(sbomFile, sourceRef) + if err != nil { + logger.Warning("Failed to collect components from %s: %v", filepath.Base(sbomFile), err) + continue + } + + logger.Info("Processing %s: %d components (source: %s)", + filepath.Base(sbomFile), len(components), sourceRef) + + allComponents = append(allComponents, components...) + } + + // Deduplicate components + uniqueComponents := DeduplicateComponents(allComponents) + mergedSBOM["components"] = uniqueComponents + + // Write merged SBOM to file + data, err := json.MarshalIndent(mergedSBOM, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal merged SBOM: %w", err) + } + + if err := os.WriteFile(outputFile, data, 0644); err != nil { + return fmt.Errorf("failed to write merged SBOM: %w", err) + } + + logger.Success("Successfully merged %d SBOMs into one with %d unique components", + len(inputFiles), len(uniqueComponents)) + + // Log summary + logger.Info("Merge summary with source tracking:") + for _, sbomFile := range inputFiles { + sourceRef, _ := ExtractSourceReference(sbomFile) + data, _ := os.ReadFile(sbomFile) + var sbom map[string]interface{} + err := json.Unmarshal(data, &sbom) + if err != nil { + continue + } + compCount := 0 + if components, ok := sbom["components"].([]interface{}); ok { + compCount = len(components) + } + logger.Info(" - %s: %d components (source: %s)", + strings.TrimSuffix(filepath.Base(sbomFile), ".json"), compCount, sourceRef) + } + + return nil +} diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 2bd2cba..a26a684 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "os" + "path/filepath" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -128,3 +130,36 @@ func (s *S3Client) ListObjects(ctx context.Context, bucket, prefix string) ([]st logger.Info("Found %d objects in S3", len(keys)) return keys, nil } + +// DownloadAll downloads all files from S3 bucket to local directory. +func (s *S3Client) DownloadAll(ctx context.Context, bucket, prefix, localDir string) ([]string, error) { + logger.Info("Downloading all files from s3://%s/%s", bucket, prefix) + + // List all objects + keys, err := s.ListObjects(ctx, bucket, prefix) + if err != nil { + return nil, err + } + + downloadedFiles := make([]string, 0) + + for _, key := range keys { + // Skip directories (keys ending with /) + if strings.HasSuffix(key, "/") { + continue + } + + filename := filepath.Base(key) + localPath := filepath.Join(localDir, filename) + + if err := s.Download(ctx, bucket, key, localPath); err != nil { + logger.Warning("Failed to download %s: %v", key, err) + continue + } + + downloadedFiles = append(downloadedFiles, localPath) + } + + logger.Info("Downloaded %d files", len(downloadedFiles)) + return downloadedFiles, nil +} From c995da9094c336b8cb0924d6541a07633dd7ccaa Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 23:23:59 -0500 Subject: [PATCH 25/28] feat: add merge Signed-off-by: Julio Jimenez --- cmd/clickbom/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/clickbom/main.go b/cmd/clickbom/main.go index 359d46d..53a7168 100644 --- a/cmd/clickbom/main.go +++ b/cmd/clickbom/main.go @@ -275,6 +275,9 @@ func handleClickHouse(ctx context.Context, cfg *config.Config, sbomFile string) } func generateTableName(cfg *config.Config) string { + if cfg.Merge { + return strings.ReplaceAll(cfg.S3Key, ".", "_") + } switch cfg.SBOMSource { case "github": return strings.ReplaceAll(strings.ToLower(cfg.Repository), "/", "_") From e7a5e5e0f0fd036d4012719cbd43bd9622b395a9 Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 23:24:55 -0500 Subject: [PATCH 26/28] feat: add merge Signed-off-by: Julio Jimenez --- cmd/clickbom/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/clickbom/main.go b/cmd/clickbom/main.go index 53a7168..9335347 100644 --- a/cmd/clickbom/main.go +++ b/cmd/clickbom/main.go @@ -276,7 +276,7 @@ func handleClickHouse(ctx context.Context, cfg *config.Config, sbomFile string) func generateTableName(cfg *config.Config) string { if cfg.Merge { - return strings.ReplaceAll(cfg.S3Key, ".", "_") + return fmt.Sprintf("merged_%s", strings.ReplaceAll(cfg.S3Key, ".", "_")) } switch cfg.SBOMSource { case "github": From 4547913aa66c7f27ddf4c2adb322fdcee02054bf Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 23:27:25 -0500 Subject: [PATCH 27/28] feat: add merge Signed-off-by: Julio Jimenez --- cmd/clickbom/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/clickbom/main.go b/cmd/clickbom/main.go index 9335347..4b004b3 100644 --- a/cmd/clickbom/main.go +++ b/cmd/clickbom/main.go @@ -276,7 +276,8 @@ func handleClickHouse(ctx context.Context, cfg *config.Config, sbomFile string) func generateTableName(cfg *config.Config) string { if cfg.Merge { - return fmt.Sprintf("merged_%s", strings.ReplaceAll(cfg.S3Key, ".", "_")) + replacer := strings.NewReplacer(".", "_", "-", "_") + return fmt.Sprintf("merged_%s", replacer.Replace(cfg.S3Key)) } switch cfg.SBOMSource { case "github": From 7b39d425ff9868c6c760d6748c6cdb472696dacf Mon Sep 17 00:00:00 2001 From: Julio Jimenez Date: Fri, 14 Nov 2025 23:45:43 -0500 Subject: [PATCH 28/28] fix: lint Signed-off-by: Julio Jimenez --- internal/sbom/merge.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sbom/merge.go b/internal/sbom/merge.go index ab89e49..da2c6f1 100644 --- a/internal/sbom/merge.go +++ b/internal/sbom/merge.go @@ -114,7 +114,7 @@ func DeduplicateComponents(components []map[string]interface{}) []map[string]int for _, comp := range components { name, _ := comp["name"].(string) if name == "" { - name = "unknown" + name = "unknown" // nolint:goconst } version, _ := comp["version"].(string)