[registry] Fix ArtifactHub sourceURL to persist actual package URL

generators/artifacthub/package_manager.go

@@ -2,6 +2,7 @@ package artifacthub
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/meshery/meshkit/generators/models"
 )
@@ -12,7 +13,20 @@ type ArtifactHubPackageManager struct {
 }
 
 func (ahpm ArtifactHubPackageManager) GetPackage() (models.Package, error) {
-	// get relevant packages
+	// Check if SourceURL is an actual URL (from a previous generation)
+	// If so, create a package directly from it instead of searching
+	if ahpm.SourceURL != "" && (strings.HasPrefix(ahpm.SourceURL, "http://") || strings.HasPrefix(ahpm.SourceURL, "https://") || strings.HasPrefix(ahpm.SourceURL, "oci://")) {
+		// SourceURL is an actual URL, use it directly
+		pkg := AhPackage{
+			Name:     ahpm.PackageName,
+			ChartUrl: ahpm.SourceURL,
+		}
+		// Try to extract version from the URL or fetch it
+		_ = pkg.UpdatePackageData() // This might fail but that's okay, ChartUrl is already set
+		return pkg, nil
+	}
+
+	// get relevant packages by searching with package name
 	pkgs, err := GetAhPackagesWithName(ahpm.PackageName)
 	if err != nil {
 		return nil, err

generators/artifacthub/package_test.go

@@ -71,3 +71,55 @@ func TestGetChartUrl(t *testing.T) {
 		})
 	}
 }
+
+// TestGetPackageWithDirectURL tests that when SourceURL is an actual URL,
+// it is used directly instead of searching
+func TestGetPackageWithDirectURL(t *testing.T) {
+	tests := []struct {
+		name        string
+		packageName string
+		sourceURL   string
+		wantErr     bool
+	}{
+		{
+			name:        "Direct HTTP URL should be used",
+			packageName: "consul",
+			sourceURL:   "https://charts.bitnami.com/bitnami/consul-1.0.0.tgz",
+			wantErr:     false,
+		},
+		{
+			name:        "Direct HTTPS URL should be used",
+			packageName: "test-package",
+			sourceURL:   "https://example.com/charts/test-1.0.0.tgz",
+			wantErr:     false,
+		},
+		{
+			name:        "OCI URL should be used",
+			packageName: "test-oci",
+			sourceURL:   "oci://registry.example.com/charts/test",
+			wantErr:     false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pm := ArtifactHubPackageManager{
+				PackageName: tt.packageName,
+				SourceURL:   tt.sourceURL,
+			}
+
+			pkg, err := pm.GetPackage()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetPackage() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if err == nil {
+				// Verify the package has the correct source URL
+				if pkg.GetSourceURL() != tt.sourceURL {
+					t.Errorf("GetSourceURL() = %v, want %v", pkg.GetSourceURL(), tt.sourceURL)
+				}
+			}
+		})
+	}
+}

generators/readme.md

@@ -0,0 +1,66 @@
+# Generators
+
+Generators are responsible for creating Meshery components from various sources like ArtifactHub and GitHub.
+
+## ArtifactHub Generator
+
+The ArtifactHub generator creates Meshery components by discovering Helm charts from [ArtifactHub](https://artifacthub.io/).
+
+### How It Works
+
+#### First Generation (New Model)
+
+When a model is generated for the first time with `registrant: artifacthub`:
+
+1. **CSV Input**: The model's `SourceURL` field contains a search query (e.g., `"consul"`)
+2. **Search ArtifactHub**: The generator searches ArtifactHub for packages matching the name
+3. **Package Selection**: The best package is selected based on ranking (verified publisher, CNCF, official, etc.)
+4. **Package Resolution**: The selected package's actual chart URL is obtained (e.g., `"https://charts.bitnami.com/bitnami/consul-1.0.0.tgz"`)
+5. **Component Generation**: Components are generated from the Helm chart's CRDs
+6. **Persistence**: 
+   - Model metadata `source_uri` is set to the actual package URL
+   - CSV model's `SourceURL` is updated to the actual package URL
+   - Model definition is written to filesystem
+   - Updated CSV is sent to spreadsheet with the resolved URL
+
+#### Subsequent Updates (Existing Model)
+
+When the model is updated in subsequent runs:
+
+1. **CSV Input**: The model's `SourceURL` field contains the actual package URL (e.g., `"https://charts.bitnami.com/bitnami/consul-1.0.0.tgz"`)
+2. **Direct URL Usage**: The generator detects the URL and uses it directly (no search needed)
+3. **Package Fetch**: The package is fetched from the URL
+4. **Component Generation**: Components are generated from the Helm chart's CRDs
+
+### Benefits
+
+- **Consistency**: The actual package URL is stored and used everywhere
+- **Efficiency**: Subsequent updates don't need to search ArtifactHub again
+- **Reliability**: Using direct URLs ensures we get the same package every time
+- **Backward Compatible**: Still supports search queries for new models
+
+### URL Detection
+
+The generator automatically detects if `SourceURL` is:
+- **A search query**: Any string that doesn't start with `http://`, `https://`, or `oci://`
+- **A direct URL**: Any string starting with `http://`, `https://`, or `oci://`
+
+### Code Structure
+
+```
+generators/
+├── artifacthub/
+│   ├── package_manager.go  # Main entry point, detects URL vs search query
+│   ├── scanner.go          # Searches ArtifactHub API
+│   ├── ranker.go           # Ranks packages by quality metrics
+│   └── package.go          # Handles package data and component generation
+└── generator.go            # Factory for creating generators
+```
+
+## GitHub Generator
+
+The GitHub generator creates Meshery components from GitHub repositories containing Kubernetes manifests or Helm charts.
+
+### Usage
+
+Set `registrant: github` in the model CSV and provide a GitHub URL in the `SourceURL` field.

registry/model.go

@@ -98,7 +98,7 @@ type ModelCSV struct {
 }
 
 var modelMetadataValues = []string{
-	"primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "capabilities", "isAnnotation", "shape",
+	"primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "capabilities", "isAnnotation", "shape", "sourceURL",
 }
 
 // keep
@@ -162,6 +162,11 @@ func (m *ModelCSV) UpdateModelDefinition(modelDef *_model.ModelDefinition) error
 				isAnnotation = true
 			}
 			metadata.IsAnnotation = &isAnnotation
+		case "sourceURL":
+			// Store SourceURL as source_uri in metadata for tracking the actual package URL
+			if m.SourceURL != "" {
+				metadata.AdditionalProperties["source_uri"] = m.SourceURL
+			}
 		default:
 			// For keys that do not have a direct mapping, store them in AdditionalProperties
 			metadata.AdditionalProperties[key] = modelMetadata[key]
@@ -623,17 +628,22 @@ func GenerateComponentsFromPkg(pkg models.Package, compDirPath string, defVersio
 		return 0, 0, err
 	}
 	lengthOfComps := len(comps)
+
+	// Set the source_uri in the model metadata to the actual package URL
+	if modelDef.Metadata == nil {
+		modelDef.Metadata = &_model.ModelDefinition_Metadata{}
+	}
+	if modelDef.Metadata.AdditionalProperties == nil {
+		modelDef.Metadata.AdditionalProperties = make(map[string]interface{})
+	}
+	// Get the actual source URL from the package
+	actualSourceURL := pkg.GetSourceURL()
+	if actualSourceURL != "" {
+		modelDef.Metadata.AdditionalProperties["source_uri"] = actualSourceURL
+	}
+
 	for _, comp := range comps {
 		comp.Version = defVersion
-		if modelDef.Metadata == nil {
-			modelDef.Metadata = &_model.ModelDefinition_Metadata{}
-		}
-		if modelDef.Metadata.AdditionalProperties == nil {
-			modelDef.Metadata.AdditionalProperties = make(map[string]interface{})
-		}
-		if comp.Model != nil && comp.Model.Metadata != nil && comp.Model.Metadata.AdditionalProperties != nil {
-			modelDef.Metadata.AdditionalProperties["source_uri"] = comp.Model.Metadata.AdditionalProperties["source_uri"]
-		}
 		comp.Model = &modelDef
 
 		AssignDefaultsForCompDefs(&comp, &modelDef)
@@ -865,6 +875,7 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co
 				return
 			}
 
+			// Use the SourceURL from the CSV, which now contains the actual package URL after first generation
 			generator, err := generators.NewGenerator(model.Registrant, model.SourceURL, model.Model)
 			if err != nil {
 				err = ErrGenerateModel(err, model.Model)
@@ -903,6 +914,15 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co
 				LogError.Error(err)
 				return
 			}
+
+			// Get the actual source URL from the package and update the CSV model's SourceURL
+			// This needs to happen before writeModelDefToFileSystem so the model is written with the correct source_uri
+			actualSourceURL := pkg.GetSourceURL()
+			if actualSourceURL != "" {
+				// Update the CSV model's SourceURL to the actual package URL for spreadsheet updates
+				model.SourceURL = actualSourceURL
+			}
+
 			modelDef, alreadyExist, err := writeModelDefToFileSystem(&model, version, modelDirPath)
 			if err != nil {
 				err = ErrGenerateModel(err, model.Model)
@@ -912,21 +932,12 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co
 			if alreadyExist {
 				totalAvailableModels--
 			}
+
 			for _, comp := range comps {
 				comp.Version = defVersion
 				// Assign the component status corresponding to model status.
 				// i.e., If model is enabled, comps are also "enabled". Ultimately, all individual comps will have the ability to control their status.
 				// The status "enabled" indicates that the component will be registered inside the registry.
-				if modelDef.Metadata == nil {
-					modelDef.Metadata = &_model.ModelDefinition_Metadata{}
-				}
-				if modelDef.Metadata.AdditionalProperties == nil {
-					modelDef.Metadata.AdditionalProperties = make(map[string]interface{})
-				}
-
-				if comp.Model != nil && comp.Model.Metadata != nil && comp.Model.Metadata.AdditionalProperties != nil {
-					modelDef.Metadata.AdditionalProperties["source_uri"] = comp.Model.Metadata.AdditionalProperties["source_uri"]
-				}
 				comp.Model = modelDef
 
 				AssignDefaultsForCompDefs(&comp, modelDef)
@@ -940,6 +951,7 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co
 					return
 				}
 			}
+
 			if !alreadyExist {
 				if len(comps) == 0 {
 					err = ErrGenerateModel(fmt.Errorf("no components found for model"), model.Model)