diff --git a/generators/artifacthub/package_manager.go b/generators/artifacthub/package_manager.go index a6615899..afc01e6b 100644 --- a/generators/artifacthub/package_manager.go +++ b/generators/artifacthub/package_manager.go @@ -2,6 +2,7 @@ package artifacthub import ( "fmt" + "strings" "github.com/meshery/meshkit/generators/models" ) @@ -12,7 +13,20 @@ type ArtifactHubPackageManager struct { } func (ahpm ArtifactHubPackageManager) GetPackage() (models.Package, error) { - // get relevant packages + // Check if SourceURL is an actual URL (from a previous generation) + // If so, create a package directly from it instead of searching + if ahpm.SourceURL != "" && (strings.HasPrefix(ahpm.SourceURL, "http://") || strings.HasPrefix(ahpm.SourceURL, "https://") || strings.HasPrefix(ahpm.SourceURL, "oci://")) { + // SourceURL is an actual URL, use it directly + pkg := AhPackage{ + Name: ahpm.PackageName, + ChartUrl: ahpm.SourceURL, + } + // Try to extract version from the URL or fetch it + _ = pkg.UpdatePackageData() // This might fail but that's okay, ChartUrl is already set + return pkg, nil + } + + // get relevant packages by searching with package name pkgs, err := GetAhPackagesWithName(ahpm.PackageName) if err != nil { return nil, err diff --git a/generators/artifacthub/package_test.go b/generators/artifacthub/package_test.go index 34b24938..b515c8c8 100644 --- a/generators/artifacthub/package_test.go +++ b/generators/artifacthub/package_test.go @@ -71,3 +71,55 @@ func TestGetChartUrl(t *testing.T) { }) } } + +// TestGetPackageWithDirectURL tests that when SourceURL is an actual URL, +// it is used directly instead of searching +func TestGetPackageWithDirectURL(t *testing.T) { + tests := []struct { + name string + packageName string + sourceURL string + wantErr bool + }{ + { + name: "Direct HTTP URL should be used", + packageName: "consul", + sourceURL: "https://charts.bitnami.com/bitnami/consul-1.0.0.tgz", + wantErr: false, + }, + { + name: "Direct HTTPS URL should be used", + packageName: "test-package", + sourceURL: "https://example.com/charts/test-1.0.0.tgz", + wantErr: false, + }, + { + name: "OCI URL should be used", + packageName: "test-oci", + sourceURL: "oci://registry.example.com/charts/test", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pm := ArtifactHubPackageManager{ + PackageName: tt.packageName, + SourceURL: tt.sourceURL, + } + + pkg, err := pm.GetPackage() + if (err != nil) != tt.wantErr { + t.Errorf("GetPackage() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if err == nil { + // Verify the package has the correct source URL + if pkg.GetSourceURL() != tt.sourceURL { + t.Errorf("GetSourceURL() = %v, want %v", pkg.GetSourceURL(), tt.sourceURL) + } + } + }) + } +} diff --git a/generators/readme.md b/generators/readme.md index e69de29b..c2c71c91 100644 --- a/generators/readme.md +++ b/generators/readme.md @@ -0,0 +1,66 @@ +# Generators + +Generators are responsible for creating Meshery components from various sources like ArtifactHub and GitHub. + +## ArtifactHub Generator + +The ArtifactHub generator creates Meshery components by discovering Helm charts from [ArtifactHub](https://artifacthub.io/). + +### How It Works + +#### First Generation (New Model) + +When a model is generated for the first time with `registrant: artifacthub`: + +1. **CSV Input**: The model's `SourceURL` field contains a search query (e.g., `"consul"`) +2. **Search ArtifactHub**: The generator searches ArtifactHub for packages matching the name +3. **Package Selection**: The best package is selected based on ranking (verified publisher, CNCF, official, etc.) +4. **Package Resolution**: The selected package's actual chart URL is obtained (e.g., `"https://charts.bitnami.com/bitnami/consul-1.0.0.tgz"`) +5. **Component Generation**: Components are generated from the Helm chart's CRDs +6. **Persistence**: + - Model metadata `source_uri` is set to the actual package URL + - CSV model's `SourceURL` is updated to the actual package URL + - Model definition is written to filesystem + - Updated CSV is sent to spreadsheet with the resolved URL + +#### Subsequent Updates (Existing Model) + +When the model is updated in subsequent runs: + +1. **CSV Input**: The model's `SourceURL` field contains the actual package URL (e.g., `"https://charts.bitnami.com/bitnami/consul-1.0.0.tgz"`) +2. **Direct URL Usage**: The generator detects the URL and uses it directly (no search needed) +3. **Package Fetch**: The package is fetched from the URL +4. **Component Generation**: Components are generated from the Helm chart's CRDs + +### Benefits + +- **Consistency**: The actual package URL is stored and used everywhere +- **Efficiency**: Subsequent updates don't need to search ArtifactHub again +- **Reliability**: Using direct URLs ensures we get the same package every time +- **Backward Compatible**: Still supports search queries for new models + +### URL Detection + +The generator automatically detects if `SourceURL` is: +- **A search query**: Any string that doesn't start with `http://`, `https://`, or `oci://` +- **A direct URL**: Any string starting with `http://`, `https://`, or `oci://` + +### Code Structure + +``` +generators/ +├── artifacthub/ +│ ├── package_manager.go # Main entry point, detects URL vs search query +│ ├── scanner.go # Searches ArtifactHub API +│ ├── ranker.go # Ranks packages by quality metrics +│ └── package.go # Handles package data and component generation +└── generator.go # Factory for creating generators +``` + +## GitHub Generator + +The GitHub generator creates Meshery components from GitHub repositories containing Kubernetes manifests or Helm charts. + +### Usage + +Set `registrant: github` in the model CSV and provide a GitHub URL in the `SourceURL` field. diff --git a/registry/model.go b/registry/model.go index 6a84abed..8ad0e764 100644 --- a/registry/model.go +++ b/registry/model.go @@ -98,7 +98,7 @@ type ModelCSV struct { } var modelMetadataValues = []string{ - "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "capabilities", "isAnnotation", "shape", + "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "capabilities", "isAnnotation", "shape", "sourceURL", } // keep @@ -162,6 +162,11 @@ func (m *ModelCSV) UpdateModelDefinition(modelDef *_model.ModelDefinition) error isAnnotation = true } metadata.IsAnnotation = &isAnnotation + case "sourceURL": + // Store SourceURL as source_uri in metadata for tracking the actual package URL + if m.SourceURL != "" { + metadata.AdditionalProperties["source_uri"] = m.SourceURL + } default: // For keys that do not have a direct mapping, store them in AdditionalProperties metadata.AdditionalProperties[key] = modelMetadata[key] @@ -623,17 +628,22 @@ func GenerateComponentsFromPkg(pkg models.Package, compDirPath string, defVersio return 0, 0, err } lengthOfComps := len(comps) + + // Set the source_uri in the model metadata to the actual package URL + if modelDef.Metadata == nil { + modelDef.Metadata = &_model.ModelDefinition_Metadata{} + } + if modelDef.Metadata.AdditionalProperties == nil { + modelDef.Metadata.AdditionalProperties = make(map[string]interface{}) + } + // Get the actual source URL from the package + actualSourceURL := pkg.GetSourceURL() + if actualSourceURL != "" { + modelDef.Metadata.AdditionalProperties["source_uri"] = actualSourceURL + } + for _, comp := range comps { comp.Version = defVersion - if modelDef.Metadata == nil { - modelDef.Metadata = &_model.ModelDefinition_Metadata{} - } - if modelDef.Metadata.AdditionalProperties == nil { - modelDef.Metadata.AdditionalProperties = make(map[string]interface{}) - } - if comp.Model != nil && comp.Model.Metadata != nil && comp.Model.Metadata.AdditionalProperties != nil { - modelDef.Metadata.AdditionalProperties["source_uri"] = comp.Model.Metadata.AdditionalProperties["source_uri"] - } comp.Model = &modelDef AssignDefaultsForCompDefs(&comp, &modelDef) @@ -865,6 +875,7 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co return } + // Use the SourceURL from the CSV, which now contains the actual package URL after first generation generator, err := generators.NewGenerator(model.Registrant, model.SourceURL, model.Model) if err != nil { err = ErrGenerateModel(err, model.Model) @@ -903,6 +914,15 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co LogError.Error(err) return } + + // Get the actual source URL from the package and update the CSV model's SourceURL + // This needs to happen before writeModelDefToFileSystem so the model is written with the correct source_uri + actualSourceURL := pkg.GetSourceURL() + if actualSourceURL != "" { + // Update the CSV model's SourceURL to the actual package URL for spreadsheet updates + model.SourceURL = actualSourceURL + } + modelDef, alreadyExist, err := writeModelDefToFileSystem(&model, version, modelDirPath) if err != nil { err = ErrGenerateModel(err, model.Model) @@ -912,21 +932,12 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co if alreadyExist { totalAvailableModels-- } + for _, comp := range comps { comp.Version = defVersion // Assign the component status corresponding to model status. // i.e., If model is enabled, comps are also "enabled". Ultimately, all individual comps will have the ability to control their status. // The status "enabled" indicates that the component will be registered inside the registry. - if modelDef.Metadata == nil { - modelDef.Metadata = &_model.ModelDefinition_Metadata{} - } - if modelDef.Metadata.AdditionalProperties == nil { - modelDef.Metadata.AdditionalProperties = make(map[string]interface{}) - } - - if comp.Model != nil && comp.Model.Metadata != nil && comp.Model.Metadata.AdditionalProperties != nil { - modelDef.Metadata.AdditionalProperties["source_uri"] = comp.Model.Metadata.AdditionalProperties["source_uri"] - } comp.Model = modelDef AssignDefaultsForCompDefs(&comp, modelDef) @@ -940,6 +951,7 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co return } } + if !alreadyExist { if len(comps) == 0 { err = ErrGenerateModel(fmt.Errorf("no components found for model"), model.Model)