From 563eb416a4eebda8b8206c40eb48a139c36bf053 Mon Sep 17 00:00:00 2001 From: Sima Arfania Date: Wed, 2 Jul 2025 22:35:43 -0400 Subject: [PATCH 01/15] feat: add auto-triage via Backstage --- .github/workflows/autoassign.yml | 87 ++++++++++++ scripts/backstage-lookup/go.mod | 3 + scripts/backstage-lookup/main.go | 225 +++++++++++++++++++++++++++++++ 3 files changed, 315 insertions(+) create mode 100644 .github/workflows/autoassign.yml create mode 100644 scripts/backstage-lookup/go.mod create mode 100644 scripts/backstage-lookup/main.go diff --git a/.github/workflows/autoassign.yml b/.github/workflows/autoassign.yml new file mode 100644 index 000000000..d8e759a86 --- /dev/null +++ b/.github/workflows/autoassign.yml @@ -0,0 +1,87 @@ +name: autoassign + +on: + issues: + types: [opened] + +permissions: + issues: write + contents: read + repository-projects: write + +jobs: + auto-assign: + runs-on: ubuntu-latest + if: contains(github.event.issue.body, 'Affected Resource(s)') + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.21' + + - name: Build backstage-lookup tool + run: | + cd scripts/backstage-lookup + go build -o ../../backstage-lookup + + - name: Parse issue form + uses: stefanbuck/github-issue-parser@v3 + id: issue-parser + with: + template-path: .github/ISSUE_TEMPLATE/3-bug-report-enhanced.yml + + - name: Lookup team ownership + id: backstage-lookup + env: + BACKSTAGE_URL: ${{ vars.BACKSTAGE_URL || 'https://enghub.grafana-ops.net' }} + TERRAFORM_AUTOMATION_TOKEN: ${{ secrets.TERRAFORM_AUTOMATION_TOKEN }} # TODO: confirm this works + run: | + RESOURCES=$(echo '${{ steps.issue-parser.outputs.jsonString }}' | jq -r '.["affected-resources"] // empty' | tr '\n' ' ') + + if [[ -z "$RESOURCES" ]]; then + echo "projects=" >> $GITHUB_OUTPUT + echo "teams=" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Resources: $RESOURCES" + ./backstage-lookup $RESOURCES >> $GITHUB_OUTPUT || { + echo "projects=" >> $GITHUB_OUTPUT + echo "teams=" >> $GITHUB_OUTPUT + } + + - name: Assign to projects and add labels + if: steps.backstage-lookup.outputs.teams != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Add to projects + for project in ${{ steps.backstage-lookup.outputs.projects }}; do + [[ -n "$project" ]] && gh project item-add $project --url ${{ github.event.issue.html_url }} + done + + # Add team labels + for team in ${{ steps.backstage-lookup.outputs.teams }}; do + [[ -n "$team" ]] && gh issue edit ${{ github.event.issue.number }} --add-label "team/$team" + done + + - name: Add comment + uses: actions/github-script@v7 + with: + script: | + const teams = '${{ steps.backstage-lookup.outputs.teams }}'.split(' ').filter(t => t); + const projects = '${{ steps.backstage-lookup.outputs.projects }}'.split(' ').filter(p => p); + + const message = teams.length > 0 + ? `🤖 **Auto-assigned to:** ${teams.map(t => `@grafana/${t}`).join(' ')}\n**Projects:** ${projects.join(', ') || 'none'}` + : `🔍 **Manual triage needed** - no team ownership found. Please mention \`@grafana/platform-monitoring\``; + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: message + }); diff --git a/scripts/backstage-lookup/go.mod b/scripts/backstage-lookup/go.mod new file mode 100644 index 000000000..717e35938 --- /dev/null +++ b/scripts/backstage-lookup/go.mod @@ -0,0 +1,3 @@ +module backstage-lookup + +go 1.21 diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go new file mode 100644 index 000000000..3dcf0f014 --- /dev/null +++ b/scripts/backstage-lookup/main.go @@ -0,0 +1,225 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "regexp" + "strings" + "time" +) + +// Constants for API and configuration +const ( + defaultURL = "https://enghub.grafana-ops.net" + defaultTimeout = 30 * time.Second + groupPrefix = "group:" +) + +// Environment variable names +const ( + EnvBackstageURL = "BACKSTAGE_URL" + EnvToken = "TERRAFORM_AUTOMATION_TOKEN" +) + +// API response types +type Component struct { + Spec struct { + Owner string `json:"owner"` + } `json:"spec"` +} + +type Group struct { + Metadata struct { + Links []struct { + Type string `json:"type"` + URL string `json:"url"` + } `json:"links"` + } `json:"metadata"` +} + +// Result represents the lookup result +type Result struct { + Projects []string + Teams []string +} + +// BackstageLookup handles API interactions with Backstage +type BackstageLookup struct { + client *http.Client + baseURL string + token string + projectRegex *regexp.Regexp +} + +// NewBackstageLookup creates a new Backstage API client +func NewBackstageLookup(baseURL, token string) *BackstageLookup { + if baseURL == "" { + baseURL = defaultURL + } + return &BackstageLookup{ + client: &http.Client{Timeout: defaultTimeout}, + baseURL: baseURL, + token: token, + projectRegex: regexp.MustCompile(`/projects/(\d+)`), + } +} + +// get performs an authenticated HTTP request to Backstage API +func (b *BackstageLookup) get(url string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Authorization", "Bearer "+b.token) + req.Header.Set("Accept", "application/json") + + resp, err := b.client.Do(req) + if err != nil { + return nil, fmt.Errorf("executing request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("status %d", resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} + +// findOwner retrieves the owner of a component by trying different prefixes and namespaces +func (b *BackstageLookup) findOwner(resource string) string { + endpoints := []string{ + "default/resource-", "default/datasource-", + "backstage-catalog/resource-", "backstage-catalog/datasource-", + } + + for _, endpoint := range endpoints { + url := fmt.Sprintf("%s/api/catalog/entities/by-name/component/%s%s", b.baseURL, endpoint, resource) + + data, err := b.get(url) + if err != nil { + continue + } + + var comp Component + if json.Unmarshal(data, &comp) == nil && comp.Spec.Owner != "" { + return comp.Spec.Owner + } + } + return "" +} + +// findProject retrieves GitHub project information for a group +func (b *BackstageLookup) findProject(namespace, team string) string { + url := fmt.Sprintf("%s/api/catalog/entities/by-name/group/%s/%s", b.baseURL, namespace, team) + + data, err := b.get(url) + if err != nil { + return "" + } + + var group Group + if json.Unmarshal(data, &group) != nil { + return "" + } + + for _, link := range group.Metadata.Links { + if link.Type == "github_project" { + if matches := b.projectRegex.FindStringSubmatch(link.URL); len(matches) >= 2 { + return matches[1] + } + } + } + return "" +} + +// parseOwner parses a group owner string into namespace and name +func parseOwner(owner string) (namespace, team string) { + if !strings.HasPrefix(owner, groupPrefix) { + return "", "" + } + + parts := strings.Split(strings.TrimPrefix(owner, groupPrefix), "/") + if len(parts) == 2 { + return parts[0], parts[1] + } + return "", "" +} + +// LookupResource looks up project and team information for a Terraform resource +func (b *BackstageLookup) LookupResource(resource string) (projects, teams []string) { + if resource == "Other (please describe in the issue)" { + return nil, nil + } + + log.Printf("Processing: %s", resource) + + owner := b.findOwner(resource) + if owner == "" { + log.Printf("No owner found for %s - manual triage needed", resource) + return nil, nil + } + + namespace, team := parseOwner(owner) + if namespace == "" || team == "" { + log.Printf("Invalid owner %s for %s - manual triage needed", owner, resource) + return nil, nil + } + + log.Printf("Found owner %s for %s", owner, resource) + + if project := b.findProject(namespace, team); project != "" { + log.Printf("Found project %s for team %s", project, team) + return []string{project}, []string{team} + } + + log.Printf("No project found for team %s", team) + return nil, []string{team} +} + +func unique(slice []string) []string { + seen := make(map[string]bool) + result := make([]string, 0, len(slice)) + for _, item := range slice { + if !seen[item] { + seen[item] = true + result = append(result, item) + } + } + return result +} + +func main() { + log.SetFlags(log.LstdFlags | log.Lshortfile) + + if len(os.Args) < 2 { + log.Fatal("Usage: backstage-lookup [resource2] ...") + } + + baseURL := os.Getenv("BACKSTAGE_URL") + token := os.Getenv("TERRAFORM_AUTOMATION_TOKEN") + if token == "" { + log.Fatal("TERRAFORM_AUTOMATION_TOKEN required") + } + + lookup := NewBackstageLookup(baseURL, token) + + var allProjects, allTeams []string + for _, resource := range os.Args[1:] { + if resource = strings.TrimSpace(resource); resource != "" { + // Clean resource name + resource = strings.TrimSuffix(strings.TrimSuffix(resource, " (resource)"), " (data source)") + projects, teams := lookup.LookupResource(resource) + allProjects = append(allProjects, projects...) + allTeams = append(allTeams, teams...) + } + } + + fmt.Printf("projects=%s\n", strings.Join(unique(allProjects), " ")) + fmt.Printf("teams=%s\n", strings.Join(unique(allTeams), " ")) +} From b55e769eb94fac3f2dafa1546f3f4b8d4b2483ee Mon Sep 17 00:00:00 2001 From: spinillos Date: Thu, 3 Jul 2025 09:30:57 +0200 Subject: [PATCH 02/15] Fix security issues --- .github/workflows/autoassign.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/autoassign.yml b/.github/workflows/autoassign.yml index d8e759a86..34f43747e 100644 --- a/.github/workflows/autoassign.yml +++ b/.github/workflows/autoassign.yml @@ -28,7 +28,7 @@ jobs: go build -o ../../backstage-lookup - name: Parse issue form - uses: stefanbuck/github-issue-parser@v3 + uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e # v3.2.1 id: issue-parser with: template-path: .github/ISSUE_TEMPLATE/3-bug-report-enhanced.yml @@ -57,15 +57,17 @@ jobs: if: steps.backstage-lookup.outputs.teams != '' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HTML_URL: ${{ github.event.issue.html_url }} + ISSUE_NUMBER: ${{ github.event.issue.number }} run: | # Add to projects for project in ${{ steps.backstage-lookup.outputs.projects }}; do - [[ -n "$project" ]] && gh project item-add $project --url ${{ github.event.issue.html_url }} + [[ -n "$project" ]] && gh project item-add $project --url ${HTML_URL} done # Add team labels for team in ${{ steps.backstage-lookup.outputs.teams }}; do - [[ -n "$team" ]] && gh issue edit ${{ github.event.issue.number }} --add-label "team/$team" + [[ -n "$team" ]] && gh issue edit ${ISSUE_NUMBER} --add-label "team/$team" done - name: Add comment From cfd91cbcd9f42e1fed3172efe7a49f9a9731d01c Mon Sep 17 00:00:00 2001 From: spinillos Date: Thu, 3 Jul 2025 09:34:28 +0200 Subject: [PATCH 03/15] Update hashes --- .github/workflows/autoassign.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/autoassign.yml b/.github/workflows/autoassign.yml index 34f43747e..a600e8293 100644 --- a/.github/workflows/autoassign.yml +++ b/.github/workflows/autoassign.yml @@ -15,10 +15,10 @@ jobs: if: contains(github.event.issue.body, 'Affected Resource(s)') steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version: '1.21' @@ -71,7 +71,7 @@ jobs: done - name: Add comment - uses: actions/github-script@v7 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 with: script: | const teams = '${{ steps.backstage-lookup.outputs.teams }}'.split(' ').filter(t => t); From 4d507193c228c5da0109debe80a714fdeac1d64f Mon Sep 17 00:00:00 2001 From: spinillos Date: Thu, 3 Jul 2025 09:37:39 +0200 Subject: [PATCH 04/15] Get secrets from vault --- .github/workflows/autoassign.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/autoassign.yml b/.github/workflows/autoassign.yml index a600e8293..0e29592f9 100644 --- a/.github/workflows/autoassign.yml +++ b/.github/workflows/autoassign.yml @@ -33,11 +33,18 @@ jobs: with: template-path: .github/ISSUE_TEMPLATE/3-bug-report-enhanced.yml + - name: Get Secrets + uses: grafana/shared-workflows/actions/get-vault-secrets@75804962c1ba608148988c1e2dc35fbb0ee21746 + with: + repo_secrets: | + BACKSTAGE_URL=backstage:backstage_url + TERRAFORM_AUTOMATION_TOKEN=backstage:terraform_automation_token + - name: Lookup team ownership id: backstage-lookup env: - BACKSTAGE_URL: ${{ vars.BACKSTAGE_URL || 'https://enghub.grafana-ops.net' }} - TERRAFORM_AUTOMATION_TOKEN: ${{ secrets.TERRAFORM_AUTOMATION_TOKEN }} # TODO: confirm this works + BACKSTAGE_URL: ${{ env.BACKSTAGE_URL }} + TERRAFORM_AUTOMATION_TOKEN: ${{ env.TERRAFORM_AUTOMATION_TOKEN }} # TODO: confirm this works run: | RESOURCES=$(echo '${{ steps.issue-parser.outputs.jsonString }}' | jq -r '.["affected-resources"] // empty' | tr '\n' ' ') From f7cf66ea68da6001f4a68251d869d3a3be2aa971 Mon Sep 17 00:00:00 2001 From: spinillos Date: Thu, 3 Jul 2025 10:09:16 +0200 Subject: [PATCH 05/15] Login with service account --- .github/workflows/autoassign.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/autoassign.yml b/.github/workflows/autoassign.yml index 0e29592f9..7f22c019a 100644 --- a/.github/workflows/autoassign.yml +++ b/.github/workflows/autoassign.yml @@ -33,6 +33,23 @@ jobs: with: template-path: .github/ISSUE_TEMPLATE/3-bug-report-enhanced.yml + - uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 + name: Auth with Backstage + id: gcloud-auth + with: + token_format: access_token + workload_identity_provider: "projects/304398677251/locations/global/workloadIdentityPools/github/providers/github-provider" + service_account: "github-terraform-provider-ci@grafanalabs-workload-identity.iam.gserviceaccount.com" + create_credentials_file: true + + - name: Login to GAR + if: github.event.action == 'opened' || github.event.action == 'synchronize' + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.gcloud-auth.outputs.access_token }} + - name: Get Secrets uses: grafana/shared-workflows/actions/get-vault-secrets@75804962c1ba608148988c1e2dc35fbb0ee21746 with: From f0726901066d9dcf98c7b6f086ea72572513bd8a Mon Sep 17 00:00:00 2001 From: spinillos Date: Thu, 3 Jul 2025 11:17:02 +0200 Subject: [PATCH 06/15] Remove login to gar --- .github/workflows/autoassign.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/autoassign.yml b/.github/workflows/autoassign.yml index 7f22c019a..74f95fe26 100644 --- a/.github/workflows/autoassign.yml +++ b/.github/workflows/autoassign.yml @@ -40,15 +40,6 @@ jobs: token_format: access_token workload_identity_provider: "projects/304398677251/locations/global/workloadIdentityPools/github/providers/github-provider" service_account: "github-terraform-provider-ci@grafanalabs-workload-identity.iam.gserviceaccount.com" - create_credentials_file: true - - - name: Login to GAR - if: github.event.action == 'opened' || github.event.action == 'synchronize' - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - registry: us-docker.pkg.dev - username: oauth2accesstoken - password: ${{ steps.gcloud-auth.outputs.access_token }} - name: Get Secrets uses: grafana/shared-workflows/actions/get-vault-secrets@75804962c1ba608148988c1e2dc35fbb0ee21746 From d85eb9899bd0621950dc0ea57b44b8e04ec8bd47 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 3 Jul 2025 14:00:48 +0200 Subject: [PATCH 07/15] fix(backstage-lookup): traverse relations to look for GitHub projects --- .../appplatform/catalog-resource.yaml | 4 ++-- scripts/backstage-lookup/main.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/resources/appplatform/catalog-resource.yaml b/internal/resources/appplatform/catalog-resource.yaml index c74d37727..75b91df11 100644 --- a/internal/resources/appplatform/catalog-resource.yaml +++ b/internal/resources/appplatform/catalog-resource.yaml @@ -9,7 +9,7 @@ metadata: spec: subcomponentOf: component:default/terraform-provider-grafana type: terraform-resource - owner: group:default/grafana-app-platform + owner: group:default/grafana-app-platform-squad lifecycle: production --- apiVersion: backstage.io/v1alpha1 @@ -22,5 +22,5 @@ metadata: spec: subcomponentOf: component:default/terraform-provider-grafana type: terraform-resource - owner: group:default/grafana-app-platform + owner: group:default/grafana-app-platform-squad lifecycle: production diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go index 3dcf0f014..a921ff4bb 100644 --- a/scripts/backstage-lookup/main.go +++ b/scripts/backstage-lookup/main.go @@ -39,6 +39,11 @@ type Group struct { URL string `json:"url"` } `json:"links"` } `json:"metadata"` + + Relations []struct { + Type string `json:"type"` + TargetRef string `json:"targetRef"` + } `json:"relations"` } // Result represents the lookup result @@ -135,6 +140,20 @@ func (b *BackstageLookup) findProject(namespace, team string) string { } } } + + // Walk through parentOf relations and return first match + // Background: the teams in group:default/ are synced from GitHub, we can't add arbitrary links to these, we have added the links to teams in group:backstage-catalog: instead. The teams in the backstage-catalog namespace refer to the GitHub teams as their parent. In theory multiple children could have a GitHub project, this loop returns the first match. + for _, relation := range group.Relations { + if relation.Type == "parentOf" { + fmt.Println(relation.TargetRef) + namespace, team := parseOwner(relation.TargetRef) + project := b.findProject(namespace, team) + if project != "" { + return project + } + } + } + return "" } From d8ed195a4d988ad86f8ec2cb59634c53e39f92d1 Mon Sep 17 00:00:00 2001 From: Sima Arfania Date: Thu, 3 Jul 2025 12:32:00 -0400 Subject: [PATCH 08/15] fix: source backstage URL from secrets vault --- .github/workflows/autoassign.yml | 2 +- scripts/backstage-lookup/main.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/autoassign.yml b/.github/workflows/autoassign.yml index 74f95fe26..1d1e78f5e 100644 --- a/.github/workflows/autoassign.yml +++ b/.github/workflows/autoassign.yml @@ -51,7 +51,7 @@ jobs: - name: Lookup team ownership id: backstage-lookup env: - BACKSTAGE_URL: ${{ env.BACKSTAGE_URL }} + BACKSTAGE_URL: ${{ env.BACKSTAGE_URL }} # TODO: confirm this works TERRAFORM_AUTOMATION_TOKEN: ${{ env.TERRAFORM_AUTOMATION_TOKEN }} # TODO: confirm this works run: | RESOURCES=$(echo '${{ steps.issue-parser.outputs.jsonString }}' | jq -r '.["affected-resources"] // empty' | tr '\n' ' ') diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go index a921ff4bb..83ed8554b 100644 --- a/scripts/backstage-lookup/main.go +++ b/scripts/backstage-lookup/main.go @@ -14,7 +14,6 @@ import ( // Constants for API and configuration const ( - defaultURL = "https://enghub.grafana-ops.net" defaultTimeout = 30 * time.Second groupPrefix = "group:" ) @@ -62,9 +61,6 @@ type BackstageLookup struct { // NewBackstageLookup creates a new Backstage API client func NewBackstageLookup(baseURL, token string) *BackstageLookup { - if baseURL == "" { - baseURL = defaultURL - } return &BackstageLookup{ client: &http.Client{Timeout: defaultTimeout}, baseURL: baseURL, @@ -221,6 +217,10 @@ func main() { } baseURL := os.Getenv("BACKSTAGE_URL") + if baseURL == "" { + log.Fatal("BACKSTAGE_URL required") + } + token := os.Getenv("TERRAFORM_AUTOMATION_TOKEN") if token == "" { log.Fatal("TERRAFORM_AUTOMATION_TOKEN required") From a563d41b12c20a727d0ef6c8ad60fa034294a80a Mon Sep 17 00:00:00 2001 From: spinillos Date: Thu, 3 Jul 2025 20:18:15 +0200 Subject: [PATCH 09/15] Pass access token to the script as env --- .github/workflows/autoassign.yml | 5 +++-- scripts/backstage-lookup/go.mod | 4 +++- scripts/backstage-lookup/go.sum | 0 scripts/backstage-lookup/main.go | 13 +++++++++---- 4 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 scripts/backstage-lookup/go.sum diff --git a/.github/workflows/autoassign.yml b/.github/workflows/autoassign.yml index 1d1e78f5e..9515908c2 100644 --- a/.github/workflows/autoassign.yml +++ b/.github/workflows/autoassign.yml @@ -46,13 +46,14 @@ jobs: with: repo_secrets: | BACKSTAGE_URL=backstage:backstage_url - TERRAFORM_AUTOMATION_TOKEN=backstage:terraform_automation_token + AUDIENCE=backstage:audience - name: Lookup team ownership id: backstage-lookup env: BACKSTAGE_URL: ${{ env.BACKSTAGE_URL }} # TODO: confirm this works - TERRAFORM_AUTOMATION_TOKEN: ${{ env.TERRAFORM_AUTOMATION_TOKEN }} # TODO: confirm this works + AUDIENCE: ${{ env.AUDIENCE }} # TODO: confirm this works + ACCESS_TOKEN: ${{ steps.gcloud-auth.outputs.access_token }} run: | RESOURCES=$(echo '${{ steps.issue-parser.outputs.jsonString }}' | jq -r '.["affected-resources"] // empty' | tr '\n' ' ') diff --git a/scripts/backstage-lookup/go.mod b/scripts/backstage-lookup/go.mod index 717e35938..a8136dfca 100644 --- a/scripts/backstage-lookup/go.mod +++ b/scripts/backstage-lookup/go.mod @@ -1,3 +1,5 @@ module backstage-lookup -go 1.21 +go 1.23.0 + +toolchain go1.24.4 diff --git a/scripts/backstage-lookup/go.sum b/scripts/backstage-lookup/go.sum new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go index 83ed8554b..281d708c9 100644 --- a/scripts/backstage-lookup/main.go +++ b/scripts/backstage-lookup/main.go @@ -221,12 +221,17 @@ func main() { log.Fatal("BACKSTAGE_URL required") } - token := os.Getenv("TERRAFORM_AUTOMATION_TOKEN") - if token == "" { - log.Fatal("TERRAFORM_AUTOMATION_TOKEN required") + audience := os.Getenv("AUDIENCE") + if audience == "" { + log.Fatal("AUDIENCE required") } - lookup := NewBackstageLookup(baseURL, token) + accessToken := os.Getenv("ACCESS_TOKEN") + if accessToken == "" { + log.Fatal("ACCESS_TOKEN required") + } + + lookup := NewBackstageLookup(baseURL, accessToken) var allProjects, allTeams []string for _, resource := range os.Args[1:] { From dea3970c2120a18b3e33bfb1ca578945d721f02c Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 10 Jul 2025 10:23:52 +0200 Subject: [PATCH 10/15] feat: add functions to add issues to GitHub projects --- scripts/backstage-lookup/README.md | 22 ++++++++ scripts/backstage-lookup/github.go | 88 ++++++++++++++++++++++++++++++ scripts/backstage-lookup/go.mod | 7 +++ scripts/backstage-lookup/go.sum | 6 ++ scripts/backstage-lookup/main.go | 15 +++-- 5 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 scripts/backstage-lookup/README.md create mode 100644 scripts/backstage-lookup/github.go diff --git a/scripts/backstage-lookup/README.md b/scripts/backstage-lookup/README.md new file mode 100644 index 000000000..a02e92a90 --- /dev/null +++ b/scripts/backstage-lookup/README.md @@ -0,0 +1,22 @@ +## Development + +Configure an access token for Backstage: + +```console +export ACCESS_TOKEN= +``` + +Set up a port-forward with kubectl or k9s: + +```console +kubectl port-forward -n backstage service/backstage-ingress 8080 +export BACKSTAGE_URL=http://localhost:8080 +``` + +Get a GitHub token with the 'project' scope: + +```console +gh auth login -s 'project' +export GITHUB_TOKEN=$(gh auth token) +``` + diff --git a/scripts/backstage-lookup/github.go b/scripts/backstage-lookup/github.go new file mode 100644 index 000000000..2a57e715d --- /dev/null +++ b/scripts/backstage-lookup/github.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "os" + + "github.com/shurcooL/githubv4" + "golang.org/x/oauth2" +) + +type GitHubClient struct { + Client *githubv4.Client +} + +func NewGitHubClient() GitHubClient { + src := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + ) + ctx := context.Background() + httpClient := oauth2.NewClient(ctx, src) + + return GitHubClient{ + Client: githubv4.NewClient(httpClient), + } +} + +func (g *GitHubClient) AddIssueToProject(org, repo string, issueNumber, projectNumber int) error { + ctx := context.Background() + + contentId, err := g.findIssueId(ctx, org, repo, issueNumber) + if err != nil { + return err + } + + projectId, err := g.findProjectId(ctx, org, projectNumber) + if err != nil { + return err + } + + return g.addIssueToProject(ctx, contentId, projectId) +} + +func (g *GitHubClient) findIssueId(ctx context.Context, org, repo string, number int) (string, error) { + var query struct { + Repository struct { + Issue struct { + Id string + } `graphql:"issue(number: $number)"` + } `graphql:"repository(owner: $owner, name: $name)"` + } + + err := g.Client.Query(ctx, &query, map[string]any{ + "owner": githubv4.String(org), + "name": githubv4.String(repo), + "number": githubv4.Int(number), + }) + return query.Repository.Issue.Id, err +} + +func (g *GitHubClient) findProjectId(ctx context.Context, org string, number int) (string, error) { + var query struct { + Organization struct { + ProjectV2 struct { + Id string + } `graphql:"projectV2(number: $number)"` + } `graphql:"organization(login: $owner)"` + } + + err := g.Client.Query(ctx, &query, map[string]any{ + "owner": githubv4.String(org), + "number": githubv4.Int(number), + }) + return query.Organization.ProjectV2.Id, err +} + +func (g *GitHubClient) addIssueToProject(ctx context.Context, contentId, projectId string) error { + var mutation struct { + AddProjectV2ItemById struct { + ClientMutationId string + } `graphql:"addProjectV2ItemById(input: $input)"` + } + input := githubv4.AddProjectV2ItemByIdInput{ + ContentID: githubv4.ID(contentId), + ProjectID: githubv4.ID(projectId), + } + + return g.Client.Mutate(ctx, &mutation, input, nil) +} diff --git a/scripts/backstage-lookup/go.mod b/scripts/backstage-lookup/go.mod index a8136dfca..b55a0a41e 100644 --- a/scripts/backstage-lookup/go.mod +++ b/scripts/backstage-lookup/go.mod @@ -3,3 +3,10 @@ module backstage-lookup go 1.23.0 toolchain go1.24.4 + +require ( + github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 + golang.org/x/oauth2 v0.30.0 +) + +require github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect diff --git a/scripts/backstage-lookup/go.sum b/scripts/backstage-lookup/go.sum index e69de29bb..96c5b5dae 100644 --- a/scripts/backstage-lookup/go.sum +++ b/scripts/backstage-lookup/go.sum @@ -0,0 +1,6 @@ +github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M= +github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= +github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go index 281d708c9..36949290b 100644 --- a/scripts/backstage-lookup/main.go +++ b/scripts/backstage-lookup/main.go @@ -221,10 +221,10 @@ func main() { log.Fatal("BACKSTAGE_URL required") } - audience := os.Getenv("AUDIENCE") - if audience == "" { - log.Fatal("AUDIENCE required") - } + //audience := os.Getenv("AUDIENCE") + //if audience == "" { + // log.Fatal("AUDIENCE required") + //} accessToken := os.Getenv("ACCESS_TOKEN") if accessToken == "" { @@ -246,4 +246,11 @@ func main() { fmt.Printf("projects=%s\n", strings.Join(unique(allProjects), " ")) fmt.Printf("teams=%s\n", strings.Join(unique(allTeams), " ")) + + client := NewGitHubClient() + + err := client.AddIssueToProject("grafana", "terraform-provider-grafana", 2239, 513) + if err != nil { + panic(err) + } } From 92685f867b7940476aebdbb94ed79af666ecb94c Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 10 Jul 2025 16:28:34 +0200 Subject: [PATCH 11/15] refactor: use go-backstage lib instead of homegrown --- scripts/backstage-lookup/backstage.go | 182 ++++++++++++++++++ scripts/backstage-lookup/github.go | 82 +++++++- scripts/backstage-lookup/go.mod | 4 +- scripts/backstage-lookup/go.sum | 16 ++ scripts/backstage-lookup/main.go | 261 +++++--------------------- 5 files changed, 323 insertions(+), 222 deletions(-) create mode 100644 scripts/backstage-lookup/backstage.go diff --git a/scripts/backstage-lookup/backstage.go b/scripts/backstage-lookup/backstage.go new file mode 100644 index 000000000..2bfccf56f --- /dev/null +++ b/scripts/backstage-lookup/backstage.go @@ -0,0 +1,182 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "regexp" + "strings" + + "github.com/datolabs-io/go-backstage/v3" + "github.com/mitchellh/mapstructure" + "golang.org/x/oauth2" +) + +type BackstageClient struct { + Client *backstage.Client + Filters func(string) []string +} + +func NewBackstageClient() (*BackstageClient, error) { + baseURL := os.Getenv("BACKSTAGE_URL") + if baseURL == "" { + return nil, fmt.Errorf("BACKSTAGE_URL required") + } + + accessToken := os.Getenv("BACKSTAGE_TOKEN") + if accessToken == "" { + return nil, fmt.Errorf("BACKSTAGE_TOKEN required") + } + + src := oauth2.StaticTokenSource( + &oauth2.Token{ + AccessToken: accessToken, + TokenType: "Bearer", + }) + ctx := context.Background() + httpClient := oauth2.NewClient(ctx, src) + + client, err := backstage.NewClient(baseURL, "default", httpClient) + if err != nil { + return nil, err + } + + return &BackstageClient{ + Client: client, + Filters: func(resourceName string) []string { + return []string{ + fmt.Sprintf("kind=Component,metadata.name=%s", resourceName), + } + }, + }, nil +} + +func (b *BackstageClient) FindProjectsForResource(resourceName string) ([]string, error) { + resources, err := b.findComponents(resourceName) + if err != nil { + return nil, err + } + if len(resources) > 1 { + log.Printf("Multiple components found, using first %s.", resources[0].Metadata.Name) + } + + projects, err := b.findProjectsForGroup(resources[0].Spec.Owner) + if err != nil { + return nil, err + } + + if len(projects) == 0 { + return nil, fmt.Errorf("FindProjectForResource: no projects found") + } + + // URL must look like https://github.com/orgs//projects/ + re := regexp.MustCompile(`https://github.com/orgs/.*/projects/(\d+).*`) + + var ids []string + for _, project := range projects { + ids = append(ids, string(re.FindSubmatch([]byte(project))[1])) + } + return ids, nil +} + +func (b *BackstageClient) findComponents(resourceName string) ([]backstage.ComponentEntityV1alpha1, error) { + ctx := context.Background() + entities, _, err := b.Client.Catalog.Entities.List(ctx, &backstage.ListEntityOptions{ + Filters: b.Filters(resourceName), + }) + if err != nil { + return nil, err + } + if len(entities) == 0 { + return nil, fmt.Errorf("findComponents: No entities found.") + } + if len(entities) > 1 { + log.Printf("Multiple entities found.") + } + + components := make([]backstage.ComponentEntityV1alpha1, len(entities)) + if err := mapstructure.Decode(entities, &components); err != nil { + return nil, err + } + + return components, nil +} + +func (b *BackstageClient) findGroupByRef(ref string) (*backstage.GroupEntityV1alpha1, error) { + entityRef, err := parseEntityRef(ref) + if err != nil { + return nil, err + } + + ctx := context.Background() + entities, _, err := b.Client.Catalog.Entities.List(ctx, &backstage.ListEntityOptions{ + Filters: []string{ + fmt.Sprintf("kind=Group,metadata.name=%s,metadata.namespace=%s", entityRef.Name, entityRef.Namespace), + }, + }) + if err != nil { + return nil, err + } + if len(entities) == 0 { + return nil, fmt.Errorf("findGroupByRef: No entities found.") + } + if len(entities) > 1 { + return nil, fmt.Errorf("findGroupByRef: Multiple entities found.") + } + var group backstage.GroupEntityV1alpha1 + if err := mapstructure.Decode(entities[0], &group); err != nil { + return nil, err + } + + group.Metadata = entities[0].Metadata + group.Relations = entities[0].Relations + + return &group, nil +} + +func (b *BackstageClient) findProjectsForGroup(groupRef string) ([]string, error) { + group, err := b.findGroupByRef(groupRef) + if err != nil { + return nil, err + } + + var githubProjects []string + for _, link := range group.Metadata.Links { + if link.Type == "github_project" { + githubProjects = append(githubProjects, link.URL) + } + } + if len(githubProjects) == 0 { + for _, relation := range group.Relations { + if relation.Type == "parentOf" { + projects, _ := b.findProjectsForGroup(relation.TargetRef) + githubProjects = append(githubProjects, projects...) + } + } + } + return githubProjects, nil +} + +type EntityRef struct { + Kind string + Namespace string + Name string +} + +func parseEntityRef(ref string) (*EntityRef, error) { + kindParts := strings.Split(ref, ":") + if len(kindParts) != 2 { + return nil, fmt.Errorf("Could not parse entityRef.") + } + + parts := strings.Split(kindParts[1], "/") + if len(parts) != 2 { + return nil, fmt.Errorf("Could not parse entityRef.") + } + return &EntityRef{ + Kind: kindParts[0], + Namespace: parts[0], + Name: parts[1], + }, nil +} diff --git a/scripts/backstage-lookup/github.go b/scripts/backstage-lookup/github.go index 2a57e715d..b54119fbb 100644 --- a/scripts/backstage-lookup/github.go +++ b/scripts/backstage-lookup/github.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "os" "github.com/shurcooL/githubv4" @@ -12,16 +13,20 @@ type GitHubClient struct { Client *githubv4.Client } -func NewGitHubClient() GitHubClient { +func NewGitHubClient() (*GitHubClient, error) { + accessToken := os.Getenv("GITHUB_TOKEN") + if accessToken == "" { + return nil, fmt.Errorf("GITHUB_TOKEN required") + } src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + &oauth2.Token{AccessToken: accessToken}, ) ctx := context.Background() httpClient := oauth2.NewClient(ctx, src) - return GitHubClient{ + return &GitHubClient{ Client: githubv4.NewClient(httpClient), - } + }, nil } func (g *GitHubClient) AddIssueToProject(org, repo string, issueNumber, projectNumber int) error { @@ -40,6 +45,21 @@ func (g *GitHubClient) AddIssueToProject(org, repo string, issueNumber, projectN return g.addIssueToProject(ctx, contentId, projectId) } +func (g *GitHubClient) RemoveIssueFromProject(org, repo string, issueNumber, projectNumber int) error { + ctx := context.Background() + + itemId, projectId, err := g.findProjectItemId(ctx, org, repo, projectNumber, issueNumber) + if err != nil { + return err + } + + if itemId == "" { + return nil + } + + return g.removeIssueFromProject(ctx, itemId, projectId) +} + func (g *GitHubClient) findIssueId(ctx context.Context, org, repo string, number int) (string, error) { var query struct { Repository struct { @@ -57,6 +77,45 @@ func (g *GitHubClient) findIssueId(ctx context.Context, org, repo string, number return query.Repository.Issue.Id, err } +func (g *GitHubClient) findProjectItemId(ctx context.Context, org, repo string, projectNumber, issueNumber int) (string, string, error) { + var query struct { + Repository struct { + Issue struct { + ProjectItems struct { + Edges []struct { + Node struct { + Id string + ProjectV2Item struct { + Project struct { + Id string + Number int + } + } `graphql:"... on ProjectV2Item"` + } + } + } `graphql:"projectItems(first: 100)"` // assumes issues don't have more than 100 projects + } `graphql:"issue(number: $number)"` + } `graphql:"repository(owner: $owner, name: $name)"` + } + + if err := g.Client.Query(ctx, &query, map[string]any{ + "owner": githubv4.String(org), + "name": githubv4.String(repo), + "number": githubv4.Int(issueNumber), + }); err != nil { + return "", "", err + } + + for _, edge := range query.Repository.Issue.ProjectItems.Edges { + if edge.Node.ProjectV2Item.Project.Number == projectNumber { + return edge.Node.Id, edge.Node.ProjectV2Item.Project.Id, nil + } + } + + // not sure if we should return an error, not finding a project could be expected + return "", "", nil //fmt.Errorf("findProjectItemId: issue not found on project") +} + func (g *GitHubClient) findProjectId(ctx context.Context, org string, number int) (string, error) { var query struct { Organization struct { @@ -86,3 +145,18 @@ func (g *GitHubClient) addIssueToProject(ctx context.Context, contentId, project return g.Client.Mutate(ctx, &mutation, input, nil) } + +func (g *GitHubClient) removeIssueFromProject(ctx context.Context, itemId, projectId string) error { + var mutation struct { + DeleteProjectV2Item struct { + ClientMutationId string + } `graphql:"deleteProjectV2Item(input: $input)"` + } + + input := githubv4.DeleteProjectV2ItemInput{ + ItemID: githubv4.ID(itemId), + ProjectID: githubv4.ID(projectId), + } + + return g.Client.Mutate(ctx, &mutation, input, nil) +} diff --git a/scripts/backstage-lookup/go.mod b/scripts/backstage-lookup/go.mod index b55a0a41e..dac6ca302 100644 --- a/scripts/backstage-lookup/go.mod +++ b/scripts/backstage-lookup/go.mod @@ -1,10 +1,12 @@ module backstage-lookup -go 1.23.0 +go 1.24.0 toolchain go1.24.4 require ( + github.com/datolabs-io/go-backstage/v3 v3.1.0 + github.com/mitchellh/mapstructure v1.5.0 github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 golang.org/x/oauth2 v0.30.0 ) diff --git a/scripts/backstage-lookup/go.sum b/scripts/backstage-lookup/go.sum index 96c5b5dae..af0d84cf6 100644 --- a/scripts/backstage-lookup/go.sum +++ b/scripts/backstage-lookup/go.sum @@ -1,6 +1,22 @@ +github.com/datolabs-io/go-backstage/v3 v3.1.0 h1:gkcYDsss1DAEpN3p/nIQCY9dpMmDsG3YgRJTZZje5j4= +github.com/datolabs-io/go-backstage/v3 v3.1.0/go.mod h1:8Xt7Q+A8dUQvgidXII+Wj6UmRmbuj/YPiAzUbzeRcvY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE= +github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M= github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go index 36949290b..3d435bd58 100644 --- a/scripts/backstage-lookup/main.go +++ b/scripts/backstage-lookup/main.go @@ -1,202 +1,14 @@ package main import ( - "encoding/json" "fmt" - "io" "log" - "net/http" "os" - "regexp" + "slices" + "strconv" "strings" - "time" ) -// Constants for API and configuration -const ( - defaultTimeout = 30 * time.Second - groupPrefix = "group:" -) - -// Environment variable names -const ( - EnvBackstageURL = "BACKSTAGE_URL" - EnvToken = "TERRAFORM_AUTOMATION_TOKEN" -) - -// API response types -type Component struct { - Spec struct { - Owner string `json:"owner"` - } `json:"spec"` -} - -type Group struct { - Metadata struct { - Links []struct { - Type string `json:"type"` - URL string `json:"url"` - } `json:"links"` - } `json:"metadata"` - - Relations []struct { - Type string `json:"type"` - TargetRef string `json:"targetRef"` - } `json:"relations"` -} - -// Result represents the lookup result -type Result struct { - Projects []string - Teams []string -} - -// BackstageLookup handles API interactions with Backstage -type BackstageLookup struct { - client *http.Client - baseURL string - token string - projectRegex *regexp.Regexp -} - -// NewBackstageLookup creates a new Backstage API client -func NewBackstageLookup(baseURL, token string) *BackstageLookup { - return &BackstageLookup{ - client: &http.Client{Timeout: defaultTimeout}, - baseURL: baseURL, - token: token, - projectRegex: regexp.MustCompile(`/projects/(\d+)`), - } -} - -// get performs an authenticated HTTP request to Backstage API -func (b *BackstageLookup) get(url string) ([]byte, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("creating request: %w", err) - } - - req.Header.Set("Authorization", "Bearer "+b.token) - req.Header.Set("Accept", "application/json") - - resp, err := b.client.Do(req) - if err != nil { - return nil, fmt.Errorf("executing request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("status %d", resp.StatusCode) - } - - return io.ReadAll(resp.Body) -} - -// findOwner retrieves the owner of a component by trying different prefixes and namespaces -func (b *BackstageLookup) findOwner(resource string) string { - endpoints := []string{ - "default/resource-", "default/datasource-", - "backstage-catalog/resource-", "backstage-catalog/datasource-", - } - - for _, endpoint := range endpoints { - url := fmt.Sprintf("%s/api/catalog/entities/by-name/component/%s%s", b.baseURL, endpoint, resource) - - data, err := b.get(url) - if err != nil { - continue - } - - var comp Component - if json.Unmarshal(data, &comp) == nil && comp.Spec.Owner != "" { - return comp.Spec.Owner - } - } - return "" -} - -// findProject retrieves GitHub project information for a group -func (b *BackstageLookup) findProject(namespace, team string) string { - url := fmt.Sprintf("%s/api/catalog/entities/by-name/group/%s/%s", b.baseURL, namespace, team) - - data, err := b.get(url) - if err != nil { - return "" - } - - var group Group - if json.Unmarshal(data, &group) != nil { - return "" - } - - for _, link := range group.Metadata.Links { - if link.Type == "github_project" { - if matches := b.projectRegex.FindStringSubmatch(link.URL); len(matches) >= 2 { - return matches[1] - } - } - } - - // Walk through parentOf relations and return first match - // Background: the teams in group:default/ are synced from GitHub, we can't add arbitrary links to these, we have added the links to teams in group:backstage-catalog: instead. The teams in the backstage-catalog namespace refer to the GitHub teams as their parent. In theory multiple children could have a GitHub project, this loop returns the first match. - for _, relation := range group.Relations { - if relation.Type == "parentOf" { - fmt.Println(relation.TargetRef) - namespace, team := parseOwner(relation.TargetRef) - project := b.findProject(namespace, team) - if project != "" { - return project - } - } - } - - return "" -} - -// parseOwner parses a group owner string into namespace and name -func parseOwner(owner string) (namespace, team string) { - if !strings.HasPrefix(owner, groupPrefix) { - return "", "" - } - - parts := strings.Split(strings.TrimPrefix(owner, groupPrefix), "/") - if len(parts) == 2 { - return parts[0], parts[1] - } - return "", "" -} - -// LookupResource looks up project and team information for a Terraform resource -func (b *BackstageLookup) LookupResource(resource string) (projects, teams []string) { - if resource == "Other (please describe in the issue)" { - return nil, nil - } - - log.Printf("Processing: %s", resource) - - owner := b.findOwner(resource) - if owner == "" { - log.Printf("No owner found for %s - manual triage needed", resource) - return nil, nil - } - - namespace, team := parseOwner(owner) - if namespace == "" || team == "" { - log.Printf("Invalid owner %s for %s - manual triage needed", owner, resource) - return nil, nil - } - - log.Printf("Found owner %s for %s", owner, resource) - - if project := b.findProject(namespace, team); project != "" { - log.Printf("Found project %s for team %s", project, team) - return []string{project}, []string{team} - } - - log.Printf("No project found for team %s", team) - return nil, []string{team} -} - func unique(slice []string) []string { seen := make(map[string]bool) result := make([]string, 0, len(slice)) @@ -212,45 +24,60 @@ func unique(slice []string) []string { func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) - if len(os.Args) < 2 { - log.Fatal("Usage: backstage-lookup [resource2] ...") + if len(os.Args) < 3 { + log.Fatal("Usage: backstage-lookup [resource2] ...") } - baseURL := os.Getenv("BACKSTAGE_URL") - if baseURL == "" { - log.Fatal("BACKSTAGE_URL required") + backstage, err := NewBackstageClient() + if err != nil { + log.Fatal(err) } - - //audience := os.Getenv("AUDIENCE") - //if audience == "" { - // log.Fatal("AUDIENCE required") - //} - - accessToken := os.Getenv("ACCESS_TOKEN") - if accessToken == "" { - log.Fatal("ACCESS_TOKEN required") + backstage.Filters = func(resourceName string) []string { + return []string{ + fmt.Sprintf("kind=Component,metadata.name=resource-%s", resourceName), + fmt.Sprintf("kind=Component,metadata.name=datasource-%s", resourceName), + } } - lookup := NewBackstageLookup(baseURL, accessToken) + issueNumber, err := strconv.Atoi(os.Args[1]) + if err != nil { + log.Fatal(err) + } - var allProjects, allTeams []string - for _, resource := range os.Args[1:] { + var allProjects []string + for _, resource := range os.Args[2:] { if resource = strings.TrimSpace(resource); resource != "" { - // Clean resource name - resource = strings.TrimSuffix(strings.TrimSuffix(resource, " (resource)"), " (data source)") - projects, teams := lookup.LookupResource(resource) + projects, err := backstage.FindProjectsForResource(resource) + if err != nil { + log.Fatal(err) + } allProjects = append(allProjects, projects...) - allTeams = append(allTeams, teams...) } } - fmt.Printf("projects=%s\n", strings.Join(unique(allProjects), " ")) - fmt.Printf("teams=%s\n", strings.Join(unique(allTeams), " ")) - - client := NewGitHubClient() + allProjects = unique(allProjects) + fmt.Printf("Assigning issue #%d to projects=%s\n", issueNumber, strings.Join(allProjects, " ")) - err := client.AddIssueToProject("grafana", "terraform-provider-grafana", 2239, 513) + github, err := NewGitHubClient() if err != nil { - panic(err) + log.Fatal(err) + } + + // If the resource is not owned by monitoring and there are other projects claiming ownership, then remove monitoring. + resourceIsOwnedByPlatformMonitoring := -1 != slices.IndexFunc(allProjects, func(p string) bool { return p == "513" }) + if len(allProjects) > 0 && !resourceIsOwnedByPlatformMonitoring { + if err := github.RemoveIssueFromProject("grafana", "terraform-provider-grafana", issueNumber, 513); err != nil { + log.Fatal(err) + } + } + + for _, projectNumber := range allProjects { + projectNumberInt, err := strconv.Atoi(projectNumber) + if err != nil { + log.Fatal(err) + } + if err := github.AddIssueToProject("grafana", "terraform-provider-grafana", issueNumber, projectNumberInt); err != nil { + log.Fatal(err) + } } } From 574b39362d7fb036815146e9d53c741583245560 Mon Sep 17 00:00:00 2001 From: Duologic Date: Thu, 10 Jul 2025 16:29:27 +0200 Subject: [PATCH 12/15] docs: fix variable name --- scripts/backstage-lookup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/backstage-lookup/README.md b/scripts/backstage-lookup/README.md index a02e92a90..3f0ac513e 100644 --- a/scripts/backstage-lookup/README.md +++ b/scripts/backstage-lookup/README.md @@ -3,7 +3,7 @@ Configure an access token for Backstage: ```console -export ACCESS_TOKEN= +export BACKSTAGE_TOKEN= ``` Set up a port-forward with kubectl or k9s: From 51f823eb4bb8e510d75124398974de8201198c2d Mon Sep 17 00:00:00 2001 From: Sima Arfania Date: Thu, 10 Jul 2025 16:02:09 -0400 Subject: [PATCH 13/15] feat: add verbose error handling --- scripts/backstage-lookup/main.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go index 3d435bd58..418128b0b 100644 --- a/scripts/backstage-lookup/main.go +++ b/scripts/backstage-lookup/main.go @@ -47,9 +47,11 @@ func main() { var allProjects []string for _, resource := range os.Args[2:] { if resource = strings.TrimSpace(resource); resource != "" { + fmt.Printf("Looking up resource: %s\n", resource) projects, err := backstage.FindProjectsForResource(resource) if err != nil { - log.Fatal(err) + log.Printf("Warning: failed to find projects for resource %s: %v", resource, err) + continue } allProjects = append(allProjects, projects...) } @@ -66,18 +68,21 @@ func main() { // If the resource is not owned by monitoring and there are other projects claiming ownership, then remove monitoring. resourceIsOwnedByPlatformMonitoring := -1 != slices.IndexFunc(allProjects, func(p string) bool { return p == "513" }) if len(allProjects) > 0 && !resourceIsOwnedByPlatformMonitoring { + fmt.Printf("Removing issue #%d from platform-monitoring project (513)\n", issueNumber) if err := github.RemoveIssueFromProject("grafana", "terraform-provider-grafana", issueNumber, 513); err != nil { - log.Fatal(err) + log.Printf("Warning: failed to remove from platform-monitoring project: %v", err) } } for _, projectNumber := range allProjects { projectNumberInt, err := strconv.Atoi(projectNumber) if err != nil { - log.Fatal(err) + log.Printf("Warning: invalid project number %s: %v", projectNumber, err) + continue } + fmt.Printf("Adding issue #%d to project %d\n", issueNumber, projectNumberInt) if err := github.AddIssueToProject("grafana", "terraform-provider-grafana", issueNumber, projectNumberInt); err != nil { - log.Fatal(err) + log.Printf("Warning: failed to add to project %d: %v", projectNumberInt, err) } } } From 935113376cb03fb4abf17925767a243fce3a810c Mon Sep 17 00:00:00 2001 From: Duologic Date: Fri, 11 Jul 2025 14:59:08 +0200 Subject: [PATCH 14/15] feat: add dry-run flag --- scripts/backstage-lookup/main.go | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go index 418128b0b..b65b55fa9 100644 --- a/scripts/backstage-lookup/main.go +++ b/scripts/backstage-lookup/main.go @@ -1,9 +1,9 @@ package main import ( + "flag" "fmt" "log" - "os" "slices" "strconv" "strings" @@ -24,7 +24,17 @@ func unique(slice []string) []string { func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) - if len(os.Args) < 3 { + dryRun := flag.Bool("dry-run", true, "Dry-run prints the intended actions.") + + flag.Parse() + + if *dryRun { + fmt.Println("Dry-run: use --dry-run=false to turn off") + } + + positionalArgs := flag.Args() + + if len(positionalArgs) < 2 { log.Fatal("Usage: backstage-lookup [resource2] ...") } @@ -39,13 +49,13 @@ func main() { } } - issueNumber, err := strconv.Atoi(os.Args[1]) + issueNumber, err := strconv.Atoi(positionalArgs[0]) if err != nil { log.Fatal(err) } var allProjects []string - for _, resource := range os.Args[2:] { + for _, resource := range positionalArgs[1:] { if resource = strings.TrimSpace(resource); resource != "" { fmt.Printf("Looking up resource: %s\n", resource) projects, err := backstage.FindProjectsForResource(resource) @@ -58,6 +68,7 @@ func main() { } allProjects = unique(allProjects) + fmt.Printf("Assigning issue #%d to projects=%s\n", issueNumber, strings.Join(allProjects, " ")) github, err := NewGitHubClient() @@ -69,8 +80,10 @@ func main() { resourceIsOwnedByPlatformMonitoring := -1 != slices.IndexFunc(allProjects, func(p string) bool { return p == "513" }) if len(allProjects) > 0 && !resourceIsOwnedByPlatformMonitoring { fmt.Printf("Removing issue #%d from platform-monitoring project (513)\n", issueNumber) - if err := github.RemoveIssueFromProject("grafana", "terraform-provider-grafana", issueNumber, 513); err != nil { - log.Printf("Warning: failed to remove from platform-monitoring project: %v", err) + if !*dryRun { + if err := github.RemoveIssueFromProject("grafana", "terraform-provider-grafana", issueNumber, 513); err != nil { + log.Printf("Warning: failed to remove from platform-monitoring project: %v", err) + } } } @@ -81,8 +94,10 @@ func main() { continue } fmt.Printf("Adding issue #%d to project %d\n", issueNumber, projectNumberInt) - if err := github.AddIssueToProject("grafana", "terraform-provider-grafana", issueNumber, projectNumberInt); err != nil { - log.Printf("Warning: failed to add to project %d: %v", projectNumberInt, err) + if !*dryRun { + if err := github.AddIssueToProject("grafana", "terraform-provider-grafana", issueNumber, projectNumberInt); err != nil { + log.Printf("Warning: failed to add to project %d: %v", projectNumberInt, err) + } } } } From c00e772a526d90f780c128f72df2a5366296fa23 Mon Sep 17 00:00:00 2001 From: Duologic Date: Fri, 1 Aug 2025 13:36:10 +0200 Subject: [PATCH 15/15] fix: allow overriding the groupRef --- scripts/backstage-lookup/backstage.go | 19 +++++++++++-------- scripts/backstage-lookup/main.go | 23 ++++++++++++++--------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/scripts/backstage-lookup/backstage.go b/scripts/backstage-lookup/backstage.go index 2bfccf56f..125313d75 100644 --- a/scripts/backstage-lookup/backstage.go +++ b/scripts/backstage-lookup/backstage.go @@ -52,16 +52,19 @@ func NewBackstageClient() (*BackstageClient, error) { }, nil } -func (b *BackstageClient) FindProjectsForResource(resourceName string) ([]string, error) { - resources, err := b.findComponents(resourceName) - if err != nil { - return nil, err - } - if len(resources) > 1 { - log.Printf("Multiple components found, using first %s.", resources[0].Metadata.Name) +func (b *BackstageClient) FindProjectsForResource(resourceName, groupRef string) ([]string, error) { + if groupRef == "" { + resources, err := b.findComponents(resourceName) + if err != nil { + return nil, err + } + if len(resources) > 1 { + log.Printf("Multiple components found, using first %s.", resources[0].Metadata.Name) + } + groupRef = resources[0].Spec.Owner } - projects, err := b.findProjectsForGroup(resources[0].Spec.Owner) + projects, err := b.findProjectsForGroup(groupRef) if err != nil { return nil, err } diff --git a/scripts/backstage-lookup/main.go b/scripts/backstage-lookup/main.go index b65b55fa9..46a8c72ce 100644 --- a/scripts/backstage-lookup/main.go +++ b/scripts/backstage-lookup/main.go @@ -25,6 +25,7 @@ func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) dryRun := flag.Bool("dry-run", true, "Dry-run prints the intended actions.") + groupRef := flag.String("group-ref", "", "Assign to project for this Backstage groupRef.") flag.Parse() @@ -38,6 +39,15 @@ func main() { log.Fatal("Usage: backstage-lookup [resource2] ...") } + issueNumber, err := strconv.Atoi(positionalArgs[0]) + if err != nil { + log.Fatal(err) + } + + do(issueNumber, positionalArgs[1:], *dryRun, *groupRef) +} + +func do(issueNumber int, resources []string, dryRun bool, groupRef string) { backstage, err := NewBackstageClient() if err != nil { log.Fatal(err) @@ -49,16 +59,11 @@ func main() { } } - issueNumber, err := strconv.Atoi(positionalArgs[0]) - if err != nil { - log.Fatal(err) - } - var allProjects []string - for _, resource := range positionalArgs[1:] { + for _, resource := range resources { if resource = strings.TrimSpace(resource); resource != "" { fmt.Printf("Looking up resource: %s\n", resource) - projects, err := backstage.FindProjectsForResource(resource) + projects, err := backstage.FindProjectsForResource(resource, groupRef) if err != nil { log.Printf("Warning: failed to find projects for resource %s: %v", resource, err) continue @@ -80,7 +85,7 @@ func main() { resourceIsOwnedByPlatformMonitoring := -1 != slices.IndexFunc(allProjects, func(p string) bool { return p == "513" }) if len(allProjects) > 0 && !resourceIsOwnedByPlatformMonitoring { fmt.Printf("Removing issue #%d from platform-monitoring project (513)\n", issueNumber) - if !*dryRun { + if !dryRun { if err := github.RemoveIssueFromProject("grafana", "terraform-provider-grafana", issueNumber, 513); err != nil { log.Printf("Warning: failed to remove from platform-monitoring project: %v", err) } @@ -94,7 +99,7 @@ func main() { continue } fmt.Printf("Adding issue #%d to project %d\n", issueNumber, projectNumberInt) - if !*dryRun { + if !dryRun { if err := github.AddIssueToProject("grafana", "terraform-provider-grafana", issueNumber, projectNumberInt); err != nil { log.Printf("Warning: failed to add to project %d: %v", projectNumberInt, err) }