Skip to content

feat(perf): Cache changed files during event processing #2317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat(perf): Cache changed files during event processing #2317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
658148f
feat(perf): cache `vcs.GetFiles()` to reduce redundant VCS API volume
aThorp96 Nov 5, 2025
838d438
chore(lint): set golangci default switch case as exhaustive
aThorp96 Nov 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .golangci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ linters:
- zerologlint
settings:
misspell:
ignore-words:
ignore-rules:
- cancelled
extra-words:
- typo: "canceled"
Expand All @@ -72,6 +72,8 @@ linters:
- fmt.Fprintln
- (io.Closer).Close
- updateConfigMap
exhaustive:
default-signifies-exhaustive: true
gocritic:
disabled-checks:
- unlambda
Expand Down
126 changes: 65 additions & 61 deletions pkg/provider/bitbucketdatacenter/bitbucketdatacenter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type Provider struct {
projectKey string
repo *v1alpha1.Repository
triggerEvent string
changedFiles *changedfiles.ChangedFiles
}

func (v Provider) Client() *scm.Client {
Expand Down Expand Up @@ -371,79 +372,82 @@ func (v *Provider) GetConfig() *info.ProviderConfig {
}

func (v *Provider) GetFiles(ctx context.Context, runevent *info.Event) (changedfiles.ChangedFiles, error) {
OrgAndRepo := fmt.Sprintf("%s/%s", runevent.Organization, runevent.Repository)
if runevent.TriggerTarget == triggertype.PullRequest {
opts := &scm.ListOptions{Page: 1, Size: apiResponseLimit}
changedFiles := changedfiles.ChangedFiles{}
for {
changes, _, err := v.Client().PullRequests.ListChanges(ctx, OrgAndRepo, runevent.PullRequestNumber, opts)
if err != nil {
return changedfiles.ChangedFiles{}, fmt.Errorf("failed to list changes for pull request: %w", err)
}

for _, c := range changes {
changedFiles.All = append(changedFiles.All, c.Path)
if c.Added {
changedFiles.Added = append(changedFiles.Added, c.Path)
}
if c.Modified {
changedFiles.Modified = append(changedFiles.Modified, c.Path)
}
if c.Renamed {
changedFiles.Renamed = append(changedFiles.Renamed, c.Path)
if v.changedFiles == nil {
Copy link
Contributor

@zakisk zakisk Nov 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aThorp96 another approach is instead of making changes in every provider, I see that wherever in a func GetFiles is called, there has event reference. so can you please check in the code for changedFiles where the GetFiles is called, making changedFiles a field in event struct?

e.g.

if event.changedFiles == nil {
    event.changedFiles = v.GetFiles(ctx, event)
}

OrgAndRepo := fmt.Sprintf("%s/%s", runevent.Organization, runevent.Repository)
switch runevent.TriggerTarget {
case triggertype.PullRequest:
opts := &scm.ListOptions{Page: 1, Size: apiResponseLimit}
changedFiles := changedfiles.ChangedFiles{}
for {
changes, _, err := v.Client().PullRequests.ListChanges(ctx, OrgAndRepo, runevent.PullRequestNumber, opts)
if err != nil {
return changedfiles.ChangedFiles{}, fmt.Errorf("failed to list changes for pull request: %w", err)
}
if c.Deleted {
changedFiles.Deleted = append(changedFiles.Deleted, c.Path)
}
}

// In the Jenkins-x/go-scm package, the `isLastPage` field is not available, and the value of
// `response.Page.Last` is set to `0`. Therefore, to determine if there are more items to fetch,
// we can check if the length of the currently fetched items is less than the specified limit.
// If the length is less than the limit, it indicates that there are no more items to retrieve.
if len(changes) < apiResponseLimit {
break
}
for _, c := range changes {
changedFiles.All = append(changedFiles.All, c.Path)
if c.Added {
changedFiles.Added = append(changedFiles.Added, c.Path)
}
if c.Modified {
changedFiles.Modified = append(changedFiles.Modified, c.Path)
}
if c.Renamed {
changedFiles.Renamed = append(changedFiles.Renamed, c.Path)
}
if c.Deleted {
changedFiles.Deleted = append(changedFiles.Deleted, c.Path)
}
}

opts.Page++
}
return changedFiles, nil
}
// In the Jenkins-x/go-scm package, the `isLastPage` field is not available, and the value of
// `response.Page.Last` is set to `0`. Therefore, to determine if there are more items to fetch,
// we can check if the length of the currently fetched items is less than the specified limit.
// If the length is less than the limit, it indicates that there are no more items to retrieve.
if len(changes) < apiResponseLimit {
break
}

if runevent.TriggerTarget == triggertype.Push {
opts := &scm.ListOptions{Page: 1, Size: apiResponseLimit}
changedFiles := changedfiles.ChangedFiles{}
for {
changes, _, err := v.Client().Git.ListChanges(ctx, OrgAndRepo, runevent.SHA, opts)
if err != nil {
return changedfiles.ChangedFiles{}, fmt.Errorf("failed to list changes for commit %s: %w", runevent.SHA, err)
opts.Page++
}

for _, c := range changes {
changedFiles.All = append(changedFiles.All, c.Path)
if c.Added {
changedFiles.Added = append(changedFiles.Added, c.Path)
}
if c.Modified {
changedFiles.Modified = append(changedFiles.Modified, c.Path)
v.changedFiles = &changedFiles
case triggertype.Push:
opts := &scm.ListOptions{Page: 1, Size: apiResponseLimit}
changedFiles := changedfiles.ChangedFiles{}
for {
changes, _, err := v.Client().Git.ListChanges(ctx, OrgAndRepo, runevent.SHA, opts)
if err != nil {
return changedfiles.ChangedFiles{}, fmt.Errorf("failed to list changes for commit %s: %w", runevent.SHA, err)
}
if c.Renamed {
changedFiles.Renamed = append(changedFiles.Renamed, c.Path)

for _, c := range changes {
changedFiles.All = append(changedFiles.All, c.Path)
if c.Added {
changedFiles.Added = append(changedFiles.Added, c.Path)
}
if c.Modified {
changedFiles.Modified = append(changedFiles.Modified, c.Path)
}
if c.Renamed {
changedFiles.Renamed = append(changedFiles.Renamed, c.Path)
}
if c.Deleted {
changedFiles.Deleted = append(changedFiles.Deleted, c.Path)
}
}
if c.Deleted {
changedFiles.Deleted = append(changedFiles.Deleted, c.Path)

if len(changes) < apiResponseLimit {
break
}
}

if len(changes) < apiResponseLimit {
break
opts.Page++
}

opts.Page++
v.changedFiles = &changedFiles
default:
v.changedFiles = &changedfiles.ChangedFiles{}
}
Comment on lines +377 to 448
Copy link

@gemini-code-assist gemini-code-assist bot Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The logic for processing the []*scm.Change slice is duplicated in the PullRequest and Push cases. This could be refactored into a helper function to reduce code duplication and improve maintainability.

For example, you could create a helper function like this:

func processScmChanges(changes []*scm.Change, changedFiles *changedfiles.ChangedFiles) {
	for _, c := range changes {
		changedFiles.All = append(changedFiles.All, c.Path)
		if c.Added {
			changedFiles.Added = append(changedFiles.Added, c.Path)
		}
		if c.Modified {
			changedFiles.Modified = append(changedFiles.Modified, c.Path)
		}
		if c.Renamed {
			changedFiles.Renamed = append(changedFiles.Renamed, c.Path)
		}
		if c.Deleted {
			changedFiles.Deleted = append(changedFiles.Deleted, c.Path)
		}
	}
}

And then call it from both case blocks.

return changedFiles, nil
}
return changedfiles.ChangedFiles{}, nil
return *v.changedFiles, nil
}

func (v *Provider) CreateToken(_ context.Context, _ []string, _ *info.Event) (string, error) {
Expand Down
25 changes: 22 additions & 3 deletions pkg/provider/bitbucketdatacenter/bitbucketdatacenter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@ import (
"github.com/openshift-pipelines/pipelines-as-code/pkg/params/triggertype"
"github.com/openshift-pipelines/pipelines-as-code/pkg/provider"
bbtest "github.com/openshift-pipelines/pipelines-as-code/pkg/provider/bitbucketdatacenter/test"
"github.com/openshift-pipelines/pipelines-as-code/pkg/test/metrics"

"github.com/jenkins-x/go-scm/scm"
"go.uber.org/zap"
zapobserver "go.uber.org/zap/zaptest/observer"
"gotest.tools/v3/assert"
"knative.dev/pkg/metrics/metricstest"
_ "knative.dev/pkg/metrics/testing"
rtesting "knative.dev/pkg/reconciler/testing"
)

Expand Down Expand Up @@ -796,6 +799,7 @@ func TestGetFiles(t *testing.T) {
ctx, _ := rtesting.SetupFakeContext(t)
client, mux, tearDown, tURL := bbtest.SetupBBDataCenterClient()
defer tearDown()
metrics.ResetMetrics()

stats := &bbtest.DiffStats{
Values: tt.changeFiles,
Expand All @@ -821,13 +825,17 @@ func TestGetFiles(t *testing.T) {
}
})
}
v := &Provider{client: client, baseURL: tURL}

metricsTags := map[string]string{"provider": "bitbucket-datacenter", "event-type": string(tt.event.TriggerTarget)}
metricstest.CheckStatsNotReported(t, "pipelines_as_code_git_provider_api_request_count")

v := &Provider{client: client, baseURL: tURL, triggerEvent: string(tt.event.TriggerTarget)}
changedFiles, err := v.GetFiles(ctx, tt.event)
if tt.wantError {
assert.Equal(t, err.Error(), tt.errMsg)
return
} else {
assert.NilError(t, err, nil)
}
assert.NilError(t, err, nil)
assert.Equal(t, tt.wantAddedFilesCount, len(changedFiles.Added))
assert.Equal(t, tt.wantDeletedFilesCount, len(changedFiles.Deleted))
assert.Equal(t, tt.wantModifiedFilesCount, len(changedFiles.Modified))
Expand All @@ -844,6 +852,17 @@ func TestGetFiles(t *testing.T) {
assert.Equal(t, tt.changeFiles[i].Path.ToString, changedFiles.All[i])
}
}

// Check caching
metricstest.CheckCountData(t, "pipelines_as_code_git_provider_api_request_count", metricsTags, 1)
_, _ = v.GetFiles(ctx, tt.event)
if tt.wantError {
// No caching on error
metricstest.CheckCountData(t, "pipelines_as_code_git_provider_api_request_count", metricsTags, 2)
} else {
// Cache API results on success
metricstest.CheckCountData(t, "pipelines_as_code_git_provider_api_request_count", metricsTags, 1)
}
})
}
}
102 changes: 53 additions & 49 deletions pkg/provider/github/github.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type Provider struct {
userType string // The type of user i.e bot or not
skippedRun
triggerEvent string
changedFiles *changedfiles.ChangedFiles
}

type skippedRun struct {
Expand Down Expand Up @@ -517,67 +518,70 @@ func (v *Provider) getPullRequest(ctx context.Context, runevent *info.Event) (*i
return runevent, nil
}

// GetFiles get a files from pull request.
// GetFiles gets the files changed by a given event.
func (v *Provider) GetFiles(ctx context.Context, runevent *info.Event) (changedfiles.ChangedFiles, error) {
if runevent.TriggerTarget == triggertype.PullRequest {
opt := &github.ListOptions{PerPage: v.PaginedNumber}
changedFiles := changedfiles.ChangedFiles{}
for {
repoCommit, resp, err := wrapAPI(v, "list_pull_request_files", func() ([]*github.CommitFile, *github.Response, error) {
return v.Client().PullRequests.ListFiles(ctx, runevent.Organization, runevent.Repository, runevent.PullRequestNumber, opt)
if v.changedFiles == nil {
switch runevent.TriggerTarget {
case triggertype.PullRequest:
opt := &github.ListOptions{PerPage: v.PaginedNumber}
changedFiles := changedfiles.ChangedFiles{}
for {
repoCommit, resp, err := wrapAPI(v, "list_pull_request_files", func() ([]*github.CommitFile, *github.Response, error) {
return v.Client().PullRequests.ListFiles(ctx, runevent.Organization, runevent.Repository, runevent.PullRequestNumber, opt)
})
if err != nil {
return changedfiles.ChangedFiles{}, err
}
for j := range repoCommit {
changedFiles.All = append(changedFiles.All, *repoCommit[j].Filename)
if *repoCommit[j].Status == "added" {
changedFiles.Added = append(changedFiles.Added, *repoCommit[j].Filename)
}
if *repoCommit[j].Status == "removed" {
changedFiles.Deleted = append(changedFiles.Deleted, *repoCommit[j].Filename)
}
if *repoCommit[j].Status == "modified" {
changedFiles.Modified = append(changedFiles.Modified, *repoCommit[j].Filename)
}
if *repoCommit[j].Status == "renamed" {
changedFiles.Renamed = append(changedFiles.Renamed, *repoCommit[j].Filename)
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
v.changedFiles = &changedFiles
case triggertype.Push:
changedFiles := changedfiles.ChangedFiles{}
rC, _, err := wrapAPI(v, "get_commit_files", func() (*github.RepositoryCommit, *github.Response, error) {
return v.Client().Repositories.GetCommit(ctx, runevent.Organization, runevent.Repository, runevent.SHA, &github.ListOptions{})
})
if err != nil {
return changedfiles.ChangedFiles{}, err
}
for j := range repoCommit {
changedFiles.All = append(changedFiles.All, *repoCommit[j].Filename)
if *repoCommit[j].Status == "added" {
changedFiles.Added = append(changedFiles.Added, *repoCommit[j].Filename)
for i := range rC.Files {
changedFiles.All = append(changedFiles.All, *rC.Files[i].Filename)
if *rC.Files[i].Status == "added" {
changedFiles.Added = append(changedFiles.Added, *rC.Files[i].Filename)
}
if *repoCommit[j].Status == "removed" {
changedFiles.Deleted = append(changedFiles.Deleted, *repoCommit[j].Filename)
if *rC.Files[i].Status == "removed" {
changedFiles.Deleted = append(changedFiles.Deleted, *rC.Files[i].Filename)
}
if *repoCommit[j].Status == "modified" {
changedFiles.Modified = append(changedFiles.Modified, *repoCommit[j].Filename)
if *rC.Files[i].Status == "modified" {
changedFiles.Modified = append(changedFiles.Modified, *rC.Files[i].Filename)
}
if *repoCommit[j].Status == "renamed" {
changedFiles.Renamed = append(changedFiles.Renamed, *repoCommit[j].Filename)
if *rC.Files[i].Status == "renamed" {
changedFiles.Renamed = append(changedFiles.Renamed, *rC.Files[i].Filename)
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return changedFiles, nil
}

if runevent.TriggerTarget == "push" {
changedFiles := changedfiles.ChangedFiles{}
rC, _, err := wrapAPI(v, "get_commit_files", func() (*github.RepositoryCommit, *github.Response, error) {
return v.Client().Repositories.GetCommit(ctx, runevent.Organization, runevent.Repository, runevent.SHA, &github.ListOptions{})
})
if err != nil {
return changedfiles.ChangedFiles{}, err
}
for i := range rC.Files {
changedFiles.All = append(changedFiles.All, *rC.Files[i].Filename)
if *rC.Files[i].Status == "added" {
changedFiles.Added = append(changedFiles.Added, *rC.Files[i].Filename)
}
if *rC.Files[i].Status == "removed" {
changedFiles.Deleted = append(changedFiles.Deleted, *rC.Files[i].Filename)
}
if *rC.Files[i].Status == "modified" {
changedFiles.Modified = append(changedFiles.Modified, *rC.Files[i].Filename)
}
if *rC.Files[i].Status == "renamed" {
changedFiles.Renamed = append(changedFiles.Renamed, *rC.Files[i].Filename)
}
v.changedFiles = &changedFiles
default:
v.changedFiles = &changedfiles.ChangedFiles{}
}
Comment on lines +524 to 582
Copy link

@gemini-code-assist gemini-code-assist bot Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The logic for processing []*github.CommitFile is duplicated for PullRequest and Push cases. This can be refactored into a helper function. Additionally, using a switch statement on *repoCommit[j].Status would be more readable and slightly more efficient than multiple if statements.

Example helper function:

func processGithubCommitFiles(files []*github.CommitFile, changedFiles *changedfiles.ChangedFiles) {
	for i := range files {
		filename := *files[i].Filename
		changedFiles.All = append(changedFiles.All, filename)
		switch *files[i].Status {
		case "added":
			changedFiles.Added = append(changedFiles.Added, filename)
		case "removed":
			changedFiles.Deleted = append(changedFiles.Deleted, filename)
		case "modified":
			changedFiles.Modified = append(changedFiles.Modified, filename)
		case "renamed":
			changedFiles.Renamed = append(changedFiles.Renamed, filename)
		}
	}
}

return changedFiles, nil
}
return changedfiles.ChangedFiles{}, nil
return *v.changedFiles, nil
}

// getObject Get an object from a repository.
Expand Down
Loading