diff --git a/internal/api/fileupload/filters/client.go b/internal/api/fileupload/filters/client.go new file mode 100644 index 000000000..78ed2faba --- /dev/null +++ b/internal/api/fileupload/filters/client.go @@ -0,0 +1,78 @@ +package filters + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/google/uuid" +) + +// AllowList represents the response structure from the deeproxy filters API. +type AllowList struct { + ConfigFiles []string `json:"configFiles"` + Extensions []string `json:"extensions"` +} + +// Client defines the interface for the filters client. +type Client interface { + GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) +} + +// DeeproxyClient is the deeproxy implementation of the Client interface. +type DeeproxyClient struct { + httpClient *http.Client + cfg Config +} + +// Config contains the configuration for the filters client. +type Config struct { + BaseURL string + IsFedRamp bool +} + +var _ Client = (*DeeproxyClient)(nil) + +// NewDeeproxyClient creates a new DeeproxyClient with the given configuration and options. +func NewDeeproxyClient(cfg Config, opts ...Opt) *DeeproxyClient { + c := &DeeproxyClient{ + cfg: cfg, + httpClient: http.DefaultClient, + } + + for _, opt := range opts { + opt(c) + } + + return c +} + +// GetFilters returns the deeproxy filters in the form of an AllowList. +func (c *DeeproxyClient) GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + var allowList AllowList + + url := getFilterURL(c.cfg.BaseURL, orgID, c.cfg.IsFedRamp) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) + if err != nil { + return allowList, fmt.Errorf("failed to create deeproxy filters request: %w", err) + } + + req.Header.Set("snyk-org-name", orgID.String()) + + resp, err := c.httpClient.Do(req) + if err != nil { + return allowList, fmt.Errorf("error making deeproxy filters request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return allowList, fmt.Errorf("unexpected response code: %s", resp.Status) + } + + if err := json.NewDecoder(resp.Body).Decode(&allowList); err != nil { + return allowList, fmt.Errorf("failed to decode deeproxy filters response: %w", err) + } + + return allowList, nil +} diff --git a/internal/api/fileupload/filters/client_test.go b/internal/api/fileupload/filters/client_test.go new file mode 100644 index 000000000..2a30d8cde --- /dev/null +++ b/internal/api/fileupload/filters/client_test.go @@ -0,0 +1,77 @@ +package filters //nolint:testpackage // Testing private utility functions. + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClients(t *testing.T) { + tests := []struct { + getClient func(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) (Client, func()) + clientName string + }{ + { + clientName: "deeproxyClient", + getClient: func(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) (Client, func()) { + t.Helper() + + s := setupServer(t, orgID, expectedAllow) + cleanup := func() { + s.Close() + } + c := NewDeeproxyClient(Config{BaseURL: s.URL, IsFedRamp: true}, WithHTTPClient(s.Client())) + + return c, cleanup + }, + }, + { + clientName: "fakeClient", + getClient: func(t *testing.T, _ uuid.UUID, expectedAllow AllowList) (Client, func()) { + t.Helper() + + c := NewFakeClient(expectedAllow) + + return c, func() {} + }, + }, + } + + for _, testData := range tests { + t.Run(testData.clientName+": GetFilters", func(t *testing.T) { + orgID := uuid.New() + expectedAllow := AllowList{ + ConfigFiles: []string{"package.json"}, + Extensions: []string{".ts", ".js"}, + } + client, cleanup := testData.getClient(t, orgID, expectedAllow) + defer cleanup() + + allow, err := client.GetFilters(t.Context(), orgID) + require.NoError(t, err) + + assert.Equal(t, expectedAllow, allow) + }) + } +} + +func setupServer(t *testing.T, orgID uuid.UUID, expectedAllow AllowList) *httptest.Server { + t.Helper() + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expectedURL := getFilterURL("", orgID, true) + assert.Equal(t, expectedURL, r.URL.Path) + assert.Equal(t, orgID.String(), r.Header.Get("snyk-org-name")) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(expectedAllow); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + return + } + })) + ts.Start() + return ts +} diff --git a/internal/api/fileupload/filters/fake_client.go b/internal/api/fileupload/filters/fake_client.go new file mode 100644 index 000000000..4d2d26d71 --- /dev/null +++ b/internal/api/fileupload/filters/fake_client.go @@ -0,0 +1,25 @@ +package filters + +import ( + "context" + + "github.com/google/uuid" +) + +type FakeClient struct { + getFilters func(ctx context.Context, orgID uuid.UUID) (AllowList, error) +} + +var _ Client = (*FakeClient)(nil) + +func NewFakeClient(allowList AllowList) *FakeClient { + return &FakeClient{ + getFilters: func(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + return allowList, nil + }, + } +} + +func (f *FakeClient) GetFilters(ctx context.Context, orgID uuid.UUID) (AllowList, error) { + return f.getFilters(ctx, orgID) +} diff --git a/internal/api/fileupload/filters/opts.go b/internal/api/fileupload/filters/opts.go new file mode 100644 index 000000000..0ce885788 --- /dev/null +++ b/internal/api/fileupload/filters/opts.go @@ -0,0 +1,13 @@ +package filters + +import "net/http" + +// Opt is a function that configures an deeproxyClient instance. +type Opt func(*DeeproxyClient) + +// WithHTTPClient sets a custom HTTP client for the filters client. +func WithHTTPClient(httpClient *http.Client) Opt { + return func(c *DeeproxyClient) { + c.httpClient = httpClient + } +} diff --git a/internal/api/fileupload/filters/utils.go b/internal/api/fileupload/filters/utils.go new file mode 100644 index 000000000..457998f00 --- /dev/null +++ b/internal/api/fileupload/filters/utils.go @@ -0,0 +1,17 @@ +package filters + +import ( + "fmt" + "strings" + + "github.com/google/uuid" +) + +func getFilterURL(baseURL string, orgID uuid.UUID, isFedRamp bool) string { + if isFedRamp { + return fmt.Sprintf("%s/hidden/orgs/%s/code/filters", baseURL, orgID) + } + + deeproxyURL := strings.ReplaceAll(baseURL, "api", "deeproxy") + return fmt.Sprintf("%s/filters", deeproxyURL) +} diff --git a/internal/api/fileupload/filters/utils_test.go b/internal/api/fileupload/filters/utils_test.go new file mode 100644 index 000000000..62f68fef4 --- /dev/null +++ b/internal/api/fileupload/filters/utils_test.go @@ -0,0 +1,20 @@ +package filters //nolint:testpackage // Testing private utility functions. + +import ( + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +var orgID = uuid.MustParse("738ef92e-21cc-4a11-8c13-388d89272f4b") + +func Test_getBaseUrl_notFedramp(t *testing.T) { + actualURL := getFilterURL("https://api.snyk.io", orgID, false) + assert.Equal(t, "https://deeproxy.snyk.io/filters", actualURL) +} + +func Test_getBaseUrl_fedramp(t *testing.T) { + actualURL := getFilterURL("https://api.snyk.io", orgID, true) + assert.Equal(t, "https://api.snyk.io/hidden/orgs/738ef92e-21cc-4a11-8c13-388d89272f4b/code/filters", actualURL) +} diff --git a/internal/api/fileupload/types.go b/internal/api/fileupload/types.go new file mode 100644 index 000000000..de71702cc --- /dev/null +++ b/internal/api/fileupload/types.go @@ -0,0 +1,15 @@ +package fileupload + +import ( + "sync" + + "github.com/puzpuzpuz/xsync" +) + +// Filters holds the filtering configuration for file uploads with thread-safe maps. +type Filters struct { + SupportedExtensions *xsync.MapOf[string, bool] + SupportedConfigFiles *xsync.MapOf[string, bool] + Once sync.Once + InitErr error +} diff --git a/internal/api/fileupload/uploadrevision/client.go b/internal/api/fileupload/uploadrevision/client.go new file mode 100644 index 000000000..95d2dd2e0 --- /dev/null +++ b/internal/api/fileupload/uploadrevision/client.go @@ -0,0 +1,300 @@ +package uploadrevision + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + + "github.com/google/uuid" + "github.com/snyk/error-catalog-golang-public/snyk_errors" +) + +// SealableClient defines the interface for file upload API operations. +type SealableClient interface { + CreateRevision(ctx context.Context, orgID OrgID) (*ResponseBody, error) + UploadFiles(ctx context.Context, orgID OrgID, revisionID RevisionID, files []UploadFile) error + SealRevision(ctx context.Context, orgID OrgID, revisionID RevisionID) (*SealResponseBody, error) + + GetLimits() Limits +} + +// This will force go to complain if the type doesn't satisfy the interface. +var _ SealableClient = (*HTTPSealableClient)(nil) + +// Config contains the configuration for the file upload client. +type Config struct { + BaseURL string +} + +// HTTPSealableClient implements the SealableClient interface for file upload operations via HTTP API. +type HTTPSealableClient struct { + cfg Config + httpClient *http.Client +} + +// apiVersion specifies the API version to use for requests. +const apiVersion = "2024-10-15" + +const ( + fileSizeLimit = 50_000_000 // 50MB - maximum size per individual file + fileCountLimit = 300_000 // 300,000 - maximum number of files per request + totalPayloadSizeLimit = 200_000_000 // 200MB - maximum total uncompressed payload size per request + filePathLengthLimit = 256 // 256 - maximum length of file names +) + +// NewClient creates a new file upload client with the given configuration and options. +func NewClient(cfg Config, opts ...Opt) *HTTPSealableClient { + httpClient := &http.Client{ + Transport: http.DefaultTransport, + } + c := HTTPSealableClient{cfg, httpClient} + + for _, opt := range opts { + opt(&c) + } + + return &c +} + +// CreateRevision creates a new upload revision for the specified organization. +func (c *HTTPSealableClient) CreateRevision(ctx context.Context, orgID OrgID) (*ResponseBody, error) { + if orgID == uuid.Nil { + return nil, ErrEmptyOrgID + } + + body := RequestBody{ + Data: RequestData{ + Attributes: RequestAttributes{ + RevisionType: RevisionTypeSnapshot, + }, + Type: ResourceTypeUploadRevision, + }, + } + buff := bytes.NewBuffer(nil) + if err := json.NewEncoder(buff).Encode(body); err != nil { + return nil, fmt.Errorf("failed to encode request body: %w", err) + } + + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions?version=%s", c.cfg.BaseURL, orgID, apiVersion) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, buff) + if err != nil { + return nil, fmt.Errorf("failed to create revision request: %w", err) + } + req.Header.Set(ContentType, "application/vnd.api+json") + + res, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error making create revision request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusCreated { + return nil, handleUnexpectedStatusCodes(res.Body, res.StatusCode, res.Status, "create upload revision") + } + + var respBody ResponseBody + if err := json.NewDecoder(res.Body).Decode(&respBody); err != nil { + return nil, fmt.Errorf("failed to decode upload revision response: %w", err) + } + + return &respBody, nil +} + +// UploadFiles uploads the provided files to the specified revision. It will not close the file descriptors. +func (c *HTTPSealableClient) UploadFiles(ctx context.Context, orgID OrgID, revisionID RevisionID, files []UploadFile) error { + if orgID == uuid.Nil { + return ErrEmptyOrgID + } + + if revisionID == uuid.Nil { + return ErrEmptyRevisionID + } + + if err := validateFiles(files); err != nil { + return err + } + + // Create pipe for multipart data + pipeReader, pipeWriter := io.Pipe() + defer pipeReader.Close() + + mpartWriter := multipart.NewWriter(pipeWriter) + + go streamFilesToPipe(pipeWriter, mpartWriter, files) + body := compressRequestBody(pipeReader) + + // Load body bytes into memmory so go can determine the Content-Length + // and not send the request chunked + bts, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("failed to create upload files request: %w", err) + } + + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions/%s/files?version=%s", c.cfg.BaseURL, orgID, revisionID, apiVersion) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bts)) + if err != nil { + return fmt.Errorf("failed to create upload files request: %w", err) + } + req.Header.Set(ContentType, mpartWriter.FormDataContentType()) + req.Header.Set(ContentEncoding, "gzip") + + res, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("error making upload files request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusNoContent { + return handleUnexpectedStatusCodes(res.Body, res.StatusCode, res.Status, "upload files") + } + + return nil +} + +// streamFilesToPipe writes files to the multipart form. +func streamFilesToPipe(pipeWriter *io.PipeWriter, mpartWriter *multipart.Writer, files []UploadFile) { + var streamError error + defer func() { + if closeErr := mpartWriter.Close(); closeErr != nil && streamError == nil { + streamError = closeErr + } + pipeWriter.CloseWithError(streamError) + }() + + for _, file := range files { + // Create form file part + part, err := mpartWriter.CreateFormFile(file.Path, file.Path) + if err != nil { + streamError = NewMultipartError(file.Path, err) + return + } + + if _, err := io.Copy(part, file.File); err != nil { + streamError = fmt.Errorf("failed to copy file content for %s: %w", file.Path, err) + return + } + } +} + +// validateFiles validates the files before upload. +func validateFiles(files []UploadFile) error { + if len(files) > fileCountLimit { + return NewFileCountLimitError(len(files), fileCountLimit) + } + + if len(files) == 0 { + return ErrNoFilesProvided + } + + var totalPayloadSize int64 + for _, file := range files { + if len(file.Path) > filePathLengthLimit { + return NewFilePathLengthLimitError(file.Path, len(file.Path), filePathLengthLimit) + } + + fileInfo, err := file.File.Stat() + if err != nil { + return NewFileAccessError(file.Path, err) + } + + if !fileInfo.Mode().IsRegular() { + return NewSpecialFileError(file.Path, fileInfo.Mode()) + } + + if fileInfo.Size() > fileSizeLimit { + return NewFileSizeLimitError(file.Path, fileInfo.Size(), fileSizeLimit) + } + + totalPayloadSize += fileInfo.Size() + } + + if totalPayloadSize > totalPayloadSizeLimit { + return NewTotalPayloadSizeLimitError(totalPayloadSize, totalPayloadSizeLimit) + } + + return nil +} + +// SealRevision seals the specified upload revision, marking it as complete. +func (c *HTTPSealableClient) SealRevision(ctx context.Context, orgID OrgID, revisionID RevisionID) (*SealResponseBody, error) { + if orgID == uuid.Nil { + return nil, ErrEmptyOrgID + } + + if revisionID == uuid.Nil { + return nil, ErrEmptyRevisionID + } + + body := SealRequestBody{ + Data: SealRequestData{ + ID: revisionID, + Attributes: SealRequestAttributes{ + Sealed: true, + }, + Type: ResourceTypeUploadRevision, + }, + } + buff := bytes.NewBuffer(nil) + if err := json.NewEncoder(buff).Encode(body); err != nil { + return nil, fmt.Errorf("failed to encode request body: %w", err) + } + + url := fmt.Sprintf("%s/hidden/orgs/%s/upload_revisions/%s?version=%s", c.cfg.BaseURL, orgID, revisionID, apiVersion) + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, url, buff) + if err != nil { + return nil, fmt.Errorf("failed to create seal request: %w", err) + } + req.Header.Set(ContentType, "application/vnd.api+json") + + res, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error making seal revision request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, handleUnexpectedStatusCodes(res.Body, res.StatusCode, res.Status, "seal upload revision") + } + + var respBody SealResponseBody + if err := json.NewDecoder(res.Body).Decode(&respBody); err != nil { + return nil, fmt.Errorf("failed to decode upload revision response: %w", err) + } + + return &respBody, nil +} + +func handleUnexpectedStatusCodes(body io.ReadCloser, statusCode int, status, operation string) error { + bts, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + if len(bts) > 0 { + snykErrorList, parseErr := snyk_errors.FromJSONAPIErrorBytes(bts) + if parseErr == nil && len(snykErrorList) > 0 && snykErrorList[0].Title != "" { + errsToJoin := []error{} + for i := range snykErrorList { + errsToJoin = append(errsToJoin, snykErrorList[i]) + } + return fmt.Errorf("api error during %s: %w", operation, errors.Join(errsToJoin...)) + } + } + + return NewHTTPError(statusCode, status, operation, bts) +} + +// GetLimits returns the upload Limits defined in the low level client. +func (c *HTTPSealableClient) GetLimits() Limits { + return Limits{ + FileCountLimit: fileCountLimit, + FileSizeLimit: fileSizeLimit, + TotalPayloadSizeLimit: totalPayloadSizeLimit, + FilePathLengthLimit: filePathLengthLimit, + } +} diff --git a/internal/api/fileupload/uploadrevision/client_test.go b/internal/api/fileupload/uploadrevision/client_test.go new file mode 100644 index 000000000..70d7efb8c --- /dev/null +++ b/internal/api/fileupload/uploadrevision/client_test.go @@ -0,0 +1,576 @@ +package uploadrevision_test + +import ( + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "io/fs" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path" + "runtime" + "strings" + "testing" + "testing/fstest" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) + +var ( + orgID = uuid.MustParse("9102b78b-c28d-4392-a39f-08dd26fd9622") + revID = uuid.MustParse("ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f") +) + +func TestClient_CreateRevision(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + resp, err := c.CreateRevision(context.Background(), orgID) + + require.NoError(t, err) + expectedID := uuid.MustParse("a7d975fb-2076-49b7-bc1f-31c395c3ce93") + assert.Equal(t, expectedID, resp.Data.ID) +} + +func TestClient_CreateRevision_EmptyOrgID(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + resp, err := c.CreateRevision(context.Background(), uuid.Nil) + + assert.Error(t, err) + assert.Nil(t, resp) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyOrgID) +} + +func TestClient_CreateRevision_ServerError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + c := uploadrevision2.NewClient(uploadrevision2.Config{ + BaseURL: srv.URL, + }) + + resp, err := c.CreateRevision(context.Background(), orgID) + + assert.Nil(t, resp) + var httpErr *uploadrevision2.HTTPError + assert.ErrorAs(t, err, &httpErr) + assert.Equal(t, http.StatusInternalServerError, httpErr.StatusCode) + assert.Equal(t, "create upload revision", httpErr.Operation) +} + +func TestClient_UploadFiles(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + mockFS := fstest.MapFS{ + "foo/bar": {Data: []byte("asdf")}, + } + fd, err := mockFS.Open("foo/bar") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision2.UploadFile{ + {Path: "foo/bar", File: fd}, + }) + + require.NoError(t, err) +} + +func TestClient_UploadFiles_MultipleFiles(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + mockFS := fstest.MapFS{ + "file1.txt": {Data: []byte("content1")}, + "file2.json": {Data: []byte("content2")}, + } + + file1, err := mockFS.Open("file1.txt") + require.NoError(t, err) + file2, err := mockFS.Open("file2.json") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision2.UploadFile{ + {Path: "file1.txt", File: file1}, + {Path: "file2.json", File: file2}, + }) + + require.NoError(t, err) +} + +func TestClient_UploadFiles_EmptyOrgID(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + mockFS := fstest.MapFS{ + "test.txt": {Data: []byte("content")}, + } + file, err := mockFS.Open("test.txt") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + uuid.Nil, // empty orgID + revID, + []uploadrevision2.UploadFile{ + {Path: "test.txt", File: file}, + }) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyOrgID) +} + +func TestClient_UploadFiles_EmptyRevisionID(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + mockFS := fstest.MapFS{ + "test.txt": {Data: []byte("content")}, + } + file, err := mockFS.Open("test.txt") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + uuid.Nil, // empty revisionID + []uploadrevision2.UploadFile{ + {Path: "test.txt", File: file}, + }) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyRevisionID) +} + +func TestClient_UploadFiles_FileSizeLimit(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + largeContent := make([]byte, c.GetLimits().FileSizeLimit+1) + mockFS := fstest.MapFS{ + "large_file.txt": {Data: largeContent}, + } + + file, err := mockFS.Open("large_file.txt") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision2.UploadFile{ + {Path: "large_file.txt", File: file}, + }) + + assert.Error(t, err) + var fileSizeErr *uploadrevision2.FileSizeLimitError + assert.ErrorAs(t, err, &fileSizeErr) + assert.Equal(t, "large_file.txt", fileSizeErr.FilePath) + assert.Equal(t, c.GetLimits().FileSizeLimit+1, fileSizeErr.FileSize) + assert.Equal(t, c.GetLimits().FileSizeLimit, fileSizeErr.Limit) +} + +func TestClient_UploadFiles_FileCountLimit(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + files := make([]uploadrevision2.UploadFile, c.GetLimits().FileCountLimit+1) + mockFS := fstest.MapFS{} + + for i := range c.GetLimits().FileCountLimit + 1 { + filename := fmt.Sprintf("file%d.txt", i) + mockFS[filename] = &fstest.MapFile{Data: []byte("content")} + + file, err := mockFS.Open(filename) + require.NoError(t, err) + + files[i] = uploadrevision2.UploadFile{ + Path: filename, + File: file, + } + } + + err := c.UploadFiles(context.Background(), orgID, revID, files) + + assert.Error(t, err) + var fileCountErr *uploadrevision2.FileCountLimitError + assert.ErrorAs(t, err, &fileCountErr) + assert.Equal(t, c.GetLimits().FileCountLimit+1, fileCountErr.Count) + assert.Equal(t, c.GetLimits().FileCountLimit, fileCountErr.Limit) +} + +func TestClient_UploadFiles_FilePathLengthLimit(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + // Create a file path that exceeds the limit + longFilePath := strings.Repeat("a", c.GetLimits().FilePathLengthLimit+1) + + mockFS := fstest.MapFS{ + "short_file.txt": {Data: []byte("content")}, + } + + file, err := mockFS.Open("short_file.txt") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision2.UploadFile{ + {Path: longFilePath, File: file}, + }) + + assert.Error(t, err) + var filePathLengthErr *uploadrevision2.FilePathLengthLimitError + assert.ErrorAs(t, err, &filePathLengthErr) + assert.Equal(t, longFilePath, filePathLengthErr.FilePath) + assert.Equal(t, c.GetLimits().FilePathLengthLimit+1, filePathLengthErr.Length) + assert.Equal(t, c.GetLimits().FilePathLengthLimit, filePathLengthErr.Limit) +} + +func TestClient_UploadFiles_FilePathLengthExactlyAtLimit(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + // Create a file name that is exactly at the limit + filePathAtLimit := strings.Repeat("a", c.GetLimits().FilePathLengthLimit) + + mockFS := fstest.MapFS{ + "short_file.txt": {Data: []byte("content")}, + } + + file, err := mockFS.Open("short_file.txt") + require.NoError(t, err) + + // This should not error since the file path is exactly at the limit + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision2.UploadFile{ + {Path: filePathAtLimit, File: file}, + }) + + assert.NoError(t, err) +} + +func TestClient_UploadFiles_TotalPayloadSizeLimit(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + // Create multiple files that individually are under the size limit, + // but together exceed the total payload size limit + mockFS := fstest.MapFS{} + files := []uploadrevision2.UploadFile{} + + // Use files that are 30MB each (under the 50MB individual limit) + // 8 files = 240MB > 200MB total limit + fileSize := int64(30_000_000) + numFiles := 8 + + for i := range numFiles { + filename := fmt.Sprintf("file%d.txt", i) + mockFS[filename] = &fstest.MapFile{Data: make([]byte, fileSize)} + + file, err := mockFS.Open(filename) + require.NoError(t, err) + + files = append(files, uploadrevision2.UploadFile{ + Path: filename, + File: file, + }) + } + + err := c.UploadFiles(context.Background(), orgID, revID, files) + + assert.Error(t, err) + var totalSizeErr *uploadrevision2.TotalPayloadSizeLimitError + assert.ErrorAs(t, err, &totalSizeErr) + assert.Equal(t, fileSize*int64(numFiles), totalSizeErr.TotalSize) + assert.Equal(t, c.GetLimits().TotalPayloadSizeLimit, totalSizeErr.Limit) +} + +func TestClient_UploadFiles_TotalPayloadSizeExactlyAtLimit(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + // Test boundary: exactly 200MB (should succeed) + mockFS := fstest.MapFS{} + files := []uploadrevision2.UploadFile{} + + // Create files that sum exactly to 200MB + // 4 files of 50MB each = 200MB exactly + fileSize := int64(50_000_000) + numFiles := 4 + + for i := 0; i < numFiles; i++ { + filename := fmt.Sprintf("file%d.txt", i) + mockFS[filename] = &fstest.MapFile{Data: make([]byte, fileSize)} + + file, err := mockFS.Open(filename) + require.NoError(t, err) + + files = append(files, uploadrevision2.UploadFile{ + Path: filename, + File: file, + }) + } + + err := c.UploadFiles(context.Background(), orgID, revID, files) + + // Should succeed - exactly at limit is allowed + assert.NoError(t, err) +} + +func TestClient_UploadFiles_IndividualFileSizeExactlyAtLimit(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + // Test boundary: individual file exactly 50MB (should succeed) + mockFS := fstest.MapFS{ + "exact_limit.bin": {Data: make([]byte, c.GetLimits().FileSizeLimit)}, + } + + file, err := mockFS.Open("exact_limit.bin") + require.NoError(t, err) + + err = c.UploadFiles(context.Background(), orgID, revID, []uploadrevision2.UploadFile{ + {Path: "exact_limit.bin", File: file}, + }) + + // Should succeed - exactly at limit is allowed + assert.NoError(t, err) +} + +func TestClient_UploadFiles_SpecialFileError(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + tests := []struct { + name string + setupFS func() (fstest.MapFS, string) + setupRealFile func() string + }{ + { + name: "directory file", + setupFS: func() (fstest.MapFS, string) { + return fstest.MapFS{ + "test-directory": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, "test-directory" + }, + }, + } + + // on non windows os test this case + if runtime.GOOS != "windows" { + tests = append(tests, struct { + name string + setupFS func() (fstest.MapFS, string) + setupRealFile func() string + }{ + name: "device file", + setupRealFile: func() string { + return "/dev/null" + }, + }) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var file fs.File + var filePath string + var err error + + if tt.setupFS != nil { + mockFS, path := tt.setupFS() + filePath = path + file, err = mockFS.Open(path) + require.NoError(t, err) + } else if tt.setupRealFile != nil { + filePath = tt.setupRealFile() + + realFile, openErr := os.Open(filePath) + require.NoError(t, openErr) + defer realFile.Close() + file = realFile + } + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision2.UploadFile{ + {Path: filePath, File: file}, + }) + + assert.Error(t, err) + + var sfe *uploadrevision2.SpecialFileError + assert.ErrorAs(t, err, &sfe) + assert.Equal(t, filePath, sfe.FilePath) + }) + } +} + +func TestClient_UploadFiles_Symlink(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + tmpDir := t.TempDir() + tmpFile := path.Join(tmpDir, "temp-regular-file") + + err := os.WriteFile(tmpFile, []byte("foo bar"), 0o600) + require.NoError(t, err) + + tmpSlnPth := path.Join(tmpDir, "temp-symlink") + err = os.Symlink(tmpFile, tmpSlnPth) + require.NoError(t, err) + + tmpSln, err := os.Open(tmpSlnPth) + require.NoError(t, err) + defer tmpSln.Close() + + err = c.UploadFiles(context.Background(), + orgID, + revID, + []uploadrevision2.UploadFile{ + {Path: tmpSlnPth, File: tmpSln}, + }) + + assert.NoError(t, err) +} + +func TestClient_UploadFiles_EmptyFileList(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + err := c.UploadFiles(context.Background(), orgID, revID, []uploadrevision2.UploadFile{}) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision2.ErrNoFilesProvided) +} + +func TestClient_SealRevision(t *testing.T) { + srv, c := setupTestServer(t) + defer srv.Close() + + resp, err := c.SealRevision(context.Background(), orgID, revID) + + require.NoError(t, err) + assert.Equal(t, revID, resp.Data.ID) + assert.True(t, resp.Data.Attributes.Sealed) +} + +func TestClient_SealRevision_EmptyOrgID(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + resp, err := c.SealRevision(context.Background(), + uuid.Nil, // empty orgID + revID, + ) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyOrgID) + assert.Nil(t, resp) +} + +func TestClient_SealRevision_EmptyRevisionID(t *testing.T) { + c := uploadrevision2.NewClient(uploadrevision2.Config{}) + + resp, err := c.SealRevision(context.Background(), + orgID, + uuid.Nil, // empty revisionID + ) + + assert.Error(t, err) + assert.ErrorIs(t, err, uploadrevision2.ErrEmptyRevisionID) + assert.Nil(t, resp) +} + +func setupTestServer(t *testing.T) (*httptest.Server, *uploadrevision2.HTTPSealableClient) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "2024-10-15", r.URL.Query().Get("version")) + + switch { + // Create revision + case r.Method == http.MethodPost && + r.URL.Path == "/hidden/orgs/9102b78b-c28d-4392-a39f-08dd26fd9622/upload_revisions": + + assert.Equal(t, "application/vnd.api+json", r.Header.Get("Content-Type")) + + w.WriteHeader(http.StatusCreated) + _, err := w.Write([]byte(`{ + "data": { + "attributes": { + "revision_type": "snapshot", + "sealed": false + }, + "id": "a7d975fb-2076-49b7-bc1f-31c395c3ce93", + "type": "upload_revision" + } + }`)) + assert.NoError(t, err) + + // Upload files + case r.Method == http.MethodPost && + r.URL.Path == "/hidden/orgs/9102b78b-c28d-4392-a39f-08dd26fd9622/upload_revisions/ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f/files": + + assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + assert.Contains(t, r.Header.Get("Content-Type"), "multipart/form-data") + + contentType := r.Header.Get("Content-Type") + _, params, err := mime.ParseMediaType(contentType) + require.NoError(t, err) + boundary := params["boundary"] + require.NotEmpty(t, boundary, "multipart boundary should be present") + + gzipReader, err := gzip.NewReader(r.Body) + require.NoError(t, err) + reader := multipart.NewReader(gzipReader, boundary) + + for { + _, err := reader.NextPart() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + } + + w.WriteHeader(http.StatusNoContent) + + // Seal revision + case r.Method == http.MethodPatch && + r.URL.Path == "/hidden/orgs/9102b78b-c28d-4392-a39f-08dd26fd9622/upload_revisions/ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f": + + assert.Equal(t, "application/vnd.api+json", r.Header.Get("Content-Type")) + + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{ + "data": { + "attributes": { + "revision_type": "snapshot", + "sealed": true + }, + "id": "ff1bd2c6-7a5f-48fb-9a5b-52d711c8b47f", + "type": "upload_revision" + } + }`)) + assert.NoError(t, err) + + default: + t.Errorf("Unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + + client := uploadrevision2.NewClient(uploadrevision2.Config{ + BaseURL: srv.URL, + }) + + return srv, client +} diff --git a/internal/api/fileupload/uploadrevision/compression.go b/internal/api/fileupload/uploadrevision/compression.go new file mode 100644 index 000000000..04899883c --- /dev/null +++ b/internal/api/fileupload/uploadrevision/compression.go @@ -0,0 +1,67 @@ +package uploadrevision + +import ( + "compress/gzip" + "io" + "net/http" +) + +// CompressionRoundTripper is an http.RoundTripper that automatically compresses +// request bodies using gzip compression. It wraps another RoundTripper and adds +// Content-Encoding: gzip header while removing Content-Length to allow proper +// compression handling. +type CompressionRoundTripper struct { + defaultRoundTripper http.RoundTripper +} + +// NewCompressionRoundTripper creates a new CompressionRoundTripper that wraps +// the provided RoundTripper. If drt is nil, http.DefaultTransport is used. +// All HTTP requests with a body will be automatically compressed using gzip. +func NewCompressionRoundTripper(drt http.RoundTripper) *CompressionRoundTripper { + rt := drt + if rt == nil { + rt = http.DefaultTransport + } + return &CompressionRoundTripper{rt} +} + +// compressRequestBody wraps the given reader with gzip compression. +func compressRequestBody(body io.Reader) io.ReadCloser { + pipeReader, pipeWriter := io.Pipe() + + go func() { + var err error + gzWriter := gzip.NewWriter(pipeWriter) + + _, err = io.Copy(gzWriter, body) + + if closeErr := gzWriter.Close(); closeErr != nil && err == nil { + err = closeErr + } + pipeWriter.CloseWithError(err) + }() + + return pipeReader +} + +// RoundTrip implements the http.RoundTripper interface. It compresses the request +// body using gzip if a body is present, sets the Content-Encoding header to "gzip", +// and removes the Content-Length header to allow Go's HTTP client to calculate +// the correct length after compression. Requests without a body are passed through +// unchanged to the wrapped RoundTripper. +func (crt *CompressionRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + if r.Body == nil || r.Body == http.NoBody { + //nolint:wrapcheck // No need to wrap the error here. + return crt.defaultRoundTripper.RoundTrip(r) + } + + compressedBody := compressRequestBody(r.Body) + + r.Body = compressedBody + r.Header.Set(ContentEncoding, "gzip") + r.Header.Del(ContentLength) + r.ContentLength = -1 // Let Go calculate the length + + //nolint:wrapcheck // No need to wrap the error here. + return crt.defaultRoundTripper.RoundTrip(r) +} diff --git a/internal/api/fileupload/uploadrevision/compression_test.go b/internal/api/fileupload/uploadrevision/compression_test.go new file mode 100644 index 000000000..892c4fba0 --- /dev/null +++ b/internal/api/fileupload/uploadrevision/compression_test.go @@ -0,0 +1,120 @@ +package uploadrevision_test + +import ( + "compress/gzip" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) + +func TestCompressionRoundTripper_RoundTrip(t *testing.T) { + t.Run("request without body", func(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify no compression headers are set + assert.Empty(t, r.Header.Get("Content-Encoding")) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, http.NoBody) + require.NoError(t, err) + + resp, err := crt.RoundTrip(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + }) + + t.Run("request with body gets compressed", func(t *testing.T) { + ctx := context.Background() + originalBody := "Hello, World! This is some test data that should be compressed." + var receivedBody []byte + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + assert.Empty(t, r.Header.Get("Content-Length")) + assert.Equal(t, int64(-1), r.ContentLength) + + gzipReader, err := gzip.NewReader(r.Body) + require.NoError(t, err) + defer gzipReader.Close() + + receivedBody, err = io.ReadAll(gzipReader) + require.NoError(t, err) + + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, server.URL, strings.NewReader(originalBody)) + require.NoError(t, err) + + resp, err := crt.RoundTrip(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + + assert.Equal(t, originalBody, string(receivedBody)) + }) + + t.Run("preserves existing headers except Content-Length", func(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + assert.Equal(t, "Bearer token123", r.Header.Get("Authorization")) + assert.Equal(t, "gzip", r.Header.Get("Content-Encoding")) + assert.Empty(t, r.Header.Get("Content-Length")) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + crt := uploadrevision.NewCompressionRoundTripper(http.DefaultTransport) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, server.URL, strings.NewReader(`{"key":"value"}`)) + require.NoError(t, err) + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer token123") + req.Header.Set("Content-Length", "15") + + resp, err := crt.RoundTrip(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + resp.Body.Close() + }) + + t.Run("wraps underlying transport errors", func(t *testing.T) { + ctx := context.Background() + failingTransport := &failingRoundTripper{err: assert.AnError} + crt := uploadrevision.NewCompressionRoundTripper(failingTransport) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://example.com", strings.NewReader("test")) + require.NoError(t, err) + + _, err = crt.RoundTrip(req) + assert.Error(t, err) + assert.ErrorIs(t, err, assert.AnError) + }) +} + +// failingRoundTripper is a test helper that always returns an error. +type failingRoundTripper struct { + err error +} + +func (f *failingRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, f.err +} diff --git a/internal/api/fileupload/uploadrevision/errors.go b/internal/api/fileupload/uploadrevision/errors.go new file mode 100644 index 000000000..7c96d44f8 --- /dev/null +++ b/internal/api/fileupload/uploadrevision/errors.go @@ -0,0 +1,174 @@ +package uploadrevision + +import ( + "errors" + "fmt" + "os" +) + +// Sentinel errors for common conditions. +var ( + ErrNoFilesProvided = errors.New("no files provided for upload") + ErrEmptyOrgID = errors.New("organization ID cannot be empty") + ErrEmptyRevisionID = errors.New("revision ID cannot be empty") +) + +// FileSizeLimitError indicates a file exceeds the maximum allowed size. +type FileSizeLimitError struct { + FilePath string + FileSize int64 + Limit int64 +} + +func (e *FileSizeLimitError) Error() string { + return fmt.Sprintf("file %s size %d exceeds limit of %d bytes", e.FilePath, e.FileSize, e.Limit) +} + +// FileCountLimitError indicates too many files were provided. +type FileCountLimitError struct { + Count int + Limit int +} + +func (e *FileCountLimitError) Error() string { + return fmt.Sprintf("too many files: %d exceeds limit of %d", e.Count, e.Limit) +} + +// TotalPayloadSizeLimitError indicates the total size of all files exceeds the maximum allowed payload size. +type TotalPayloadSizeLimitError struct { + TotalSize int64 + Limit int64 +} + +func (e *TotalPayloadSizeLimitError) Error() string { + return fmt.Sprintf("total payload size %d bytes exceeds limit of %d bytes", e.TotalSize, e.Limit) +} + +// FilePathLengthLimitError indicates a file path exceeds the maximum allowed length. +type FilePathLengthLimitError struct { + FilePath string + Length int + Limit int +} + +func (e *FilePathLengthLimitError) Error() string { + return fmt.Sprintf("file name %s length %d exceeds limit of %d characters", e.FilePath, e.Length, e.Limit) +} + +// FileAccessError indicates a file cannot be accessed or read. +type FileAccessError struct { + FilePath string + Err error +} + +func (e *FileAccessError) Error() string { + return fmt.Sprintf("file %s cannot be accessed: %v", e.FilePath, e.Err) +} + +func (e *FileAccessError) Unwrap() error { + return e.Err +} + +// HTTPError represents an HTTP error response. +type HTTPError struct { + StatusCode int + Status string + Operation string + Body []byte +} + +func (e *HTTPError) Error() string { + return fmt.Sprintf("unsuccessful request to %s: %s", e.Operation, e.Status) +} + +// MultipartError indicates an error creating multipart form data. +type MultipartError struct { + FilePath string + Err error +} + +func (e *MultipartError) Error() string { + return fmt.Sprintf("failed to create multipart form for %s: %v", e.FilePath, e.Err) +} + +func (e *MultipartError) Unwrap() error { + return e.Err +} + +// SpecialFileError indicates a path points to a special file (device, pipe, socket, etc.) instead of a regular file. +type SpecialFileError struct { + FilePath string + Mode os.FileMode +} + +func (e *SpecialFileError) Error() string { + return fmt.Sprintf("path %s is not a regular file (mode: %s)", e.FilePath, e.Mode) +} + +// NewFileSizeLimitError creates a new FileSizeLimitError with the given parameters. +func NewFileSizeLimitError(filePath string, fileSize, limit int64) *FileSizeLimitError { + return &FileSizeLimitError{ + FilePath: filePath, + FileSize: fileSize, + Limit: limit, + } +} + +// NewFileCountLimitError creates a new FileCountLimitError with the given parameters. +func NewFileCountLimitError(count, limit int) *FileCountLimitError { + return &FileCountLimitError{ + Count: count, + Limit: limit, + } +} + +// NewTotalPayloadSizeLimitError creates a new TotalPayloadSizeLimitError with the given parameters. +func NewTotalPayloadSizeLimitError(totalSize, limit int64) *TotalPayloadSizeLimitError { + return &TotalPayloadSizeLimitError{ + TotalSize: totalSize, + Limit: limit, + } +} + +// NewFileAccessError creates a new FileAccessError with the given parameters. +func NewFileAccessError(filePath string, err error) *FileAccessError { + return &FileAccessError{ + FilePath: filePath, + Err: err, + } +} + +// NewHTTPError creates a new HTTPError with the given parameters. +func NewHTTPError(statusCode int, status, operation string, body []byte) *HTTPError { + return &HTTPError{ + StatusCode: statusCode, + Status: status, + Operation: operation, + Body: body, + } +} + +// NewMultipartError creates a new MultipartError with the given parameters. +func NewMultipartError(filePath string, err error) *MultipartError { + return &MultipartError{ + FilePath: filePath, + Err: err, + } +} + +// NewSpecialFileError creates a new SpecialFileError with the given path and mode. +func NewSpecialFileError(path string, mode os.FileMode) *SpecialFileError { + return &SpecialFileError{ + FilePath: path, + Mode: mode, + } +} + +// NewFilePathLengthLimitError creates a new FilePathLengthLimitError with the given parameters. +func NewFilePathLengthLimitError(filePath string, length, limit int) *FilePathLengthLimitError { + return &FilePathLengthLimitError{ + FilePath: filePath, + Length: length, + Limit: limit, + } +} diff --git a/internal/api/fileupload/uploadrevision/fake_client.go b/internal/api/fileupload/uploadrevision/fake_client.go new file mode 100644 index 000000000..0a1ed46d7 --- /dev/null +++ b/internal/api/fileupload/uploadrevision/fake_client.go @@ -0,0 +1,146 @@ +package uploadrevision + +import ( + "context" + "fmt" + "io" + + "github.com/google/uuid" +) + +type LoadedFile struct { + Path string + Content string +} + +// revisionState holds the in-memory state for a single revision. +type revisionState struct { + orgID OrgID + sealed bool + files []LoadedFile +} + +// FakeSealableClient is a mock implementation of the SealableClient for testing. +// It tracks revisions in memory and enforces the revision lifecycle (create -> upload -> seal). +type FakeSealableClient struct { + cfg FakeClientConfig + revisions map[RevisionID]*revisionState +} + +type FakeClientConfig struct { + Limits +} + +var _ SealableClient = (*FakeSealableClient)(nil) + +// NewFakeSealableClient creates a new instance of the fake client. +func NewFakeSealableClient(cfg FakeClientConfig) *FakeSealableClient { + return &FakeSealableClient{ + cfg: cfg, + revisions: make(map[RevisionID]*revisionState), + } +} + +func (f *FakeSealableClient) CreateRevision(_ context.Context, orgID OrgID) (*ResponseBody, error) { + newRevisionID := uuid.New() + f.revisions[newRevisionID] = &revisionState{ + orgID: orgID, + sealed: false, + } + + return &ResponseBody{ + Data: ResponseData{ + ID: newRevisionID, + }, + }, nil +} + +func (f *FakeSealableClient) UploadFiles(_ context.Context, orgID OrgID, revisionID RevisionID, files []UploadFile) error { + rev, ok := f.revisions[revisionID] + if !ok { + return fmt.Errorf("revision %s not found", revisionID) + } + + if rev.orgID != orgID { + return fmt.Errorf("orgID mismatch for revision %s", revisionID) + } + + if rev.sealed { + return fmt.Errorf("revision %s is sealed and cannot be modified", revisionID) + } + + if len(files) > f.cfg.FileCountLimit { + return NewFileCountLimitError(len(files), f.cfg.FileCountLimit) + } + + if len(files) == 0 { + return ErrNoFilesProvided + } + + var totalPayloadSize int64 + for _, file := range files { + fileInfo, err := file.File.Stat() + if err != nil { + return NewFileAccessError(file.Path, err) + } + + if !fileInfo.Mode().IsRegular() { + return NewSpecialFileError(file.Path, fileInfo.Mode()) + } + + if fileInfo.Size() > f.cfg.FileSizeLimit { + return NewFileSizeLimitError(file.Path, fileInfo.Size(), f.cfg.FileSizeLimit) + } + + totalPayloadSize += fileInfo.Size() + } + + if totalPayloadSize > f.cfg.TotalPayloadSizeLimit { + return NewTotalPayloadSizeLimitError(totalPayloadSize, f.cfg.TotalPayloadSizeLimit) + } + + for _, file := range files { + bts, err := io.ReadAll(file.File) + if err != nil { + return err + } + rev.files = append(rev.files, LoadedFile{ + Path: file.Path, + Content: string(bts), + }) + } + return nil +} + +func (f *FakeSealableClient) SealRevision(_ context.Context, orgID OrgID, revisionID RevisionID) (*SealResponseBody, error) { + rev, ok := f.revisions[revisionID] + if !ok { + return nil, fmt.Errorf("revision %s not found", revisionID) + } + + if rev.orgID != orgID { + return nil, fmt.Errorf("orgID mismatch for revision %s", revisionID) + } + + rev.sealed = true + return &SealResponseBody{}, nil +} + +// GetSealedRevisionFiles is a test helper to retrieve files for a sealed revision. +// It is not part of the SealableClient interface. +func (f *FakeSealableClient) GetSealedRevisionFiles(revisionID RevisionID) ([]LoadedFile, error) { + rev, ok := f.revisions[revisionID] + if !ok { + return nil, fmt.Errorf("revision %s not found", revisionID) + } + + if !rev.sealed { + return nil, fmt.Errorf("revision %s is not sealed", revisionID) + } + + return rev.files, nil +} + +func (f *FakeSealableClient) GetLimits() Limits { + return f.cfg.Limits +} diff --git a/internal/api/fileupload/uploadrevision/opts.go b/internal/api/fileupload/uploadrevision/opts.go new file mode 100644 index 000000000..9b1cfbbf0 --- /dev/null +++ b/internal/api/fileupload/uploadrevision/opts.go @@ -0,0 +1,13 @@ +package uploadrevision + +import "net/http" + +// Opt is a function that configures an HTTPSealableClient instance. +type Opt func(*HTTPSealableClient) + +// WithHTTPClient sets a custom HTTP client for the file upload client. +func WithHTTPClient(httpClient *http.Client) Opt { + return func(c *HTTPSealableClient) { + c.httpClient = httpClient + } +} diff --git a/internal/api/fileupload/uploadrevision/opts_test.go b/internal/api/fileupload/uploadrevision/opts_test.go new file mode 100644 index 000000000..99d201996 --- /dev/null +++ b/internal/api/fileupload/uploadrevision/opts_test.go @@ -0,0 +1,47 @@ +package uploadrevision_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) + +type CustomRoundTripper struct{} + +func (crt *CustomRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + r.Header.Set("foo", "bar") + return http.DefaultTransport.RoundTrip(r) +} + +func Test_WithHTTPClient(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fooValue := r.Header.Get("foo") + assert.Equal(t, "bar", fooValue) + + resp, err := json.Marshal(uploadrevision2.ResponseBody{}) + require.NoError(t, err) + + w.WriteHeader(http.StatusCreated) + _, err = w.Write(resp) + assert.NoError(t, err) + })) + defer srv.Close() + customClient := srv.Client() + customClient.Transport = &CustomRoundTripper{} + + llc := uploadrevision2.NewClient(uploadrevision2.Config{ + BaseURL: srv.URL, + }, uploadrevision2.WithHTTPClient(customClient)) + + _, err := llc.CreateRevision(context.Background(), uuid.New()) + + require.NoError(t, err) +} diff --git a/internal/api/fileupload/uploadrevision/types.go b/internal/api/fileupload/uploadrevision/types.go new file mode 100644 index 000000000..9d4563e14 --- /dev/null +++ b/internal/api/fileupload/uploadrevision/types.go @@ -0,0 +1,138 @@ +package uploadrevision + +import ( + "io/fs" + + "github.com/google/uuid" +) + +// OrgID represents an organization identifier. +type OrgID = uuid.UUID + +// RevisionID represents a revision identifier. +type RevisionID = uuid.UUID + +// RevisionType represents the type of revision being created. +type RevisionType string + +const ( + // RevisionTypeSnapshot represents a snapshot revision type. + RevisionTypeSnapshot RevisionType = "snapshot" +) + +// ResourceType represents the type of resource in API requests. +type ResourceType string + +const ( + // ResourceTypeUploadRevision represents an upload revision resource type. + ResourceTypeUploadRevision ResourceType = "upload_revision" +) + +// RequestAttributes contains the attributes for creating an upload revision. +type RequestAttributes struct { + RevisionType RevisionType `json:"revision_type"` //nolint:tagliatelle // API expects snake_case +} + +// RequestData contains the data payload for creating an upload revision. +type RequestData struct { + Attributes RequestAttributes `json:"attributes"` + Type ResourceType `json:"type"` +} + +// RequestBody contains the complete request body for creating an upload revision. +type RequestBody struct { + Data RequestData `json:"data"` +} + +// ResponseAttributes contains the attributes returned when creating an upload revision. +type ResponseAttributes struct { + RevisionType RevisionType `json:"revision_type"` //nolint:tagliatelle // API expects snake_case + Sealed bool `json:"sealed"` +} + +// ResponseData contains the data returned when creating an upload revision. +type ResponseData struct { + ID RevisionID `json:"id"` + Type ResourceType `json:"type"` + Attributes ResponseAttributes `json:"attributes"` +} + +// ResponseBody contains the complete response body when creating an upload revision. +type ResponseBody struct { + Data ResponseData `json:"data"` +} + +// SealRequestAttributes contains the attributes for sealing an upload revision. +type SealRequestAttributes struct { + Sealed bool `json:"sealed"` +} + +// SealRequestData contains the data payload for sealing an upload revision. +type SealRequestData struct { + ID RevisionID `json:"id"` + Type ResourceType `json:"type"` + Attributes SealRequestAttributes `json:"attributes"` +} + +// SealRequestBody contains the complete request body for sealing an upload revision. +type SealRequestBody struct { + Data SealRequestData `json:"data"` +} + +// SealResponseAttributes contains the attributes returned when sealing an upload revision. +type SealResponseAttributes struct { + RevisionType RevisionType `json:"revision_type"` //nolint:tagliatelle // API expects snake_case + Sealed bool `json:"sealed"` +} + +// SealResponseData contains the data returned when sealing an upload revision. +type SealResponseData struct { + ID RevisionID `json:"id"` + Type ResourceType `json:"type"` + Attributes SealResponseAttributes `json:"attributes"` +} + +// SealResponseBody contains the complete response body when sealing an upload revision. +type SealResponseBody struct { + Data SealResponseData `json:"data"` +} + +// ResponseError represents an error in an API response. +type ResponseError struct { + ID string `json:"id"` + Title string `json:"title"` + Status string `json:"status"` + Detail string `json:"detail"` +} + +// ErrorResponseBody contains the complete error response body. +type ErrorResponseBody struct { + Errors []ResponseError `json:"errors"` +} + +// UploadFile represents a file to be uploaded, containing both the path and file handle. +type UploadFile struct { + Path string // The path of the uploaded file, relative to the root directory. + File fs.File +} + +const ( + // ContentType is the HTTP header name for content type. + ContentType = "Content-Type" + // ContentEncoding is the HTTP header name for content encoding. + ContentEncoding = "Content-Encoding" + // ContentLength is the HTTP header name for content length. + ContentLength = "Content-Length" +) + +// Limits contains the limits enforced by the low level client. +type Limits struct { + // FileCountLimit specifies the maximum number of files allowed in a single upload. + FileCountLimit int + // FileSizeLimit specifies the maximum allowed file size in bytes. + FileSizeLimit int64 + // TotalPayloadSizeLimit specifies the maximum total uncompressed payload size in bytes. + TotalPayloadSizeLimit int64 + // FilePathLengthLimit specifies the maximum allowed file name length in characters. + FilePathLengthLimit int +} diff --git a/pkg/apiclients/fileupload/batch.go b/pkg/apiclients/fileupload/batch.go new file mode 100644 index 000000000..0153a6b84 --- /dev/null +++ b/pkg/apiclients/fileupload/batch.go @@ -0,0 +1,115 @@ +package fileupload + +import ( + "fmt" + "iter" + "os" + "path/filepath" + + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) + +// uploadBatch manages a batch of files for upload. +type uploadBatch struct { + files []uploadrevision.UploadFile + currentSize int64 + limits uploadrevision.Limits +} + +func newUploadBatch(limits uploadrevision.Limits) *uploadBatch { + return &uploadBatch{ + files: make([]uploadrevision.UploadFile, 0, limits.FileCountLimit), + limits: limits, + } +} + +func (b *uploadBatch) addFile(file uploadrevision.UploadFile, fileSize int64) { + b.files = append(b.files, file) + b.currentSize += fileSize +} + +func (b *uploadBatch) wouldExceedLimits(fileSize int64) bool { + wouldExceedCount := len(b.files) >= b.limits.FileCountLimit + wouldExceedSize := b.currentSize+fileSize > b.limits.TotalPayloadSizeLimit + return wouldExceedCount || wouldExceedSize +} + +func (b *uploadBatch) isEmpty() bool { + return len(b.files) == 0 +} + +func (b *uploadBatch) closeRemainingFiles() { + for _, file := range b.files { + file.File.Close() + } +} + +type batchingResult struct { + batch *uploadBatch + filteredFiles []FilteredFile +} + +func batchPaths(rootPath string, paths <-chan string, limits uploadrevision.Limits, filters ...filter) iter.Seq2[*batchingResult, error] { + return func(yield func(*batchingResult, error) bool) { + batch := newUploadBatch(limits) + filtered := []FilteredFile{} + for path := range paths { + relPath, err := filepath.Rel(rootPath, path) + if err != nil { + if !yield(nil, fmt.Errorf("failed to get relative path of file %s: %w", path, err)) { + return + } + } + + f, err := os.Open(path) + if err != nil { + f.Close() + if !yield(nil, fmt.Errorf("failed to open file %s: %w", path, err)) { + return + } + } + + fstat, err := f.Stat() + if err != nil { + f.Close() + if !yield(nil, fmt.Errorf("failed to stat file %s: %w", path, err)) { + return + } + } + + ff := applyFilters(fileToFilter{Path: relPath, Stat: fstat}, filters...) + if ff != nil { + f.Close() + filtered = append(filtered, *ff) + continue + } + + if batch.wouldExceedLimits(fstat.Size()) { + if !yield(&batchingResult{batch: batch, filteredFiles: filtered}, nil) { + return + } + batch = newUploadBatch(limits) + filtered = []FilteredFile{} + } + + batch.addFile(uploadrevision.UploadFile{ + Path: relPath, + File: f, + }, fstat.Size()) + } + + if !batch.isEmpty() || len(filtered) > 0 { + yield(&batchingResult{batch: batch, filteredFiles: filtered}, nil) + } + } +} + +func applyFilters(ff fileToFilter, filters ...filter) *FilteredFile { + for _, filter := range filters { + if ff := filter(ff); ff != nil { + return ff + } + } + + return nil +} diff --git a/pkg/apiclients/fileupload/client.go b/pkg/apiclients/fileupload/client.go new file mode 100644 index 000000000..a8b6876e1 --- /dev/null +++ b/pkg/apiclients/fileupload/client.go @@ -0,0 +1,333 @@ +package fileupload + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "runtime" + + "github.com/google/uuid" + "github.com/puzpuzpuz/xsync" + "github.com/rs/zerolog" + + fileuploadinternal "github.com/snyk/go-application-framework/internal/api/fileupload" + "github.com/snyk/go-application-framework/internal/api/fileupload/filters" + uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/pkg/utils" +) + +// Config contains configuration for the file upload client. +type Config struct { + BaseURL string + OrgID OrgID +} + +// HTTPClient provides high-level file upload functionality. +type HTTPClient struct { + uploadRevisionSealableClient uploadrevision2.SealableClient + filtersClient filters.Client + cfg Config + filters fileuploadinternal.Filters + logger *zerolog.Logger +} + +// Client defines the interface for the high level file upload client. +type Client interface { + CreateRevisionFromPaths(ctx context.Context, paths []string) (UploadResult, error) + CreateRevisionFromDir(ctx context.Context, dirPath string) (UploadResult, error) + CreateRevisionFromFile(ctx context.Context, filePath string) (UploadResult, error) +} + +var _ Client = (*HTTPClient)(nil) + +// NewClient creates a new high-level file upload client. +func NewClient(httpClient *http.Client, cfg Config, opts ...Option) Client { + client := &HTTPClient{ + cfg: cfg, + filters: fileuploadinternal.Filters{ + SupportedExtensions: xsync.NewMapOf[bool](), + SupportedConfigFiles: xsync.NewMapOf[bool](), + }, + } + + for _, opt := range opts { + opt(client) + } + + if client.logger == nil { + client.logger = utils.Ptr(zerolog.Nop()) + } + + if client.uploadRevisionSealableClient == nil { + client.uploadRevisionSealableClient = uploadrevision2.NewClient(uploadrevision2.Config{ + BaseURL: cfg.BaseURL, + }, uploadrevision2.WithHTTPClient(httpClient)) + } + + if client.filtersClient == nil { + client.filtersClient = filters.NewDeeproxyClient(filters.Config{ + BaseURL: cfg.BaseURL, + IsFedRamp: false, //cfg.IsFedRamp, + }, filters.WithHTTPClient(httpClient)) + } + + return client +} + +func (c *HTTPClient) loadFilters(ctx context.Context) error { + c.filters.Once.Do(func() { + filtersResp, err := c.filtersClient.GetFilters(ctx, c.cfg.OrgID) + if err != nil { + c.filters.InitErr = err + return + } + + for _, ext := range filtersResp.Extensions { + c.filters.SupportedExtensions.Store(ext, true) + } + for _, configFile := range filtersResp.ConfigFiles { + // .gitignore and .dcignore should not be uploaded + // (https://github.com/snyk/code-client/blob/d6f6a2ce4c14cb4b05aa03fb9f03533d8cf6ca4a/src/files.ts#L138) + if configFile == ".gitignore" || configFile == ".dcignore" { + continue + } + c.filters.SupportedConfigFiles.Store(configFile, true) + } + }) + return c.filters.InitErr +} + +// createDeeproxyFilter creates a filter function based on the current deeproxy filtering configuration. +func (c *HTTPClient) createDeeproxyFilter(ctx context.Context) (filter, error) { + if err := c.loadFilters(ctx); err != nil { + return nil, fmt.Errorf("failed to load deeproxy filters: %w", err) + } + + return func(ff fileToFilter) *FilteredFile { + fileExt := filepath.Ext(ff.Stat.Name()) + fileName := filepath.Base(ff.Stat.Name()) + _, isSupportedExtension := c.filters.SupportedExtensions.Load(fileExt) + _, isSupportedConfigFile := c.filters.SupportedConfigFiles.Load(fileName) + + if !isSupportedExtension && !isSupportedConfigFile { + var reason error + if !isSupportedConfigFile { + reason = errors.Join(reason, fmt.Errorf("file name is not a part of the supported config files: %s", fileName)) + } + if !isSupportedExtension { + reason = errors.Join(reason, fmt.Errorf("file extension is not supported: %s", fileExt)) + } + return &FilteredFile{ + Path: ff.Path, + Reason: reason, + } + } + + return nil + }, nil +} + +func (c *HTTPClient) uploadBatch(ctx context.Context, revID RevisionID, batch *uploadBatch) error { + defer batch.closeRemainingFiles() + + if batch.isEmpty() { + return nil + } + + err := c.uploadRevisionSealableClient.UploadFiles(ctx, c.cfg.OrgID, revID, batch.files) + if err != nil { + return fmt.Errorf("failed to upload files: %w", err) + } + + return nil +} + +// addPathsToRevision adds multiple file paths to an existing revision. +func (c *HTTPClient) addPathsToRevision( + ctx context.Context, + revisionID RevisionID, + rootPath string, + pathsChan <-chan string, + opts uploadOptions, +) (UploadResult, error) { + res := UploadResult{ + RevisionID: revisionID, + FilteredFiles: make([]FilteredFile, 0), + } + + fileSizeFilter := func(ff fileToFilter) *FilteredFile { + fileSizeLimit := c.uploadRevisionSealableClient.GetLimits().FileSizeLimit + if ff.Stat.Size() > fileSizeLimit { + return &FilteredFile{ + Path: ff.Path, + Reason: uploadrevision2.NewFileSizeLimitError(ff.Stat.Name(), ff.Stat.Size(), fileSizeLimit), + } + } + + return nil + } + + filePathLengthFilter := func(ff fileToFilter) *FilteredFile { + filePathLengthLimit := c.uploadRevisionSealableClient.GetLimits().FilePathLengthLimit + if len(ff.Path) > filePathLengthLimit { + return &FilteredFile{ + Path: ff.Path, + Reason: uploadrevision2.NewFilePathLengthLimitError(ff.Path, len(ff.Path), filePathLengthLimit), + } + } + + return nil + } + + filters := []filter{ + fileSizeFilter, + filePathLengthFilter, + } + if !opts.SkipDeeproxyFiltering { + deeproxyFilter, err := c.createDeeproxyFilter(ctx) + if err != nil { + return res, err + } + + filters = append(filters, deeproxyFilter) + } + + for batchResult, err := range batchPaths(rootPath, pathsChan, c.uploadRevisionSealableClient.GetLimits(), filters...) { + if err != nil { + return res, fmt.Errorf("failed to batch files: %w", err) + } + + res.FilteredFiles = append(res.FilteredFiles, batchResult.filteredFiles...) + + err = c.uploadBatch(ctx, revisionID, batchResult.batch) + if err != nil { + return res, err + } + + res.UploadedFilesCount += len(batchResult.batch.files) + } + + return res, nil +} + +// createRevision creates a new revision and returns its ID. +func (c *HTTPClient) createRevision(ctx context.Context) (RevisionID, error) { + revision, err := c.uploadRevisionSealableClient.CreateRevision(ctx, c.cfg.OrgID) + if err != nil { + return uuid.Nil, fmt.Errorf("failed to create revision: %w", err) + } + return revision.Data.ID, nil +} + +// addFileToRevision adds a single file to an existing revision. +func (c *HTTPClient) addFileToRevision(ctx context.Context, revisionID RevisionID, filePath string, opts uploadOptions) (UploadResult, error) { + writableChan := make(chan string, 1) + writableChan <- filePath + close(writableChan) + + return c.addPathsToRevision(ctx, revisionID, filepath.Dir(filePath), writableChan, opts) +} + +// addDirToRevision adds a directory and all its contents to an existing revision. +func (c *HTTPClient) addDirToRevision(ctx context.Context, revisionID RevisionID, dirPath string, opts uploadOptions) (UploadResult, error) { + //nolint:contextcheck // will be considered later + sources, err := forPath(dirPath, c.logger, runtime.NumCPU()) + if err != nil { + return UploadResult{}, fmt.Errorf("failed to list files in directory %s: %w", dirPath, err) + } + + return c.addPathsToRevision(ctx, revisionID, dirPath, sources, opts) +} + +// sealRevision seals a revision, making it immutable. +func (c *HTTPClient) sealRevision(ctx context.Context, revisionID RevisionID) error { + _, err := c.uploadRevisionSealableClient.SealRevision(ctx, c.cfg.OrgID, revisionID) + if err != nil { + return fmt.Errorf("failed to seal revision: %w", err) + } + return nil +} + +// CreateRevisionFromPaths uploads multiple paths (files or directories), returning a revision ID. +// This is a convenience method that creates, uploads, and seals a revision. +func (c *HTTPClient) CreateRevisionFromPaths(ctx context.Context, paths []string) (UploadResult, error) { + opts := uploadOptions{ + SkipDeeproxyFiltering: true, + } + + res := UploadResult{ + FilteredFiles: make([]FilteredFile, 0), + } + + revisionID, err := c.createRevision(ctx) + if err != nil { + return res, err + } + res.RevisionID = revisionID + + for _, pth := range paths { + info, err := os.Stat(pth) + if err != nil { + return UploadResult{}, uploadrevision2.NewFileAccessError(pth, err) + } + + if info.IsDir() { + dirUploadRes, err := c.addDirToRevision(ctx, revisionID, pth, opts) + if err != nil { + return res, fmt.Errorf("failed to add directory %s: %w", pth, err) + } + res.FilteredFiles = append(res.FilteredFiles, dirUploadRes.FilteredFiles...) + res.UploadedFilesCount += dirUploadRes.UploadedFilesCount + } else { + fileUploadRes, err := c.addFileToRevision(ctx, revisionID, pth, opts) + if err != nil { + return res, fmt.Errorf("failed to add file %s: %w", pth, err) + } + res.FilteredFiles = append(res.FilteredFiles, fileUploadRes.FilteredFiles...) + res.UploadedFilesCount += fileUploadRes.UploadedFilesCount + } + } + + if res.UploadedFilesCount == 0 && len(res.FilteredFiles) == 0 { + return res, ErrNoFilesProvided + } + + if err := c.sealRevision(ctx, revisionID); err != nil { + return res, err + } + + return res, nil +} + +// CreateRevisionFromDir uploads a directory and all its contents, returning a revision ID. +// This is a convenience method for validating the directory path and calling CreateRevisionFromPaths with a single directory path. +func (c *HTTPClient) CreateRevisionFromDir(ctx context.Context, dirPath string) (UploadResult, error) { + info, err := os.Stat(dirPath) + if err != nil { + return UploadResult{}, uploadrevision2.NewFileAccessError(dirPath, err) + } + + if !info.IsDir() { + return UploadResult{}, fmt.Errorf("the provided path is not a directory: %s", dirPath) + } + + return c.CreateRevisionFromPaths(ctx, []string{dirPath}) +} + +// CreateRevisionFromFile uploads a single file, returning a revision ID. +// This is a convenience method for validating the file path and calling CreateRevisionFromPaths with a single file path. +func (c *HTTPClient) CreateRevisionFromFile(ctx context.Context, filePath string) (UploadResult, error) { + info, err := os.Stat(filePath) + if err != nil { + return UploadResult{}, uploadrevision2.NewFileAccessError(filePath, err) + } + + if !info.Mode().IsRegular() { + return UploadResult{}, fmt.Errorf("the provided path is not a regular file: %s", filePath) + } + + return c.CreateRevisionFromPaths(ctx, []string{filePath}) +} diff --git a/pkg/apiclients/fileupload/client_test.go b/pkg/apiclients/fileupload/client_test.go new file mode 100644 index 000000000..726137909 --- /dev/null +++ b/pkg/apiclients/fileupload/client_test.go @@ -0,0 +1,613 @@ +package fileupload_test + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/snyk/go-application-framework/internal/api/fileupload/filters" + uploadrevision2 "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" +) + +var mainpath = filepath.Join("src", "main.go") +var utilspath = filepath.Join("src", "utils.go") +var helperpath = filepath.Join("src", "utils", "helper.go") +var docpath = filepath.Join("docs", "README.md") +var gomodpath = filepath.Join("src", "go.mod") +var scriptpath = filepath.Join("src", "script.js") +var packagelockpath = filepath.Join("src", "package.json") +var nonexistpath = filepath.Join("nonexistent", "file.go") +var missingpath = filepath.Join("another", "missing", "path.txt") + +// CreateTmpFiles is an utility function used to create temporary files in tests. +func createTmpFiles(t *testing.T, files []uploadrevision2.LoadedFile) (dir *os.File) { + t.Helper() + + tempDir := t.TempDir() + dir, err := os.Open(tempDir) + if err != nil { + panic(err) + } + + for _, file := range files { + fullPath := filepath.Join(tempDir, file.Path) + + parentDir := filepath.Dir(fullPath) + if err := os.MkdirAll(parentDir, 0o755); err != nil { + panic(err) + } + + f, err := os.Create(fullPath) + if err != nil { + panic(err) + } + + if _, err := f.WriteString(file.Content); err != nil { + f.Close() + panic(err) + } + f.Close() + } + + t.Cleanup(func() { + if dir != nil { + dir.Close() + } + }) + + return dir +} + +func Test_CreateRevisionFromPaths(t *testing.T) { + llcfg := uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 10, + FileSizeLimit: 100, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 20, + }, + } + + allowList := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("mixed files and directories", func(t *testing.T) { + allFiles := []uploadrevision2.LoadedFile{ + {Path: mainpath, Content: "package main"}, + {Path: utilspath, Content: "package utils"}, + {Path: "config.yaml", Content: "version: 1"}, + {Path: "README.md", Content: "# Project"}, + } + + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList) + + paths := []string{ + filepath.Join(dir.Name(), "src"), // Directory + filepath.Join(dir.Name(), "README.md"), // Individual file + } + + res, err := client.CreateRevisionFromPaths(ctx, paths) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + require.Len(t, uploadedFiles, 3) // 2 from src/ + 1 README.md + + uploadedPaths := make([]string, len(uploadedFiles)) + for i, f := range uploadedFiles { + uploadedPaths[i] = f.Path + } + assert.Contains(t, uploadedPaths, "main.go") + assert.Contains(t, uploadedPaths, "utils.go") + assert.Contains(t, uploadedPaths, "README.md") + }) + + t.Run("error handling with better context", func(t *testing.T) { + ctx, _, client, _ := setupTest(t, llcfg, []uploadrevision2.LoadedFile{}, allowList) + + paths := []string{ + nonexistpath, + missingpath, + } + + _, err := client.CreateRevisionFromPaths(ctx, paths) + require.Error(t, err) + var fileAccessErr *uploadrevision2.FileAccessError + assert.ErrorAs(t, err, &fileAccessErr) + assert.Equal(t, nonexistpath, fileAccessErr.FilePath) + }) +} + +func Test_CreateRevisionFromDir(t *testing.T) { + llcfg := uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 2, + FileSizeLimit: 100, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 20, + }, + } + + allowList := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("uploading a shallow directory", func(t *testing.T) { + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "file1.txt", + Content: "content1", + }, + { + Path: "file2.txt", + Content: "content2", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with nested files", func(t *testing.T) { + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: filepath.Join("src", "main.go"), + Content: "package main\n\nfunc main() {}", + }, + { + Path: filepath.Join("src", "utils", "helper.go"), + Content: "package utils\n\nfunc Helper() {}", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory exceeding the file count limit for a single upload", func(t *testing.T) { + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "file1.txt", + Content: "root level file", + }, + { + Path: mainpath, + Content: "package main\n\nfunc main() {}", + }, + { + Path: helperpath, + Content: "package utils\n\nfunc Helper() {}", + }, + { + Path: docpath, + Content: "# Project Documentation", + }, + { + Path: gomodpath, + Content: "foo bar", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with file exceeding the file size limit", func(t *testing.T) { + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "file2.txt", + Content: "foo", + }, + } + additionalFiles := []uploadrevision2.LoadedFile{ + { + Path: "file1.txt", + Content: "foo bar", + }, + } + + allFiles := make([]uploadrevision2.LoadedFile, 0, 2) + allFiles = append(allFiles, expectedFiles...) + allFiles = append(allFiles, additionalFiles...) + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 2, + FileSizeLimit: 6, + TotalPayloadSizeLimit: 100, + FilePathLengthLimit: 20, + }, + }, allFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + var fileSizeErr *uploadrevision2.FileSizeLimitError + assert.Len(t, res.FilteredFiles, 1) + ff := res.FilteredFiles[0] + assert.Contains(t, ff.Path, "file1.txt") + assert.ErrorAs(t, ff.Reason, &fileSizeErr) + assert.Equal(t, "file1.txt", fileSizeErr.FilePath) + assert.Equal(t, int64(6), fileSizeErr.Limit) + assert.Equal(t, int64(7), fileSizeErr.FileSize) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory exceeding total payload size limit triggers batching", func(t *testing.T) { + // Create files that together exceed the payload size limit but not the count limit + // Each file is 30 bytes, limit is 70 bytes, so 3 files (90 bytes) should be split into 2 batches + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "file1.txt", + Content: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", // 30 bytes + }, + { + Path: "file2.txt", + Content: "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyy", // 30 bytes + }, + { + Path: "file3.txt", + Content: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", // 30 bytes + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 10, // High enough to not trigger count-based batching + FileSizeLimit: 50, // Each file is under this + TotalPayloadSizeLimit: 70, // 70 bytes - forces batching by size + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + // Success proves size-based batching works - without it, the low-level client + // would reject the 90-byte payload (limit: 70 bytes). + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading large individual files near payload limit", func(t *testing.T) { + // Tests edge case where individual files are large relative to the payload limit. + // File1: 150 bytes, File2: 80 bytes, File3: 60 bytes; Limit: 200 bytes + // Expected batches: [File1], [File2], [File3] - each file in its own batch + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "large1.txt", + Content: string(make([]byte, 150)), + }, + { + Path: "large2.txt", + Content: string(make([]byte, 80)), + }, + { + Path: "large3.txt", + Content: string(make([]byte, 60)), + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 10, + FileSizeLimit: 160, + TotalPayloadSizeLimit: 200, + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading files with variable sizes triggers optimal batching", func(t *testing.T) { + // Tests realistic scenario with mixed file sizes. + // Files: 10, 60, 5, 70, 45 bytes; Limit: 100 bytes + // Expected batching: [10+60+5=75], [70], [45] + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "tiny.txt", + Content: string(make([]byte, 10)), + }, + { + Path: "medium.txt", + Content: string(make([]byte, 60)), + }, + { + Path: "small.txt", + Content: string(make([]byte, 5)), + }, + { + Path: "large.txt", + Content: string(make([]byte, 70)), + }, + { + Path: "mid.txt", + Content: string(make([]byte, 45)), + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 10, + FileSizeLimit: 80, + TotalPayloadSizeLimit: 100, + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading directory where both size and count limits would be reached", func(t *testing.T) { + // Tests scenario where both limits are approached. + // 8 files of 30 bytes each = 240 bytes total + // FileCountLimit: 10, TotalPayloadSizeLimit: 200 bytes + // Should batch by size first: [file1-6=180], [file7-8=60] + expectedFiles := make([]uploadrevision2.LoadedFile, 8) + for i := 0; i < 8; i++ { + expectedFiles[i] = uploadrevision2.LoadedFile{ + Path: fmt.Sprintf("file%d.txt", i), + Content: string(make([]byte, 30)), + } + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 10, + FileSizeLimit: 50, + TotalPayloadSizeLimit: 200, + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a directory with filtering disabled", func(t *testing.T) { + allFiles := []uploadrevision2.LoadedFile{ + { + Path: mainpath, + Content: "package main\n\nfunc main() {}", + }, + { + Path: helperpath, + Content: "package utils\n\nfunc Helper() {}", + }, + { + Path: gomodpath, + Content: "foo bar", + }, + { + Path: scriptpath, + Content: "console.log('hi')", + }, + { + Path: packagelockpath, + Content: "{}", + }, + } + + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, allFiles, allowList) + + res, err := client.CreateRevisionFromDir(ctx, dir.Name()) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, allFiles, uploadedFiles) + }) +} + +func Test_CreateRevisionFromFile(t *testing.T) { + llcfg := uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 2, + FileSizeLimit: 100, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 20, + }, + } + + allowList := filters.AllowList{ + ConfigFiles: []string{"go.mod"}, + Extensions: []string{".txt", ".go", ".md"}, + } + + t.Run("uploading a file", func(t *testing.T) { + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "file1.txt", + Content: "content1", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) + + t.Run("uploading a file exceeding the file size limit", func(t *testing.T) { + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "file1.txt", + Content: "foo bar", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 1, + FileSizeLimit: 6, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 20, + }, + }, expectedFiles, allowList) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) + require.NoError(t, err) + + var fileSizeErr *uploadrevision2.FileSizeLimitError + assert.Len(t, res.FilteredFiles, 1) + ff := res.FilteredFiles[0] + assert.Contains(t, ff.Path, "file1.txt") + assert.ErrorAs(t, ff.Reason, &fileSizeErr) + assert.Equal(t, "file1.txt", fileSizeErr.FilePath) + assert.Equal(t, int64(6), fileSizeErr.Limit) + assert.Equal(t, int64(7), fileSizeErr.FileSize) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, nil, uploadedFiles) + }) + + t.Run("uploading a file exceeding the file path limit", func(t *testing.T) { + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "file1.txt", + Content: "foo bar", + }, + } + ctx, fakeSealableClient, client, dir := setupTest(t, uploadrevision2.FakeClientConfig{ + Limits: uploadrevision2.Limits{ + FileCountLimit: 1, + FileSizeLimit: 10, + TotalPayloadSizeLimit: 10_000, + FilePathLengthLimit: 5, + }, + }, expectedFiles, allowList) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "file1.txt")) + require.NoError(t, err) + + var filePathErr *uploadrevision2.FilePathLengthLimitError + assert.Len(t, res.FilteredFiles, 1) + ff := res.FilteredFiles[0] + assert.Contains(t, ff.Path, "file1.txt") + assert.ErrorAs(t, ff.Reason, &filePathErr) + assert.Equal(t, "file1.txt", filePathErr.FilePath) + assert.Equal(t, 5, filePathErr.Limit) + + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, nil, uploadedFiles) + }) + + t.Run("uploading a file with filtering disabled", func(t *testing.T) { + expectedFiles := []uploadrevision2.LoadedFile{ + { + Path: "script.js", + Content: "console.log('hi')", + }, + } + + ctx, fakeSealableClient, client, dir := setupTest(t, llcfg, expectedFiles, allowList) + + res, err := client.CreateRevisionFromFile(ctx, path.Join(dir.Name(), "script.js")) + require.NoError(t, err) + + assert.Empty(t, res.FilteredFiles) + uploadedFiles, err := fakeSealableClient.GetSealedRevisionFiles(res.RevisionID) + require.NoError(t, err) + expectEqualFiles(t, expectedFiles, uploadedFiles) + }) +} + +func expectEqualFiles(t *testing.T, expectedFiles, uploadedFiles []uploadrevision2.LoadedFile) { + t.Helper() + + require.Equal(t, len(expectedFiles), len(uploadedFiles)) + + slices.SortFunc(expectedFiles, func(fileA, fileB uploadrevision2.LoadedFile) int { + return strings.Compare(fileA.Path, fileB.Path) + }) + + slices.SortFunc(uploadedFiles, func(fileA, fileB uploadrevision2.LoadedFile) int { + return strings.Compare(fileA.Path, fileB.Path) + }) + + for i := range uploadedFiles { + assert.Equal(t, expectedFiles[i].Path, uploadedFiles[i].Path) + assert.Equal(t, expectedFiles[i].Content, uploadedFiles[i].Content) + } +} + +func setupTest( + t *testing.T, + llcfg uploadrevision2.FakeClientConfig, + files []uploadrevision2.LoadedFile, + _ filters.AllowList, +) (context.Context, *uploadrevision2.FakeSealableClient, fileupload.Client, *os.File) { + t.Helper() + + ctx := context.Background() + orgID := uuid.New() + + fakeSealeableClient := uploadrevision2.NewFakeSealableClient(llcfg) + client := fileupload.NewClient( + nil, + fileupload.Config{ + OrgID: orgID, + }, + fileupload.WithUploadRevisionSealableClient(fakeSealeableClient), + ) + + dir := createTmpFiles(t, files) + + return ctx, fakeSealeableClient, client, dir +} diff --git a/pkg/apiclients/fileupload/errors.go b/pkg/apiclients/fileupload/errors.go new file mode 100644 index 000000000..0c35bc39a --- /dev/null +++ b/pkg/apiclients/fileupload/errors.go @@ -0,0 +1,38 @@ +package fileupload + +import ( + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) + +// Aliasing uploadRevisionSealableClient errors so that they're scoped to the fileupload package as well. + +// Sentinel errors for common conditions. +var ( + ErrNoFilesProvided = uploadrevision.ErrNoFilesProvided + ErrEmptyOrgID = uploadrevision.ErrEmptyOrgID + ErrEmptyRevisionID = uploadrevision.ErrEmptyRevisionID +) + +// FileSizeLimitError indicates a file exceeds the maximum allowed size. +type FileSizeLimitError = uploadrevision.FileSizeLimitError + +// FilePathLengthLimitError indicates a file's path exceeds the maximum allowed size. +type FilePathLengthLimitError = uploadrevision.FilePathLengthLimitError + +// FileCountLimitError indicates too many files were provided. +type FileCountLimitError = uploadrevision.FileCountLimitError + +// TotalPayloadSizeLimitError indicates the total size of all files exceeds the maximum allowed payload size. +type TotalPayloadSizeLimitError = uploadrevision.TotalPayloadSizeLimitError + +// FileAccessError indicates a file access permission issue. +type FileAccessError = uploadrevision.FileAccessError + +// SpecialFileError indicates a path points to a special file (device, pipe, socket, etc.) instead of a regular file. +type SpecialFileError = uploadrevision.SpecialFileError + +// HTTPError indicates an HTTP request/response error. +type HTTPError = uploadrevision.HTTPError + +// MultipartError indicates an issue with multipart request handling. +type MultipartError = uploadrevision.MultipartError diff --git a/pkg/apiclients/fileupload/fake_client.go b/pkg/apiclients/fileupload/fake_client.go new file mode 100644 index 000000000..4cd59a087 --- /dev/null +++ b/pkg/apiclients/fileupload/fake_client.go @@ -0,0 +1,99 @@ +package fileupload + +import ( + "context" + "fmt" + "os" + + "github.com/google/uuid" + + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) + +type FakeClient struct { + revisions map[RevisionID][]string + err error + uploadCount int // Tracks how many uploads have occurred + lastRevision RevisionID +} + +var _ Client = (*FakeClient)(nil) + +// NewFakeClient creates a new fake client. +func NewFakeClient() *FakeClient { + return &FakeClient{ + revisions: make(map[RevisionID][]string), + } +} + +// WithError configures the fake to return an error. +func (f *FakeClient) WithError(err error) *FakeClient { + f.err = err + return f +} + +func (f *FakeClient) CreateRevisionFromDir(ctx context.Context, dirPath string) (UploadResult, error) { + if f.err != nil { + return UploadResult{}, f.err + } + + info, err := os.Stat(dirPath) + if err != nil { + return UploadResult{}, uploadrevision.NewFileAccessError(dirPath, err) + } + + if !info.IsDir() { + return UploadResult{}, fmt.Errorf("the provided path is not a directory: %s", dirPath) + } + + return f.CreateRevisionFromPaths(ctx, []string{dirPath}) +} + +func (f *FakeClient) CreateRevisionFromFile(ctx context.Context, filePath string) (UploadResult, error) { + if f.err != nil { + return UploadResult{}, f.err + } + + info, err := os.Stat(filePath) + if err != nil { + return UploadResult{}, uploadrevision.NewFileAccessError(filePath, err) + } + + if !info.Mode().IsRegular() { + return UploadResult{}, fmt.Errorf("the provided path is not a regular file: %s", filePath) + } + + return f.CreateRevisionFromPaths(ctx, []string{filePath}) +} + +func (f *FakeClient) CreateRevisionFromPaths(ctx context.Context, paths []string) (UploadResult, error) { + if f.err != nil { + return UploadResult{}, f.err + } + + revID := uuid.New() + f.revisions[revID] = append([]string(nil), paths...) + f.uploadCount++ + f.lastRevision = revID + + return UploadResult{RevisionID: revID, UploadedFilesCount: len(paths)}, nil +} + +func (f *FakeClient) GetRevisionPaths(revID RevisionID) []string { + return f.revisions[revID] +} + +// UploadOccurred returns true if at least one upload has been performed. +func (f *FakeClient) UploadOccurred() bool { + return f.uploadCount > 0 +} + +// GetUploadCount returns the number of uploads that have occurred. +func (f *FakeClient) GetUploadCount() int { + return f.uploadCount +} + +// GetLastRevisionID returns the ID of the most recent revision created. +func (f *FakeClient) GetLastRevisionID() RevisionID { + return f.lastRevision +} diff --git a/pkg/apiclients/fileupload/filter.go b/pkg/apiclients/fileupload/filter.go new file mode 100644 index 000000000..3ad664d09 --- /dev/null +++ b/pkg/apiclients/fileupload/filter.go @@ -0,0 +1,17 @@ +package fileupload + +import "os" + +type fileToFilter struct { + Path string + Stat os.FileInfo +} + +// FilteredFile represents a file that was filtered. +// It includes the filtered file's path and the reason it was filtered. +type FilteredFile struct { + Path string + Reason error +} + +type filter func(fileToFilter) *FilteredFile diff --git a/pkg/apiclients/fileupload/list_sources.go b/pkg/apiclients/fileupload/list_sources.go new file mode 100644 index 000000000..9bb0689c4 --- /dev/null +++ b/pkg/apiclients/fileupload/list_sources.go @@ -0,0 +1,21 @@ +package fileupload + +import ( + "fmt" + + "github.com/rs/zerolog" + + "github.com/snyk/go-application-framework/pkg/utils" +) + +// forPath returns a channel that notifies each file in the path that doesn't match the filter rules. +func forPath(path string, logger *zerolog.Logger, maxThreads int) (<-chan string, error) { + filter := utils.NewFileFilter(path, logger, utils.WithThreadNumber(maxThreads)) + rules, err := filter.GetRules([]string{".gitignore", ".dcignore", ".snyk"}) + if err != nil { + return nil, fmt.Errorf("failed to get rules: %w", err) + } + + results := filter.GetFilteredFiles(filter.GetAllFiles(), rules) + return results, nil +} diff --git a/pkg/apiclients/fileupload/list_sources_test.go b/pkg/apiclients/fileupload/list_sources_test.go new file mode 100644 index 000000000..a473a2090 --- /dev/null +++ b/pkg/apiclients/fileupload/list_sources_test.go @@ -0,0 +1,49 @@ +package fileupload + +import ( + "fmt" + "io" + "path/filepath" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_ListsSources_Simplest(t *testing.T) { + sourcesDir := filepath.Join("testdata", "simplest") + + files, err := listSourcesForPath(sourcesDir) + require.NoError(t, err) + assert.Len(t, files, 2, "Expecting 2 files") + assert.Contains(t, files, filepath.Join(sourcesDir, "package.json")) + assert.Contains(t, files, filepath.Join(sourcesDir, "src", "index.js")) +} + +func Test_ListsSources_WithIgnores(t *testing.T) { + sourcesDir := filepath.Join("testdata", "with-ignores") + + files, err := listSourcesForPath(sourcesDir) + require.NoError(t, err) + + assert.Len(t, files, 3, "Expecting 3 files") + assert.Contains(t, files, filepath.Join(sourcesDir, ".gitignore")) + assert.Contains(t, files, filepath.Join(sourcesDir, "package.json")) + assert.Contains(t, files, filepath.Join(sourcesDir, "src", "with-ignores.js")) +} + +func listSourcesForPath(sourcesDir string) ([]string, error) { + mockLogger := zerolog.New(io.Discard) + filesCh, err := forPath(sourcesDir, &mockLogger, 2) + if err != nil { + return nil, fmt.Errorf("failed to list sources: %w", err) + } + + files := []string{} + for file := range filesCh { + files = append(files, file) + } + + return files, nil +} diff --git a/pkg/apiclients/fileupload/opts.go b/pkg/apiclients/fileupload/opts.go new file mode 100644 index 000000000..e9403d1a4 --- /dev/null +++ b/pkg/apiclients/fileupload/opts.go @@ -0,0 +1,24 @@ +package fileupload + +import ( + "github.com/rs/zerolog" + + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) + +// Option allows customizing the Client during construction. +type Option func(*HTTPClient) + +// WithUploadRevisionSealableClient allows injecting a custom low-level client (primarily for testing). +func WithUploadRevisionSealableClient(client uploadrevision.SealableClient) Option { + return func(c *HTTPClient) { + c.uploadRevisionSealableClient = client + } +} + +// WithLogger allows injecting a custom logger instance. +func WithLogger(logger *zerolog.Logger) Option { + return func(h *HTTPClient) { + h.logger = logger + } +} diff --git a/pkg/apiclients/fileupload/testdata/simplest/package.json b/pkg/apiclients/fileupload/testdata/simplest/package.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/pkg/apiclients/fileupload/testdata/simplest/package.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/pkg/apiclients/fileupload/testdata/simplest/src/index.js b/pkg/apiclients/fileupload/testdata/simplest/src/index.js new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/apiclients/fileupload/testdata/with-ignores/.gitignore b/pkg/apiclients/fileupload/testdata/with-ignores/.gitignore new file mode 100644 index 000000000..344b2f249 --- /dev/null +++ b/pkg/apiclients/fileupload/testdata/with-ignores/.gitignore @@ -0,0 +1 @@ +**/gitignored.js diff --git a/pkg/apiclients/fileupload/testdata/with-ignores/package.json b/pkg/apiclients/fileupload/testdata/with-ignores/package.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/pkg/apiclients/fileupload/testdata/with-ignores/package.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/pkg/apiclients/fileupload/testdata/with-ignores/src/with-ignores.js b/pkg/apiclients/fileupload/testdata/with-ignores/src/with-ignores.js new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/apiclients/fileupload/types.go b/pkg/apiclients/fileupload/types.go new file mode 100644 index 000000000..2618f4b48 --- /dev/null +++ b/pkg/apiclients/fileupload/types.go @@ -0,0 +1,23 @@ +package fileupload + +import ( + "github.com/snyk/go-application-framework/internal/api/fileupload/uploadrevision" +) + +// OrgID represents an organization identifier. +type OrgID = uploadrevision.OrgID + +// RevisionID represents a revision identifier. +type RevisionID = uploadrevision.RevisionID + +// UploadOptions configures the behavior of file upload operations. +type uploadOptions struct { + SkipDeeproxyFiltering bool +} + +// UploadResult respresents the result of the upload. +type UploadResult struct { + RevisionID RevisionID // The ID of the revision which was created. + UploadedFilesCount int // The number of uploaded files. + FilteredFiles []FilteredFile // The list of files which were filtered. +}