diff --git a/README.md b/README.md index b093ae0d..1dc5974b 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,20 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos Number of repository commits to process (default 500) -debug Print debugging information +-enterprise-upload-url string + Upload URL for Github Enterprise (defaults to the URL set in -enterprise-url if any) +-enterprise-url string + URL for Github Enterprise +-enterprise-user string + Username for Github Enterprise (defaults to first target) -github-access-token string GitHub access token to use for API requests -load string Load session file -no-expand-orgs Don't add members to targets when processing organizations +-no-server + Disables web server -port int Port to run web server on (default 9393) -save string @@ -54,6 +62,14 @@ A session stored in a file can be loaded with the `-load` option: Gitrob will start its web interface and serve the results for analysis. +### Use with Github Enterprise + +To configure Gitrob for Github Enterprise, the following switches can be used: + +- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github web interface can be found. Example: `-enterprise-url=https://github.yourcompany.com` +- `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. Example: `-enterprise-upload-url=https://github.yourcompany.com/api/v3/upload` +- `enterprise-user`: Optional, defaults to the first target. Example: `-enterprise-user=your.username` + ## Installation A [precompiled version is available](https://github.com/michenriksen/gitrob/releases) for each release, alternatively you can use the latest version of the source code from this repository in order to build your own binary. diff --git a/core/git.go b/core/git.go index f5abbc5c..831ba8fe 100644 --- a/core/git.go +++ b/core/git.go @@ -1,120 +1,137 @@ package core import ( - "fmt" - "io/ioutil" + "fmt" + "io/ioutil" - "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/object" - "gopkg.in/src-d/go-git.v4/utils/merkletrie" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) const ( - EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" ) -func CloneRepository(url *string, branch *string, depth int) (*git.Repository, string, error) { - urlVal := *url - branchVal := *branch - dir, err := ioutil.TempDir("", "gitrob") - if err != nil { - return nil, "", err - } - repository, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: urlVal, - Depth: depth, - ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), - SingleBranch: true, - Tags: git.NoTags, - }) - if err != nil { - return nil, dir, err - } - return repository, dir, nil +func CloneRepository(url *string, branch *string, sess *Session) (*git.Repository, string, error) { + urlVal := *url + branchVal := *branch + dir, err := ioutil.TempDir("", "gitrob") + if err != nil { + return nil, "", err + } + + options := &git.CloneOptions{ + URL: urlVal, + Depth: *sess.Options.CommitDepth, + ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), + SingleBranch: true, + Tags: git.NoTags, + } + + if sess.GithubAccessToken != "" && *sess.Options.EnterpriseUser != "" { + options.Auth = &http.BasicAuth{Username: *sess.Options.EnterpriseUser, Password: sess.GithubAccessToken} + } + + repository, err := git.PlainClone(dir, false, options) + if err != nil { + return nil, dir, err + } + return repository, dir, nil } func GetRepositoryHistory(repository *git.Repository) ([]*object.Commit, error) { - var commits []*object.Commit - ref, err := repository.Head() - if err != nil { - return nil, err - } - cIter, err := repository.Log(&git.LogOptions{From: ref.Hash()}) - if err != nil { - return nil, err - } - cIter.ForEach(func(c *object.Commit) error { - commits = append(commits, c) - return nil - }) - return commits, nil + var commits []*object.Commit + ref, err := repository.Head() + if err != nil { + return nil, err + } + cIter, err := repository.Log(&git.LogOptions{From: ref.Hash()}) + if err != nil { + return nil, err + } + cIter.ForEach(func(c *object.Commit) error { + commits = append(commits, c) + return nil + }) + return commits, nil } func GetChanges(commit *object.Commit, repo *git.Repository) (object.Changes, error) { - parentCommit, err := GetParentCommit(commit, repo) - if err != nil { - return nil, err - } - - commitTree, err := commit.Tree() - if err != nil { - return nil, err - } - - parentCommitTree, err := parentCommit.Tree() - if err != nil { - return nil, err - } - - changes, err := object.DiffTree(parentCommitTree, commitTree) - if err != nil { - return nil, err - } - return changes, nil + parentCommit, err := GetParentCommit(commit, repo) + if err != nil { + //this may be the parent commit + parentCommit = commit + //return nil, err + } + + commitTree, err := commit.Tree() + if err != nil { + return nil, err + } + + parentCommitTree, err := parentCommit.Tree() + if err != nil { + return nil, err + } + + //changes, err := object.DiffTree(parentCommitTree, commitTree) + var changes object.Changes + if parentCommit == commit { + changes, err = object.DiffTree(nil, parentCommitTree) + } else { + changes, err = object.DiffTree(parentCommitTree, commitTree) + } + + if err != nil { + return nil, err + } + return changes, nil } func GetParentCommit(commit *object.Commit, repo *git.Repository) (*object.Commit, error) { - if commit.NumParents() == 0 { - parentCommit, err := repo.CommitObject(plumbing.NewHash(EmptyTreeCommitId)) - if err != nil { - return nil, err - } - return parentCommit, nil - } - parentCommit, err := commit.Parents().Next() - if err != nil { - return nil, err - } - return parentCommit, nil + if commit.NumParents() == 0 { + parentCommit, err := repo.CommitObject(plumbing.NewHash(EmptyTreeCommitId)) + if err != nil { + return nil, err + } + return parentCommit, nil + } + parentCommit, err := commit.Parents().Next() + if err != nil { + return nil, err + } + return parentCommit, nil } func GetChangeAction(change *object.Change) string { - action, err := change.Action() - if err != nil { - return "Unknown" - } - switch action { - case merkletrie.Insert: - return "Insert" - case merkletrie.Modify: - return "Modify" - case merkletrie.Delete: - return "Delete" - default: - return "Unknown" - } + action, err := change.Action() + if err != nil { + return "Unknown" + } + switch action { + case merkletrie.Insert: + return "Insert" + case merkletrie.Modify: + return "Modify" + case merkletrie.Delete: + return "Delete" + default: + return "Unknown" + } } func GetChangePath(change *object.Change) string { - action, err := change.Action() - if err != nil { - return change.To.Name - } - - if action == merkletrie.Delete { - return change.From.Name - } else { - return change.To.Name - } + action, err := change.Action() + if err != nil { + return change.To.Name + } + + if action == merkletrie.Delete { + return change.From.Name + } else { + return change.To.Name + } } diff --git a/core/github.go b/core/github.go index a1941701..a32552be 100644 --- a/core/github.go +++ b/core/github.go @@ -1,113 +1,151 @@ package core import ( - "context" + "context" - "github.com/google/go-github/github" + "github.com/google/go-github/github" ) type GithubOwner struct { - Login *string - ID *int64 - Type *string - Name *string - AvatarURL *string - URL *string - Company *string - Blog *string - Location *string - Email *string - Bio *string + Login *string + ID *int64 + Type *string + Name *string + AvatarURL *string + URL *string + Company *string + Blog *string + Location *string + Email *string + Bio *string } type GithubRepository struct { - Owner *string - ID *int64 - Name *string - FullName *string - CloneURL *string - URL *string - DefaultBranch *string - Description *string - Homepage *string + Owner *string + ID *int64 + Name *string + FullName *string + CloneURL *string + URL *string + DefaultBranch *string + Description *string + Homepage *string } func GetUserOrOrganization(login string, client *github.Client) (*GithubOwner, error) { - ctx := context.Background() - user, _, err := client.Users.Get(ctx, login) - if err != nil { - return nil, err - } - return &GithubOwner{ - Login: user.Login, - ID: user.ID, - Type: user.Type, - Name: user.Name, - AvatarURL: user.AvatarURL, - URL: user.HTMLURL, - Company: user.Company, - Blog: user.Blog, - Location: user.Location, - Email: user.Email, - Bio: user.Bio, - }, nil + ctx := context.Background() + user, _, err := client.Users.Get(ctx, login) + if err != nil { + return nil, err + } + return &GithubOwner{ + Login: user.Login, + ID: user.ID, + Type: user.Type, + Name: user.Name, + AvatarURL: user.AvatarURL, + URL: user.HTMLURL, + Company: user.Company, + Blog: user.Blog, + Location: user.Location, + Email: user.Email, + Bio: user.Bio, + }, nil } func GetRepositoriesFromOwner(login *string, client *github.Client) ([]*GithubRepository, error) { - var allRepos []*GithubRepository - loginVal := *login - ctx := context.Background() - opt := &github.RepositoryListOptions{ - Type: "sources", - } + var allRepos []*GithubRepository + loginVal := *login + ctx := context.Background() + opt := &github.RepositoryListOptions{ + Type: "sources", + } - for { - repos, resp, err := client.Repositories.List(ctx, loginVal, opt) - if err != nil { - return allRepos, err - } - for _, repo := range repos { - if !*repo.Fork { - r := GithubRepository{ - Owner: repo.Owner.Login, - ID: repo.ID, - Name: repo.Name, - FullName: repo.FullName, - CloneURL: repo.CloneURL, - URL: repo.HTMLURL, - DefaultBranch: repo.DefaultBranch, - Description: repo.Description, - Homepage: repo.Homepage, - } - allRepos = append(allRepos, &r) - } - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } + for { + repos, resp, err := client.Repositories.List(ctx, loginVal, opt) + if err != nil { + return allRepos, err + } + for _, repo := range repos { + if !*repo.Fork { + r := GithubRepository{ + Owner: repo.Owner.Login, + ID: repo.ID, + Name: repo.Name, + FullName: repo.FullName, + CloneURL: repo.CloneURL, + URL: repo.HTMLURL, + DefaultBranch: repo.DefaultBranch, + Description: repo.Description, + Homepage: repo.Homepage, + } + allRepos = append(allRepos, &r) + } + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } - return allRepos, nil + return allRepos, nil +} + +func GetRepositoriesFromOrganization(login *string, client *github.Client) ([]*GithubRepository, error) { + var allRepos []*GithubRepository + loginVal := *login + ctx := context.Background() + opt := &github.RepositoryListByOrgOptions{ + Type: "sources", + } + + for { + repos, resp, err := client.Repositories.ListByOrg(ctx, loginVal, opt) + if err != nil { + return allRepos, err + } + for _, repo := range repos { + if !*repo.Fork { + r := GithubRepository{ + Owner: repo.Owner.Login, + ID: repo.ID, + Name: repo.Name, + FullName: repo.FullName, + CloneURL: repo.SSHURL, + URL: repo.HTMLURL, + DefaultBranch: repo.DefaultBranch, + Description: repo.Description, + Homepage: repo.Homepage, + } + allRepos = append(allRepos, &r) + } + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + + return allRepos, nil } func GetOrganizationMembers(login *string, client *github.Client) ([]*GithubOwner, error) { - var allMembers []*GithubOwner - loginVal := *login - ctx := context.Background() - opt := &github.ListMembersOptions{} - for { - members, resp, err := client.Organizations.ListMembers(ctx, loginVal, opt) - if err != nil { - return allMembers, err - } - for _, member := range members { - allMembers = append(allMembers, &GithubOwner{Login: member.Login, ID: member.ID, Type: member.Type}) - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } - return allMembers, nil + var allMembers []*GithubOwner + loginVal := *login + ctx := context.Background() + opt := &github.ListMembersOptions{} + for { + members, resp, err := client.Organizations.ListMembers(ctx, loginVal, opt) + if err != nil { + return allMembers, err + } + for _, member := range members { + allMembers = append(allMembers, &GithubOwner{Login: member.Login, ID: member.ID, Type: member.Type}) + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return allMembers, nil } diff --git a/core/options.go b/core/options.go index cf610816..6caf6472 100644 --- a/core/options.go +++ b/core/options.go @@ -7,6 +7,10 @@ import ( type Options struct { CommitDepth *int GithubAccessToken *string `json:"-"` + EnterpriseURL *string + EnterpriseAPI *string + EnterpriseUpload *string + EnterpriseUser *string NoExpandOrgs *bool Threads *int Save *string `json:"-"` @@ -15,6 +19,7 @@ type Options struct { Port *int Silent *bool Debug *bool + NoServer *bool Logins []string } @@ -22,6 +27,9 @@ func ParseOptions() (Options, error) { options := Options{ CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + EnterpriseURL: flag.String("enterprise-url", "", "URL of the GitHub Enterprise instance, e.g. https://github.yourcompany.com"), + EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise, e.g. https://github.yourcompany.com/api/v3/upload"), + EnterpriseUser: flag.String("enterprise-user", "", "Username for your GitHub Enterprise account"), NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), Save: flag.String("save", "", "Save session to file"), @@ -30,6 +38,7 @@ func ParseOptions() (Options, error) { Port: flag.Int("port", 9393, "Port to run web server on"), Silent: flag.Bool("silent", false, "Suppress all output except for errors"), Debug: flag.Bool("debug", false, "Print debugging information"), + NoServer: flag.Bool("no-server", false, "Disables web server"), } flag.Parse() diff --git a/core/router.go b/core/router.go index 227e29bb..56d6fcde 100644 --- a/core/router.go +++ b/core/router.go @@ -1,8 +1,8 @@ package core import ( + "context" "fmt" - "io/ioutil" "net/http" "strings" @@ -10,10 +10,12 @@ import ( "github.com/gin-contrib/secure" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" + "github.com/google/go-github/github" ) const ( - GithubBaseUri = "https://raw.githubusercontent.com" + contextKeyGithubClient = "kGithubClient" + MaximumFileSize = 102400 CspPolicy = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'" ReferrerPolicy = "no-referrer" @@ -74,36 +76,26 @@ func NewRouter(s *Session) *gin.Engine { router.GET("/repositories", func(c *gin.Context) { c.JSON(200, s.Repositories) }) - router.GET("/files/:owner/:repo/:commit/*path", fetchFile) + + router.GET("/files/:owner/:repo/:commit/*path", func (c *gin.Context) { + c.Set(contextKeyGithubClient, s.GithubClient) + fetchFile(c) + }) return router } func fetchFile(c *gin.Context) { - fileUrl := fmt.Sprintf("%s/%s/%s/%s%s", GithubBaseUri, c.Param("owner"), c.Param("repo"), c.Param("commit"), c.Param("path")) - resp, err := http.Head(fileUrl) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "message": err, - }) - return + client, _ := c.Get(contextKeyGithubClient) + githubClient := client.(*github.Client) + + ctx := context.Background() + options := &github.RepositoryContentGetOptions{ + Ref: c.Param("commit"), } - if resp.StatusCode == http.StatusNotFound { - c.JSON(http.StatusNotFound, gin.H{ - "message": "No content", - }) - return - } + fileResponse, _, _, err := githubClient.Repositories.GetContents(ctx, c.Param("owner"), c.Param("repo"), c.Param("path"), options) - if resp.ContentLength > MaximumFileSize { - c.JSON(http.StatusUnprocessableEntity, gin.H{ - "message": fmt.Sprintf("File size exceeds maximum of %d bytes", MaximumFileSize), - }) - return - } - - resp, err = http.Get(fileUrl) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{ "message": err, @@ -111,14 +103,14 @@ func fetchFile(c *gin.Context) { return } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "message": err, + if fileResponse.GetSize() > MaximumFileSize { + c.JSON(http.StatusUnprocessableEntity, gin.H{ + "message": fmt.Sprintf("File size exceeds maximum of %d bytes", MaximumFileSize), }) return } + + content, _ := fileResponse.GetContent() - c.String(http.StatusOK, string(body[:])) + c.String(http.StatusOK, content) } diff --git a/core/session.go b/core/session.go index bf58134e..5721e483 100644 --- a/core/session.go +++ b/core/session.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "runtime" + "strings" "sync" "time" @@ -23,6 +24,9 @@ const ( StatusGathering = "gathering" StatusAnalyzing = "analyzing" StatusFinished = "finished" + + githubDotComURL = "https://github.com" + githubAPIPath = "/api/v3/" ) type Stats struct { @@ -59,8 +63,11 @@ func (s *Session) Start() { s.InitLogger() s.InitThreads() s.InitGithubAccessToken() + s.initEnterpriseConfig() s.InitGithubClient() - s.InitRouter() + if !*s.Options.NoServer { + s.InitRouter() + } } func (s *Session) Finish() { @@ -130,13 +137,61 @@ func (s *Session) InitGithubAccessToken() { } } +func (s *Session) initEnterpriseConfig() { + apiURL := *s.Options.EnterpriseURL + + if apiURL == "" { + return + } + + apiURL = strings.TrimSuffix(apiURL, "/") + + *s.Options.EnterpriseURL = apiURL + apiPath := apiURL + githubAPIPath + s.Options.EnterpriseAPI = &apiPath + + uploadURL := *s.Options.EnterpriseUpload + + if uploadURL == "" { + uploadURL = *s.Options.EnterpriseAPI + } else { + if !strings.HasSuffix(uploadURL, "/") { + uploadURL += "/" + *s.Options.EnterpriseUpload = uploadURL + } + } + + if *s.Options.EnterpriseUser == "" && len(s.Options.Logins) > 0 { + *s.Options.EnterpriseUser = s.Options.Logins[0] + } +} + +func (s *Session) GithubURL() string { + if s.Options.EnterpriseURL != nil && *s.Options.EnterpriseURL != "" { + return *s.Options.EnterpriseURL + } + + return githubDotComURL +} + func (s *Session) InitGithubClient() { ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: s.GithubAccessToken}, ) tc := oauth2.NewClient(ctx, ts) - s.GithubClient = github.NewClient(tc) + + if s.Options.EnterpriseAPI != nil && *s.Options.EnterpriseAPI != "" { + enterpriseClient, err := github.NewEnterpriseClient(*s.Options.EnterpriseAPI, *s.Options.EnterpriseUpload, tc) + if err != nil { + s.Out.Fatal("Error creating GitHub Enterprise client: %s\n", err) + } + + s.GithubClient = enterpriseClient + } else { + s.GithubClient = github.NewClient(tc) + } + s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) } diff --git a/core/signatures.go b/core/signatures.go index fe36cc03..65b91e7d 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -59,8 +59,8 @@ type Finding struct { RepositoryUrl string } -func (f *Finding) setupUrls() { - f.RepositoryUrl = fmt.Sprintf("https://github.com/%s/%s", f.RepositoryOwner, f.RepositoryName) +func (f *Finding) setupUrls(githubURL string) { + f.RepositoryUrl = strings.Join([]string {githubURL, f.RepositoryOwner, f.RepositoryName}, "/") f.FileUrl = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryUrl, f.CommitHash, f.FilePath) f.CommitUrl = fmt.Sprintf("%s/commit/%s", f.RepositoryUrl, f.CommitHash) } @@ -77,8 +77,8 @@ func (f *Finding) generateID() { f.Id = fmt.Sprintf("%x", h.Sum(nil)) } -func (f *Finding) Initialize() { - f.setupUrls() +func (f *Finding) Initialize(githubURL string) { + f.setupUrls(githubURL) f.generateID() } diff --git a/main.go b/main.go index a693ad89..b11a330e 100644 --- a/main.go +++ b/main.go @@ -57,12 +57,18 @@ func GatherRepositories(sess *core.Session) { for i := 0; i < threadNum; i++ { go func() { for { + var repos []*core.GithubRepository + var err error target, ok := <-ch if !ok { wg.Done() return } - repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient) + if *target.Type == "Organization" { + repos, err = core.GetRepositoriesFromOrganization(target.Login, sess.GithubClient) + } else { + repos, err = core.GetRepositoriesFromOwner(target.Login, sess.GithubClient) + } if err != nil { sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err) } @@ -103,6 +109,8 @@ func AnalyzeRepositories(sess *core.Session) { sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) + githubURL := sess.GithubURL() + for i := 0; i < threadNum; i++ { go func(tid int) { for { @@ -115,7 +123,7 @@ func AnalyzeRepositories(sess *core.Session) { } sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) - clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, *sess.Options.CommitDepth) + clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, sess) if err != nil { if err.Error() != "remote repository is empty" { sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) @@ -163,7 +171,7 @@ func AnalyzeRepositories(sess *core.Session) { CommitMessage: strings.TrimSpace(commit.Message), CommitAuthor: commit.Author.String(), } - finding.Initialize() + finding.Initialize(githubURL) sess.AddFinding(finding) sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description) @@ -218,7 +226,9 @@ func main() { sess.Out.Info("%s\n\n", core.ASCIIBanner) sess.Out.Important("%s v%s started at %s\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339)) sess.Out.Important("Loaded %d signatures\n", len(core.Signatures)) - sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port) + if !*sess.Options.NoServer { + sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port) + } if sess.Stats.Status == "finished" { sess.Out.Important("Loaded session file: %s\n", *sess.Options.Load) @@ -242,6 +252,8 @@ func main() { } PrintSessionStats(sess) - sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n") - select {} + if !*sess.Options.NoServer { + sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n") + select {} + } }