Reduce memory usage of go/analysis

Author: jirfag

README.md

@@ -560,6 +560,7 @@ Global Flags:
   -j, --concurrency int           Concurrency (default NumCPU) (default 8)
       --cpu-profile-path string   Path to CPU profile output file
       --mem-profile-path string   Path to memory profile output file
+      --trace-path string         Path to trace output file
   -v, --verbose                   verbose output
 
 ```

pkg/commands/root.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"runtime"
 	"runtime/pprof"
+	"runtime/trace"
 	"strconv"
 
 	"github.com/spf13/cobra"
@@ -37,6 +38,16 @@ func (e *Executor) persistentPreRun(_ *cobra.Command, _ []string) {
 			runtime.MemProfileRate, _ = strconv.Atoi(rate)
 		}
 	}
+
+	if e.cfg.Run.TracePath != "" {
+		f, err := os.Create(e.cfg.Run.TracePath)
+		if err != nil {
+			e.log.Fatalf("Can't create file %s: %s", e.cfg.Run.TracePath, err)
+		}
+		if err = trace.Start(f); err != nil {
+			e.log.Fatalf("Can't start tracing: %s", err)
+		}
+	}
 }
 
 func (e *Executor) persistentPostRun(_ *cobra.Command, _ []string) {
@@ -58,6 +69,9 @@ func (e *Executor) persistentPostRun(_ *cobra.Command, _ []string) {
 		}
 		f.Close()
 	}
+	if e.cfg.Run.TracePath != "" {
+		trace.Stop()
+	}
 
 	os.Exit(e.exitCode)
 }
@@ -136,6 +150,7 @@ func initRootFlagSet(fs *pflag.FlagSet, cfg *config.Config, needVersionOption bo
 
 	fs.StringVar(&cfg.Run.CPUProfilePath, "cpu-profile-path", "", wh("Path to CPU profile output file"))
 	fs.StringVar(&cfg.Run.MemProfilePath, "mem-profile-path", "", wh("Path to memory profile output file"))
+	fs.StringVar(&cfg.Run.TracePath, "trace-path", "", wh("Path to trace output file"))
 	fs.IntVarP(&cfg.Run.Concurrency, "concurrency", "j", getDefaultConcurrency(), wh("Concurrency (default NumCPU)"))
 	if needVersionOption {
 		fs.BoolVar(&cfg.Run.PrintVersion, "version", false, wh("Print version"))

pkg/commands/run.go

@@ -451,21 +451,30 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
 	startedAt := time.Now()
 	debugf("Started tracking time")
 
-	var rssValues []uint64
-	ticker := time.NewTicker(10 * time.Millisecond)
+	var maxRSSMB, totalRSSMB float64
+	var iterationsCount int
+	ticker := time.NewTicker(100 * time.Millisecond)
 	defer ticker.Stop()
 
 	logEveryRecord := os.Getenv("GL_MEM_LOG_EVERY") == "1"
+	const MB = 1024 * 1024
 
 	track := func() {
+		debugf("Starting memory tracing iteration ...")
 		var m runtime.MemStats
 		runtime.ReadMemStats(&m)
 
 		if logEveryRecord {
+			debugf("Stopping memory tracing iteration, printing ...")
 			printMemStats(&m, logger)
 		}
 
-		rssValues = append(rssValues, m.Sys)
+		rssMB := float64(m.Sys) / MB
+		if rssMB > maxRSSMB {
+			maxRSSMB = rssMB
+		}
+		totalRSSMB += rssMB
+		iterationsCount++
 	}
 
 	for {
@@ -476,7 +485,7 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
 		case <-ctx.Done():
 			stop = true
 			debugf("Stopped resources tracking")
-		case <-ticker.C: // track every second
+		case <-ticker.C:
 		}
 
 		if stop {
@@ -485,19 +494,10 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
 	}
 	track()
 
-	var avg, max uint64
-	for _, v := range rssValues {
-		avg += v
-		if v > max {
-			max = v
-		}
-	}
-	avg /= uint64(len(rssValues))
+	avgRSSMB := totalRSSMB / float64(iterationsCount)
 
-	const MB = 1024 * 1024
-	maxMB := float64(max) / MB
 	logger.Infof("Memory: %d samples, avg is %.1fMB, max is %.1fMB",
-		len(rssValues), float64(avg)/MB, maxMB)
+		iterationsCount, avgRSSMB, maxRSSMB)
 	logger.Infof("Execution took %s", time.Since(startedAt))
 	close(done)
 }

pkg/config/config.go

@@ -102,6 +102,7 @@ type Run struct {
 	Silent              bool
 	CPUProfilePath      string
 	MemProfilePath      string
+	TracePath           string
 	Concurrency         int
 	PrintResourcesUsage bool `mapstructure:"print-resources-usage"`
 

pkg/config/reader.go

@@ -102,6 +102,10 @@ func (r *FileReader) validateConfig() error {
 		return errors.New("option run.memprofilepath in config isn't allowed")
 	}
 
+	if c.Run.TracePath != "" {
+		return errors.New("option run.tracepath in config isn't allowed")
+	}
+
 	if c.Run.IsVerbose {
 		return errors.New("can't set run.verbose option with config: only on command-line")
 	}

pkg/golinters/goanalysis/linter.go

@@ -114,12 +114,10 @@ func (lnt Linter) Run(ctx context.Context, lintCtx *linter.Context) ([]result.Is
 		return nil, errors.Wrap(err, "failed to configure analyzers")
 	}
 
-	runner := newRunner(lnt.name, lintCtx.Log.Child("goanalysis"), lintCtx.PkgCache, lintCtx.LoadGuard)
+	runner := newRunner(lnt.name, lintCtx.Log.Child("goanalysis"), lintCtx.PkgCache, lintCtx.LoadGuard, lintCtx.NeedWholeProgram)
 
 	diags, errs := runner.run(lnt.analyzers, lintCtx.Packages)
-	for i := 1; i < len(errs); i++ {
-		lintCtx.Log.Warnf("%s error: %s", lnt.Name(), errs[i])
-	}
+	// Don't print all errs: they can duplicate.
 	if len(errs) != 0 {
 		return nil, errs[0]
 	}

pkg/golinters/goanalysis/metalinter.go

@@ -46,12 +46,10 @@ func (ml MetaLinter) Run(ctx context.Context, lintCtx *linter.Context) ([]result
 		allAnalyzers = append(allAnalyzers, linter.analyzers...)
 	}
 
-	runner := newRunner("metalinter", lintCtx.Log.Child("goanalysis"), lintCtx.PkgCache, lintCtx.LoadGuard)
+	runner := newRunner("metalinter", lintCtx.Log.Child("goanalysis"), lintCtx.PkgCache, lintCtx.LoadGuard, lintCtx.NeedWholeProgram)
 
 	diags, errs := runner.run(allAnalyzers, lintCtx.Packages)
-	for i := 1; i < len(errs); i++ {
-		lintCtx.Log.Warnf("go/analysis metalinter error: %s", errs[i])
-	}
+	// Don't print all errs: they can duplicate.
 	if len(errs) != 0 {
 		return nil, errs[0]
 	}