Reduce memory consumption when loading large number of commits (#3687)

stefanhaller · web-flow · commit a171ec42943c · 2024-06-23T12:09:16.000+02:00
(Github decided to auto-close #2533, and I don't see any way to reopen it, so opening a new one here. Please see there for discussion and review.) When pressing `>` in the commits panel to trigger loading all the remaining commits past the initial 300, memory consumption is a pretty big problem for larger repositories. The two main causes seem to be 1. the cell memory from rendering the entire list of commits into the gocui view 2. the pipe sets when git.log.showGraph is on This PR addresses only the first of these problems, by not rendering the entire view, but only the visible portion of it. Since we already re-render the visible portion of the view on every layout call, this was relatively easy to do. Below are some measurements for our repository at work (261.985 commits): | | master | this PR | | ------------- | ------ | ------- | | without Graph | 855 MB | 360 MB | | with Graph | 3.1 GB | 770 MB | And for the linux kernel repo (1.170.387 commits): | | master | this PR | | ------------- | ----------------------------------------- | ------- | | without Graph | 5.8 GB | 1.2G | | with Graph | Killed by the OS after it reached 86.9 GB | 39.9 GB | The measurements were taken after scrolling all the way down in the list of commits. They have to be taken with a grain of salt, as memory consumption fluctuates quite a bit in ways that I find hard to make sense of. As you can see, there's more work to do to reduce the memory usage for the graph, but for our repo at work this PR makes it usable already, which it wasn't really before.
diff --git a/go.mod b/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/integrii/flaggy v1.4.0
 	github.com/jesseduffield/generics v0.0.0-20220320043834-727e535cbe68
 	github.com/jesseduffield/go-git/v5 v5.1.2-0.20221018185014-fdd53fef665d
-	github.com/jesseduffield/gocui v0.3.1-0.20240623092910-a42926c14fc9
+	github.com/jesseduffield/gocui v0.3.1-0.20240623095254-05e1204c2454
 	github.com/jesseduffield/kill v0.0.0-20220618033138-bfbe04675d10
 	github.com/jesseduffield/lazycore v0.0.0-20221012050358-03d2e40243c5
 	github.com/jesseduffield/minimal/gitignore v0.3.3-0.20211018110810-9cde264e6b1e
diff --git a/go.sum b/go.sum
@@ -188,8 +188,8 @@ github.com/jesseduffield/generics v0.0.0-20220320043834-727e535cbe68 h1:EQP2Tv8T
 github.com/jesseduffield/generics v0.0.0-20220320043834-727e535cbe68/go.mod h1:+LLj9/WUPAP8LqCchs7P+7X0R98HiFujVFANdNaxhGk=
 github.com/jesseduffield/go-git/v5 v5.1.2-0.20221018185014-fdd53fef665d h1:bO+OmbreIv91rCe8NmscRwhFSqkDJtzWCPV4Y+SQuXE=
 github.com/jesseduffield/go-git/v5 v5.1.2-0.20221018185014-fdd53fef665d/go.mod h1:nGNEErzf+NRznT+N2SWqmHnDnF9aLgANB1CUNEan09o=
-github.com/jesseduffield/gocui v0.3.1-0.20240623092910-a42926c14fc9 h1:JJ0DrXgAUpGBGV5w8nzrQLMWTgcTvf745IKAk08qjcM=
-github.com/jesseduffield/gocui v0.3.1-0.20240623092910-a42926c14fc9/go.mod h1:XtEbqCbn45keRXEu+OMZkjN5gw6AEob59afsgHjokZ8=
+github.com/jesseduffield/gocui v0.3.1-0.20240623095254-05e1204c2454 h1:rTPA5WiPM1SPUA3r2kSb3RiILC93am6irMvOLjO7JNA=
+github.com/jesseduffield/gocui v0.3.1-0.20240623095254-05e1204c2454/go.mod h1:XtEbqCbn45keRXEu+OMZkjN5gw6AEob59afsgHjokZ8=
 github.com/jesseduffield/kill v0.0.0-20220618033138-bfbe04675d10 h1:jmpr7KpX2+2GRiE91zTgfq49QvgiqB0nbmlwZ8UnOx0=
 github.com/jesseduffield/kill v0.0.0-20220618033138-bfbe04675d10/go.mod h1:aA97kHeNA+sj2Hbki0pvLslmE4CbDyhBeSSTUUnOuVo=
 github.com/jesseduffield/lazycore v0.0.0-20221012050358-03d2e40243c5 h1:CDuQmfOjAtb1Gms6a1p5L2P8RhbLUq5t8aL7PiQd2uY=
diff --git a/pkg/gui/background.go b/pkg/gui/background.go
@@ -1,6 +1,8 @@
 package gui
 
 import (
+	"fmt"
+	"runtime"
 	"strings"
 	"time"
 
@@ -46,6 +48,29 @@ func (self *BackgroundRoutineMgr) startBackgroundRoutines() {
 				refreshInterval)
 		}
 	}
+
+	if self.gui.Config.GetDebug() {
+		self.goEvery(time.Second*time.Duration(10), self.gui.stopChan, func() error {
+			formatBytes := func(b uint64) string {
+				const unit = 1000
+				if b < unit {
+					return fmt.Sprintf("%d B", b)
+				}
+				div, exp := uint64(unit), 0
+				for n := b / unit; n >= unit; n /= unit {
+					div *= unit
+					exp++
+				}
+				return fmt.Sprintf("%.1f %cB",
+					float64(b)/float64(div), "kMGTPE"[exp])
+			}
+
+			m := runtime.MemStats{}
+			runtime.ReadMemStats(&m)
+			self.gui.c.Log.Infof("Heap memory in use: %s", formatBytes(m.HeapAlloc))
+			return nil
+		})
+	}
 }
 
 func (self *BackgroundRoutineMgr) startBackgroundFetch() {
diff --git a/pkg/gui/context/base_context.go b/pkg/gui/context/base_context.go
@@ -20,11 +20,12 @@ type BaseContext struct {
 	onFocusFn           onFocusFn
 	onFocusLostFn       onFocusLostFn
 
-	focusable                  bool
-	transient                  bool
-	hasControlledBounds        bool
-	needsRerenderOnWidthChange bool
-	highlightOnFocus           bool
+	focusable                   bool
+	transient                   bool
+	hasControlledBounds         bool
+	needsRerenderOnWidthChange  bool
+	needsRerenderOnHeightChange bool
+	highlightOnFocus            bool
 
 	*ParentContextMgr
 }
@@ -37,15 +38,16 @@ type (
 var _ types.IBaseContext = &BaseContext{}
 
 type NewBaseContextOpts struct {
-	Kind                       types.ContextKind
-	Key                        types.ContextKey
-	View                       *gocui.View
-	WindowName                 string
-	Focusable                  bool
-	Transient                  bool
-	HasUncontrolledBounds      bool // negating for the sake of making false the default
-	HighlightOnFocus           bool
-	NeedsRerenderOnWidthChange bool
+	Kind                        types.ContextKind
+	Key                         types.ContextKey
+	View                        *gocui.View
+	WindowName                  string
+	Focusable                   bool
+	Transient                   bool
+	HasUncontrolledBounds       bool // negating for the sake of making false the default
+	HighlightOnFocus            bool
+	NeedsRerenderOnWidthChange  bool
+	NeedsRerenderOnHeightChange bool
 
 	OnGetOptionsMap func() map[string]string
 }
@@ -56,18 +58,19 @@ func NewBaseContext(opts NewBaseContextOpts) *BaseContext {
 	hasControlledBounds := !opts.HasUncontrolledBounds
 
 	return &BaseContext{
-		kind:                       opts.Kind,
-		key:                        opts.Key,
-		view:                       opts.View,
-		windowName:                 opts.WindowName,
-		onGetOptionsMap:            opts.OnGetOptionsMap,
-		focusable:                  opts.Focusable,
-		transient:                  opts.Transient,
-		hasControlledBounds:        hasControlledBounds,
-		highlightOnFocus:           opts.HighlightOnFocus,
-		needsRerenderOnWidthChange: opts.NeedsRerenderOnWidthChange,
-		ParentContextMgr:           &ParentContextMgr{},
-		viewTrait:                  viewTrait,
+		kind:                        opts.Kind,
+		key:                         opts.Key,
+		view:                        opts.View,
+		windowName:                  opts.WindowName,
+		onGetOptionsMap:             opts.OnGetOptionsMap,
+		focusable:                   opts.Focusable,
+		transient:                   opts.Transient,
+		hasControlledBounds:         hasControlledBounds,
+		highlightOnFocus:            opts.HighlightOnFocus,
+		needsRerenderOnWidthChange:  opts.NeedsRerenderOnWidthChange,
+		needsRerenderOnHeightChange: opts.NeedsRerenderOnHeightChange,
+		ParentContextMgr:            &ParentContextMgr{},
+		viewTrait:                   viewTrait,
 	}
 }
 
@@ -197,6 +200,10 @@ func (self *BaseContext) NeedsRerenderOnWidthChange() bool {
 	return self.needsRerenderOnWidthChange
 }
 
+func (self *BaseContext) NeedsRerenderOnHeightChange() bool {
+	return self.needsRerenderOnHeightChange
+}
+
 func (self *BaseContext) Title() string {
 	return ""
 }
diff --git a/pkg/gui/context/list_context_trait.go b/pkg/gui/context/list_context_trait.go
@@ -18,14 +18,18 @@ type ListContextTrait struct {
 	// we should find out exactly which lines are now part of the path and refresh those.
 	// We should also keep track of the previous path and refresh those lines too.
 	refreshViewportOnChange bool
+	// If this is true, we only render the visible lines of the list. Useful for lists that can
+	// get very long, because it can save a lot of memory
+	renderOnlyVisibleLines bool
 }
 
 func (self *ListContextTrait) IsListContext() {}
 
 func (self *ListContextTrait) FocusLine() {
 	// Doing this at the end of the layout function because we need the view to be
 	// resized before we focus the line, otherwise if we're in accordion mode
-	// the view could be squashed and won't how to adjust the cursor/origin
+	// the view could be squashed and won't how to adjust the cursor/origin.
+	// Also, refreshing the viewport needs to happen after the view has been resized.
 	self.c.AfterLayout(func() error {
 		oldOrigin, _ := self.GetViewTrait().ViewPortYBounds()
 
@@ -40,22 +44,18 @@ func (self *ListContextTrait) FocusLine() {
 			self.GetViewTrait().CancelRangeSelect()
 		}
 
-		// If FocusPoint() caused the view to scroll (because the selected line
-		// was out of view before), we need to rerender the view port again.
-		// This can happen when pressing , or . to scroll by pages, or < or > to
-		// jump to the top or bottom.
-		newOrigin, _ := self.GetViewTrait().ViewPortYBounds()
-		if self.refreshViewportOnChange && oldOrigin != newOrigin {
+		if self.refreshViewportOnChange {
 			self.refreshViewport()
+		} else if self.renderOnlyVisibleLines {
+			newOrigin, _ := self.GetViewTrait().ViewPortYBounds()
+			if oldOrigin != newOrigin {
+				return self.HandleRender()
+			}
 		}
 		return nil
 	})
 
 	self.setFooter()
-
-	if self.refreshViewportOnChange {
-		self.refreshViewport()
-	}
 }
 
 func (self *ListContextTrait) refreshViewport() {
@@ -93,8 +93,21 @@ func (self *ListContextTrait) HandleFocusLost(opts types.OnFocusLostOpts) error
 // OnFocus assumes that the content of the context has already been rendered to the view. OnRender is the function which actually renders the content to the view
 func (self *ListContextTrait) HandleRender() error {
 	self.list.ClampSelection()
-	content := self.renderLines(-1, -1)
-	self.GetViewTrait().SetContent(content)
+	if self.renderOnlyVisibleLines {
+		// Rendering only the visible area can save a lot of cell memory for
+		// those views that support it.
+		totalLength := self.list.Len()
+		if self.getNonModelItems != nil {
+			totalLength += len(self.getNonModelItems())
+		}
+		self.GetViewTrait().SetContentLineCount(totalLength)
+		startIdx, length := self.GetViewTrait().ViewPortYBounds()
+		content := self.renderLines(startIdx, startIdx+length)
+		self.GetViewTrait().SetViewPortContentAndClearEverythingElse(content)
+	} else {
+		content := self.renderLines(-1, -1)
+		self.GetViewTrait().SetContent(content)
+	}
 	self.c.Render()
 	self.setFooter()
 
@@ -123,3 +136,7 @@ func (self *ListContextTrait) IsItemVisible(item types.HasUrn) bool {
 func (self *ListContextTrait) RangeSelectEnabled() bool {
 	return true
 }
+
+func (self *ListContextTrait) RenderOnlyVisibleLines() bool {
+	return self.renderOnlyVisibleLines
+}
diff --git a/pkg/gui/context/local_commits_context.go b/pkg/gui/context/local_commits_context.go
@@ -72,19 +72,21 @@ func NewLocalCommitsContext(c *ContextCommon) *LocalCommitsContext {
 		SearchTrait:           NewSearchTrait(c),
 		ListContextTrait: &ListContextTrait{
 			Context: NewSimpleContext(NewBaseContext(NewBaseContextOpts{
-				View:                       c.Views().Commits,
-				WindowName:                 "commits",
-				Key:                        LOCAL_COMMITS_CONTEXT_KEY,
-				Kind:                       types.SIDE_CONTEXT,
-				Focusable:                  true,
-				NeedsRerenderOnWidthChange: true,
+				View:                        c.Views().Commits,
+				WindowName:                  "commits",
+				Key:                         LOCAL_COMMITS_CONTEXT_KEY,
+				Kind:                        types.SIDE_CONTEXT,
+				Focusable:                   true,
+				NeedsRerenderOnWidthChange:  true,
+				NeedsRerenderOnHeightChange: true,
 			})),
 			ListRenderer: ListRenderer{
 				list:              viewModel,
 				getDisplayStrings: getDisplayStrings,
 			},
 			c:                       c,
 			refreshViewportOnChange: true,
+			renderOnlyVisibleLines:  true,
 		},
 	}
 
diff --git a/pkg/gui/context/remote_branches_context.go b/pkg/gui/context/remote_branches_context.go
@@ -37,13 +37,14 @@ func NewRemoteBranchesContext(
 		DynamicTitleBuilder:   NewDynamicTitleBuilder(c.Tr.RemoteBranchesDynamicTitle),
 		ListContextTrait: &ListContextTrait{
 			Context: NewSimpleContext(NewBaseContext(NewBaseContextOpts{
-				View:                       c.Views().RemoteBranches,
-				WindowName:                 "branches",
-				Key:                        REMOTE_BRANCHES_CONTEXT_KEY,
-				Kind:                       types.SIDE_CONTEXT,
-				Focusable:                  true,
-				Transient:                  true,
-				NeedsRerenderOnWidthChange: true,
+				View:                        c.Views().RemoteBranches,
+				WindowName:                  "branches",
+				Key:                         REMOTE_BRANCHES_CONTEXT_KEY,
+				Kind:                        types.SIDE_CONTEXT,
+				Focusable:                   true,
+				Transient:                   true,
+				NeedsRerenderOnWidthChange:  true,
+				NeedsRerenderOnHeightChange: true,
 			})),
 			ListRenderer: ListRenderer{
 				list:              viewModel,
diff --git a/pkg/gui/context/sub_commits_context.go b/pkg/gui/context/sub_commits_context.go
@@ -115,13 +115,14 @@ func NewSubCommitsContext(
 		DynamicTitleBuilder: NewDynamicTitleBuilder(c.Tr.SubCommitsDynamicTitle),
 		ListContextTrait: &ListContextTrait{
 			Context: NewSimpleContext(NewBaseContext(NewBaseContextOpts{
-				View:                       c.Views().SubCommits,
-				WindowName:                 "branches",
-				Key:                        SUB_COMMITS_CONTEXT_KEY,
-				Kind:                       types.SIDE_CONTEXT,
-				Focusable:                  true,
-				Transient:                  true,
-				NeedsRerenderOnWidthChange: true,
+				View:                        c.Views().SubCommits,
+				WindowName:                  "branches",
+				Key:                         SUB_COMMITS_CONTEXT_KEY,
+				Kind:                        types.SIDE_CONTEXT,
+				Focusable:                   true,
+				Transient:                   true,
+				NeedsRerenderOnWidthChange:  true,
+				NeedsRerenderOnHeightChange: true,
 			})),
 			ListRenderer: ListRenderer{
 				list:              viewModel,
@@ -130,6 +131,7 @@ func NewSubCommitsContext(
 			},
 			c:                       c,
 			refreshViewportOnChange: true,
+			renderOnlyVisibleLines:  true,
 		},
 	}
 
diff --git a/pkg/gui/context/view_trait.go b/pkg/gui/context/view_trait.go
@@ -34,6 +34,15 @@ func (self *ViewTrait) SetViewPortContent(content string) {
 	self.view.OverwriteLines(y, content)
 }
 
+func (self *ViewTrait) SetViewPortContentAndClearEverythingElse(content string) {
+	_, y := self.view.Origin()
+	self.view.OverwriteLinesAndClearEverythingElse(y, content)
+}
+
+func (self *ViewTrait) SetContentLineCount(lineCount int) {
+	self.view.SetContentLineCount(lineCount)
+}
+
 func (self *ViewTrait) SetContent(content string) {
 	self.view.SetContent(content)
 }
diff --git a/pkg/gui/controllers/list_controller.go b/pkg/gui/controllers/list_controller.go
@@ -53,13 +53,19 @@ func (self *ListController) HandleScrollRight() error {
 func (self *ListController) HandleScrollUp() error {
 	scrollHeight := self.c.UserConfig.Gui.ScrollHeight
 	self.context.GetViewTrait().ScrollUp(scrollHeight)
+	if self.context.RenderOnlyVisibleLines() {
+		return self.context.HandleRender()
+	}
 
 	return nil
 }
 
 func (self *ListController) HandleScrollDown() error {
 	scrollHeight := self.c.UserConfig.Gui.ScrollHeight
 	self.context.GetViewTrait().ScrollDown(scrollHeight)
+	if self.context.RenderOnlyVisibleLines() {
+		return self.context.HandleRender()
+	}
 
 	return nil
 }
diff --git a/pkg/gui/layout.go b/pkg/gui/layout.go
@@ -72,14 +72,26 @@ func (gui *Gui) layout(g *gocui.Gui) error {
 			frameOffset = 0
 		}
 
+		mustRerender := false
 		if context.NeedsRerenderOnWidthChange() {
 			// view.Width() returns the width -1 for some reason
 			oldWidth := view.Width() + 1
 			newWidth := dimensionsObj.X1 - dimensionsObj.X0 + 2*frameOffset
 			if oldWidth != newWidth {
-				contextsToRerender = append(contextsToRerender, context)
+				mustRerender = true
 			}
 		}
+		if context.NeedsRerenderOnHeightChange() {
+			// view.Height() returns the height -1 for some reason
+			oldHeight := view.Height() + 1
+			newHeight := dimensionsObj.Y1 - dimensionsObj.Y0 + 2*frameOffset
+			if oldHeight != newHeight {
+				mustRerender = true
+			}
+		}
+		if mustRerender {
+			contextsToRerender = append(contextsToRerender, context)
+		}
 
 		_, err = g.SetView(
 			viewName,
diff --git a/pkg/gui/types/context.go b/pkg/gui/types/context.go
@@ -63,6 +63,9 @@ type IBaseContext interface {
 	// true if the view needs to be rerendered when its width changes
 	NeedsRerenderOnWidthChange() bool
 
+	// true if the view needs to be rerendered when its height changes
+	NeedsRerenderOnHeightChange() bool
+
 	// returns the desired title for the view upon activation. If there is no desired title (returns empty string), then
 	// no title will be set
 	Title() string
@@ -150,6 +153,7 @@ type IListContext interface {
 	FocusLine()
 	IsListContext() // used for type switch
 	RangeSelectEnabled() bool
+	RenderOnlyVisibleLines() bool
 }
 
 type IPatchExplorerContext interface {
@@ -172,6 +176,8 @@ type IViewTrait interface {
 	SetRangeSelectStart(yIdx int)
 	CancelRangeSelect()
 	SetViewPortContent(content string)
+	SetViewPortContentAndClearEverythingElse(content string)
+	SetContentLineCount(lineCount int)
 	SetContent(content string)
 	SetFooter(value string)
 	SetOriginX(value int)
diff --git a/pkg/integration/components/view_driver.go b/pkg/integration/components/view_driver.go
diff --git a/vendor/github.com/jesseduffield/gocui/view.go b/vendor/github.com/jesseduffield/gocui/view.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
