Merge pull request ActiveState#3660 from ActiveState/mitchell/dx-3213

Implement an artifact cache in the depot.

Author: mitchell-as

internal/constants/constants.go

@@ -493,3 +493,6 @@ const IgnoreEnvEnvVarName = "ACTIVESTATE_CLI_IGNORE_ENV"
 
 // ProgressUrlPathName is the trailing path for a project's build progress.
 const BuildProgressUrlPathName = "distributions"
+
+// RuntimeCacheSizeConfigKey is the config key for the runtime cache size.
+const RuntimeCacheSizeConfigKey = "runtime.cache.size"

internal/fileutils/fileutils.go

@@ -1299,3 +1299,22 @@ func windowsPathToPosixPath(path string) string {
 func posixPathToWindowsPath(path string) string {
 	return strings.ReplaceAll(path, "/", "\\")
 }
+
+// GetDirSize returns the on-disk size of the given directory and all of its contents.
+// Does not follow symlinks.
+func GetDirSize(dir string) (int64, error) {
+	var size int64
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if !info.IsDir() {
+			size += info.Size()
+		}
+		return nil
+	})
+	if err != nil {
+		return size, errs.Wrap(err, "Unable to compute size")
+	}
+	return size, nil
+}

internal/runbits/runtime/runtime.go

@@ -281,6 +281,7 @@ func Update(
 	if proj.IsPortable() {
 		rtOpts = append(rtOpts, runtime.WithPortable())
 	}
+	rtOpts = append(rtOpts, runtime.WithCacheSize(prime.Config().GetInt(constants.RuntimeCacheSizeConfigKey)))
 
 	if isArmPlatform(buildPlan) {
 		prime.Output().Notice(locale.Tl("warning_arm_unstable", "[WARNING]Warning:[/RESET] You are using an ARM64 architecture, which is currently unstable. While it may work, you might encounter issues."))

pkg/runtime/depot.go

@@ -7,14 +7,18 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"sync"
+	"time"
 
 	"github.com/go-openapi/strfmt"
 
+	"github.com/ActiveState/cli/internal/constants"
 	"github.com/ActiveState/cli/internal/errs"
 	"github.com/ActiveState/cli/internal/fileutils"
 	"github.com/ActiveState/cli/internal/installation/storage"
 	"github.com/ActiveState/cli/internal/logging"
+	configMediator "github.com/ActiveState/cli/internal/mediators/config"
 	"github.com/ActiveState/cli/internal/sliceutils"
 	"github.com/ActiveState/cli/internal/smartlink"
 )
@@ -24,7 +28,8 @@ const (
 )
 
 type depotConfig struct {
-	Deployments map[strfmt.UUID][]deployment `json:"deployments"`
+	Deployments map[strfmt.UUID][]deployment  `json:"deployments"`
+	Cache       map[strfmt.UUID]*artifactInfo `json:"cache"`
 }
 
 type deployment struct {
@@ -41,6 +46,14 @@ const (
 	deploymentTypeCopy                = "copy"
 )
 
+type artifactInfo struct {
+	InUse          bool  `json:"inUse"`
+	Size           int64 `json:"size"`
+	LastAccessTime int64 `json:"lastAccessTime"`
+
+	id strfmt.UUID // for convenience when removing stale artifacts; should NOT have json tag
+}
+
 type ErrVolumeMismatch struct {
 	DepotVolume string
 	PathVolume  string
@@ -55,8 +68,15 @@ type depot struct {
 	depotPath string
 	artifacts map[strfmt.UUID]struct{}
 	fsMutex   *sync.Mutex
+	cacheSize int64
+}
+
+func init() {
+	configMediator.RegisterOption(constants.RuntimeCacheSizeConfigKey, configMediator.Int, 500)
 }
 
+const MB int64 = 1024 * 1024
+
 func newDepot(runtimePath string) (*depot, error) {
 	depotPath := filepath.Join(storage.CachePath(), depotName)
 
@@ -73,6 +93,7 @@ func newDepot(runtimePath string) (*depot, error) {
 	result := &depot{
 		config: depotConfig{
 			Deployments: map[strfmt.UUID][]deployment{},
+			Cache:       map[strfmt.UUID]*artifactInfo{},
 		},
 		depotPath: depotPath,
 		artifacts: map[strfmt.UUID]struct{}{},
@@ -122,6 +143,10 @@ func newDepot(runtimePath string) (*depot, error) {
 	return result, nil
 }
 
+func (d *depot) SetCacheSize(mb int) {
+	d.cacheSize = int64(mb) * MB
+}
+
 func (d *depot) Exists(id strfmt.UUID) bool {
 	_, ok := d.artifacts[id]
 	return ok
@@ -196,6 +221,10 @@ func (d *depot) DeployViaLink(id strfmt.UUID, relativeSrc, absoluteDest string)
 		Files:       files.RelativePaths(),
 		RelativeSrc: relativeSrc,
 	})
+	err = d.recordUse(id)
+	if err != nil {
+		return errs.Wrap(err, "Could not record artifact use")
+	}
 
 	return nil
 }
@@ -253,7 +282,25 @@ func (d *depot) DeployViaCopy(id strfmt.UUID, relativeSrc, absoluteDest string)
 		Files:       files.RelativePaths(),
 		RelativeSrc: relativeSrc,
 	})
+	err = d.recordUse(id)
+	if err != nil {
+		return errs.Wrap(err, "Could not record artifact use")
+	}
+
+	return nil
+}
 
+func (d *depot) recordUse(id strfmt.UUID) error {
+	// Ensure a cache entry for this artifact exists and then update its last access time.
+	if _, exists := d.config.Cache[id]; !exists {
+		size, err := fileutils.GetDirSize(d.Path(id))
+		if err != nil {
+			return errs.Wrap(err, "Could not get artifact size on disk")
+		}
+		d.config.Cache[id] = &artifactInfo{Size: size, id: id}
+	}
+	d.config.Cache[id].InUse = true
+	d.config.Cache[id].LastAccessTime = time.Now().Unix()
 	return nil
 }
 
@@ -372,14 +419,17 @@ func (d *depot) getSharedFilesToRedeploy(id strfmt.UUID, deploy deployment, path
 // Save will write config changes to disk (ie. links between depot artifacts and runtimes that use it).
 // It will also delete any stale artifacts which are not used by any runtime.
 func (d *depot) Save() error {
-	// Delete artifacts that are no longer used
+	// Mark artifacts that are no longer used and remove the old ones.
 	for id := range d.artifacts {
 		if deployments, ok := d.config.Deployments[id]; !ok || len(deployments) == 0 {
-			if err := os.RemoveAll(d.Path(id)); err != nil {
-				return errs.Wrap(err, "failed to remove stale artifact")
-			}
+			d.config.Cache[id].InUse = false
+			logging.Debug("Artifact '%s' is no longer in use", id.String())
 		}
 	}
+	err := d.removeStaleArtifacts()
+	if err != nil {
+		return errs.Wrap(err, "Could not remove stale artifacts")
+	}
 
 	// Write config file changes to disk
 	configFile := filepath.Join(d.depotPath, depotFile)
@@ -422,3 +472,40 @@ func someFilesExist(filePaths []string, basePath string) bool {
 	}
 	return false
 }
+
+// removeStaleArtifacts iterates over all unused artifacts in the depot, sorts them by last access
+// time, and removes them until the size of cached artifacts is under the limit.
+func (d *depot) removeStaleArtifacts() error {
+	var totalSize int64
+	unusedArtifacts := make([]*artifactInfo, 0)
+
+	for _, info := range d.config.Cache {
+		if !info.InUse {
+			totalSize += info.Size
+			unusedArtifacts = append(unusedArtifacts, info)
+		}
+	}
+	logging.Debug("There are %d unused artifacts totaling %.1f MB in size", len(unusedArtifacts), float64(totalSize)/float64(MB))
+
+	sort.Slice(unusedArtifacts, func(i, j int) bool {
+		return unusedArtifacts[i].LastAccessTime < unusedArtifacts[j].LastAccessTime
+	})
+
+	var rerr error
+	for _, artifact := range unusedArtifacts {
+		if totalSize <= d.cacheSize {
+			break // done
+		}
+		if err := os.RemoveAll(d.Path(artifact.id)); err == nil {
+			totalSize -= artifact.Size
+		} else {
+			if err := errs.Wrap(err, "Could not delete old artifact"); rerr == nil {
+				rerr = err
+			} else {
+				rerr = errs.Pack(rerr, err)
+			}
+		}
+		delete(d.config.Cache, artifact.id)
+	}
+	return rerr
+}

pkg/runtime/options.go

@@ -25,6 +25,10 @@ func WithPortable() SetOpt {
 	return func(opts *Opts) { opts.Portable = true }
 }
 
+func WithCacheSize(mb int) SetOpt {
+	return func(opts *Opts) { opts.CacheSize = mb }
+}
+
 func WithArchive(dir string, platformID strfmt.UUID, ext string) SetOpt {
 	return func(opts *Opts) {
 		opts.FromArchive = &fromArchive{dir, platformID, ext}

pkg/runtime/setup.go

@@ -48,6 +48,7 @@ type Opts struct {
 	BuildlogFilePath     string
 	BuildProgressUrl     string
 	Portable             bool
+	CacheSize            int
 
 	FromArchive *fromArchive
 
@@ -93,6 +94,7 @@ type setup struct {
 }
 
 func newSetup(path string, bp *buildplan.BuildPlan, env *envdef.Collection, depot *depot, opts *Opts) (*setup, error) {
+	depot.SetCacheSize(opts.CacheSize)
 	installedArtifacts := depot.List(path)
 
 	var platformID strfmt.UUID

test/integration/runtime_int_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/ActiveState/cli/internal/constants"
+	"github.com/ActiveState/cli/internal/fileutils"
 	"github.com/ActiveState/cli/internal/osutils"
 	"github.com/ActiveState/cli/internal/testhelpers/e2e"
 	"github.com/ActiveState/cli/internal/testhelpers/osutil"
@@ -212,6 +213,39 @@ func (suite *RuntimeIntegrationTestSuite) TestIgnoreEnvironmentVars() {
 	cp.ExpectExitCode(0)
 }
 
+func (suite *RuntimeIntegrationTestSuite) TestRuntimeCache() {
+	suite.OnlyRunForTags(tagsuite.Critical)
+	ts := e2e.New(suite.T(), false)
+	defer ts.Close()
+
+	ts.PrepareEmptyProject()
+
+	cp := ts.Spawn("install", "shared/zlib")
+	cp.Expect("Downloading")
+	cp.ExpectExitCode(0)
+
+	depot := filepath.Join(ts.Dirs.Cache, "depot")
+	artifacts, err := fileutils.ListDirSimple(depot, true)
+	suite.Require().NoError(err)
+
+	cp = ts.Spawn("switch", "mingw") // should not remove cached shared/zlib artifact
+	cp.ExpectExitCode(0)
+
+	artifacts2, err := fileutils.ListDirSimple(depot, true)
+	suite.Require().NoError(err)
+	suite.Assert().Equal(artifacts, artifacts2, "shared/zlib should have remained in the cache")
+
+	cp = ts.Spawn("config", "set", constants.RuntimeCacheSizeConfigKey, "0")
+	cp.ExpectExitCode(0)
+
+	cp = ts.Spawn("switch", "main") // should remove cached shared/zlib artifact
+	cp.ExpectExitCode(0)
+
+	artifacts3, err := fileutils.ListDirSimple(depot, true)
+	suite.Require().NoError(err)
+	suite.Assert().NotEqual(artifacts, artifacts3, "shared/zlib should have been removed from the cache")
+}
+
 func TestRuntimeIntegrationTestSuite(t *testing.T) {
 	suite.Run(t, new(RuntimeIntegrationTestSuite))
 }