Mark package-configs as active when writing (#4592)

Author: sigurdm

doc/cache_layout.md

@@ -34,6 +34,7 @@ Here are the top-level folders you can find in a Pub cache.
 ```plaintext
 $PUB_CACHE/
 ├── global_packages/  # Globally activated packages
+├── active_roots/     # Information about packages that this cache caches for.
 ├── bin/              # Executables compiled from globally activated packages.
 ├── git/              # Cloned git packages
 ├── hosted/           # Hosted package downloads
@@ -233,9 +234,30 @@ $PUB_CACHE/bin/
 └── stagehand
 ```
 
+# Active roots
+$PUB_CACHE/active_roots/
+
+In order to be able to prune the cache (`dart pub cache gc`) pub keeps a tally of
+each time it writes a `.dart_tool/package_config.json` file (an activation).
+
+The directory is laid out such that each file-name is the hex-encoded sha256
+hash of the absolute file-uri of the path of the package config.
+
+The first two bytes are used for a subdirectory, to prevent too many files in
+one directory.
+
+When implemented `dart pub cache gc` will look through all the package configs,
+and mark all cached packages in the cache used by those projects `alive`. If a
+package config doesn't exist, it is ignored, and the file marking it is deleted.
+
+All other packages in the cache are removed.
+
+Packages that are installed in the cache within 1 day are not deleted. This is
+to minimize the risk of race-conditions.  
+
 ## Logs
 
 When pub crashes or is run with `--verbose` it will create a
 `$PUB_CACHE/log/pub_log.txt` with the dart sdk version, platform, `$PUB_CACHE`,
-`$PUB_HOSTED_URL`, `pubspec.yaml`, `pubspec.lock`, current command, verbose log and
-stack-trace.
+`$PUB_HOSTED_URL`, `pubspec.yaml`, `pubspec.lock`, current command, verbose log
+and stack-trace.

lib/src/entrypoint.dart

@@ -402,6 +402,8 @@ See $workspacesDocUrl for more information.''',
   /// If the workspace is non-trivial: For each package in the workspace write:
   /// `.dart_tool/pub/workspace_ref.json` with a pointer to the workspace root
   /// package dir.
+  ///
+  /// Also marks the package active in `PUB_CACHE/active_roots/`.
   Future<void> writePackageConfigFiles() async {
     ensureDir(p.dirname(packageConfigPath));
 
@@ -433,6 +435,9 @@ See $workspacesDocUrl for more information.''',
         writeTextFileIfDifferent(workspaceRefPath, '$workspaceRef\n');
       }
     }
+    if (lockFile.packages.values.any((id) => id.source is CachedSource)) {
+      cache.markRootActive(packageConfigPath);
+    }
   }
 
   Future<String> _packageGraphFile(SystemCache cache) async {

lib/src/global_packages.dart

@@ -293,10 +293,14 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam
 
       lockFile.writeToFile(p.join(tempDir, 'pubspec.lock'), cache);
 
+      final packageDir = _packageDir(name);
+      tryDeleteEntry(packageDir);
+      tryRenameDir(tempDir, packageDir);
+
       // Load the package graph from [result] so we don't need to re-parse all
       // the pubspecs.
       final entrypoint = Entrypoint.global(
-        root,
+        packageForConstraint(dep, packageDir),
         lockFile,
         cache,
         solveResult: result,
@@ -305,9 +309,6 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam
       await entrypoint.writePackageConfigFiles();
 
       await entrypoint.precompileExecutables();
-
-      tryDeleteEntry(_packageDir(name));
-      tryRenameDir(tempDir, _packageDir(name));
     }
 
     final entrypoint = Entrypoint.global(

lib/src/io.dart

@@ -257,6 +257,10 @@ void writeTextFile(
   File(file).writeAsStringSync(contents, encoding: encoding);
 }
 
+/// Reads the file at [path] and writes [newContent] to it, if it is different
+/// from [newContent].
+///
+/// If the file doesn't exist it is always written.
 void writeTextFileIfDifferent(String path, String newContent) {
   // Compare to the present package_config.json
   // For purposes of equality we don't care about the `generated` timestamp.

lib/src/system_cache.dart

@@ -2,8 +2,10 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
 
+import 'package:crypto/crypto.dart';
 import 'package:path/path.dart' as p;
 import 'package:pub_semver/pub_semver.dart';
 
@@ -80,6 +82,8 @@ Consider setting the `PUB_CACHE` variable manually.
 
   Source get defaultSource => hosted;
 
+  late final Iterable<CachedSource> cachedSources = [hosted, git];
+
   /// The built-in Git source.
   GitSource get git => GitSource.instance;
 
@@ -400,6 +404,74 @@ https://dart.dev/go/pub-cache
   }
 
   bool _hasMaintainedCache = false;
+
+  late final _activeRootsDir = p.join(rootDir, 'active_roots');
+
+  /// Returns the paths of all packages_configs registered in
+  /// [_activeRootsDir].
+  List<String> activeRoots() {
+    final List<String> files;
+    try {
+      files = listDir(_activeRootsDir, includeDirs: false, recursive: true);
+    } on IOException {
+      return [];
+    }
+    final activeRoots = <String>[];
+    for (final file in files) {
+      final Object? decoded;
+      try {
+        decoded = jsonDecode(readTextFile(file));
+      } on IOException catch (e) {
+        log.fine('Could not read $file $e - deleting');
+        tryDeleteEntry(file);
+        continue;
+      } on FormatException catch (e) {
+        log.fine('Could not decode $file $e - deleting');
+        tryDeleteEntry(file);
+        continue;
+      }
+      if (decoded is! Map<String, Object?>) {
+        log.fine('Faulty $file - deleting');
+        tryDeleteEntry(file);
+        continue;
+      }
+      final uriText = decoded['package_config'];
+      if (uriText is! String) {
+        log.fine('Faulty $file - deleting');
+        tryDeleteEntry(file);
+        continue;
+      }
+      final uri = Uri.tryParse(uriText);
+      if (uri == null || !uri.isScheme('file')) {
+        log.fine('Faulty $file - deleting');
+        tryDeleteEntry(file);
+        continue;
+      }
+      activeRoots.add(uri.toFilePath());
+    }
+    return activeRoots;
+  }
+
+  /// Adds a file to the `PUB_CACHE/active_roots/` dir indicating
+  /// [packageConfigPath] is active.
+  void markRootActive(String packageConfigPath) {
+    final canonicalFileUri =
+        p.toUri(p.canonicalize(packageConfigPath)).toString();
+
+    final hash = hexEncode(sha256.convert(utf8.encode(canonicalFileUri)).bytes);
+
+    final firstTwo = hash.substring(0, 2);
+    final theRest = hash.substring(2);
+
+    final dir = p.join(_activeRootsDir, firstTwo);
+    ensureDir(dir);
+
+    final filename = p.join(dir, theRest);
+    writeTextFileIfDifferent(
+      filename,
+      '${jsonEncode({'package_config': canonicalFileUri})}\n',
+    );
+  }
 }
 
 typedef SourceRegistry = Source Function(String? name);

test/cache/gc_test.dart

@@ -0,0 +1,93 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:crypto/crypto.dart';
+import 'package:path/path.dart' as p;
+import 'package:pub/src/system_cache.dart';
+import 'package:pub/src/utils.dart';
+import 'package:test/test.dart';
+
+import '../descriptor.dart' as d;
+import '../test_pub.dart';
+
+void main() async {
+  test('marks a package active on pub get and global activate', () async {
+    final server = await servePackages();
+
+    server.serve('foo', '1.0.0');
+    server.serve('bar', '1.0.0');
+
+    await runPub(args: ['global', 'activate', 'foo']);
+
+    // Without cached dependencies we don't register the package
+    await d.dir('app_none', [d.appPubspec()]).create();
+
+    await d.appDir(dependencies: {'bar': '1.0.0'}).create();
+
+    await d.dir('app_hosted', [
+      d.appPubspec(dependencies: {'bar': '^1.0.0'}),
+    ]).create();
+
+    await d.git('lib', [d.libPubspec('lib', '1.0.0')]).create();
+
+    await d.dir('app_git', [
+      d.appPubspec(
+        dependencies: {
+          'lib': {'git': '../lib'},
+        },
+      ),
+    ]).create();
+
+    await d.dir('app_path', [
+      d.appPubspec(
+        dependencies: {
+          'lib': {'path': '../lib'},
+        },
+      ),
+    ]).create();
+
+    await pubGet(workingDirectory: p.join(d.sandbox, 'app_none'));
+    await pubGet(workingDirectory: p.join(d.sandbox, 'app_hosted'));
+    await pubGet(workingDirectory: p.join(d.sandbox, 'app_git'));
+    await pubGet(workingDirectory: p.join(d.sandbox, 'app_path'));
+
+    final markingFiles =
+        Directory(
+          p.join(d.sandbox, cachePath, 'active_roots'),
+        ).listSync(recursive: true).whereType<File>().toList();
+
+    expect(markingFiles, hasLength(3));
+
+    for (final file in markingFiles) {
+      final uri =
+          (jsonDecode(file.readAsStringSync()) as Map)['package_config']
+              as String;
+      final hash = hexEncode(sha256.convert(utf8.encode(uri)).bytes);
+      final hashFileName =
+          '${p.basename(p.dirname(file.path))}${p.basename(file.path)}';
+      expect(hashFileName, hash);
+    }
+
+    expect(markingFiles, hasLength(3));
+
+    expect(SystemCache(rootDir: p.join(d.sandbox, cachePath)).activeRoots(), {
+      p.canonicalize(
+        p.join(d.sandbox, 'app_hosted', '.dart_tool', 'package_config.json'),
+      ),
+      p.canonicalize(
+        p.join(d.sandbox, 'app_git', '.dart_tool', 'package_config.json'),
+      ),
+
+      p.canonicalize(
+        p.join(
+          d.sandbox,
+          cachePath,
+          'global_packages',
+          'foo',
+          '.dart_tool',
+          'package_config.json',
+        ),
+      ),
+    });
+  });
+}

test/embedding/embedding_test.dart

@@ -437,7 +437,13 @@ main() {
 
 String _filter(String input) {
   return input
-      .replaceAll(p.toUri(d.sandbox).toString(), r'file://$SANDBOX')
+      .replaceAll(
+        RegExp(
+          RegExp.escape(p.toUri(d.sandbox).toString()),
+          caseSensitive: false,
+        ),
+        r'file://$SANDBOX',
+      )
       .replaceAll(d.sandbox, r'$SANDBOX')
       .replaceAll(Platform.pathSeparator, '/')
       .replaceAll(Platform.operatingSystem, r'$OS')
@@ -546,6 +552,10 @@ String _filter(String input) {
         RegExp(r'"archive_sha256":"[0-9a-f]{64}"', multiLine: true),
         r'"archive_sha256":"$SHA256"',
       )
+      .replaceAll(
+        RegExp(r'active_roots/[0-9a-f]{2}/[0-9a-f]{62}', multiLine: true),
+        r'active_roots/$HH/$HASH',
+      )
       /// TODO(sigurdm): This hack suppresses differences in stack-traces
       /// between dart 2.17 and 2.18. Remove when 2.18 is stable.
       .replaceAllMapped(

test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt

@@ -149,6 +149,9 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt.
 [E]    |   ],
 [E]    |   "configVersion": 1
 [E]    | }
+[E] IO  : Writing $N characters to text file $SANDBOX/cache/active_roots/$HH/$HASH.
+[E] FINE: Contents:
+[E]    | {"package_config":"file://$SANDBOX/myapp/.dart_tool/package_config.json"}
 [E] IO  : Writing $N characters to text file $SANDBOX/cache/log/pub_log.txt.
 
 -------------------------------- END OF OUTPUT ---------------------------------
@@ -335,6 +338,9 @@ FINE: Contents:
    |   ],
    |   "configVersion": 1
    | }
+IO  : Writing $N characters to text file $SANDBOX/cache/active_roots/$HH/$HASH.
+FINE: Contents:
+   | {"package_config":"file://$SANDBOX/myapp/.dart_tool/package_config.json"}
 ---- End log transcript ----
 -------------------------------- END OF OUTPUT ---------------------------------