file_packager: Set data package_uuid to sha256 hash of preloaded contents (#16807)

Author: connorjclark

ChangeLog.md

@@ -33,6 +33,8 @@ See docs/process.md for more on how version tagging works.
   are now exposed to native code and can be used to keep the runtime alive
   without immediately unwinding the event loop (as
   `emscripten_exit_with_live_runtime()` does). (#17160)
+- The file packager option `--use-preload-cache` now only invalidates the
+  cache if the data contents has changed. (#16807)
 
 3.1.13 - 06/02/2022
 -------------------

tests/test_other.py

@@ -21,7 +21,6 @@
 import time
 import tempfile
 import unittest
-import uuid
 from pathlib import Path
 from subprocess import PIPE, STDOUT
 
@@ -2688,15 +2687,15 @@ def clean(txt):
     # verify '--separate-metadata' option produces separate metadata file
     os.chdir('..')
 
-    self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'data1.txt', '--preload', 'subdir/data2.txt', '--js-output=immutable.js', '--separate-metadata'])
+    self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'data1.txt', '--preload', 'subdir/data2.txt', '--js-output=immutable.js', '--separate-metadata', '--use-preload-cache'])
     self.assertExists('immutable.js.metadata')
     # verify js output JS file is not touched when the metadata is separated
     orig_timestamp = os.path.getmtime('immutable.js')
     orig_content = read_file('immutable.js')
     # ensure some time passes before running the packager again so that if it does touch the
     # js file it will end up with the different timestamp.
     time.sleep(1.0)
-    self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'data1.txt', '--preload', 'subdir/data2.txt', '--js-output=immutable.js', '--separate-metadata'])
+    self.run_process([FILE_PACKAGER, 'test.data', '--preload', 'data1.txt', '--preload', 'subdir/data2.txt', '--js-output=immutable.js', '--separate-metadata', '--use-preload-cache'])
     # assert both file content and timestamp are the same as reference copy
     self.assertTextDataIdentical(orig_content, read_file('immutable.js'))
     self.assertEqual(orig_timestamp, os.path.getmtime('immutable.js'))
@@ -2707,8 +2706,7 @@ def clean(txt):
     assert metadata['files'][1]['start'] == len('data1') and metadata['files'][1]['end'] == len('data1') + len('data2') and metadata['files'][1]['filename'] == '/subdir/data2.txt'
     assert metadata['remote_package_size'] == len('data1') + len('data2')
 
-    # can only assert the uuid format is correct, the uuid's value is expected to differ in between invocation
-    uuid.UUID(metadata['package_uuid'], version=4)
+    self.assertEqual(metadata['package_uuid'], 'sha256-53ddc03623f867c7d4a631ded19c2613f2cb61d47b6aa214f47ff3cc15445bcd')
 
   def test_file_packager_unicode(self):
     unicode_name = 'unicode…☃'

tools/file_packager.py

@@ -64,13 +64,13 @@
 import base64
 import ctypes
 import fnmatch
+import hashlib
 import json
 import os
 import posixpath
 import random
 import shutil
 import sys
-import uuid
 from subprocess import PIPE
 from textwrap import dedent
 
@@ -600,7 +600,6 @@ def generate_js(data_target, data_files, metadata):
         start += len(curr)
         data.write(curr)
 
-    # TODO: sha256sum on data_target
     if start > 256 * 1024 * 1024:
       err('warning: file packager is creating an asset bundle of %d MB. '
           'this is very large, and browsers might have trouble loading it. '
@@ -735,7 +734,6 @@ def generate_js(data_target, data_files, metadata):
             Module['LZ4'].loadPackage({ 'metadata': metadata, 'compressedData': compressedData }, %s);
             Module['removeRunDependency']('datafile_%s');''' % (meta, "true" if options.use_preload_plugins else "false", js_manipulation.escape_for_js_string(data_target))
 
-    package_uuid = uuid.uuid4()
     package_name = data_target
     remote_package_size = os.path.getsize(package_name)
     remote_package_name = os.path.basename(package_name)
@@ -755,13 +753,17 @@ def generate_js(data_target, data_files, metadata):
       }
       var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;\n''' % (js_manipulation.escape_for_js_string(data_target), js_manipulation.escape_for_js_string(remote_package_name))
     metadata['remote_package_size'] = remote_package_size
-    metadata['package_uuid'] = str(package_uuid)
-    ret += '''
-      var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];
-      var PACKAGE_UUID = metadata['package_uuid'];\n'''
+    ret += '''var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n'''
 
     if options.use_preload_cache:
+      # Set the id to a hash of the preloaded data, so that caches survive over multiple builds
+      # if the data has not changed.
+      data = utils.read_binary(data_target)
+      package_uuid = 'sha256-' + hashlib.sha256(data).hexdigest()
+      metadata['package_uuid'] = str(package_uuid)
+
       code += r'''
+        var PACKAGE_UUID = metadata['package_uuid'];
         var indexedDB;
         if (typeof window === 'object') {
           indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB;