Use lockfile on read too

We need to use locks also on read, and not only on write
Windows expereinces race conditions otherwise

Reference: #215
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/_packagedcode/pypi.py

@@ -457,7 +457,7 @@ def parse_metadata(location, datasource_id, package_type):
         name=name,
         version=version,
         description=get_description(metainfo=meta, location=str(location)),
-        #TODO: https://github.com/aboutcode-org/scancode-toolkit/issues/3014
+        # TODO: https://github.com/aboutcode-org/scancode-toolkit/issues/3014
         declared_license=get_declared_license(meta),
         keywords=get_keywords(meta),
         parties=get_parties(meta),
@@ -521,19 +521,25 @@ class PypiWheelHandler(BasePypiHandler):
 
     @classmethod
     def parse(cls, location):
-        with zipfile.ZipFile(location) as zf:
-            for path in ZipPath(zf).iterdir():
-                if not path.name.endswith(META_DIR_SUFFIXES):
-                    continue
-                for metapath in path.iterdir():
-                    if not metapath.name.endswith('METADATA'):
-                        continue
 
-                    yield parse_metadata(
-                        location=metapath,
-                        datasource_id=cls.datasource_id,
-                        package_type=cls.default_package_type,
-                    )
+        from python_inspector import lockfile
+        from python_inspector.utils_pypi import PYINSP_CACHE_LOCK_TIMEOUT
+        lock_file = os.path.join(f"{location}.lockfile")
+        with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT):
+
+            with zipfile.ZipFile(location) as zf:
+                for path in ZipPath(zf).iterdir():
+                    if not path.name.endswith(META_DIR_SUFFIXES):
+                        continue
+                    for metapath in path.iterdir():
+                        if not metapath.name.endswith('METADATA'):
+                            continue
+    
+                        yield parse_metadata(
+                            location=metapath,
+                            datasource_id=cls.datasource_id,
+                            package_type=cls.default_package_type,
+                        )
 
 
 class PypiEggHandler(BasePypiHandler):
@@ -547,20 +553,26 @@ class PypiEggHandler(BasePypiHandler):
 
     @classmethod
     def parse(cls, location):
-        with zipfile.ZipFile(location) as zf:
-            for path in ZipPath(zf).iterdir():
-                if not path.name.endswith(META_DIR_SUFFIXES):
-                    continue
 
-                for metapath in path.iterdir():
-                    if not metapath.name.endswith('PKG-INFO'):
+        from python_inspector import lockfile
+        from python_inspector.utils_pypi import PYINSP_CACHE_LOCK_TIMEOUT
+        lock_file = os.path.join(f"{location}.lockfile")
+        with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT):
+
+            with zipfile.ZipFile(location) as zf:
+                for path in ZipPath(zf).iterdir():
+                    if not path.name.endswith(META_DIR_SUFFIXES):
                         continue
 
-                    yield parse_metadata(
-                        location=metapath,
-                        datasource_id=cls.datasource_id,
-                        package_type=cls.default_package_type,
-                    )
+                    for metapath in path.iterdir():
+                        if not metapath.name.endswith('PKG-INFO'):
+                            continue
+
+                        yield parse_metadata(
+                            location=metapath,
+                            datasource_id=cls.datasource_id,
+                            package_type=cls.default_package_type,
+                        )
 
 
 class PypiSdistArchiveHandler(BasePypiHandler):
@@ -765,7 +777,6 @@ def parse(cls, location):
                 )
             ]
 
-
         yield models.PackageData(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
@@ -818,6 +829,7 @@ def get_resolved_purl(purl: PackageURL, specifiers: SpecifierSet):
         is_resolved=is_resolved,
     )
 
+
 class PipfileHandler(BaseDependencyFileHandler):
     datasource_id = 'pipfile'
     path_patterns = ('*Pipfile',)

src/python_inspector/utils_pypi.py

@@ -196,7 +196,6 @@ def get_python_dot_version(version):
 DEFAULT_PYTHON_VERSION = settings.DEFAULT_PYTHON_VERSION
 CACHE_THIRDPARTY_DIR = settings.CACHE_THIRDPARTY_DIR
 
-
 ################################################################################
 
 EXTENSIONS_SDIST = (
@@ -316,9 +315,10 @@ async def get_supported_and_valid_wheels(
         ):
             continue
         if TRACE_DEEP:
+            durl = await wheel.download_url(repo)
             print(
                 f"""    get_supported_and_valid_wheels: Getting wheel from index (or cache):
-                {await wheel.download_url(repo)}"""
+                {durl}"""
             )
         wheels.append(wheel)
     return wheels
@@ -1685,6 +1685,7 @@ async def get(
         # the cache key is a hash of the normalized path
         cache_key = self.sha256_hash(quote_plus(path_or_url.strip("/")))
         cached = os.path.join(self.directory, cache_key)
+        lock_file = f"{cached}.lockfile"
 
         if force or not os.path.exists(cached):
             if TRACE_DEEP:
@@ -1699,16 +1700,16 @@ async def get(
             wmode = "w" if as_text else "wb"
 
             # acquire lock and wait until timeout to get a lock or die
-            lock_file = os.path.join(self.directory, f"{cache_key}.lockfile")
-
             with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT):
                 async with aiofiles.open(cached, mode=wmode) as fo:
                     await fo.write(content)
             return content, cached
         else:
             if TRACE_DEEP:
                 print(f"        FILE CACHE HIT: {path_or_url}")
-            return await get_local_file_content(path=cached, as_text=as_text), cached
+            # also lock on read to avoid race conditions
+            with lockfile.FileLock(lock_file).locked(timeout=PYINSP_CACHE_LOCK_TIMEOUT):
+                return await get_local_file_content(path=cached, as_text=as_text), cached
 
 
 CACHE = Cache()