aboutcode-org
diff --git a/‎.gitignore‎
Lines changed: 1 addition & 0 deletions b/‎.gitignore‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎CHANGELOG.rst‎
Lines changed: 4 additions & 0 deletions b/‎CHANGELOG.rst‎
Lines changed: 4 additions & 0 deletions
diff --git a/‎scanpipe/migrations/0016_discoveredpackage_package_uid.py‎
Lines changed: 18 additions & 0 deletions b/‎scanpipe/migrations/0016_discoveredpackage_package_uid.py‎
Lines changed: 18 additions & 0 deletions
diff --git a/‎scanpipe/models.py‎
Lines changed: 6 additions & 1 deletion b/‎scanpipe/models.py‎
Lines changed: 6 additions & 1 deletion
diff --git a/‎scanpipe/pipelines/docker.py‎
Lines changed: 2 additions & 2 deletions b/‎scanpipe/pipelines/docker.py‎
Lines changed: 2 additions & 2 deletions
diff --git a/‎scanpipe/pipelines/load_inventory.py‎
Lines changed: 4 additions & 5 deletions b/‎scanpipe/pipelines/load_inventory.py‎
Lines changed: 4 additions & 5 deletions
diff --git a/‎scanpipe/pipelines/scan_codebase.py‎
Lines changed: 0 additions & 1 deletion b/‎scanpipe/pipelines/scan_codebase.py‎
Lines changed: 0 additions & 1 deletion
diff --git a/‎scanpipe/pipelines/scan_package.py‎
Lines changed: 7 additions & 13 deletions b/‎scanpipe/pipelines/scan_package.py‎
Lines changed: 7 additions & 13 deletions
diff --git a/‎scanpipe/pipes/__init__.py‎
Lines changed: 9 additions & 4 deletions b/‎scanpipe/pipes/__init__.py‎
Lines changed: 9 additions & 4 deletions
diff --git a/‎scanpipe/pipes/alpine.py‎
Lines changed: 0 additions & 32 deletions b/‎scanpipe/pipes/alpine.py‎
Lines changed: 0 additions & 32 deletions
@@ -40,6 +40,7 @@ local
 policies.yml
 *.rdb
 *.aof
+.vscode
 
 # This is only created when packaging for external redistribution
 /thirdparty/
@@ -7,6 +7,10 @@ v31.0.0 (next)
 - WARNING: Drop support for Python 3.6 and 3.7. Add support for Python 3.10.
   Upgrade Django to version 4.x series.
 
+- Upgrade ScanCode-toolkit to version v31.
+  See https://github.com/nexB/scancode-toolkit/blob/develop/CHANGELOG.rst for an
+  overview of the changes in v31 compared to v30.
+
 - Implement run status auto-refresh using the htmx JavaScript library.
   The statuses of queued and running pipeline are now automatically refreshed
   in the project list and project details views every 10 seconds.
 
@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-06-09 18:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0015_alter_codebaseresource_project_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='discoveredpackage',
+            name='package_uid',
+            field=models.CharField(blank=True, help_text='Unique identifier for this package.', max_length=1024),
+        ),
+    ]
@@ -1726,6 +1726,11 @@ class DiscoveredPackage(
         blank=True,
         help_text=_("A list of dependencies for this package."),
     )
+    package_uid = models.CharField(
+        max_length=1024,
+        blank=True,
+        help_text=_("Unique identifier for this package."),
+    )
 
     # `AbstractPackage` model overrides:
     keywords = models.JSONField(default=list, blank=True)
@@ -1769,7 +1774,7 @@ def create_from_data(cls, project, package_data):
         If one of the values of the required fields is not available, a "ProjectError"
         is created instead of a new DiscoveredPackage instance.
         """
-        required_fields = ["type", "name", "version"]
+        required_fields = ["type", "name"]
         missing_values = [
             field_name
             for field_name in required_fields
 
@@ -20,12 +20,12 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-from scanpipe.pipelines import root_filesystems
+from scanpipe.pipelines.root_filesystems import RootFS
 from scanpipe.pipes import docker
 from scanpipe.pipes import rootfs
 
 
-class Docker(root_filesystems.RootFS):
+class Docker(RootFS):
     """
     A pipeline to analyze Docker images.
     """
 
@@ -42,15 +42,14 @@ def get_scan_json_input(self):
         Locates a JSON scan input from a project's input/ directory.
         """
         inputs = list(self.project.inputs(pattern="*.json"))
+
         if len(inputs) != 1:
             raise Exception("Only 1 JSON input file supported")
+
         self.input_location = str(inputs[0].absolute())
 
     def build_inventory_from_scan(self):
         """
-        Processes a given JSON scan input to populate codebase resources and packages.
+        Processes a JSON Scan results file to populate codebase resources and packages.
         """
-        project = self.project
-        scanned_codebase = scancode.get_virtual_codebase(project, self.input_location)
-        scancode.create_codebase_resources(project, scanned_codebase)
-        scancode.create_discovered_packages(project, scanned_codebase)
+        scancode.create_inventory_from_scan(self.project, self.input_location)
@@ -22,7 +22,6 @@
 
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
-from scanpipe.pipes import output
 from scanpipe.pipes import rootfs
 from scanpipe.pipes import scancode
 from scanpipe.pipes.input import copy_inputs
 
@@ -56,13 +56,9 @@ def steps(cls):
         "--license-text",
         "--package",
         "--url",
-    ] + [
         "--classify",
-        "--consolidate",
         "--is-license-text",
-        "--license-clarity-score",
         "--summary",
-        "--summary-key-files",
     ]
 
     def get_package_archive_input(self):
@@ -102,33 +98,31 @@ def run_scancode(self):
         """
         Scans extracted codebase/ content.
         """
-        self.scan_output = self.project.get_output_file_path("scancode", "json")
+        scan_output_path = self.project.get_output_file_path("scancode", "json")
+        self.scan_output_location = str(scan_output_path.absolute())
 
         with self.save_errors(scancode.ScancodeError):
             scancode.run_scancode(
                 location=str(self.project.codebase_path),
-                output_file=str(self.scan_output),
+                output_file=self.scan_output_location,
                 options=self.scancode_options,
                 raise_on_error=True,
             )
 
-        if not self.scan_output.exists():
+        if not scan_output_path.exists():
             raise FileNotFoundError("ScanCode output not available.")
 
     def build_inventory_from_scan(self):
         """
-        Processes the JSON scan results to determine resources and packages.
+        Processes a JSON Scan results file to populate codebase resources and packages.
         """
-        project = self.project
-        scanned_codebase = scancode.get_virtual_codebase(project, str(self.scan_output))
-        scancode.create_codebase_resources(project, scanned_codebase)
-        scancode.create_discovered_packages(project, scanned_codebase)
+        scancode.create_inventory_from_scan(self.project, self.scan_output_location)
 
     def make_summary_from_scan_results(self):
         """
         Builds a summary in JSON format from the generated scan results.
         """
-        summary = scancode.make_results_summary(self.project, str(self.scan_output))
+        summary = scancode.make_results_summary(self.project, self.scan_output_location)
         output_file = self.project.get_output_file_path("summary", "json")
 
         with output_file.open("w") as summary_file:
 
@@ -29,8 +29,6 @@
 
 from django.db.models import Count
 
-from packageurl import normalize_qualifiers
-
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredPackage
 from scanpipe.pipes import scancode
@@ -73,12 +71,19 @@ def update_or_create_package(project, package_data, codebase_resource=None):
     """
     Gets, updates or creates a DiscoveredPackage then returns it.
     Uses the `project` and `package_data` mapping to lookup and creates the
-    DiscoveredPackage using its Package URL as a unique key.
+    DiscoveredPackage using its Package URL and package_uid as a unique key.
     """
     purl_data = DiscoveredPackage.extract_purl_data(package_data)
+    package_uid = package_data.get("package_uid")
+    purl_data_and_package_uid = {
+        **purl_data,
+        "package_uid": package_uid,
+    }
 
     try:
-        package = DiscoveredPackage.objects.get(project=project, **purl_data)
+        package = DiscoveredPackage.objects.get(
+            project=project, **purl_data_and_package_uid
+        )
     except DiscoveredPackage.DoesNotExist:
         package = None