Merge branch 'main' of nexB/scancode.io into lf32-licensetext-devel

Author: lf32

scanpipe/migrations/0017_alter_discoveredpackage_package_uid_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.0.5 on 2022-06-17 06:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0016_discoveredpackage_package_uid'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='discoveredpackage',
+            name='package_uid',
+            field=models.CharField(blank=True, db_index=True, help_text='Unique identifier for this package.', max_length=1024),
+        ),
+        migrations.AddConstraint(
+            model_name='discoveredpackage',
+            constraint=models.UniqueConstraint(condition=models.Q(('package_uid', ''), _negated=True), fields=('project', 'package_uid'), name='scanpipe_discoveredpackage_unique_package_uid_within_project'),
+        ),
+    ]

scanpipe/models.py

@@ -1729,6 +1729,7 @@ class DiscoveredPackage(
     package_uid = models.CharField(
         max_length=1024,
         blank=True,
+        db_index=True,
         help_text=_("Unique identifier for this package."),
     )
 
@@ -1740,6 +1741,13 @@ class DiscoveredPackage(
 
     class Meta:
         ordering = ["uuid"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["project", "package_uid"],
+                condition=~Q(package_uid=""),
+                name="%(app_label)s_%(class)s_unique_package_uid_within_project",
+            ),
+        ]
 
     def __str__(self):
         return self.package_url or str(self.uuid)
@@ -1807,7 +1815,7 @@ def create_from_data(cls, project, package_data):
         discovered_package.save(save_error=False, capture_exception=False)
         return discovered_package
 
-    def update_from_data(self, package_data):
+    def update_from_data(self, package_data, override=False):
         """
         Update this discovered package instance with the provided `package_data`.
         The `save()` is called only if at least one field was modified.
@@ -1825,11 +1833,9 @@ def update_from_data(self, package_data):
                 continue
 
             current_value = getattr(self, field_name, None)
-            if not current_value:
+            if not current_value or (current_value != value and override):
                 setattr(self, field_name, value)
                 updated_fields.append(field_name)
-            elif current_value != value:
-                pass  # TODO: handle this case
 
         if updated_fields:
             self.save()

scanpipe/pipes/__init__.py

@@ -74,22 +74,18 @@ def update_or_create_package(project, package_data, codebase_resource=None):
     DiscoveredPackage using its Package URL and package_uid as a unique key.
     """
     purl_data = DiscoveredPackage.extract_purl_data(package_data)
-    package_uid = package_data.get("package_uid")
-    purl_data_and_package_uid = {
-        **purl_data,
-        "package_uid": package_uid,
-    }
 
     try:
         package = DiscoveredPackage.objects.get(
-            project=project, **purl_data_and_package_uid
+            project=project,
+            package_uid=package_data.get("package_uid"),
+            **purl_data,
         )
     except DiscoveredPackage.DoesNotExist:
         package = None
 
     if package:
         package.update_from_data(package_data)
-
     else:
         if codebase_resource:
             package = codebase_resource.create_and_add_package(package_data)

scanpipe/pipes/docker.py

@@ -25,10 +25,10 @@
 from pathlib import Path
 
 from container_inspector.image import Image
+from container_inspector.utils import extract_tar
 
 from scanpipe import pipes
 from scanpipe.pipes import rootfs
-from scanpipe.pipes.scancode import extract_archive
 
 logger = logging.getLogger(__name__)
 
@@ -62,7 +62,7 @@ def extract_image_from_tarball(input_tarball, extract_target, verify=True):
     Returns the `images` and an `errors` list of error messages that may have
     happen during the extraction.
     """
-    errors = list(extract_archive(location=input_tarball, target=extract_target))
+    errors = extract_tar(location=input_tarball, target_dir=extract_target)
     images = Image.get_images_from_dir(
         extracted_location=str(extract_target),
         verify=verify,
@@ -101,9 +101,9 @@ def extract_layers_from_images_to_base_path(base_path, images):
 
         for layer in image.layers:
             extract_target = target_path / layer.layer_id
-            extract_errors = extract_archive(
+            extract_errors = extract_tar(
                 location=layer.archive_location,
-                target=extract_target,
+                target_dir=extract_target,
             )
             errors.extend(extract_errors)
             layer.extracted_location = str(extract_target)

scanpipe/pipes/scancode.py

@@ -43,7 +43,6 @@
 
 from scanpipe import pipes
 from scanpipe.models import CodebaseResource
-from scanpipe.models import DiscoveredPackage
 
 logger = logging.getLogger("scanpipe.pipes")
 
@@ -392,13 +391,14 @@ def create_codebase_resources(project, scanned_codebase):
             defaults=resource_data,
         )
 
-        # Associate DiscoveredPackage to CodebaseResource, if applicable
-        if hasattr(scanned_resource, "for_packages"):
-            for package_uid in scanned_resource.for_packages:
-                package = DiscoveredPackage.objects.get(package_uid=package_uid)
-                set_codebase_resource_for_package(
-                    codebase_resource=codebase_resource, discovered_package=package
-                )
+        for_packages = getattr(scanned_resource, "for_packages", [])
+        for package_uid in for_packages:
+            logger.debug(f"Assign {package_uid} to {codebase_resource}")
+            package = project.discoveredpackages.get(package_uid=package_uid)
+            set_codebase_resource_for_package(
+                codebase_resource=codebase_resource,
+                discovered_package=package,
+            )
 
 
 def create_discovered_packages(project, scanned_codebase):

scanpipe/templates/scanpipe/package_list.html

@@ -27,7 +27,7 @@
         <tbody>
           {% for package in object_list %}
             <tr class="break-word">
-              <td style="min-width: 500px;">
+              <td style="min-width: 500px;" title="{{ package.package_uid }}">
                 {{ package.package_url }}
               </td>
               <td style="min-width: 300px; max-width: 400px;">

scanpipe/tests/data/debian_scan_codebase.json

@@ -123,7 +123,41 @@
       "manifest_path": "",
       "contains_source_code": null,
       "extra_data": {
-        "multi_arch": "same"
+        "multi_arch": "same",
+        "missing_file_references": [
+          {
+            "md5": "23c8a935fa4fc7290d55cc5df3ef56b1",
+            "path": "lib/x86_64-linux-gnu/libncurses.so.5.9",
+            "sha1": null,
+            "sha256": null,
+            "sha512": null,
+            "extra_data": {}
+          },
+          {
+            "md5": "98b70f283324e89db5787a018a54adf4",
+            "path": "usr/lib/x86_64-linux-gnu/libform.so.5.9",
+            "sha1": null,
+            "sha256": null,
+            "sha512": null,
+            "extra_data": {}
+          },
+          {
+            "md5": "e3a0f5154928da2da234920343ac14b2",
+            "path": "usr/lib/x86_64-linux-gnu/libmenu.so.5.9",
+            "sha1": null,
+            "sha256": null,
+            "sha512": null,
+            "extra_data": {}
+          },
+          {
+            "md5": "a927e7d76753bb85f5a784b653d337d2",
+            "path": "usr/lib/x86_64-linux-gnu/libpanel.so.5.9",
+            "sha1": null,
+            "sha256": null,
+            "sha512": null,
+            "extra_data": {}
+          }
+        ]
       },
       "missing_resources": [],
       "modified_resources": [],
@@ -161,7 +195,25 @@
       "manifest_path": "",
       "contains_source_code": null,
       "extra_data": {
-        "multi_arch": "same"
+        "multi_arch": "same",
+        "missing_file_references": [
+          {
+            "md5": "5d26434efecc08048ab72357af804ef7",
+            "path": "usr/lib/x86_64-linux-gnu/libndp.so.0.0.2",
+            "sha1": null,
+            "sha256": null,
+            "sha512": null,
+            "extra_data": {}
+          },
+          {
+            "md5": "60d977e0c9a9fb07c1f8ae3090ea6f48",
+            "path": "usr/share/doc/libndp0/changelog.Debian.gz",
+            "sha1": null,
+            "sha256": null,
+            "sha512": null,
+            "extra_data": {}
+          }
+        ]
       },
       "missing_resources": [],
       "modified_resources": [],
@@ -195,7 +247,6 @@
       "extension": "",
       "programming_language": "",
       "mime_type": "text/plain",
-      "file_type": "ASCII text",
       "is_binary": false,
       "is_text": true,
       "is_archive": false,
@@ -223,7 +274,6 @@
       "extension": "",
       "programming_language": "",
       "mime_type": "text/plain",
-      "file_type": "UTF-8 Unicode text",
       "is_binary": false,
       "is_text": true,
       "is_archive": false,
@@ -251,16 +301,17 @@
       "extension": "",
       "programming_language": "Haxe",
       "mime_type": "text/plain",
-      "file_type": "ASCII text",
       "is_binary": false,
       "is_text": true,
       "is_archive": false,
       "is_key_file": false,
       "is_media": false
     },
     {
-      "for_packages": [],
-      "path": "debian.tar.gz-extract/8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66/var/lib/dpkg/info/libncurses5_amd64.md5sums",
+      "for_packages": [
+        "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
+      ],
+      "path": "debian.tar.gz-extract/8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66/var/lib/dpkg/info/libncurses5:amd64.md5sums",
       "sha1": "e5ff875218d4f909576575b0471feb0e5230a861",
       "md5": "9d18792b91935a5849328cb368005ec9",
       "extra_data": {},
@@ -271,22 +322,23 @@
       "license_expressions": [],
       "emails": [],
       "urls": [],
-      "status": "no-licenses",
+      "status": "system-package",
       "type": "file",
-      "name": "libncurses5_amd64",
+      "name": "libncurses5:amd64",
       "extension": ".md5sums",
       "programming_language": "",
       "mime_type": "text/plain",
-      "file_type": "ASCII text",
       "is_binary": false,
       "is_text": true,
       "is_archive": false,
       "is_key_file": false,
       "is_media": false
     },
     {
-      "for_packages": [],
-      "path": "debian.tar.gz-extract/8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66/var/lib/dpkg/info/libndp0_amd64.md5sums",
+      "for_packages": [
+        "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
+      ],
+      "path": "debian.tar.gz-extract/8a63761caf6d45e65b8e6cdc2e0c03c55625fd142ec3356b80a9ea4a34b11b66/var/lib/dpkg/info/libndp0:amd64.md5sums",
       "sha1": "c212d44c6649df5ff13ec447f4fa30faf81fc490",
       "md5": "7cb818062922c437df1902c18862455a",
       "extra_data": {},
@@ -297,13 +349,12 @@
       "license_expressions": [],
       "emails": [],
       "urls": [],
-      "status": "no-licenses",
+      "status": "system-package",
       "type": "file",
-      "name": "libndp0_amd64",
+      "name": "libndp0:amd64",
       "extension": ".md5sums",
       "programming_language": "",
       "mime_type": "text/plain",
-      "file_type": "ASCII text",
       "is_binary": false,
       "is_text": true,
       "is_archive": false,
@@ -356,7 +407,6 @@
       "extension": "",
       "programming_language": "Haxe",
       "mime_type": "text/plain",
-      "file_type": "ASCII text",
       "is_binary": false,
       "is_text": true,
       "is_archive": false,