Skip to content

Commit 00c92aa

Browse files
keshav-spacekeshav-space
committed
Add pipeline to compute Advisory ToDos
Signed-off-by: Keshav Priyadarshi <git@keshav.space>
1 parent 8f86f46 commit 00c92aa

File tree

2 files changed

+273
-0
lines changed

2 files changed

+273
-0
lines changed

‎vulnerabilities/improvers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from vulnerabilities.pipelines import VulnerableCodePipeline
1313
from vulnerabilities.pipelines import add_cvss31_to_CVEs
1414
from vulnerabilities.pipelines import collect_commits
15+
from vulnerabilities.pipelines import compute_advisory_todo
1516
from vulnerabilities.pipelines import compute_package_risk
1617
from vulnerabilities.pipelines import compute_package_version_rank
1718
from vulnerabilities.pipelines import enhance_with_exploitdb
@@ -49,6 +50,7 @@
4950
add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline,
5051
remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline,
5152
populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
53+
compute_advisory_todo.ComputeToDo,
5254
]
5355

5456
IMPROVERS_REGISTRY = {

‎vulnerabilities/pipelines/compute_advisory_todo.py‎

Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,271 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
11+
from aboutcode.pipeline import LoopProgress
12+
13+
from vulnerabilities.models import Advisory
14+
from vulnerabilities.models import AdvisoryToDo
15+
from vulnerabilities.models import Alias
16+
from vulnerabilities.pipelines import VulnerableCodePipeline
17+
from vulnerabilities.pipes import fetchcode_utils
18+
from vulnerabilities.pipes.advisory import advisories_checksum
19+
20+
21+
class ComputeToDo(VulnerableCodePipeline):
22+
"""Compute advisory AdvisoryToDo."""
23+
24+
pipeline_id = "compute_advisory_todo"
25+
26+
@classmethod
27+
def steps(cls):
28+
return (
29+
cls.compute_individual_advisory_todo,
30+
cls.detect_conflicting_advisories,
31+
)
32+
33+
def compute_individual_advisory_todo(self):
34+
advisories = Advisory.objects.all().paginated()
35+
advisories_count = Advisory.objects.all().count()
36+
37+
self.log(
38+
f"Checking missing summary, affected and fixed packages in {advisories_count} Advisories"
39+
)
40+
progress = LoopProgress(
41+
total_iterations=advisories_count,
42+
logger=self.log,
43+
progress_step=1,
44+
)
45+
for advisory in progress.iter(advisories):
46+
advisory_todo_id = advisories_checksum(advisories=advisory)
47+
check_missing_summary(
48+
advisory=advisory,
49+
todo_id=advisory_todo_id,
50+
logger=self.log,
51+
)
52+
check_missing_affected_and_fixed_by_packages(
53+
advisory=advisory,
54+
todo_id=advisory_todo_id,
55+
logger=self.log,
56+
)
57+
58+
def detect_conflicting_advisories(self):
59+
PACKAGE_VERSIONS = {}
60+
aliases = Alias.objects.filter(alias__istartswith="cve")
61+
aliases_count = aliases.count()
62+
63+
self.log(f"Cross validating advisory affected and fixed package for {aliases_count} CVEs")
64+
65+
progress = LoopProgress(total_iterations=aliases_count, logger=self.log)
66+
for alias in progress.iter(aliases.paginated()):
67+
advisories = (
68+
Advisory.objects.filter(aliases__contains=alias.alias)
69+
.exclude(advisory_todos__issue_type="MISSING_AFFECTED_AND_FIXED_BY_PACKAGES")
70+
.distinct()
71+
)
72+
purls = get_advisories_purls(advisories=advisories)
73+
get_package_versions(
74+
purls=purls,
75+
package_versions=PACKAGE_VERSIONS,
76+
logger=self.log,
77+
)
78+
check_conflicting_affected_and_fixed_by_packages(
79+
advisories=advisories,
80+
package_versions=PACKAGE_VERSIONS,
81+
purls=purls,
82+
cve=alias,
83+
logger=self.log,
84+
)
85+
86+
87+
def check_missing_summary(advisory, todo_id, logger=None):
88+
if not advisory.summary:
89+
todo, created = AdvisoryToDo.objects.get_or_create(
90+
unique_todo_id=todo_id,
91+
issue_type="MISSING_SUMMARY",
92+
issue_detail="",
93+
)
94+
if created:
95+
todo.advisories.add(advisory)
96+
97+
98+
def check_missing_affected_and_fixed_by_packages(advisory, todo_id, logger=None):
99+
"""
100+
Check for missing affected or fixed-by packages in the advisory
101+
and create appropriate AdvisoryToDo.
102+
103+
- If both affected and fixed packages are missing add `MISSING_AFFECTED_AND_FIXED_BY_PACKAGES`.
104+
- If only the affected package is missing add `MISSING_AFFECTED_PACKAGE`.
105+
- If only the fixed package is missing add `MISSING_FIXED_BY_PACKAGE`.
106+
"""
107+
has_affected_package = False
108+
has_fixed_package = False
109+
for affected in advisory.to_advisory_data().affected_packages or []:
110+
if has_affected_package and has_fixed_package:
111+
break
112+
if not has_affected_package and affected.affected_version_range:
113+
has_affected_package = True
114+
if not has_fixed_package and affected.fixed_version:
115+
has_fixed_package = True
116+
117+
if has_affected_package and has_fixed_package:
118+
return
119+
120+
if not has_affected_package and not has_fixed_package:
121+
issue_type = "MISSING_AFFECTED_AND_FIXED_BY_PACKAGES"
122+
elif not has_affected_package:
123+
issue_type = "MISSING_AFFECTED_PACKAGE"
124+
elif has_fixed_package:
125+
issue_type = "MISSING_FIXED_BY_PACKAGE"
126+
todo, created = AdvisoryToDo.objects.get_or_create(
127+
unique_todo_id=todo_id,
128+
issue_type=issue_type,
129+
issue_detail="",
130+
)
131+
if created:
132+
todo.advisories.add(advisory)
133+
134+
135+
def get_package_versions(purls, package_versions, logger=None):
136+
for purl in purls:
137+
if purl in package_versions:
138+
continue
139+
versions = fetchcode_utils.versions(purl=purl, logger=logger)
140+
package_versions[purl] = versions
141+
142+
143+
def get_advisories_purls(advisories):
144+
purls = set()
145+
for advisory in advisories:
146+
advisory_obj = advisory.to_advisory_data()
147+
purls.update([str(i.package) for i in advisory_obj.affected_packages])
148+
return purls
149+
150+
151+
def check_conflicting_affected_and_fixed_by_packages(
152+
advisories, package_versions, purls, cve, logger=None
153+
):
154+
"""
155+
Add appropriate AdvisoryToDo for conflicting affected/fixed packages.
156+
157+
Compute the comparison matrix for the given set of advisories. Iterate through each advisory
158+
and compute and store fixed versions and normalized affected versions for each advisory,
159+
keyed by purl.
160+
161+
Use the matrix to determine conflicts in affected/fixed versions for each purl. If for any purl
162+
there is more than one set of fixed versions or more than one set of affected versions,
163+
it means the advisories have conflicting opinions on the fixed or affected packages.
164+
165+
Example of comparison matrix:
166+
{
167+
"pkg:npm/foo/bar": {
168+
"affected": {
169+
Advisory1: frozenset(NormalizedVersionRange1, NormalizedVersionRange2),
170+
Advisory2: frozenset(...),
171+
},
172+
"fixed": {
173+
Advisory1: frozenset(Version1, Version2),
174+
Advisory2: frozenset(...),
175+
},
176+
},
177+
"pkg:pypi/foobar": {
178+
"affected": {
179+
Advisory1: frozenset(...),
180+
Advisory2: frozenset(...),
181+
},
182+
"fixed": {
183+
Advisory1: frozenset(...),
184+
Advisory2: frozenset(...),
185+
},
186+
},
187+
...
188+
}
189+
"""
190+
matrix = {}
191+
for advisory in advisories:
192+
advisory_obj = advisory.to_advisory_data()
193+
for affected in advisory_obj.affected_packages or []:
194+
affected_purl = str(affected.package)
195+
196+
if affected_purl not in purls or not purls[affected_purl]:
197+
continue
198+
199+
initialize_sub_matrix(
200+
matrix=matrix,
201+
affected_purl=affected_purl,
202+
advisory=advisory,
203+
)
204+
205+
if fixed_version := affected.fixed_version:
206+
matrix[affected_purl]["fixed"][advisory].add(fixed_version)
207+
208+
if affected.affected_version_range:
209+
normalized_vers = affected.affected_version_range.normalize(
210+
known_versions=package_versions[affected_purl],
211+
)
212+
matrix[affected_purl]["affected"][advisory].add(normalized_vers)
213+
214+
has_conflicting_affected_packages = False
215+
has_conflicting_fixed_package = False
216+
messages = []
217+
for purl, board in matrix.items():
218+
fixed = board.get("fixed", {}).values()
219+
affected = board.get("affected", {}).values()
220+
221+
# Compare affected_vers set across different advisories.
222+
unique_set_of_affected_vers = {frozenset(vers) for vers in affected}
223+
224+
# Compare fixed_version set across different advisories.
225+
unique_set_of_fixed_versions = {frozenset(versions) for versions in fixed}
226+
227+
if len(unique_set_of_affected_vers) > 1:
228+
has_conflicting_affected_packages = True
229+
messages.append(
230+
f"{cve}: {purl} with conflicting affected versions {unique_set_of_affected_vers}"
231+
)
232+
if len(unique_set_of_fixed_versions) > 1:
233+
has_conflicting_fixed_package = True
234+
messages.append(
235+
f"{cve}: {purl} with conflicting fixed version {unique_set_of_fixed_versions}"
236+
)
237+
238+
if not has_conflicting_affected_packages and not has_conflicting_fixed_package:
239+
return
240+
241+
issue_type = "CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES"
242+
if not has_conflicting_fixed_package:
243+
issue_type = "CONFLICTING_AFFECTED_PACKAGES"
244+
elif not has_conflicting_affected_packages:
245+
issue_type = "CONFLICTING_FIXED_BY_PACKAGES"
246+
247+
todo_id = advisories_checksum(advisories)
248+
todo, created = AdvisoryToDo.objects.get_or_create(
249+
unique_todo_id=todo_id,
250+
issue_type=issue_type,
251+
issue_detail="\n".join(messages),
252+
)
253+
if created:
254+
todo.advisories.add(*advisories)
255+
256+
257+
def initialize_sub_matrix(matrix, affected_purl, advisory):
258+
if affected_purl not in matrix:
259+
matrix[affected_purl] = {
260+
"affected": {
261+
advisory: set(),
262+
},
263+
"fixed": {
264+
advisory: set(),
265+
},
266+
}
267+
else:
268+
if advisory not in matrix[affected_purl]["affected"]:
269+
matrix[affected_purl]["affected"] = set()
270+
if advisory not in matrix[affected_purl]["fixed"]:
271+
matrix[affected_purl]["fixed"] = set()

0 commit comments

Comments
 (0)