Cam Proudlock
7ddfe4d3ef
The rollup took the highest fix version across a component's advisories, which is wrong whenever an advisory is patched on more than one release branch. nanoid is fixed on both 3.3.x and 5.x, and it is transitive under a postcss range of ^3.3.11 that no 5.x release can satisfy - so the recommendation was unreachable rather than merely large. Per advisory the target is now the lowest fix above the version in use; per component, the highest of those. Fixes on older branches are skipped rather than offered as a downgrade.
226 lines
9.6 KiB
Python
226 lines
9.6 KiB
Python
"""The vulnerability scanner's judgement, without touching the network.
|
|
|
|
scan_sbom.py decides three things that matter: whether a finding can stop a
|
|
release, which version clears it, and whether somebody already accepted it.
|
|
Each is pure given an OSV entry, so each is tested against a hand-written entry
|
|
rather than a live query. The network call itself is not tested - if OSV is
|
|
down the script says so and exits 2, which is the only sensible behaviour and
|
|
not worth a mock.
|
|
"""
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
SCANNER = REPO / 'scripts' / 'scan_sbom.py'
|
|
|
|
|
|
def load():
|
|
"""Import the script by path - scripts/ is not a package."""
|
|
spec = importlib.util.spec_from_file_location('scan_sbom', SCANNER)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture(scope='module')
|
|
def scan():
|
|
return load()
|
|
|
|
|
|
class TestSeverity:
|
|
def test_word_is_taken_from_the_database(self, scan):
|
|
entry = {'database_specific': {'severity': 'HIGH'}}
|
|
assert scan.severity_of(entry) == 'HIGH'
|
|
|
|
def test_lowercase_is_accepted(self, scan):
|
|
assert scan.severity_of(
|
|
{'database_specific': {'severity': 'critical'}}) == 'CRITICAL'
|
|
|
|
def test_a_cvss_vector_alone_is_unknown_not_guessed(self, scan):
|
|
# Deliberate: parsing a vector into a score we might get wrong is worse
|
|
# than an honest gap, because somebody would act on the wrong number.
|
|
entry = {'severity': [{'type': 'CVSS_V3',
|
|
'score': 'CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/'
|
|
'C:H/I:H/A:H'}]}
|
|
assert scan.severity_of(entry) == 'UNKNOWN'
|
|
|
|
def test_missing_severity_is_unknown(self, scan):
|
|
assert scan.severity_of({}) == 'UNKNOWN'
|
|
|
|
def test_unknown_never_reaches_the_gate(self, scan):
|
|
assert not scan.at_or_above('UNKNOWN', scan.GATE_AT)
|
|
|
|
def test_gate_floor_and_above(self, scan):
|
|
assert scan.at_or_above('HIGH', 'HIGH')
|
|
assert scan.at_or_above('CRITICAL', 'HIGH')
|
|
assert not scan.at_or_above('MODERATE', 'HIGH')
|
|
assert not scan.at_or_above('LOW', 'HIGH')
|
|
|
|
|
|
class TestPurl:
|
|
def test_npm(self, scan):
|
|
assert scan.parse_purl('pkg:npm/axios@1.13.2') == ('npm', 'axios')
|
|
|
|
def test_pypi_maps_to_the_osv_name(self, scan):
|
|
assert scan.parse_purl('pkg:pypi/pyjwt@2.12.1') == ('PyPI', 'pyjwt')
|
|
|
|
def test_scoped_npm_name_survives(self, scan):
|
|
assert scan.parse_purl('pkg:npm/%40vue/compiler-sfc@3.5.0') == (
|
|
'npm', '@vue/compiler-sfc')
|
|
|
|
def test_no_version_is_fine(self, scan):
|
|
assert scan.parse_purl('pkg:npm/axios') == ('npm', 'axios')
|
|
|
|
|
|
class TestFixedVersions:
|
|
def test_reads_the_fixed_event(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'PyPI', 'name': 'pyjwt'},
|
|
'ranges': [{'events': [{'introduced': '0'}, {'fixed': '2.13.0'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:pypi/pyjwt@2.12.1') == ['2.13.0']
|
|
|
|
def test_another_packages_fix_is_not_attributed_here(self, scan):
|
|
# An advisory covering several packages must not lend one package's
|
|
# fix version to another - that would send somebody to a wrong version.
|
|
entry = {'affected': [
|
|
{'package': {'ecosystem': 'npm', 'name': 'other'},
|
|
'ranges': [{'events': [{'fixed': '9.9.9'}]}]},
|
|
{'package': {'ecosystem': 'npm', 'name': 'axios'},
|
|
'ranges': [{'events': [{'fixed': '1.16.0'}]}]},
|
|
]}
|
|
assert scan.fixed_versions(entry, 'pkg:npm/axios@1.13.2') == ['1.16.0']
|
|
|
|
def test_same_name_in_another_ecosystem_is_ignored(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'npm', 'name': 'pyjwt'},
|
|
'ranges': [{'events': [{'fixed': '9.9.9'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:pypi/pyjwt@2.12.1') == []
|
|
|
|
def test_no_fix_published_is_empty(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'npm', 'name': 'axios'},
|
|
'ranges': [{'events': [{'introduced': '0'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:npm/axios@1.13.2') == []
|
|
|
|
|
|
class TestVersionOrder:
|
|
def test_ten_is_above_nine(self, scan):
|
|
# A string compare puts 1.9.0 above 1.10.0, which would recommend
|
|
# downgrading.
|
|
assert scan.version_key('1.10.0') > scan.version_key('1.9.0')
|
|
|
|
def test_prerelease_does_not_displace_a_release(self, scan):
|
|
assert scan.version_key('2.0.0') > scan.version_key('2.0.0-rc1')
|
|
|
|
def test_versions_with_different_segment_counts_compare(self, scan):
|
|
# OSV fix versions are not normalised, so a two-segment and a
|
|
# three-segment version land in the same sort.
|
|
assert scan.version_key('2.0.1') > scan.version_key('2.0')
|
|
assert scan.version_key('1.16.0') > scan.version_key('1.9')
|
|
|
|
|
|
class TestUpgradePlan:
|
|
def _finding(self, component, version, fixedin, allowlisted=False):
|
|
return {'component': component, 'version': version, 'fixedin': fixedin,
|
|
'allowlisted': allowlisted}
|
|
|
|
def test_highest_fix_wins_and_counts_what_it_clears(self, scan):
|
|
findings = [
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.13.5']),
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.16.0']),
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.15.1']),
|
|
]
|
|
assert scan.upgrade_plan(findings) == [('axios 1.13.2', '1.16.0', 3, False)]
|
|
|
|
def test_a_major_jump_is_flagged(self, scan):
|
|
findings = [self._finding('nanoid 3.3.11', '3.3.11', ['5.1.16'])]
|
|
assert scan.upgrade_plan(findings)[0][3] is True
|
|
|
|
def test_a_branch_patch_beats_a_major_on_the_same_advisory(self, scan):
|
|
# The real case this was found on: nanoid is fixed in BOTH 3.3.18 and
|
|
# 5.1.16, and it is transitive under a postcss range of ^3.3.11 - so
|
|
# 5.x is not merely a bigger job, it is unreachable. Recommending the
|
|
# highest fix sent somebody to a major upgrade they could not make.
|
|
findings = [self._finding('nanoid 3.3.11', '3.3.11',
|
|
['3.3.18', '5.1.16'])]
|
|
component, version, _count, major = scan.upgrade_plan(findings)[0]
|
|
assert version == '3.3.18'
|
|
assert major is False
|
|
|
|
def test_component_target_satisfies_every_advisory_against_it(self, scan):
|
|
# Two advisories, minimal fixes 3.3.16 and 3.3.18: the row must be the
|
|
# higher, or the upgrade clears one and leaves the other.
|
|
findings = [
|
|
self._finding('nanoid 3.3.11', '3.3.11', ['3.3.16', '5.1.16']),
|
|
self._finding('nanoid 3.3.11', '3.3.11', ['3.3.18', '5.1.6']),
|
|
]
|
|
assert scan.upgrade_plan(findings)[0][1] == '3.3.18'
|
|
|
|
def test_a_fix_on_an_older_branch_is_never_a_downgrade(self, scan):
|
|
# axios advisories list fixes on the 0.30.x line too; on 1.13.2 those
|
|
# are behind us and must not be recommended.
|
|
findings = [self._finding('axios 1.13.2', '1.13.2',
|
|
['0.30.2', '1.15.2'])]
|
|
assert scan.upgrade_plan(findings)[0][1] == '1.15.2'
|
|
|
|
def test_only_older_branch_fixes_means_no_row(self, scan):
|
|
findings = [self._finding('axios 1.13.2', '1.13.2', ['0.30.2'])]
|
|
assert scan.upgrade_plan(findings) == []
|
|
|
|
def test_a_patch_bump_is_not_flagged(self, scan):
|
|
findings = [self._finding('pyjwt 2.12.1', '2.12.1', ['2.13.0'])]
|
|
assert scan.upgrade_plan(findings)[0][3] is False
|
|
|
|
def test_allowlisted_findings_do_not_ask_for_an_upgrade(self, scan):
|
|
findings = [self._finding('pyjwt 2.12.1', '2.12.1', ['2.13.0'],
|
|
allowlisted=True)]
|
|
assert scan.upgrade_plan(findings) == []
|
|
|
|
def test_a_finding_with_no_fix_is_omitted(self, scan):
|
|
findings = [self._finding('thing 1.0', '1.0', [])]
|
|
assert scan.upgrade_plan(findings) == []
|
|
|
|
|
|
class TestAllowlist:
|
|
def test_the_shipped_allowlist_parses_and_is_complete(self, scan):
|
|
"""Every entry must carry the reasoning, or it is just a mute."""
|
|
data = json.loads(scan.ALLOWLIST.read_text(encoding='utf8'))
|
|
for entry in data.get('accepted', []):
|
|
for field in ('id', 'reason', 'acceptedby', 'accepted'):
|
|
assert entry.get(field), '%s missing %s' % (entry, field)
|
|
assert len(entry['reason']) > 40, (
|
|
'reason for %s is too short to be a reason' % entry['id'])
|
|
|
|
def test_allowlist_is_keyed_by_advisory_id(self, scan):
|
|
allowed = scan.load_allowlist()
|
|
for key, entry in allowed.items():
|
|
assert key == entry['id']
|
|
|
|
|
|
class TestScopeSeparation:
|
|
def test_build_only_components_are_read_as_not_shipped(self, scan, tmp_path):
|
|
sbom = tmp_path / 'sbom.json'
|
|
sbom.write_text(json.dumps({'components': [
|
|
{'name': 'axios', 'version': '1.0', 'purl': 'pkg:npm/axios@1.0'},
|
|
{'name': 'vite', 'version': '5.0', 'purl': 'pkg:npm/vite@5.0',
|
|
'scope': 'excluded'},
|
|
]}), encoding='utf8')
|
|
components = scan.load_components(sbom)
|
|
shipped = {name: ship for _p, name, _v, ship in components}
|
|
assert shipped == {'axios': True, 'vite': False}
|
|
|
|
def test_a_component_without_a_purl_is_skipped(self, scan, tmp_path):
|
|
# OSV is queried by purl; a component without one cannot be looked up,
|
|
# and inventing an identifier would produce confident wrong answers.
|
|
sbom = tmp_path / 'sbom.json'
|
|
sbom.write_text(json.dumps({'components': [
|
|
{'name': 'local-thing', 'version': '1.0'},
|
|
]}), encoding='utf8')
|
|
assert scan.load_components(sbom) == []
|