1cc2a6b782
The rollup took the highest fix version across a component's advisories. That is wrong whenever an advisory is patched on more than one release branch. nanoid is the case that exposed it. Both advisories against it are fixed on two branches - 3.3.16 and 5.1.16, 3.3.18 and 5.1.6 - and the rollup recommended 5.1.16 with a MAJOR warning. But nanoid is transitive, pulled in by postcss at `^3.3.11`, a range no 5.x release can satisfy. The recommendation was not merely a bigger job than necessary, it was unreachable: the actual target is 3.3.18, a patch inside the existing range. Per advisory the target is now the lowest published fix above the version in use; per component it is the highest of those, since one upgrade has to satisfy every advisory against it. Fixes on older branches are skipped rather than offered as a downgrade - axios advisories list 0.30.x fixes that would have been a downgrade from 1.13.2. This also gives the MAJOR label meaning. It now fires only when no fix exists on the current branch, rather than whenever some branch happened to be higher.
226 lines
9.6 KiB
Python
226 lines
9.6 KiB
Python
"""The vulnerability scanner's judgement, without touching the network.
|
|
|
|
scan_sbom.py decides three things that matter: whether a finding can stop a
|
|
release, which version clears it, and whether somebody already accepted it.
|
|
Each is pure given an OSV entry, so each is tested against a hand-written entry
|
|
rather than a live query. The network call itself is not tested - if OSV is
|
|
down the script says so and exits 2, which is the only sensible behaviour and
|
|
not worth a mock.
|
|
"""
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
SCANNER = REPO / 'scripts' / 'scan_sbom.py'
|
|
|
|
|
|
def load():
|
|
"""Import the script by path - scripts/ is not a package."""
|
|
spec = importlib.util.spec_from_file_location('scan_sbom', SCANNER)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture(scope='module')
|
|
def scan():
|
|
return load()
|
|
|
|
|
|
class TestSeverity:
|
|
def test_word_is_taken_from_the_database(self, scan):
|
|
entry = {'database_specific': {'severity': 'HIGH'}}
|
|
assert scan.severity_of(entry) == 'HIGH'
|
|
|
|
def test_lowercase_is_accepted(self, scan):
|
|
assert scan.severity_of(
|
|
{'database_specific': {'severity': 'critical'}}) == 'CRITICAL'
|
|
|
|
def test_a_cvss_vector_alone_is_unknown_not_guessed(self, scan):
|
|
# Deliberate: parsing a vector into a score we might get wrong is worse
|
|
# than an honest gap, because somebody would act on the wrong number.
|
|
entry = {'severity': [{'type': 'CVSS_V3',
|
|
'score': 'CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/'
|
|
'C:H/I:H/A:H'}]}
|
|
assert scan.severity_of(entry) == 'UNKNOWN'
|
|
|
|
def test_missing_severity_is_unknown(self, scan):
|
|
assert scan.severity_of({}) == 'UNKNOWN'
|
|
|
|
def test_unknown_never_reaches_the_gate(self, scan):
|
|
assert not scan.at_or_above('UNKNOWN', scan.GATE_AT)
|
|
|
|
def test_gate_floor_and_above(self, scan):
|
|
assert scan.at_or_above('HIGH', 'HIGH')
|
|
assert scan.at_or_above('CRITICAL', 'HIGH')
|
|
assert not scan.at_or_above('MODERATE', 'HIGH')
|
|
assert not scan.at_or_above('LOW', 'HIGH')
|
|
|
|
|
|
class TestPurl:
|
|
def test_npm(self, scan):
|
|
assert scan.parse_purl('pkg:npm/axios@1.13.2') == ('npm', 'axios')
|
|
|
|
def test_pypi_maps_to_the_osv_name(self, scan):
|
|
assert scan.parse_purl('pkg:pypi/pyjwt@2.12.1') == ('PyPI', 'pyjwt')
|
|
|
|
def test_scoped_npm_name_survives(self, scan):
|
|
assert scan.parse_purl('pkg:npm/%40vue/compiler-sfc@3.5.0') == (
|
|
'npm', '@vue/compiler-sfc')
|
|
|
|
def test_no_version_is_fine(self, scan):
|
|
assert scan.parse_purl('pkg:npm/axios') == ('npm', 'axios')
|
|
|
|
|
|
class TestFixedVersions:
|
|
def test_reads_the_fixed_event(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'PyPI', 'name': 'pyjwt'},
|
|
'ranges': [{'events': [{'introduced': '0'}, {'fixed': '2.13.0'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:pypi/pyjwt@2.12.1') == ['2.13.0']
|
|
|
|
def test_another_packages_fix_is_not_attributed_here(self, scan):
|
|
# An advisory covering several packages must not lend one package's
|
|
# fix version to another - that would send somebody to a wrong version.
|
|
entry = {'affected': [
|
|
{'package': {'ecosystem': 'npm', 'name': 'other'},
|
|
'ranges': [{'events': [{'fixed': '9.9.9'}]}]},
|
|
{'package': {'ecosystem': 'npm', 'name': 'axios'},
|
|
'ranges': [{'events': [{'fixed': '1.16.0'}]}]},
|
|
]}
|
|
assert scan.fixed_versions(entry, 'pkg:npm/axios@1.13.2') == ['1.16.0']
|
|
|
|
def test_same_name_in_another_ecosystem_is_ignored(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'npm', 'name': 'pyjwt'},
|
|
'ranges': [{'events': [{'fixed': '9.9.9'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:pypi/pyjwt@2.12.1') == []
|
|
|
|
def test_no_fix_published_is_empty(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'npm', 'name': 'axios'},
|
|
'ranges': [{'events': [{'introduced': '0'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:npm/axios@1.13.2') == []
|
|
|
|
|
|
class TestVersionOrder:
|
|
def test_ten_is_above_nine(self, scan):
|
|
# A string compare puts 1.9.0 above 1.10.0, which would recommend
|
|
# downgrading.
|
|
assert scan.version_key('1.10.0') > scan.version_key('1.9.0')
|
|
|
|
def test_prerelease_does_not_displace_a_release(self, scan):
|
|
assert scan.version_key('2.0.0') > scan.version_key('2.0.0-rc1')
|
|
|
|
def test_versions_with_different_segment_counts_compare(self, scan):
|
|
# OSV fix versions are not normalised, so a two-segment and a
|
|
# three-segment version land in the same sort.
|
|
assert scan.version_key('2.0.1') > scan.version_key('2.0')
|
|
assert scan.version_key('1.16.0') > scan.version_key('1.9')
|
|
|
|
|
|
class TestUpgradePlan:
|
|
def _finding(self, component, version, fixedin, allowlisted=False):
|
|
return {'component': component, 'version': version, 'fixedin': fixedin,
|
|
'allowlisted': allowlisted}
|
|
|
|
def test_highest_fix_wins_and_counts_what_it_clears(self, scan):
|
|
findings = [
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.13.5']),
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.16.0']),
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.15.1']),
|
|
]
|
|
assert scan.upgrade_plan(findings) == [('axios 1.13.2', '1.16.0', 3, False)]
|
|
|
|
def test_a_major_jump_is_flagged(self, scan):
|
|
findings = [self._finding('nanoid 3.3.11', '3.3.11', ['5.1.16'])]
|
|
assert scan.upgrade_plan(findings)[0][3] is True
|
|
|
|
def test_a_branch_patch_beats_a_major_on_the_same_advisory(self, scan):
|
|
# The real case this was found on: nanoid is fixed in BOTH 3.3.18 and
|
|
# 5.1.16, and it is transitive under a postcss range of ^3.3.11 - so
|
|
# 5.x is not merely a bigger job, it is unreachable. Recommending the
|
|
# highest fix sent somebody to a major upgrade they could not make.
|
|
findings = [self._finding('nanoid 3.3.11', '3.3.11',
|
|
['3.3.18', '5.1.16'])]
|
|
component, version, _count, major = scan.upgrade_plan(findings)[0]
|
|
assert version == '3.3.18'
|
|
assert major is False
|
|
|
|
def test_component_target_satisfies_every_advisory_against_it(self, scan):
|
|
# Two advisories, minimal fixes 3.3.16 and 3.3.18: the row must be the
|
|
# higher, or the upgrade clears one and leaves the other.
|
|
findings = [
|
|
self._finding('nanoid 3.3.11', '3.3.11', ['3.3.16', '5.1.16']),
|
|
self._finding('nanoid 3.3.11', '3.3.11', ['3.3.18', '5.1.6']),
|
|
]
|
|
assert scan.upgrade_plan(findings)[0][1] == '3.3.18'
|
|
|
|
def test_a_fix_on_an_older_branch_is_never_a_downgrade(self, scan):
|
|
# axios advisories list fixes on the 0.30.x line too; on 1.13.2 those
|
|
# are behind us and must not be recommended.
|
|
findings = [self._finding('axios 1.13.2', '1.13.2',
|
|
['0.30.2', '1.15.2'])]
|
|
assert scan.upgrade_plan(findings)[0][1] == '1.15.2'
|
|
|
|
def test_only_older_branch_fixes_means_no_row(self, scan):
|
|
findings = [self._finding('axios 1.13.2', '1.13.2', ['0.30.2'])]
|
|
assert scan.upgrade_plan(findings) == []
|
|
|
|
def test_a_patch_bump_is_not_flagged(self, scan):
|
|
findings = [self._finding('pyjwt 2.12.1', '2.12.1', ['2.13.0'])]
|
|
assert scan.upgrade_plan(findings)[0][3] is False
|
|
|
|
def test_allowlisted_findings_do_not_ask_for_an_upgrade(self, scan):
|
|
findings = [self._finding('pyjwt 2.12.1', '2.12.1', ['2.13.0'],
|
|
allowlisted=True)]
|
|
assert scan.upgrade_plan(findings) == []
|
|
|
|
def test_a_finding_with_no_fix_is_omitted(self, scan):
|
|
findings = [self._finding('thing 1.0', '1.0', [])]
|
|
assert scan.upgrade_plan(findings) == []
|
|
|
|
|
|
class TestAllowlist:
|
|
def test_the_shipped_allowlist_parses_and_is_complete(self, scan):
|
|
"""Every entry must carry the reasoning, or it is just a mute."""
|
|
data = json.loads(scan.ALLOWLIST.read_text(encoding='utf8'))
|
|
for entry in data.get('accepted', []):
|
|
for field in ('id', 'reason', 'acceptedby', 'accepted'):
|
|
assert entry.get(field), '%s missing %s' % (entry, field)
|
|
assert len(entry['reason']) > 40, (
|
|
'reason for %s is too short to be a reason' % entry['id'])
|
|
|
|
def test_allowlist_is_keyed_by_advisory_id(self, scan):
|
|
allowed = scan.load_allowlist()
|
|
for key, entry in allowed.items():
|
|
assert key == entry['id']
|
|
|
|
|
|
class TestScopeSeparation:
|
|
def test_build_only_components_are_read_as_not_shipped(self, scan, tmp_path):
|
|
sbom = tmp_path / 'sbom.json'
|
|
sbom.write_text(json.dumps({'components': [
|
|
{'name': 'axios', 'version': '1.0', 'purl': 'pkg:npm/axios@1.0'},
|
|
{'name': 'vite', 'version': '5.0', 'purl': 'pkg:npm/vite@5.0',
|
|
'scope': 'excluded'},
|
|
]}), encoding='utf8')
|
|
components = scan.load_components(sbom)
|
|
shipped = {name: ship for _p, name, _v, ship in components}
|
|
assert shipped == {'axios': True, 'vite': False}
|
|
|
|
def test_a_component_without_a_purl_is_skipped(self, scan, tmp_path):
|
|
# OSV is queried by purl; a component without one cannot be looked up,
|
|
# and inventing an identifier would produce confident wrong answers.
|
|
sbom = tmp_path / 'sbom.json'
|
|
sbom.write_text(json.dumps({'components': [
|
|
{'name': 'local-thing', 'version': '1.0'},
|
|
]}), encoding='utf8')
|
|
assert scan.load_components(sbom) == []
|