794bb4668f
The SBOM was generated and installed onto every server, and nothing read it. "Are we exposed to this advisory" was still a person going and looking, which meant it happened when somebody remembered to. scripts/scan_sbom.py queries OSV.dev by purl - no account, no API key, no agent, no vulnerability database to maintain on a box that cannot reach the internet. The SBOM is already the input OSV wants, so this is a lookup against what shipped rather than a second scan of the tree that could disagree with it. Three decisions worth recording: Shipped and build-only findings are both reported, but only shipped ones can fail a build. Two thirds of the components here are build tooling the SBOM already marks scope 'excluded', and the highest-severity match in the whole tree today is a CRITICAL against the test runner. A gate that fails a release over that trains everybody to bypass it, and then it catches nothing. Severity is taken from the advisory, never derived. Where there is only a CVSS vector the report says UNKNOWN, which is shown and never gates. A guessed score is worse than an honest gap because somebody would act on it. The allowlist is a record rather than a mute button: each entry carries the reason, the accepter, the date, and what would invalidate the reasoning. The first entry is a worked example - a PyJWT algorithm-confusion advisory that needs both mixed algorithm families and a raw JSON JWK as the key, and this application configures neither. The report ends with a per-component upgrade rollup, since that is the part somebody acts on. A target on a different major version is labelled rather than listed beside patch bumps as though the two cost the same. Current state: 8 shipped components have fixes available, none of them exploitable through a path this application opens today.
195 lines
8.0 KiB
Python
195 lines
8.0 KiB
Python
"""The vulnerability scanner's judgement, without touching the network.
|
|
|
|
scan_sbom.py decides three things that matter: whether a finding can stop a
|
|
release, which version clears it, and whether somebody already accepted it.
|
|
Each is pure given an OSV entry, so each is tested against a hand-written entry
|
|
rather than a live query. The network call itself is not tested - if OSV is
|
|
down the script says so and exits 2, which is the only sensible behaviour and
|
|
not worth a mock.
|
|
"""
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
SCANNER = REPO / 'scripts' / 'scan_sbom.py'
|
|
|
|
|
|
def load():
|
|
"""Import the script by path - scripts/ is not a package."""
|
|
spec = importlib.util.spec_from_file_location('scan_sbom', SCANNER)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture(scope='module')
|
|
def scan():
|
|
return load()
|
|
|
|
|
|
class TestSeverity:
|
|
def test_word_is_taken_from_the_database(self, scan):
|
|
entry = {'database_specific': {'severity': 'HIGH'}}
|
|
assert scan.severity_of(entry) == 'HIGH'
|
|
|
|
def test_lowercase_is_accepted(self, scan):
|
|
assert scan.severity_of(
|
|
{'database_specific': {'severity': 'critical'}}) == 'CRITICAL'
|
|
|
|
def test_a_cvss_vector_alone_is_unknown_not_guessed(self, scan):
|
|
# Deliberate: parsing a vector into a score we might get wrong is worse
|
|
# than an honest gap, because somebody would act on the wrong number.
|
|
entry = {'severity': [{'type': 'CVSS_V3',
|
|
'score': 'CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/'
|
|
'C:H/I:H/A:H'}]}
|
|
assert scan.severity_of(entry) == 'UNKNOWN'
|
|
|
|
def test_missing_severity_is_unknown(self, scan):
|
|
assert scan.severity_of({}) == 'UNKNOWN'
|
|
|
|
def test_unknown_never_reaches_the_gate(self, scan):
|
|
assert not scan.at_or_above('UNKNOWN', scan.GATE_AT)
|
|
|
|
def test_gate_floor_and_above(self, scan):
|
|
assert scan.at_or_above('HIGH', 'HIGH')
|
|
assert scan.at_or_above('CRITICAL', 'HIGH')
|
|
assert not scan.at_or_above('MODERATE', 'HIGH')
|
|
assert not scan.at_or_above('LOW', 'HIGH')
|
|
|
|
|
|
class TestPurl:
|
|
def test_npm(self, scan):
|
|
assert scan.parse_purl('pkg:npm/axios@1.13.2') == ('npm', 'axios')
|
|
|
|
def test_pypi_maps_to_the_osv_name(self, scan):
|
|
assert scan.parse_purl('pkg:pypi/pyjwt@2.12.1') == ('PyPI', 'pyjwt')
|
|
|
|
def test_scoped_npm_name_survives(self, scan):
|
|
assert scan.parse_purl('pkg:npm/%40vue/compiler-sfc@3.5.0') == (
|
|
'npm', '@vue/compiler-sfc')
|
|
|
|
def test_no_version_is_fine(self, scan):
|
|
assert scan.parse_purl('pkg:npm/axios') == ('npm', 'axios')
|
|
|
|
|
|
class TestFixedVersions:
|
|
def test_reads_the_fixed_event(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'PyPI', 'name': 'pyjwt'},
|
|
'ranges': [{'events': [{'introduced': '0'}, {'fixed': '2.13.0'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:pypi/pyjwt@2.12.1') == ['2.13.0']
|
|
|
|
def test_another_packages_fix_is_not_attributed_here(self, scan):
|
|
# An advisory covering several packages must not lend one package's
|
|
# fix version to another - that would send somebody to a wrong version.
|
|
entry = {'affected': [
|
|
{'package': {'ecosystem': 'npm', 'name': 'other'},
|
|
'ranges': [{'events': [{'fixed': '9.9.9'}]}]},
|
|
{'package': {'ecosystem': 'npm', 'name': 'axios'},
|
|
'ranges': [{'events': [{'fixed': '1.16.0'}]}]},
|
|
]}
|
|
assert scan.fixed_versions(entry, 'pkg:npm/axios@1.13.2') == ['1.16.0']
|
|
|
|
def test_same_name_in_another_ecosystem_is_ignored(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'npm', 'name': 'pyjwt'},
|
|
'ranges': [{'events': [{'fixed': '9.9.9'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:pypi/pyjwt@2.12.1') == []
|
|
|
|
def test_no_fix_published_is_empty(self, scan):
|
|
entry = {'affected': [{
|
|
'package': {'ecosystem': 'npm', 'name': 'axios'},
|
|
'ranges': [{'events': [{'introduced': '0'}]}],
|
|
}]}
|
|
assert scan.fixed_versions(entry, 'pkg:npm/axios@1.13.2') == []
|
|
|
|
|
|
class TestVersionOrder:
|
|
def test_ten_is_above_nine(self, scan):
|
|
# A string compare puts 1.9.0 above 1.10.0, which would recommend
|
|
# downgrading.
|
|
assert scan.version_key('1.10.0') > scan.version_key('1.9.0')
|
|
|
|
def test_prerelease_does_not_displace_a_release(self, scan):
|
|
assert scan.version_key('2.0.0') > scan.version_key('2.0.0-rc1')
|
|
|
|
def test_versions_with_different_segment_counts_compare(self, scan):
|
|
# OSV fix versions are not normalised, so a two-segment and a
|
|
# three-segment version land in the same sort.
|
|
assert scan.version_key('2.0.1') > scan.version_key('2.0')
|
|
assert scan.version_key('1.16.0') > scan.version_key('1.9')
|
|
|
|
|
|
class TestUpgradePlan:
|
|
def _finding(self, component, version, fixedin, allowlisted=False):
|
|
return {'component': component, 'version': version, 'fixedin': fixedin,
|
|
'allowlisted': allowlisted}
|
|
|
|
def test_highest_fix_wins_and_counts_what_it_clears(self, scan):
|
|
findings = [
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.13.5']),
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.16.0']),
|
|
self._finding('axios 1.13.2', '1.13.2', ['1.15.1']),
|
|
]
|
|
assert scan.upgrade_plan(findings) == [('axios 1.13.2', '1.16.0', 3, False)]
|
|
|
|
def test_a_major_jump_is_flagged(self, scan):
|
|
findings = [self._finding('nanoid 3.3.11', '3.3.11', ['5.1.16'])]
|
|
assert scan.upgrade_plan(findings)[0][3] is True
|
|
|
|
def test_a_patch_bump_is_not_flagged(self, scan):
|
|
findings = [self._finding('pyjwt 2.12.1', '2.12.1', ['2.13.0'])]
|
|
assert scan.upgrade_plan(findings)[0][3] is False
|
|
|
|
def test_allowlisted_findings_do_not_ask_for_an_upgrade(self, scan):
|
|
findings = [self._finding('pyjwt 2.12.1', '2.12.1', ['2.13.0'],
|
|
allowlisted=True)]
|
|
assert scan.upgrade_plan(findings) == []
|
|
|
|
def test_a_finding_with_no_fix_is_omitted(self, scan):
|
|
findings = [self._finding('thing 1.0', '1.0', [])]
|
|
assert scan.upgrade_plan(findings) == []
|
|
|
|
|
|
class TestAllowlist:
|
|
def test_the_shipped_allowlist_parses_and_is_complete(self, scan):
|
|
"""Every entry must carry the reasoning, or it is just a mute."""
|
|
data = json.loads(scan.ALLOWLIST.read_text(encoding='utf8'))
|
|
for entry in data.get('accepted', []):
|
|
for field in ('id', 'reason', 'acceptedby', 'accepted'):
|
|
assert entry.get(field), '%s missing %s' % (entry, field)
|
|
assert len(entry['reason']) > 40, (
|
|
'reason for %s is too short to be a reason' % entry['id'])
|
|
|
|
def test_allowlist_is_keyed_by_advisory_id(self, scan):
|
|
allowed = scan.load_allowlist()
|
|
for key, entry in allowed.items():
|
|
assert key == entry['id']
|
|
|
|
|
|
class TestScopeSeparation:
|
|
def test_build_only_components_are_read_as_not_shipped(self, scan, tmp_path):
|
|
sbom = tmp_path / 'sbom.json'
|
|
sbom.write_text(json.dumps({'components': [
|
|
{'name': 'axios', 'version': '1.0', 'purl': 'pkg:npm/axios@1.0'},
|
|
{'name': 'vite', 'version': '5.0', 'purl': 'pkg:npm/vite@5.0',
|
|
'scope': 'excluded'},
|
|
]}), encoding='utf8')
|
|
components = scan.load_components(sbom)
|
|
shipped = {name: ship for _p, name, _v, ship in components}
|
|
assert shipped == {'axios': True, 'vite': False}
|
|
|
|
def test_a_component_without_a_purl_is_skipped(self, scan, tmp_path):
|
|
# OSV is queried by purl; a component without one cannot be looked up,
|
|
# and inventing an identifier would produce confident wrong answers.
|
|
sbom = tmp_path / 'sbom.json'
|
|
sbom.write_text(json.dumps({'components': [
|
|
{'name': 'local-thing', 'version': '1.0'},
|
|
]}), encoding='utf8')
|
|
assert scan.load_components(sbom) == []
|