#!/usr/bin/env python3 """End-to-end verifier for the bomsh provenance bundle. Two independent self-consistency checks on the artefacts that `make bomsh` produces. The PERSISTENT-ID assertion in the bomsh CI job only proves the gitoid externalRef *exists* in the enriched SPDX; neither of these follow-up properties is guaranteed by it: (A) Resolvability -- every gitoid in the SPDX externalRefs resolves to a blob present at omnibor/objects//. Catches the `bomsh_sbom.py` regression class that emits a syntactically well-formed gitoid which does not actually point at anything in the shipped ADG. (B) Object-store integrity -- every blob in omnibor/objects/ round-trips through sha1(b"blob \\0" + content), so a corrupt or truncated object store is caught at PR time, not by a downstream verifier weeks later. CLI form (used by `.github/workflows/sbom.yml`): python3 scripts/bomsh_verify.py \\ --spdx-glob 'omnibor.wolfssl-*.spdx.json' \\ --omnibor-dir omnibor Library form (used by scripts/test_gen_sbom.py): from scripts import bomsh_verify ok, messages = bomsh_verify.verify(...) """ import argparse import glob as _glob import hashlib import json import os import re import sys from typing import List GITOID_LOCATOR_PREFIX = 'gitoid:blob:sha1:' # An OmniBOR sha1 gitoid is exactly 40 lowercase-hex chars. Validate before # the value is used to build an object path: a crafted SPDX with a locator like # 'gitoid:blob:sha1:../../../etc/shadow' otherwise passes the prefix check and # turns the os.path.join() below into a path-traversal existence oracle. _SHA1_HEX_RE = re.compile(r'[0-9a-f]{40}\Z') def gitoid_sha1(path): """OmniBOR `gitoid:blob:sha1:` is the canonical Git blob hash: sha1(b"blob \\0" + content). Symlinks are followed transparently by `open()`, which matches what bomsh records (the trace sees the target, not the symlink).""" with open(path, 'rb') as f: data = f.read() h = hashlib.sha1() h.update(f'blob {len(data)}\0'.encode()) h.update(data) return h.hexdigest() def load_spdx_gitoids(spdx_path): """Return [(package_name, gitoid_hex), ...] for every externalRef of referenceType 'gitoid' in the SPDX document at spdx_path. Raises ValueError on a malformed locator (anything that isn't `gitoid:blob:sha1:`). An sha256 locator would land here too if bomsh ever switches; the failure is the right behaviour, since a maintainer must update the verifier in lockstep.""" with open(spdx_path) as f: spdx = json.load(f) gitoids = [] for pkg in spdx.get('packages', []): for ref in pkg.get('externalRefs', []): if ref.get('referenceType') != 'gitoid': continue loc = ref.get('referenceLocator', '') if not loc.startswith(GITOID_LOCATOR_PREFIX): raise ValueError( f'unexpected gitoid locator format: {loc!r} ' f'(expected {GITOID_LOCATOR_PREFIX}; if bomsh ' f'has switched to sha256 the verifier needs updating)') gid = loc[len(GITOID_LOCATOR_PREFIX):] if not _SHA1_HEX_RE.match(gid): raise ValueError( f'malformed gitoid {gid!r} in locator {loc!r}: expected ' f'40 lowercase-hex sha1 characters (refusing to use it in ' f'an object path)') gitoids.append((pkg.get('name', ''), gid)) return gitoids def check_resolvability(spdx_gitoids, omnibor_objects_dir): """(A) Every SPDX gitoid resolves to a file at `//`. Returns a list of (pkg_name, gitoid, expected_path) for the unresolved ones; empty list means every gitoid resolved.""" missing = [] for pkg_name, gid in spdx_gitoids: obj = os.path.join(omnibor_objects_dir, gid[:2], gid[2:]) if not os.path.isfile(obj): missing.append((pkg_name, gid, obj)) return missing _HEX_CHARS = frozenset('0123456789abcdef') def _looks_like_blob_path(parts): """True iff `parts` is the canonical `/` shape Git uses for content-addressed blob fanout: exactly two components, the first of which is a 2-char lowercase-hex prefix and the second of which is the remaining 38 lowercase-hex chars of a sha1 digest. Anything else (`info/`, `pack/...`, deeper nesting) is housekeeping and must NOT be gitoid-checked. sha1-only on purpose: check_object_store_integrity() hashes with gitoid_sha1() and load_spdx_gitoids() rejects non-sha1 locators, so the whole verifier is sha1-scoped. Admitting a 62-char (sha256) digest here would only let the integrity check compare it against an sha1 hash and always report it corrupt; a real bomsh switch to sha256 must update the hashing (and this length) in lockstep.""" if len(parts) != 2: return False aa, rest = parts if len(aa) != 2 or not all(c in _HEX_CHARS for c in aa): return False if len(rest) != 38: return False return all(c in _HEX_CHARS for c in rest) def check_object_store_integrity(omnibor_objects_dir): """(B) Every blob in round-trips through `gitoid_sha1`. Returns (count_total, [(path, expected, actual), ...] for blobs whose content does not match their expected gitoid). The directory layout is `//` (Git's standard fanout, where is the first two hex chars of the digest); files outside that shape are skipped silently (e.g. `info/` or `pack/` siblings, README files, etc.).""" bad = [] obj_count = 0 for root, _, files in os.walk(omnibor_objects_dir): for fname in files: obj = os.path.join(root, fname) rel = os.path.relpath(obj, omnibor_objects_dir) parts = rel.split(os.sep) if not _looks_like_blob_path(parts): continue expected = parts[0] + parts[1] obj_count += 1 actual = gitoid_sha1(obj) if actual != expected: bad.append((obj, expected, actual)) return obj_count, bad def verify(spdx_glob, omnibor_dir): """Orchestrate the two checks. Returns (ok: bool, messages: List[str]). `messages` is appended to in success and failure both, so callers can log the success lines ('OK: N gitoid(s) verified' + ' objects round-trip: M blobs') even when ok is True.""" messages: List[str] = [] spdx_paths = sorted(_glob.glob(spdx_glob)) if not spdx_paths: return False, [f'no SPDX matched {spdx_glob!r}'] spdx_path = spdx_paths[0] try: spdx_gitoids = load_spdx_gitoids(spdx_path) except (json.JSONDecodeError, ValueError) as e: return False, [f'could not load SPDX gitoids: {e}'] if not spdx_gitoids: return False, [f'no gitoid externalRefs in {spdx_path}'] objects_dir = os.path.join(omnibor_dir, 'objects') missing = check_resolvability(spdx_gitoids, objects_dir) if missing: for pkg_name, gid, obj in missing: messages.append( f'DANGLING: {pkg_name} gitoid {gid} -> {obj}') messages.append( f'{len(missing)} SPDX gitoid(s) not present in ' f'{objects_dir}/ (provenance bundle is broken)') return False, messages obj_count, bad = check_object_store_integrity(objects_dir) if bad: for obj, expected, actual in bad[:5]: messages.append( f'CORRUPT: {obj} expected {expected} got {actual}') messages.append( f'{len(bad)} object(s) in {objects_dir}/ failed gitoid ' f'round-trip (object store is corrupt)') return False, messages messages.append(f'OK: {len(spdx_gitoids)} gitoid(s) verified') messages.append(f' objects round-trip: {obj_count} blobs') return True, messages def main(): parser = argparse.ArgumentParser( description='End-to-end verifier for the bomsh provenance bundle.') parser.add_argument('--spdx-glob', default='omnibor.wolfssl-*.spdx.json', help='Glob matching the bomsh-enriched SPDX file ' '(default: %(default)s)') parser.add_argument('--omnibor-dir', default='omnibor', help='Path to the OmniBOR directory containing ' 'objects/ (default: %(default)s)') args = parser.parse_args() ok, messages = verify(args.spdx_glob, args.omnibor_dir) for line in messages: print(line, file=sys.stderr if not ok else sys.stdout) sys.exit(0 if ok else 1) if __name__ == '__main__': main()