Skip to content

Commit 211b559

Browse files
miss-islingtonsethmlarson
miss-islington
and
sethmlarson
authored
[3.12] gh-112844: Add SBOM for external dependencies (GH-115789) (#116128)
Co-authored-by: Seth Michael Larson <[email protected]>
1 parent 811f4bd commit 211b559

File tree

3 files changed

+287
-17
lines changed

3 files changed

+287
-17
lines changed

‎.github/CODEOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,5 +168,6 @@ Lib/ast.py @isidentical
168168
**/*zipfile/_path/* @jaraco
169169

170170
# SBOM
171+
/Misc/externals.spdx.json @sethmlarson
171172
/Misc/sbom.spdx.json @sethmlarson
172173
/Tools/build/generate_sbom.py @sethmlarson

‎Misc/externals.spdx.json

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
{
2+
"SPDXID": "SPDXRef-DOCUMENT",
3+
"packages": [
4+
{
5+
"SPDXID": "SPDXRef-PACKAGE-bzip2",
6+
"checksums": [
7+
{
8+
"algorithm": "SHA256",
9+
"checksumValue": "ab8d1b0cc087c20d4c32c0e4fcf7d0c733a95da12cedc6d63b3f0a9af07427e2"
10+
}
11+
],
12+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/bzip2-1.0.8.tar.gz",
13+
"externalRefs": [
14+
{
15+
"referenceCategory": "SECURITY",
16+
"referenceLocator": "cpe:2.3:a:bzip:bzip2:1.0.8:*:*:*:*:*:*:*",
17+
"referenceType": "cpe23Type"
18+
}
19+
],
20+
"licenseConcluded": "NOASSERTION",
21+
"name": "bzip2",
22+
"primaryPackagePurpose": "SOURCE",
23+
"versionInfo": "1.0.8"
24+
},
25+
{
26+
"SPDXID": "SPDXRef-PACKAGE-libffi",
27+
"checksums": [
28+
{
29+
"algorithm": "SHA256",
30+
"checksumValue": "9d802681adfea27d84cae0487a785fb9caa925bdad44c401b364c59ab2b8edda"
31+
}
32+
],
33+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/libffi-3.4.4.tar.gz",
34+
"externalRefs": [
35+
{
36+
"referenceCategory": "SECURITY",
37+
"referenceLocator": "cpe:2.3:a:libffi_project:libffi:3.4.4:*:*:*:*:*:*:*",
38+
"referenceType": "cpe23Type"
39+
}
40+
],
41+
"licenseConcluded": "NOASSERTION",
42+
"name": "libffi",
43+
"primaryPackagePurpose": "SOURCE",
44+
"versionInfo": "3.4.4"
45+
},
46+
{
47+
"SPDXID": "SPDXRef-PACKAGE-openssl",
48+
"checksums": [
49+
{
50+
"algorithm": "SHA256",
51+
"checksumValue": "e6a77c273ebb284fedd8ea19b081fce74a9455936ffd47215f7c24713e2614b2"
52+
}
53+
],
54+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.13.tar.gz",
55+
"externalRefs": [
56+
{
57+
"referenceCategory": "SECURITY",
58+
"referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.13:*:*:*:*:*:*:*",
59+
"referenceType": "cpe23Type"
60+
}
61+
],
62+
"licenseConcluded": "NOASSERTION",
63+
"name": "openssl",
64+
"primaryPackagePurpose": "SOURCE",
65+
"versionInfo": "3.0.13"
66+
},
67+
{
68+
"SPDXID": "SPDXRef-PACKAGE-sqlite",
69+
"checksums": [
70+
{
71+
"algorithm": "SHA256",
72+
"checksumValue": "6f0364a27375435a34137b138ca4fedef8d23eec6493ca1dfff33bfc0c34fda4"
73+
}
74+
],
75+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.45.1.0.tar.gz",
76+
"externalRefs": [
77+
{
78+
"referenceCategory": "SECURITY",
79+
"referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.45.1.0:*:*:*:*:*:*:*",
80+
"referenceType": "cpe23Type"
81+
}
82+
],
83+
"licenseConcluded": "NOASSERTION",
84+
"name": "sqlite",
85+
"primaryPackagePurpose": "SOURCE",
86+
"versionInfo": "3.45.1.0"
87+
},
88+
{
89+
"SPDXID": "SPDXRef-PACKAGE-tcl-core",
90+
"checksums": [
91+
{
92+
"algorithm": "SHA256",
93+
"checksumValue": "6e33a88f116822167734cd72b693b5d30ced130a3cae6dc2ff696042f993bb42"
94+
}
95+
],
96+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.13.0.tar.gz",
97+
"externalRefs": [
98+
{
99+
"referenceCategory": "SECURITY",
100+
"referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.0:*:*:*:*:*:*:*",
101+
"referenceType": "cpe23Type"
102+
}
103+
],
104+
"licenseConcluded": "NOASSERTION",
105+
"name": "tcl-core",
106+
"primaryPackagePurpose": "SOURCE",
107+
"versionInfo": "8.6.13.0"
108+
},
109+
{
110+
"SPDXID": "SPDXRef-PACKAGE-tk",
111+
"checksums": [
112+
{
113+
"algorithm": "SHA256",
114+
"checksumValue": "896c1f488bdd0159091bd5cce124b756dfdffa4a5350b7fd4d7d8e48421089a4"
115+
}
116+
],
117+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.13.0.tar.gz",
118+
"externalRefs": [
119+
{
120+
"referenceCategory": "SECURITY",
121+
"referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.13.0:*:*:*:*:*:*:*",
122+
"referenceType": "cpe23Type"
123+
}
124+
],
125+
"licenseConcluded": "NOASSERTION",
126+
"name": "tk",
127+
"primaryPackagePurpose": "SOURCE",
128+
"versionInfo": "8.6.13.0"
129+
},
130+
{
131+
"SPDXID": "SPDXRef-PACKAGE-tix",
132+
"checksums": [
133+
{
134+
"algorithm": "SHA256",
135+
"checksumValue": "f7b21d115867a41ae5fd7c635a4c234d3ca25126c3661eb36028c6e25601f85e"
136+
}
137+
],
138+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tix-8.4.3.6.tar.gz",
139+
"externalRefs": [
140+
{
141+
"referenceCategory": "SECURITY",
142+
"referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.4.3.6:*:*:*:*:*:*:*",
143+
"referenceType": "cpe23Type"
144+
}
145+
],
146+
"licenseConcluded": "NOASSERTION",
147+
"name": "tix",
148+
"primaryPackagePurpose": "SOURCE",
149+
"versionInfo": "8.4.3.6"
150+
},
151+
{
152+
"SPDXID": "SPDXRef-PACKAGE-xz",
153+
"checksums": [
154+
{
155+
"algorithm": "SHA256",
156+
"checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282"
157+
}
158+
],
159+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz",
160+
"externalRefs": [
161+
{
162+
"referenceCategory": "SECURITY",
163+
"referenceLocator": "cpe:2.3:a:xz_project:xz:5.2.5:*:*:*:*:*:*:*",
164+
"referenceType": "cpe23Type"
165+
}
166+
],
167+
"licenseConcluded": "NOASSERTION",
168+
"name": "xz",
169+
"primaryPackagePurpose": "SOURCE",
170+
"versionInfo": "5.2.5"
171+
},
172+
{
173+
"SPDXID": "SPDXRef-PACKAGE-zlib",
174+
"checksums": [
175+
{
176+
"algorithm": "SHA256",
177+
"checksumValue": "e3f3fb32564952006eb18b091ca8464740e5eca29d328cfb0b2da22768e0b638"
178+
}
179+
],
180+
"downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-1.3.1.tar.gz",
181+
"externalRefs": [
182+
{
183+
"referenceCategory": "SECURITY",
184+
"referenceLocator": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*",
185+
"referenceType": "cpe23Type"
186+
}
187+
],
188+
"licenseConcluded": "NOASSERTION",
189+
"name": "zlib",
190+
"primaryPackagePurpose": "SOURCE",
191+
"versionInfo": "1.3.1"
192+
}
193+
],
194+
"spdxVersion": "SPDX-2.3"
195+
}

‎Tools/build/generate_sbom.py

Lines changed: 91 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@
77
import pathlib
88
import subprocess
99
import sys
10+
import urllib.request
1011
import typing
11-
import zipfile
12-
from urllib.request import urlopen
1312

1413
CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent
1514

@@ -125,30 +124,41 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]:
125124
return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")])
126125

127126

128-
def main() -> None:
129-
sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
130-
sbom_data = json.loads(sbom_path.read_bytes())
127+
def get_externals() -> list[str]:
128+
"""
129+
Parses 'PCbuild/get_externals.bat' for external libraries.
130+
Returns a list of (git tag, name, version) tuples.
131+
"""
132+
get_externals_bat_path = CPYTHON_ROOT_DIR / "PCbuild/get_externals.bat"
133+
externals = re.findall(
134+
r"set\s+libraries\s*=\s*%libraries%\s+([a-zA-Z0-9.-]+)\s",
135+
get_externals_bat_path.read_text()
136+
)
137+
return externals
131138

132-
# We regenerate all of this information. Package information
133-
# should be preserved though since that is edited by humans.
134-
sbom_data["files"] = []
135-
sbom_data["relationships"] = []
136139

137-
# Ensure all packages in this tool are represented also in the SBOM file.
138-
actual_names = {package["name"] for package in sbom_data["packages"]}
139-
expected_names = set(PACKAGE_TO_FILES)
140-
error_if(
141-
actual_names != expected_names,
142-
f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
143-
)
140+
def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None:
141+
"""Make a bunch of assertions about the SBOM package data to ensure it's consistent."""
144142

145-
# Make a bunch of assertions about the SBOM data to ensure it's consistent.
146143
for package in sbom_data["packages"]:
147144
# Properties and ID must be properly formed.
148145
error_if(
149146
"name" not in package,
150147
"Package is missing the 'name' field"
151148
)
149+
150+
# Verify that the checksum matches the expected value
151+
# and that the download URL is valid.
152+
if "checksums" not in package or "CI" in os.environ:
153+
download_location = package["downloadLocation"]
154+
resp = urllib.request.urlopen(download_location)
155+
error_if(resp.status != 200, f"Couldn't access URL: {download_location}'")
156+
157+
package["checksums"] = [{
158+
"algorithm": "SHA256",
159+
"checksumValue": hashlib.sha256(resp.read()).hexdigest()
160+
}]
161+
152162
missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys())
153163
error_if(
154164
bool(missing_required_keys),
@@ -180,6 +190,26 @@ def main() -> None:
180190
f"License identifier must be 'NOASSERTION'"
181191
)
182192

193+
194+
def create_source_sbom() -> None:
195+
sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
196+
sbom_data = json.loads(sbom_path.read_bytes())
197+
198+
# We regenerate all of this information. Package information
199+
# should be preserved though since that is edited by humans.
200+
sbom_data["files"] = []
201+
sbom_data["relationships"] = []
202+
203+
# Ensure all packages in this tool are represented also in the SBOM file.
204+
actual_names = {package["name"] for package in sbom_data["packages"]}
205+
expected_names = set(PACKAGE_TO_FILES)
206+
error_if(
207+
actual_names != expected_names,
208+
f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
209+
)
210+
211+
check_sbom_packages(sbom_data)
212+
183213
# We call 'sorted()' here a lot to avoid filesystem scan order issues.
184214
for name, files in sorted(PACKAGE_TO_FILES.items()):
185215
package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}")
@@ -224,5 +254,49 @@ def main() -> None:
224254
sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))
225255

226256

257+
def create_externals_sbom() -> None:
258+
sbom_path = CPYTHON_ROOT_DIR / "Misc/externals.spdx.json"
259+
sbom_data = json.loads(sbom_path.read_bytes())
260+
261+
externals = get_externals()
262+
externals_name_to_version = {}
263+
externals_name_to_git_tag = {}
264+
for git_tag in externals:
265+
name, _, version = git_tag.rpartition("-")
266+
externals_name_to_version[name] = version
267+
externals_name_to_git_tag[name] = git_tag
268+
269+
# Ensure all packages in this tool are represented also in the SBOM file.
270+
actual_names = {package["name"] for package in sbom_data["packages"]}
271+
expected_names = set(externals_name_to_version)
272+
error_if(
273+
actual_names != expected_names,
274+
f"Packages defined in SBOM tool don't match those defined in SBOM file: {actual_names}, {expected_names}",
275+
)
276+
277+
# Set the versionInfo and downloadLocation fields for all packages.
278+
for package in sbom_data["packages"]:
279+
package["versionInfo"] = externals_name_to_version[package["name"]]
280+
download_location = (
281+
f"https://github.com/python/cpython-source-deps/archive/refs/tags/{externals_name_to_git_tag[package['name']]}.tar.gz"
282+
)
283+
download_location_changed = download_location != package["downloadLocation"]
284+
package["downloadLocation"] = download_location
285+
286+
# If the download URL has changed we want one to get recalulated.
287+
if download_location_changed:
288+
package.pop("checksums", None)
289+
290+
check_sbom_packages(sbom_data)
291+
292+
# Update the SBOM on disk
293+
sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))
294+
295+
296+
def main() -> None:
297+
create_source_sbom()
298+
create_externals_sbom()
299+
300+
227301
if __name__ == "__main__":
228302
main()

0 commit comments

Comments
 (0)