Skip to content

Commit 70fd9bc

Browse files
PicoCentauriPicoCentauri
committed
Add CI for building wheels
1 parent e299233 commit 70fd9bc

File tree

10 files changed

+544
-60
lines changed

10 files changed

+544
-60
lines changed

‎.github/workflows/build-wheels.yml

+226
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
name: Build wheels
2+
3+
on:
4+
push:
5+
branches: [main]
6+
tags: ["*"]
7+
pull_request:
8+
# Check all PR
9+
10+
11+
concurrency:
12+
group: python-wheels-${{ github.ref }}
13+
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
14+
15+
16+
jobs:
17+
build-wheels:
18+
runs-on: ${{ matrix.os }}
19+
name: ${{ matrix.name }} (torch v${{ matrix.torch-version }})
20+
strategy:
21+
matrix:
22+
torch-version: ["2.3", "2.4", "2.5", "2.6"]
23+
arch: ["arm64", "x86_64"]
24+
os: ["ubuntu-22.04", "ubuntu-22.04-arm", "macos-14"]
25+
exclude:
26+
# remove mismatched arch/os pairs
27+
- {os: macos-14, arch: x86_64}
28+
- {os: ubuntu-22.04, arch: arm64}
29+
- {os: ubuntu-22.04-arm, arch: x86_64}
30+
include:
31+
- name: x86_64 Linux
32+
os: ubuntu-22.04
33+
arch: x86_64
34+
cibw-arch: x86_64
35+
- name: arm64 Linux
36+
os: ubuntu-22.04-arm
37+
arch: arm64
38+
cibw-arch: aarch64
39+
- name: arm64 macOS
40+
os: macos-14
41+
arch: arm64
42+
cibw-arch: arm64
43+
44+
steps:
45+
- uses: actions/checkout@v4
46+
with:
47+
fetch-depth: 0
48+
49+
- name: Set up Python
50+
uses: actions/setup-python@v5
51+
with:
52+
python-version: "3.13"
53+
54+
- name: install dependencies
55+
run: python -m pip install cibuildwheel
56+
57+
- name: build wheel
58+
run: python -m cibuildwheel --output-dir ./wheelhouse
59+
env:
60+
CIBW_BUILD: cp312-*
61+
CIBW_SKIP: "*musllinux*"
62+
CIBW_ARCHS: "${{ matrix.cibw-arch }}"
63+
CIBW_BUILD_VERBOSITY: 1
64+
CIBW_ENVIRONMENT: >
65+
PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cpu
66+
MACOSX_DEPLOYMENT_TARGET=11
67+
PETNC_BUILD_WITH_TORCH_VERSION=${{ matrix.torch-version }}.*
68+
# do not complain for missing libtorch.so
69+
CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
70+
delocate-wheel --ignore-missing-dependencies --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
71+
CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
72+
auditwheel repair --exclude libtorch.so --exclude libtorch_cpu.so --exclude libc10.so -w {dest_dir} {wheel}
73+
74+
- uses: actions/upload-artifact@v4
75+
with:
76+
name: single-version-wheel-${{ matrix.torch-version }}-${{ matrix.os }}-${{ matrix.arch }}
77+
path: ./wheelhouse/*.whl
78+
79+
merge-wheels:
80+
needs: build-wheels
81+
runs-on: ubuntu-22.04
82+
name: merge wheels for ${{ matrix.name }}
83+
strategy:
84+
matrix:
85+
include:
86+
- name: x86_64 Linux
87+
os: ubuntu-22.04
88+
arch: x86_64
89+
- name: arm64 Linux
90+
os: ubuntu-22.04-arm
91+
arch: arm64
92+
- name: arm64 macOS
93+
os: macos-14
94+
arch: arm64
95+
steps:
96+
- uses: actions/checkout@v4
97+
98+
- name: Download wheels
99+
uses: actions/download-artifact@v4
100+
with:
101+
pattern: single-version-wheel-*-${{ matrix.os }}-${{ matrix.arch }}
102+
merge-multiple: false
103+
path: dist
104+
105+
- name: Set up Python
106+
uses: actions/setup-python@v5
107+
with:
108+
python-version: "3.13"
109+
110+
- name: install dependencies
111+
run: python -m pip install twine wheel
112+
113+
- name: merge wheels
114+
run: |
115+
# collect all torch versions used for the build
116+
REQUIRES_TORCH=$(find dist -name "*.whl" -exec unzip -p {} "pet_neighbors_convert-*.dist-info/METADATA" \; | grep "Requires-Dist: torch")
117+
MERGED_TORCH_REQUIRE=$(python scripts/create-torch-versions-range.py "$REQUIRES_TORCH")
118+
119+
echo MERGED_TORCH_REQUIRE=$MERGED_TORCH_REQUIRE
120+
121+
# unpack all single torch versions wheels in the same directory
122+
mkdir dist/unpacked
123+
find dist -name "*.whl" -print -exec python -m wheel unpack --dest dist/unpacked/ {} ';'
124+
125+
sed -i "s/Requires-Dist: torch.*/$MERGED_TORCH_REQUIRE/" dist/unpacked/pet_neighbors_convert-*/pet_neighbors_convert-*.dist-info/METADATA
126+
127+
echo "\n\n METADATA = \n\n"
128+
cat dist/unpacked/pet_neighbors_convert-*/pet_neighbors_convert-*.dist-info/METADATA
129+
130+
# check the right metadata was added to the file. grep will exit with
131+
# code `1` if the line is not found, which will stop CI
132+
grep "$MERGED_TORCH_REQUIRE" dist/unpacked/pet_neighbors_convert-*/pet_neighbors_convert-*.dist-info/METADATA
133+
134+
# repack the directory as a new wheel
135+
mkdir wheelhouse
136+
python -m wheel pack --dest wheelhouse/ dist/unpacked/*
137+
138+
- name: check wheels with twine
139+
run: twine check wheelhouse/*
140+
141+
- uses: actions/upload-artifact@v4
142+
with:
143+
name: wheel-${{ matrix.os }}-${{ matrix.arch }}
144+
path: ./wheelhouse/*.whl
145+
146+
build-sdist:
147+
name: sdist
148+
runs-on: ubuntu-22.04
149+
steps:
150+
- uses: actions/checkout@v4
151+
with:
152+
fetch-depth: 0
153+
154+
- name: Set up Python
155+
uses: actions/setup-python@v5
156+
with:
157+
python-version: "3.13"
158+
159+
- name: install dependencies
160+
run: python -m pip install build
161+
162+
- name: build sdist
163+
env:
164+
PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu"
165+
run: python -m build . --outdir=dist/
166+
167+
- uses: actions/upload-artifact@v4
168+
with:
169+
name: sdist
170+
path: dist/*.tar.gz
171+
172+
merge-and-release:
173+
name: Merge and release wheels/sdists
174+
needs: [merge-wheels, build-sdist]
175+
runs-on: ubuntu-22.04
176+
permissions:
177+
contents: write
178+
id-token: write
179+
pull-requests: write
180+
environment:
181+
name: pypi
182+
url: https://pypi.org/project/pet-neighbors-convert
183+
steps:
184+
- name: Download wheels
185+
uses: actions/download-artifact@v4
186+
with:
187+
path: wheels
188+
pattern: wheel-*
189+
merge-multiple: true
190+
191+
- name: Download sdist
192+
uses: actions/download-artifact@v4
193+
with:
194+
path: wheels
195+
name: sdist
196+
197+
- name: Re-upload a single wheels artifact
198+
uses: actions/upload-artifact@v4
199+
with:
200+
name: wheels
201+
path: wheels/*
202+
203+
- name: Comment with download link
204+
uses: PicoCentauri/comment-artifact@v1
205+
if: github.event.pull_request.head.repo.fork == false
206+
with:
207+
name: wheels
208+
description: ⚙️ Download Python wheels for this pull-request (you can install these with pip)
209+
210+
- name: Publish distribution to PyPI
211+
if: startsWith(github.ref, 'refs/tags/v')
212+
uses: pypa/gh-action-pypi-publish@release/v1
213+
with:
214+
packages-dir: wheels
215+
216+
- name: upload to GitHub release
217+
if: startsWith(github.ref, 'refs/tags/v')
218+
uses: softprops/action-gh-release@v2
219+
with:
220+
files: |
221+
wheels/*.tar.gz
222+
wheels/*.whl
223+
prerelease: ${{ contains(github.ref, '-rc') }}
224+
env:
225+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
226+

‎CMakeLists.txt

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
cmake_minimum_required(VERSION 3.27)
2+
3+
if (POLICY CMP0076)
4+
# target_sources() converts relative paths to absolute
5+
cmake_policy(SET CMP0076 NEW)
6+
endif()
7+
8+
project(neighbors_convert CXX)
9+
10+
# Set a default build type if none was specified
11+
if (${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_SOURCE_DIR})
12+
if("${CMAKE_BUILD_TYPE}" STREQUAL "" AND "${CMAKE_CONFIGURATION_TYPES}" STREQUAL "")
13+
message(STATUS "Setting build type to 'relwithdebinfo' as none was specified.")
14+
set(
15+
CMAKE_BUILD_TYPE "relwithdebinfo"
16+
CACHE STRING
17+
"Choose the type of build, options are: none(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) debug release relwithdebinfo minsizerel."
18+
FORCE
19+
)
20+
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS release debug relwithdebinfo minsizerel none)
21+
endif()
22+
endif()
23+
24+
# add path to the cmake configuration of the version of libtorch used
25+
# by the Python torch module. PYTHON_EXECUTABLE is provided by skbuild
26+
execute_process(
27+
COMMAND ${PYTHON_EXECUTABLE} -c "import torch.utils; print(torch.utils.cmake_prefix_path)"
28+
RESULT_VARIABLE TORCH_CMAKE_PATH_RESULT
29+
OUTPUT_VARIABLE TORCH_CMAKE_PATH_OUTPUT
30+
ERROR_VARIABLE TORCH_CMAKE_PATH_ERROR
31+
)
32+
33+
if (NOT ${TORCH_CMAKE_PATH_RESULT} EQUAL 0)
34+
message(FATAL_ERROR "failed to find your pytorch installation\n${TORCH_CMAKE_PATH_ERROR}")
35+
endif()
36+
37+
string(STRIP ${TORCH_CMAKE_PATH_OUTPUT} TORCH_CMAKE_PATH_OUTPUT)
38+
set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${TORCH_CMAKE_PATH_OUTPUT}")
39+
40+
find_package(Torch 2.3 REQUIRED)
41+
42+
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/_build_torch_version.py "BUILD_TORCH_VERSION = '${Torch_VERSION}'")
43+
44+
add_library(neighbors_convert SHARED
45+
"src/pet_neighbors_convert/neighbors_convert.cpp"
46+
)
47+
48+
# only link to `torch_cpu_library` instead of `torch`, which could also include
49+
# `libtorch_cuda`.
50+
target_link_libraries(neighbors_convert PUBLIC torch_cpu_library)
51+
target_include_directories(neighbors_convert PUBLIC "${TORCH_INCLUDE_DIRS}")
52+
target_compile_definitions(neighbors_convert PUBLIC "${TORCH_CXX_FLAGS}")
53+
54+
target_compile_features(neighbors_convert PUBLIC cxx_std_17)
55+
56+
install(TARGETS neighbors_convert
57+
LIBRARY DESTINATION "lib"
58+
)

‎MANIFEST.in

+4-1
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,15 @@ graft src
22

33
include LICENSE
44
include README.rst
5+
include CMakeLists.txt
56

6-
prune tests
77
prune .github
88
prune .tox
9+
prune tests
10+
prune scripts
911

1012
exclude .gitignore
1113
exclude tox.ini
14+
recursive-include build-backend *.py
1215

1316
global-exclude *.py[cod] __pycache__/* *.so *.dylib

‎build-backend/backend.py

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# This is a custom Python build backend wrapping setuptool's to add a build-time
2+
# dependencies on torch when building the wheel and not the sdist
3+
import os
4+
from setuptools import build_meta
5+
6+
FORCED_TORCH_VERSION = os.environ.get("PETNC_BUILD_WITH_TORCH_VERSION")
7+
if FORCED_TORCH_VERSION is not None:
8+
TORCH_DEP = f"torch =={FORCED_TORCH_VERSION}"
9+
else:
10+
TORCH_DEP = "torch >=2.3"
11+
12+
# ==================================================================================== #
13+
# Build backend functions definition #
14+
# ==================================================================================== #
15+
16+
# Use the default version of these
17+
prepare_metadata_for_build_wheel = build_meta.prepare_metadata_for_build_wheel
18+
get_requires_for_build_sdist = build_meta.get_requires_for_build_sdist
19+
build_wheel = build_meta.build_wheel
20+
build_sdist = build_meta.build_sdist
21+
22+
23+
# Special dependencies to build the wheels
24+
def get_requires_for_build_wheel(config_settings=None):
25+
defaults = build_meta.get_requires_for_build_wheel(config_settings)
26+
return defaults + [TORCH_DEP]

‎pyproject.toml

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
[build-system]
2-
requires = ["setuptools >= 68", "setuptools_scm>=8", "wheel", "torch >= 2.3"]
3-
build-backend = "setuptools.build_meta"
2+
requires = ["setuptools >= 68", "setuptools_scm>=8", "wheel >= 0.36"]
3+
build-backend = "backend"
4+
backend-path = ["build-backend"]
45

56
[project]
67
authors = [{name = "lab-cosmo developers"}]
@@ -15,9 +16,8 @@ classifiers = [
1516
"Topic :: Scientific/Engineering",
1617
"Topic :: Software Development :: Libraries :: Python Modules"
1718
]
18-
dependencies = ["torch >= 2.3"]
19-
description = "Extension for PET model for reordering the neighbors during message passing."
20-
dynamic = ["version"]
19+
description = "PET model extension for processing neighbor lists"
20+
dynamic = ["version", "dependencies"]
2121
license = {text = "BSD-3-Clause"}
2222
name = "pet-neighbors-convert"
2323
readme = "README.rst"

‎scripts/create-torch-versions-range.py

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/usr/bin/env python3
2+
"""
3+
This script updates the `Requires-Dist` information in the wheel METADATA
4+
to contain the range of compatible torch versions. It expects newline separated
5+
`Requires-Dist: torch ==...` information (corresponding to wheels built against a single
6+
torch version) and will print `Requires-Dist: torch >=$MIN_VERSION,<${MAX_VERSION+1}` on
7+
the standard output.
8+
9+
This output can the be used in the merged wheel containing the build against all torch
10+
versions.
11+
"""
12+
import re
13+
import sys
14+
15+
16+
if __name__ == "__main__":
17+
torch_versions_raw = sys.argv[1]
18+
19+
torch_versions = []
20+
for version in torch_versions_raw.split("\n"):
21+
if version.strip() == "":
22+
continue
23+
24+
match = re.match(r"Requires-Dist: torch[ ]?==(\d+)\.(\d+)\.\*", version)
25+
if match is None:
26+
raise ValueError(f"unexpected Requires-Dist format: {version}")
27+
28+
major, minor = match.groups()
29+
major = int(major)
30+
minor = int(minor)
31+
32+
version = (major, minor)
33+
34+
if version in torch_versions:
35+
raise ValueError(f"duplicate torch version: {version}")
36+
37+
torch_versions.append(version)
38+
39+
torch_versions = list(sorted(torch_versions))
40+
41+
min_version = f"{torch_versions[0][0]}.{torch_versions[0][1]}"
42+
max_version = f"{torch_versions[-1][0]}.{torch_versions[-1][1] + 1}"
43+
44+
print(f"Requires-Dist: torch >={min_version},<{max_version}")

0 commit comments

Comments
 (0)