Skip to content

Commit 87c18d8

Browse files
authored
Add BB downloading for OpenBLAS (#30497)
* Auto-detect binarybuilder triplet * Add OpenBLAS BinaryBuilder installation scaffolding Also make it easier to add more BB-cached versions of dependencies in the future * Enable `fixup-libgfortran.sh` to directly ask `$FC` for paths * Tell Appveyor and Travis to use BinaryBuilder OpenBLAS Also allow the build system to auto-guess the triplet
1 parent ba04451 commit 87c18d8

10 files changed

+210
-40
lines changed

.travis.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ before_install:
7676
brew tap staticfloat/julia > /dev/null;
7777
brew rm --force $(brew deps --HEAD julia);
7878
brew install -v ccache gcc gmp mpfr pcre2 staticfloat/julia/openblas-julia staticfloat/julia/suite-sparse-julia staticfloat/juliadeps/libgfortran;
79-
BUILDOPTS="-j3 USECLANG=1 USECCACHE=1 BINARYBUILDER_TRIPLET=x86_64-apple-darwin14 BINARYBUILDER_LLVM_ASSERTS=1";
80-
BUILDOPTS="$BUILDOPTS USE_BINARYBUILDER_LLVM=1 LLVM_CONFIG=$TRAVIS_BUILD_DIR/usr/tools/llvm-config LLVM_SIZE=$TRAVIS_BUILD_DIR/usr/tools/llvm-size";
79+
BUILDOPTS="-j3 USECLANG=1 USECCACHE=1 USE_BINARYBUILDER_LLVM=1 USE_BINARYBUILDER_OPENBLAS=1 BINARYBUILDER_LLVM_ASSERTS=1";
80+
BUILDOPTS="$BUILDOPTS LLVM_CONFIG=$TRAVIS_BUILD_DIR/usr/tools/llvm-config LLVM_SIZE=$TRAVIS_BUILD_DIR/usr/tools/llvm-size";
8181
BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1";
8282
BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas";
8383
for lib in SUITESPARSE BLAS LAPACK GMP MPFR LIBUNWIND; do

Make.inc

+1-2
Original file line numberDiff line numberDiff line change
@@ -232,11 +232,10 @@ INSTALL_F := $(JULIAHOME)/contrib/install.sh 644
232232
INSTALL_M := $(JULIAHOME)/contrib/install.sh 755
233233

234234
# BinaryBuilder options
235-
# TODO: Autodiscover triplet
235+
USE_BINARYBUILDER_OPENBLAS := 0
236236
USE_BINARYBUILDER_LLVM := 0
237237
# Use the Assertions build
238238
BINARYBUILDER_LLVM_ASSERTS := 0
239-
BINARYBUILDER_TRIPLET :=
240239

241240
# LLVM Options
242241
LLVMROOT := $(build_prefix)

contrib/fixup-libgfortran.sh

+21-5
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# This file is a part of Julia. License is MIT: https://julialang.org/license
33

44
# Run as: fixup-libgfortran.sh [--verbose] <$private_libdir>
5+
FC=${FC:-gfortran}
56

67
# If we're invoked with "--verbose", create a `debug` function that prints stuff out
78
if [ "$1" = "--verbose" ] || [ "$1" = "-v" ]; then
@@ -22,7 +23,7 @@ if [ "$UNAME" = "Linux" ]; then
2223
elif [ "$UNAME" = "Darwin" ]; then
2324
SHLIB_EXT="dylib"
2425
else
25-
echo "WARNING: Could not autodetect platform type ('uname -s' == $UNAME); assuming Linux" >&2
26+
echo "WARNING: Could not autodetect platform type ('uname -s' = $UNAME); assuming Linux" >&2
2627
UNAME="Linux"
2728
SHLIB_EXT="so"
2829
fi
@@ -41,6 +42,20 @@ find_shlib()
4142
fi
4243
}
4344

45+
find_shlib_dir()
46+
{
47+
# Usually, on platforms like OSX we get full paths when linking. However,
48+
# if we are inspecting, say, BinaryBuilder-built OpenBLAS libraries, we will
49+
# only get something like `@rpath/libgfortran.5.dylib` when inspecting the
50+
# libraries. We can, as a last resort, ask `$FC` directly what the full
51+
# filepath for this library is, but only if we don't have a direct path to it:
52+
if [ $(dirname "$1") = "@rpath" ]; then
53+
dirname "$($FC -print-file-name="$(basename "$1")" 2>/dev/null)"
54+
else
55+
dirname "$1" 2>/dev/null
56+
fi
57+
}
58+
4459
# First, discover all the places where libgfortran/libgcc is, as well as their true SONAMES
4560
for lib in lapack blas openblas; do
4661
for private_libname in ${private_libdir}/lib$lib*.$SHLIB_EXT*; do
@@ -51,10 +66,11 @@ for lib in lapack blas openblas; do
5166
LIBQUADMATH_PATH=$(find_shlib "$private_libname" libquadmath)
5267

5368
# Take the directories, add them onto LIBGFORTRAN_DIRS, which we use to
54-
# search for these libraries in the future.
55-
LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBGFORTRAN_PATH 2>/dev/null)"
56-
LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBGCC_PATH 2>/dev/null)"
57-
LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBQUADMATH_PATH 2>/dev/null)"
69+
# search for these libraries in the future. If there is no directory, try
70+
# asking `$FC` where such a file could be found.
71+
LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBGFORTRAN_PATH)"
72+
LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBGCC_PATH)"
73+
LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBQUADMATH_PATH)"
5874

5975
# Save the SONAMES
6076
LIBGFORTRAN_SONAMES="$LIBGFORTRAN_SONAMES $(basename "$LIBGFORTRAN_PATH")"

contrib/normalize_triplet.py

+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
#!/usr/bin/env python
2+
3+
import re, sys
4+
5+
# This script designed to mimick `src/PlatformNames.jl` in `BinaryProvider.jl`, which has
6+
# a method `platform_key_abi()` to parse uname-like output into something standarized.
7+
8+
if len(sys.argv) < 2:
9+
print("Usage: %s <host triplet> [<gcc version>]")
10+
sys.exit(1)
11+
12+
arch_mapping = {
13+
'x86_64': '(x86_|amd)64',
14+
'i686': "i\\d86",
15+
'aarch64': "aarch64",
16+
'arm': "arm(v7l)?",
17+
'powerpc64le': "p(ower)?pc64le",
18+
}
19+
platform_mapping = {
20+
'darwin': "-apple-darwin[\\d\\.]*",
21+
'freebsd': "-(.*-)?freebsd[\\d\\.]*",
22+
'windows': "-w64-mingw32",
23+
'linux': "-(.*-)?linux",
24+
}
25+
libc_mapping = {
26+
'blank_libc': "",
27+
'gnu': "-gnu",
28+
'musl': "-musl",
29+
}
30+
call_abi_mapping = {
31+
'blank_call_abi': "",
32+
'eabihf': "eabihf",
33+
}
34+
gcc_version_mapping = {
35+
'blank_gcc': "",
36+
'gcc4': "-gcc4",
37+
'gcc7': "-gcc7",
38+
'gcc8': "-gcc8",
39+
}
40+
cxx_abi_mapping = {
41+
'blank_cxx_abi': "",
42+
'cxx03': "-cxx03",
43+
'cxx11': "-cxx11",
44+
}
45+
46+
# Helper function to collapse dictionary of mappings down into a regex of
47+
# named capture groups joined by "|" operators
48+
c = lambda mapping: "("+"|".join(["(?P<%s>%s)"%(k,v) for (k, v) in mapping.items()]) + ")"
49+
mondo_regex = re.compile(
50+
"^"+
51+
c(arch_mapping)+
52+
c(platform_mapping)+
53+
c(libc_mapping)+
54+
c(call_abi_mapping)+
55+
c(gcc_version_mapping)+
56+
c(cxx_abi_mapping)+
57+
"$"
58+
)
59+
60+
# Apply our mondo regex to our input:
61+
m = mondo_regex.match(sys.argv[1])
62+
if m is None:
63+
print("ERROR: Unmatchable platform string '%s'!"%(sys.argv[1]))
64+
sys.exit(1)
65+
66+
# Helper function to find the single named field within the giant regex
67+
# that is not `nothing` for each mapping we give it.
68+
def get_field(m, mapping):
69+
g = m.groupdict()
70+
for k in mapping:
71+
if g[k] is not None:
72+
return k
73+
74+
arch = get_field(m, arch_mapping)
75+
platform = get_field(m, platform_mapping)
76+
libc = get_field(m, libc_mapping)
77+
call_abi = get_field(m, call_abi_mapping)
78+
gcc_version = get_field(m, gcc_version_mapping)
79+
cxx_abi = get_field(m, cxx_abi_mapping)
80+
81+
def r(x):
82+
x = x.replace("blank_call_abi", "")
83+
x = x.replace("blank_gcc", "")
84+
x = x.replace("blank_cxx_abi", "")
85+
x = x.replace("blank_libc", "")
86+
return x
87+
88+
def p(x):
89+
# These contain characters that can't be easily represented as
90+
# capture group names, unfortunately:
91+
os_remapping = {
92+
'darwin': 'apple-darwin14',
93+
'windows': 'w64-mingw32',
94+
'freebsd': 'unknown-freebsd11.1',
95+
}
96+
x = r(x)
97+
if x:
98+
for k in os_remapping:
99+
x = x.replace(k, os_remapping[k])
100+
return '-' + x
101+
return x
102+
103+
# If the user passes in a GCC version (like 8.2.0) use that to force a
104+
# "-gcc8" tag at the end of the triplet, but only if it has otherwise
105+
# not been specified
106+
if gcc_version == "blank_gcc":
107+
if len(sys.argv) == 3:
108+
gcc_version = {
109+
"4": "gcc4",
110+
"5": "gcc4",
111+
"6": "gcc4",
112+
"7": "gcc7",
113+
"8": "gcc8",
114+
}[sys.argv[2][0]]
115+
116+
117+
print(arch+p(platform)+p(libc)+r(call_abi)+p(gcc_version)+p(cxx_abi))
118+
119+
# Testing suite:
120+
# triplets="i686-w64-mingw32 x86_64-pc-linux-musl arm-linux-musleabihf x86_64-linux-gnu arm-linux-gnueabihf x86_64-apple-darwin14 x86_64-unknown-freebsd11.1"
121+
# for t in $triplets; do
122+
# if [[ $(./normalize_triplet.py "$t") != "$t" ]]; then
123+
# echo "ERROR: Failed test on $t"
124+
# fi
125+
# done

contrib/windows/appveyor_build.sh

+1-2
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,13 @@ if [ "$ARCH" = x86_64 ]; then
4545
echo 'USE_BLAS64 = 1' >> Make.user
4646
echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas64_' >> Make.user
4747
echo 'LIBBLASNAME = libopenblas64_' >> Make.user
48-
echo 'BINARYBUILDER_TRIPLET = x86_64-w64-mingw32' >> Make.user
4948
else
5049
bits=32
5150
archsuffix=86
5251
exc=sjlj
5352
echo "override MARCH = pentium4" >> Make.user
5453
echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas' >> Make.user
5554
echo 'LIBBLASNAME = libopenblas' >> Make.user
56-
echo 'BINARYBUILDER_TRIPLET = i686-w64-mingw32' >> Make.user
5755
fi
5856
echo "override JULIA_CPU_TARGET=generic;native" >> Make.user
5957

@@ -198,6 +196,7 @@ if [ -n "$USEMSVC" ]; then
198196
else
199197
# Use BinaryBuilder
200198
echo 'USE_BINARYBUILDER_LLVM = 1' >> Make.user
199+
echo 'USE_BINARYBUILDER_OPENBLAS = 1' >> Make.user
201200
echo 'BINARYBUILDER_LLVM_ASSERTS = 1' >> Make.user
202201
echo 'override DEP_LIBS += llvm openlibm' >> Make.user
203202
export CCACHE_DIR=/cygdrive/c/ccache

deps/Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ include $(SRCDIR)/Versions.make
1212
include $(JULIAHOME)/Make.inc
1313
include $(SRCDIR)/tools/common.mk
1414
include $(SRCDIR)/tools/git-external.mk
15+
include $(SRCDIR)/tools/bb-install.mk
1516

1617
# Special comments:
1718
#

deps/blas.mk

+20
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ endif
9292
# Do not overwrite the "-j" flag
9393
OPENBLAS_BUILD_OPTS += MAKE_NB_JOBS=0
9494

95+
ifneq ($(USE_BINARYBUILDER_OPENBLAS), 1)
96+
9597
$(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-configured: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/source-extracted
9698
perl -i -ple 's/^\s*(EXTRALIB\s*\+=\s*-lSystemStubs)\s*$$/# $$1/g' $(dir $<)/Makefile.system
9799
echo 1 > $@
@@ -194,3 +196,21 @@ configure-lapack: extract-lapack
194196
compile-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/build-compiled
195197
fastcheck-lapack: check-lapack
196198
check-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/build-checked
199+
200+
else # USE_BINARYBUILDER_OPENBLAS
201+
202+
203+
OPENBLAS_BB_URL_BASE := https://github.com/JuliaPackaging/Yggdrasil/releases/download/OpenBLAS-v$(OPENBLAS_VER)-$(OPENBLAS_BB_REL)
204+
OPENBLAS_BB_NAME := OpenBLAS.v$(OPENBLAS_VER)-$(OPENBLAS_BB_REL)
205+
206+
$(eval $(call bb-install,openblas,OPENBLAS,true))
207+
get-lapack: get-openblas
208+
extract-lapack: extract-openblas
209+
configure-lapack: configure-openblas
210+
compile-lapack: compile-openblas
211+
fastcheck-lapack: fastcheck-openblas
212+
check-lapack: check-openblas
213+
clean-lapack: clean-openblas
214+
distclean-lapack: distclean-openblas
215+
install-lapack: install-openblas
216+
endif

deps/llvm.mk

+4-29
Original file line numberDiff line numberDiff line change
@@ -513,37 +513,12 @@ ifeq ($(USE_POLLY),1)
513513
endif
514514
endif
515515
else # USE_BINARYBUILDER_LLVM
516-
LLVM_BB_URL_BASE := https://github.com/staticfloat/LLVMBuilder/releases/download
516+
LLVM_BB_URL_BASE := https://github.com/staticfloat/LLVMBuilder/releases/download/v$(LLVM_VER)-$(LLVM_BB_REL)
517517
ifneq ($(BINARYBUILDER_LLVM_ASSERTS), 1)
518-
LLVM_BB_NAME := LLVM
518+
LLVM_BB_NAME := LLVM.v$(LLVM_VER)
519519
else
520-
LLVM_BB_NAME := LLVM.asserts
520+
LLVM_BB_NAME := LLVM.asserts.v$(LLVM_VER)
521521
endif
522-
LLVM_BB_NAME := $(LLVM_BB_NAME).v$(LLVM_VER)
523-
LLVM_BB_URL := $(LLVM_BB_URL_BASE)/v$(LLVM_VER)-$(LLVM_BB_REL)/$(LLVM_BB_NAME).$(BINARYBUILDER_TRIPLET).tar.gz
524522

525-
526-
$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL):
527-
mkdir -p $@
528-
529-
$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz: | $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)
530-
$(JLDOWNLOAD) $@ $(LLVM_BB_URL)
531-
532-
$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/build-compiled: | $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz
533-
echo 1 > $@
534-
535-
$(eval $(call staged-install,llvm,llvm-$$(LLVM_VER)-$$(LLVM_BB_REL),,,,))
536-
537-
#Override provision of stage tarball
538-
$(build_staging)/llvm-$(LLVM_VER)-$(LLVM_BB_REL).tgz: $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz | $(build_staging)
539-
cp $< $@
540-
541-
clean-llvm:
542-
distclean-llvm:
543-
get-llvm: $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz
544-
extract-llvm:
545-
configure-llvm:
546-
compile-llvm:
547-
fastcheck-llvm:
548-
check-llvm:
523+
$(eval $(call bb-install,llvm,LLVM,false))
549524
endif # USE_BINARYBUILDER_LLVM

deps/openblas.version

+2
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
OPENBLAS_BRANCH=v0.3.3
22
OPENBLAS_SHA1=fd8d1868a126bb9f12bbc43b36ee30d1ba943fbb
3+
OPENBLAS_VER=0.3.3
4+
OPENBLAS_BB_REL=0

deps/tools/bb-install.mk

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
define bb-install
2+
# If the user has signified that this is a GCC-multiversioned tarball, then generate the proper tarball
3+
ifeq ($(3),true)
4+
$(2)_BB_TRIPLET := $(shell python $(call cygpath_w,$(JULIAHOME)/contrib/normalize_triplet.py) $(or $(XC_HOST),$(XC_HOST),$(BUILD_MACHINE)) $(lastword $(shell $(FC) --version | head -1)))
5+
else
6+
$(2)_BB_TRIPLET := $(shell python $(call cygpath_w,$(JULIAHOME)/contrib/normalize_triplet.py) $(or $(XC_HOST),$(XC_HOST),$(BUILD_MACHINE)))
7+
endif
8+
$(2)_BB_URL := $$($(2)_BB_URL_BASE)/$$($(2)_BB_NAME).$$($(2)_BB_TRIPLET).tar.gz
9+
10+
$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME):
11+
mkdir -p $$@
12+
13+
$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz: | $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)
14+
$$(JLDOWNLOAD) $$@ $$($(2)_BB_URL)
15+
16+
$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/build-compiled: | $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz
17+
echo 1 > $$@
18+
19+
$$(eval $$(call staged-install,$(1),$(1)-$$$$($(2)_BB_NAME),,,,))
20+
21+
#Override provision of stage tarball
22+
$$(build_staging)/$(1)-$$($(2)_BB_NAME).tgz: $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz | $$(build_staging)
23+
cp $$< $$@
24+
25+
clean-$(1):
26+
distclean-$(1):
27+
get-$(1): $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz
28+
extract-$(1):
29+
configure-$(1):
30+
compile-$(1):
31+
fastcheck-$(1):
32+
check-$(1):
33+
endef

0 commit comments

Comments
 (0)