diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 00000000..e9e3f1a7 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,3 @@ +# For testing purposes only! Note that CI passes in `--bazelrc=/dev/null` and +# does *not* use this file. +build --incompatible_enable_cc_toolchain_resolution diff --git a/.github/workflows/migration.yml b/.github/workflows/migration.yml index 4d1baf6e..5d55bf17 100644 --- a/.github/workflows/migration.yml +++ b/.github/workflows/migration.yml @@ -13,6 +13,6 @@ jobs: - uses: actions/checkout@v2 - name: test env: - BAZELISK_GITHUB_TOKEN: ${{ secrets.BAZELISK_GITHUB_TOKEN }} + BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TEST_MIGRATION: true run: tests/scripts/run_tests.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f9e31dd2..d24cb951 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v2 - name: test env: - BAZELISK_GITHUB_TOKEN: ${{ secrets.BAZELISK_GITHUB_TOKEN }} + BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: tests/scripts/${{ matrix.script }} container_test: runs-on: ubuntu-latest @@ -30,5 +30,5 @@ jobs: - uses: actions/checkout@v2 - name: test env: - BAZELISK_GITHUB_TOKEN: ${{ secrets.BAZELISK_GITHUB_TOKEN }} + BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: tests/scripts/${{ matrix.script }}_test.sh diff --git a/README.md b/README.md index 837143e1..7e26ab07 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,14 @@ For overriding toolchains on the command line, please use the `--extra_toolchains` flag in lieu of the deprecated `--crosstool_top` flag. For example, `--extra_toolchains=@llvm_toolchain//:cc-toolchain-linux`. +Note: you may need to add `build --incompatible_enable_cc_toolchain_resolution` +to your `.bazelrc` to enable toolchain resolution for `cc` toolchains (see +[this][enable-cc-toolchain-res] issue). If you do this, the +`llvm_register_toolchains` call in `WORKSPACE` shown in the example above should +be sufficient to get Bazel to use the toolchain. + +[enable-cc-toolchain-res]: https://github.com/bazelbuild/bazel/issues/7260 + If you would like to use the older method of selecting toolchains, you can continue to do so with `--crosstool_top=@llvm_toolchain//:toolchain`. @@ -78,6 +86,200 @@ Notes: depend on these tools directly in the bin directory of the toolchain. For example, `@llvm_toolchain//:bin/clang-format` is a valid and visible target. +## Setting Up Toolchains for Other Targets + +### Using `extra_targets` + +```starlark +load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain") +llvm_toolchain( + name = "llvm_toolchain", + llvm_version = "8.0.0", + extra_targets = [ + "wasm32-unknown-wasi", + ], + + # Extra targets can have their sysroots overriden too: + sysroot = { + "linux": "@some_example_sysroot_repo//:linux_sysroot", + "darwin": "@some_example_sysroot_repo//:macos_sysroot", + + "linux_wasm32-unknown-wasi": "@some_example_sysroot_repo//:wasi_sysroot", + "darwin_wasm32-unknown-wasi": "@some_example_sysroot_repo//:wasi_sysroot", + }, +) + +load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains") +llvm_register_toolchains() + +http_archive( + name = "some_example_sysroot_repo", + ... +) +``` + +The toolchain that is created will have the appropriate constraints so that Bazel +will pick it when resolving a toolchain for a particular platform. For example: + +```starlark +platform( + name = "wasi", + constraints = [ + "@platforms//os:wasi", + "@platforms//cpu:wasm32", + ] +) + +cc_library( + name = "test", + srcs = [...], +) +``` + +Running `bazel build //:test --platforms //:wasm` should use the configured +`wasm32-unknown-wasi` toolchain and produce an object file with wasm32 assembly +in it. + +Note that this should work also work with rules that apply a +[transition][transition] to require that a target be built for a particular +platform. + +[transition]: https://docs.bazel.build/versions/main/skylark/config.html#user-defined-transitions + +Also note that the order of the triples in `extra_targets` influences how toolchains will be considered during [toolchain resolution][t-res] as does using a target triple ending with `-unknown` instead of `-none` (as a rule of thumb, prefer `-none` over `-unknown` unless you have a good reason not to). See [this comment][extra-target-pitfalls-comment] for some context. + +[t-res]: https://docs.bazel.build/versions/main/toolchains.html#toolchain-resolution +[extra-target-pitfalls-comment]: WORKSPACE#L31-L63 + +Currently only the following extra targets are implemented/tested at all: + - `wasm32-unknown-wasi` + - `wasm32-unknown-none` + +Other targets *can* be specified but are unlikely to work as the glue needed to +fetch their sysroots/compiler-rt (i.e. `libclang_rt.builtins-...`) is not yet +implemented. + +[`tests/extra_targets`](tests/extra_targets) contains some examples that use the extra toolchains configured in this repo's workspace. + +### Manually + +For other targets (or if you just want to make modifications to the toolchains that the machinery in this repo produces) you can set up a toolchain manually. The process for doing so looks something like this: + +```starlark +# WORKSPACE +# (parts to set up `@llvm_toolchain` have been elided; see above) + +llvm_toolchain( + name = "llvm_toolchain", + llvm_version = "8.0.0", + + # NOTE: This is required to set up toolchains outside of `@llvm_toolchain`, unfortunately + absolute_paths = True, + ) + +# This registers the default toolchains. +load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains", "register_toolchain") + +llvm_register_toolchains() + +# Now let's make our own: +http_archive( + name = "thumbv7-sysroot", + urls = ["example.com"], +) +register_toolchain("//tests:custom_toolchain_example") + +# BUILD file: +# Example Custom Toolchain: +load("@llvm_toolchain//:cc_toolchain_config.bzl", "cc_toolchain_config") + +# Docs for this function and `overrides` are in `cc_toolchain_config.bzl.tpl`. +cc_toolchain_config( + name = "custom_toolchain_example_config", + host_platform = "linux", + custom_target_triple = "thumbv7em-unknown-none-gnueabihf", + overrides = { + "target_system_name": "thumbv7em-unknown-none-gnueabihf", + "target_cpu": "thumbv7em", + "target_libc": "unknown", + "abi_libc_version": "unknown", + + # If you omit this, be sure to depend on + # `@llvm_toolchain:host_sysroot_components`. + # "sysroot_path": "external/thumbv7-sysroot/sysroot", + + "extra_compile_flags": [ + "-mthumb", + "-mcpu=cortex-m4", + "-mfpu=fpv4-sp-d16", + "-mfloat-abi=hard", + ], + "omit_hosted_linker_flags": True, + "omit_cxx_stdlib_flag": False, + "use_llvm_ar_instead_of_libtool_on_macos": True, + } +) + +load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "conditional_cc_toolchain") +conditional_cc_toolchain( + name = "custom_toolchain", + toolchain_config = ":custom_toolchain_example_config", + host_is_darwin = False, + + sysroot_label = "@llvm_toolchain//:host_sysroot_components", # use this if not overriding + # sysroot_label = "@thumbv7-sysroot//:sysroot", # override + + absolute_paths = True, # this is required for toolchains set up outside of `@llvm_toolchain`, unfortunately + llvm_repo_label_prefix = "@llvm_toolchain//", +) + +# Constraints come from here: https://github.com/bazelbuild/platforms +toolchain( + name = "custom_toolchain_example", + exec_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + target_compatible_with = [ + "@platforms//cpu:armv7", # `v7e-mf` has not yet made it to stable Bazel? + # "@platforms//os:none", + ], + toolchain = ":custom_toolchain", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) +``` + +As with the `wasm32-unknown-wasi` example above, you can "use" the toolchain by +creating a platform matching the constraints which the toolchain satisfies and +then either specifying that platform globally (on the command line) or for a +particular target via a transition. + +Here's an example of using `target_compatible_with` on a target to get it to +only build when an appropriate target platform is specified: + +```starlark +platform( + name = "arm", + constraint_values = [ + "@platforms//cpu:armv7", + # "@platforms//os:none", + ] +) + +cc_library( + name = "custom_target_test", + srcs = ["test.cc"], + target_compatible_with = [ + "@platforms//cpu:armv7", + ] +) +``` + +Ultimately the goal is to add support for extra targets directly in this repo; +PRs are very welcome :-). + +## Misc + Other examples of toolchain configuration: https://github.com/bazelbuild/bazel/wiki/Building-with-a-custom-toolchain diff --git a/WORKSPACE b/WORKSPACE index 9af38ca0..5efc4ee3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -24,14 +24,51 @@ load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain") llvm_toolchain( name = "llvm_toolchain", - # LLVM 9.0.0 needs /usr/lib/libtinfo.so.5 that is not very straightforward - # to set up in all linux distros we test. - llvm_version = "8.0.0", + llvm_version = "12.0.0", + extra_targets = [ + # NOTE: we do *not* use `wasm32-unknown-unknown` here; using `unknown` + # makes the generated toolchain have no OS constraint which will result + # in toolchain resolution matching the toolchain even for targets that + # do have an OS. + # + # For example `wasm-unknown-unknown` which has no OS constraint will + # match `wasm-unknown-wasi` which has an `os:wasi` constraint: even + # though we'd _rather_ use `wasm32-unknown-wasi` when targeting + # `wasm32-unknown-wasi`, it's not _wrong_ to use `wasm32-unknown-none` + # since it *will* produce code that can run on `wasm32-unknown-wasi` + # systems. + # + # In other words, the target platform has to satisfy the toolchain's + # constraints, not the other way around. + # + # What we'd really like is for individual *targets* that require stdlib + # and "platform" functionality to be able to say they need "os:wasi" + # and thus need to be built with `wasm32-unknown-wasi` but this isn't + # how toolchain resolution works; targets only filter the execution + # platforms. + # + # The real solution is to actually specify an OS constraint even for + # `unknown` target triple toolchains OR to always put `unknown` target + # triple toolchains behind their OS-constraint-having peers in this + # list so toolchain resolution will pick them as a _last_ resort. + # + # For now we don't map `unknown` to `os:none` in case there are + # situations where this behavior is desirable (i.e. you want to fall + # back to `thumbv7em-unknown-unknown` when you're asked to build for + # a triple like `thumbv7em-unknown-netbsd`). + # + # Note that there is no default constraint for `//platforms/os`: + # https://github.com/bazelbuild/platforms/blob/98939346da932eef0b54cf808622f5bb0928f00b/os/BUILD#L14 + "wasm32-unknown-none", + "wasm32-unknown-wasi", + ], + # absolute_paths = True, ) -load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains") +load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains", "register_toolchain") llvm_register_toolchains() +register_toolchain("//tests:custom_toolchain_example") ## Toolchain example with a sysroot. load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") @@ -58,8 +95,20 @@ llvm_toolchain( }, ) -# Well known repos; present here only for testing. +# `bazel_skylib`; we're using its `build_test` test +http_archive( + name = "bazel_skylib", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + ], + sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", +) + +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") +bazel_skylib_workspace() +# Well known repos; present here only for testing. http_archive( name = "com_google_googletest", sha256 = "9dc9157a9a1551ec7a7e43daea9a694a0bb5fb8bec81235d8a1e6ef64c716dcb", @@ -89,14 +138,12 @@ http_archive( urls = ["https://www.openssl.org/source/openssl-1.1.1c.tar.gz"], ) -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - http_archive( name = "io_bazel_rules_go", - sha256 = "e6a6c016b0663e06fa5fccf1cd8152eab8aa8180c583ec20c872f4f9953a7ac5", + sha256 = "7904dbecbaffd068651916dce77ff3437679f9d20e1a7956bff43826e7645fcc", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.22.1/rules_go-v0.22.1.tar.gz", - "https://github.com/bazelbuild/rules_go/releases/download/v0.22.1/rules_go-v0.22.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.25.1/rules_go-v0.25.1.tar.gz", + "https://github.com/bazelbuild/rules_go/releases/download/v0.25.1/rules_go-v0.25.1.tar.gz", ], ) @@ -104,4 +151,4 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe go_rules_dependencies() -go_register_toolchains() +go_register_toolchains(version = "1.15.7") diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 219a3ee2..472e5012 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -14,12 +14,30 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +package(default_visibility = [":__subpackages__"]) + +cc_binary( + name = "simple_binary", + srcs = ["simple.cc"], +) + +cc_test( + name = "simple_test", + srcs = ["simple.cc"], +) + cc_library( name = "stdlib", srcs = ["stdlib.cc"], hdrs = ["stdlib.h"], ) +cc_binary( + name = "stdlib_binary", + srcs = ["stdlib_test.cc"], + deps = [":stdlib"], +) + cc_test( name = "stdlib_test", srcs = ["stdlib_test.cc"], @@ -34,3 +52,82 @@ sh_test( "@llvm_toolchain//:lib/libc++.a", ], ) + +# TODO: Eventually expose a nicer function for this that's less boilerplate-y +# +# It should be generated and do things like fail immediately if absolute_paths != True, etc. + +# Example Custom Toolchain: +load("@llvm_toolchain//:cc_toolchain_config.bzl", "cc_toolchain_config") + +# Docs for this function and `overrides` are in `cc_toolchain_config.bzl.tpl`. +cc_toolchain_config( + name = "custom_toolchain_example_config", + host_platform = "darwin", + custom_target_triple = "thumbv7em-unknown-none-gnueabihf", + overrides = { + "target_system_name": "thumbv7em-unknown-none-gnueabihf", + "target_cpu": "thumbv7em", + "target_libc": "unknown", + "abi_libc_version": "unknown", + + # If you omit this, be sure to depend on + # `@llvm_toolchain:host_sysroot_components`. + # "sysroot_path": "external/thumbv7-sysroot/sysroot", # TODO: check + + "extra_compile_flags": [ + "-mthumb", + "-mcpu=cortex-m4", + "-mfpu=fpv4-sp-d16", + "-mfloat-abi=hard", + ], + "omit_hosted_linker_flags": True, + "omit_cxx_stdlib_flag": False, + "use_llvm_ar_instead_of_libtool_on_macos": True, + } +) + +load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "conditional_cc_toolchain") +conditional_cc_toolchain( + name = "custom_toolchain", + toolchain_config = ":custom_toolchain_example_config", + host_is_darwin = True, # TODO + + sysroot_label = "@llvm_toolchain//:host_sysroot_components", # use this if not overriding + # sysroot_label = "@thumbv7-sysroot//:sysroot", # override + + absolute_paths = True, # this is required for toolchains set up outside of `@llvm_toolchain`, unfortunately + llvm_repo_label_prefix = "@llvm_toolchain//", +) + +# Constraints come from here: https://github.com/bazelbuild/platforms +toolchain( + name = "custom_toolchain_example", + exec_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:osx", + ], + target_compatible_with = [ + "@platforms//cpu:armv7", # `v7e-mf` has not yet made it to stable Bazel? + # "@platforms//os:none", + ], + toolchain = ":custom_toolchain", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +platform( + name = "arm", + constraint_values = [ + "@platforms//cpu:armv7", + # "@platforms//os:none", + ] +) + +cc_library( + name = "custom_target_test", + srcs = ["stdlib.cc"], + hdrs = ["stdlib.h"], + target_compatible_with = [ + "@platforms//cpu:armv7", + ] +) diff --git a/tests/extra_targets/BUILD.bazel b/tests/extra_targets/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/tests/extra_targets/wasm/BUILD.bazel b/tests/extra_targets/wasm/BUILD.bazel new file mode 100644 index 00000000..eae8483d --- /dev/null +++ b/tests/extra_targets/wasm/BUILD.bazel @@ -0,0 +1,73 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load(":transitions.bzl", "wasm_file", "wasi_file") + +# config_setting( +# name = "no_shared_objects", +# values = { +# "features": "-supports_dynamic_linker", +# } +# ) + +# cc_library( +# name = "stdlib-wasm", +# srcs = ["stdlib.cc"], +# hdrs = ["stdlib.h"], +# features = [ +# "-supports_dynamic_linker", +# ], +# target_compatible_with = [ +# "@platforms//cpu:wasm32", +# "@platforms//os:wasi", +# ], +# ) + +# TODO: add a rule that transitions --platforms to `//test:wasm` to ensure that +# toolchain resolution picks `@llvm_toolchain//:cc-clang-*_wasm32-unknown-unknown`. +# +# also have it set `--features=-supports_dynamic_linker`; ideally this would be in +# the toolchain definition but the downside of switching to using +# `unix_cc_toolchain_config.bzl` is that we lose this knob +# +# it's not clear yet if supporting different targets runs counter to using +# `unix_cc_toolchain_config.bzl` altogether – so far things are manageable but +# it's already apparent that we'll be sacrificing some power/configurability + +############################################################################### + +platform( + name = "wasm", + constraint_values = [ + "@platforms//cpu:wasm32", + "@platforms//os:none", + ], +) + +wasm_file( + name = "test.wasm", + src = "//tests:simple_binary", +) + +build_test( + name = "wasm-build-chk", + targets = [":test.wasm"], +) + +############################################################################### + +platform( + name = "wasi", + constraint_values = [ + "@platforms//cpu:wasm32", + "@platforms//os:wasi", + ], +) + +wasi_file( + name = "test.wasi", + src = "//tests:stdlib_binary", +) + +build_test( + name = "wasi-build-chk", + targets = [":test.wasi"], +) diff --git a/tests/extra_targets/wasm/transitions.bzl b/tests/extra_targets/wasm/transitions.bzl new file mode 100644 index 00000000..9366bc29 --- /dev/null +++ b/tests/extra_targets/wasm/transitions.bzl @@ -0,0 +1,83 @@ +"""Some alias rules that transition `--platforms`.""" + +def _wasm_platform_transition(_settings, _attr): + return { + "//command_line_option:platforms": + str(Label("//tests/extra_targets/wasm:wasm")), + } + +wasm_platform_transition = transition( + implementation = _wasm_platform_transition, + inputs = [], + outputs = ["//command_line_option:platforms"], +) + +def _wasm_file_impl(ctx): + wasm_file = ctx.attr.name + wasm_file = ctx.actions.declare_file(wasm_file) + ctx.actions.symlink( + output = wasm_file, + target_file = ctx.executable.src, + is_executable = True, + ) + + return [DefaultInfo(files = depset([wasm_file]))] + +wasm_file = rule( + implementation = _wasm_file_impl, + attrs = { + "src": attr.label( + executable = True, + cfg = wasm_platform_transition, + mandatory = True, + doc = "TODO", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist" + ), + }, + incompatible_use_toolchain_transition = True, + doc = "TODO", +) + +############################################################################### + +def _wasi_platform_transition(_settings, _attr): + return { + "//command_line_option:platforms": + str(Label("//tests/extra_targets/wasm:wasi")), + } + +wasi_platform_transition = transition( + implementation = _wasi_platform_transition, + inputs = [], + outputs = ["//command_line_option:platforms"], +) + +def _wasi_file_impl(ctx): + wasi_file = ctx.attr.name + wasi_file = ctx.actions.declare_file(wasi_file) + ctx.actions.symlink( + output = wasi_file, + target_file = ctx.executable.src, + is_executable = True, + ) + + return [DefaultInfo(files = depset([wasi_file]))] + +wasi_file = rule( + implementation = _wasi_file_impl, + attrs = { + "src": attr.label( + executable = True, + cfg = wasi_platform_transition, + mandatory = True, + doc = "TODO", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist" + ), + }, + incompatible_use_toolchain_transition = True, + doc = "TODO", +) diff --git a/tests/scripts/archlinux_test.sh b/tests/scripts/archlinux_test.sh index 1638ef7e..bcaef499 100755 --- a/tests/scripts/archlinux_test.sh +++ b/tests/scripts/archlinux_test.sh @@ -16,7 +16,7 @@ set -euo pipefail images=( -"archlinux/base:latest" +"archlinux:latest" ) git_root=$(git rev-parse --show-toplevel) @@ -28,7 +28,8 @@ for image in "${images[@]}"; do set -exuo pipefail # Install dependencies -pacman -Syu --noconfirm --quiet python gcc tar +pacman -Syu --noconfirm --quiet python gcc tar ncurses +link /lib/libtinfo.so.6 /lib/libtinfo.so.5 # Run tests cd /src diff --git a/tests/scripts/run_external_tests.sh b/tests/scripts/run_external_tests.sh index 5c6139e7..c4447957 100755 --- a/tests/scripts/run_external_tests.sh +++ b/tests/scripts/run_external_tests.sh @@ -19,7 +19,7 @@ os="$(uname -s | tr "[:upper:]" "[:lower:]")" readonly os # Use bazelisk for latest bazel version. -# Value of BAZELISK_GITHUB_TOKEN is set as a secret on Travis. +# Value of BAZELISK_GITHUB_TOKEN is set from a GitHub Actions secret (tests.yml/migration.yml). readonly url="https://github.com/bazelbuild/bazelisk/releases/download/v1.0/bazelisk-${os}-amd64" bazel="${TMPDIR:-/tmp}/bazelisk" readonly bazel diff --git a/tests/scripts/run_tests.sh b/tests/scripts/run_tests.sh index b06ffaa1..a782d415 100755 --- a/tests/scripts/run_tests.sh +++ b/tests/scripts/run_tests.sh @@ -32,7 +32,7 @@ os="$(uname -s | tr "[:upper:]" "[:lower:]")" readonly os # Use bazelisk to catch migration problems. -# Value of BAZELISK_GITHUB_TOKEN is set as a secret on Travis. +# Value of BAZELISK_GITHUB_TOKEN is set from a GitHub Actions secret (tests.yml/migration.yml). readonly bazelisk_version="v1.6.1" readonly url="https://github.com/bazelbuild/bazelisk/releases/download/${bazelisk_version}/bazelisk-${os}-amd64" bazel="${TMPDIR:-/tmp}/bazelisk" diff --git a/tests/simple.cc b/tests/simple.cc new file mode 100644 index 00000000..b5c86614 --- /dev/null +++ b/tests/simple.cc @@ -0,0 +1,12 @@ +#include +#include + +int main() { + std::array v = { 1, 2, 3, 4, 5 }; + + auto sum = 0; + for (const auto& i : v) { sum += i; } + assert(sum == 15); + + return 0; +} diff --git a/toolchain/BUILD.tpl b/toolchain/BUILD.tpl index bc9280f3..77b61fea 100644 --- a/toolchain/BUILD.tpl +++ b/toolchain/BUILD.tpl @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -package(default_visibility = ["//visibility:public"]) - load("@rules_cc//cc:defs.bzl", "cc_toolchain_suite") +package(default_visibility = ["//visibility:public"]) + exports_files(["Makevars"]) # Some targets may need to directly depend on these files. @@ -32,8 +32,8 @@ filegroup( ) filegroup( - name = "sysroot_components", - srcs = [%{sysroot_label}], + name = "host_sysroot_components", + srcs = [%{host_sysroot_label}], ) cc_toolchain_suite( @@ -43,6 +43,7 @@ cc_toolchain_suite( "darwin|clang": ":cc-clang-darwin", "k8": ":cc-clang-linux", "darwin": ":cc-clang-darwin", + %{extra_toolchains_for_toolchain_suite} }, ) @@ -50,46 +51,50 @@ load(":cc_toolchain_config.bzl", "cc_toolchain_config") cc_toolchain_config( name = "local_linux", - cpu = "k8", + host_platform = "k8", ) cc_toolchain_config( name = "local_darwin", - cpu = "darwin", + host_platform = "darwin", ) toolchain( - name = "cc-toolchain-darwin", + name = "cc-toolchain-linux", exec_compatible_with = [ "@platforms//cpu:x86_64", - "@platforms//os:osx", + "@platforms//os:linux", ], target_compatible_with = [ "@platforms//cpu:x86_64", - "@platforms//os:osx", + "@platforms//os:linux", ], - toolchain = ":cc-clang-darwin", + toolchain = ":cc-clang-linux", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", ) toolchain( - name = "cc-toolchain-linux", + name = "cc-toolchain-darwin", exec_compatible_with = [ "@platforms//cpu:x86_64", - "@platforms//os:linux", + "@platforms//os:osx", ], target_compatible_with = [ "@platforms//cpu:x86_64", - "@platforms//os:linux", + "@platforms//os:osx", ], - toolchain = ":cc-clang-linux", + toolchain = ":cc-clang-darwin", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", ) +%{extra_cc_toolchain_config} + + load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "conditional_cc_toolchain") -conditional_cc_toolchain("cc-clang-linux", False, %{absolute_paths}) -conditional_cc_toolchain("cc-clang-darwin", True, %{absolute_paths}) +conditional_cc_toolchain("cc-clang-linux", ":local_linux", False, ":host_sysroot_components", %{absolute_paths}) +conditional_cc_toolchain("cc-clang-darwin", ":local_darwin", True, ":host_sysroot_components", %{absolute_paths}) +%{extra_conditional_cc_toolchain_config} ## LLVM toolchain files # Needed when not using absolute paths. @@ -109,6 +114,9 @@ filegroup( "bin/ld.lld", "bin/ld", "bin/ld.gold", # Dummy file on non-linux. + "bin/wasm-ld", + "bin/lld", + "bin/lld-link", ], ) @@ -136,11 +144,11 @@ filegroup( ) filegroup( - name = "compiler_components", + name = "host_compiler_components", srcs = [ ":clang", ":include", - ":sysroot_components", + ":host_sysroot_components", ], ) @@ -198,21 +206,21 @@ filegroup( ) filegroup( - name = "linker_components", + name = "host_linker_components", srcs = [ ":clang", ":ld", ":ar", ":lib", - ":sysroot_components", + ":host_sysroot_components", ], ) filegroup( - name = "all_components", + name = "all_host_components", srcs = [ ":binutils_components", - ":compiler_components", - ":linker_components", + ":host_compiler_components", + ":host_linker_components", ], ) diff --git a/toolchain/cc_toolchain_config.bzl.tpl b/toolchain/cc_toolchain_config.bzl.tpl index 3c149ff3..1854b7d8 100644 --- a/toolchain/cc_toolchain_config.bzl.tpl +++ b/toolchain/cc_toolchain_config.bzl.tpl @@ -13,508 +13,253 @@ # limitations under the License. load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "action_config", - "artifact_name_pattern", - "env_entry", - "env_set", - "feature", - "feature_set", - "flag_group", - "flag_set", - "make_variable", - "tool", - "tool_path", - "variable_with_value", - "with_feature_set", + "@bazel_tools//tools/cpp:unix_cc_toolchain_config.bzl", + bazel_cc_toolchain_config = "cc_toolchain_config", ) -load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") -def _impl(ctx): - if (ctx.attr.cpu == "darwin"): - toolchain_identifier = "clang-darwin" - elif (ctx.attr.cpu == "k8"): - toolchain_identifier = "clang-linux" - else: - fail("Unreachable") - - if (ctx.attr.cpu == "k8"): - host_system_name = "x86_64" - elif (ctx.attr.cpu == "darwin"): - host_system_name = "x86_64-apple-macosx" - else: - fail("Unreachable") - - if (ctx.attr.cpu == "darwin"): - target_system_name = "x86_64-apple-macosx" - elif (ctx.attr.cpu == "k8"): - target_system_name = "x86_64-unknown-linux-gnu" - else: - fail("Unreachable") - - if (ctx.attr.cpu == "darwin"): - target_cpu = "darwin" - elif (ctx.attr.cpu == "k8"): - target_cpu = "k8" - else: - fail("Unreachable") - - if (ctx.attr.cpu == "k8"): - target_libc = "glibc_unknown" - elif (ctx.attr.cpu == "darwin"): - target_libc = "macosx" - else: - fail("Unreachable") - - if (ctx.attr.cpu == "darwin" or - ctx.attr.cpu == "k8"): - compiler = "clang" - else: - fail("Unreachable") - - if (ctx.attr.cpu == "k8"): - abi_version = "clang" - elif (ctx.attr.cpu == "darwin"): - abi_version = "darwin_x86_64" - else: - fail("Unreachable") - - if (ctx.attr.cpu == "darwin"): - abi_libc_version = "darwin_x86_64" - elif (ctx.attr.cpu == "k8"): - abi_libc_version = "glibc_unknown" - else: - fail("Unreachable") - - cc_target_os = None - - if (ctx.attr.cpu == "darwin" or - ctx.attr.cpu == "k8"): - builtin_sysroot = "%{sysroot_path}" - else: +def cc_toolchain_config(name, host_platform, custom_target_triple = None, overrides = {}): + """Creates a `clang` based `CcToolchain`. + + Args: + name: name of the toolchain to create + host_platform: execution target for the toolchain + This can be "darwin" (macOS) or "k8" (Linux). + custom_target_triple: the target to compile for + If not specified the toolchain created will target the host + platform. + overrides: + ``` + { + "target_system_name": string, + "target_cpu": string, + "target_libc": string, + "abi_version": string, + "abi_libc_version": string, + "sysroot_path": string, + + "extra_compile_flags": list[string], + "extra_linker_flags": list[string], + "custom_linker_tool": { + "darwin": string, + "k8": string, + }, + "omit_hosted_linker_flags": bool = False, + + "linker_include_build_id_on_linux_hosts": bool = True, + "linker_use_gnu_hash_style_on_linux_hosts": bool = True, + "linker_use_elf_hardening_so_flags_on_linux_hosts": bool = True, + + "prefer_static_cxx_libs_on_linux_hosts": bool = True, + "omit_cxx_stdlib_flag": bool = False, + + "use_llvm_ar_instead_of_libtool_on_macos": bool = False, + } + ``` + """ + if not (host_platform == "darwin" or host_platform == "k8"): fail("Unreachable") - all_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.lto_backend, - ] - - all_cpp_compile_actions = [ - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ] - - preprocessor_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ] - - codegen_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ] - - all_link_actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ] - - action_configs = [] - - if ctx.attr.cpu == "k8": + # `builtin_sysroot` support – required to use the `%sysroot%` prefix – was + # only added in bazel v4.0.0. + # + # `native.bazel_version` might give us back an empty string if a local dev build + # of bazel is being used; in this case we'll assume the version is at least + # 4.0.0. + # + # See: https://github.com/bazelbuild/bazel/commit/da345f1f249ebf28bec88c6e0d63260dfaef14e9 + sysroot_prefix_supported = int(("%{bazel_version}" or "4.0.0").split(".")[0]) >= 4 + + # A bunch of variables that get passed straight through to + # `create_cc_toolchain_config_info`. + toolchain_identifier, host_system_name, target_system_name, target_cpu, \ + target_libc, compiler, abi_version, abi_libc_version, builtin_sysroot = { + "darwin": ( + "clang-darwin-" + (custom_target_triple or "x86_64"), + "x86_64-apple-macosx", + "x86_64-apple-macosx", + "darwin", + "macosx", + "clang", + "darwin_x86_64", + "darwin_x86_64", + "%{host_sysroot_path}", + ), + "k8": ( + "clang-linux-" + (custom_target_triple or "x86_64"), + "x86_64", + "x86_64-unknown-linux-gnu", + "k8", + "glibc_unknown", + "clang", + "clang", + "glibc_unknown", + "%{host_sysroot_path}", + ), + }[host_platform] + + # Overrides: + target_system_name = overrides.get("target_system_name", target_system_name) + target_cpu = overrides.get("target_cpu", target_cpu) + target_libc = overrides.get("target_libc", target_libc) + abi_version = overrides.get("abi_version", abi_version) + abi_libc_version = overrides.get("abi_libc_version", abi_libc_version) + builtin_sysroot = overrides.get("sysroot_path", builtin_sysroot) + + # Unfiltered compiler flags: + unfiltered_compile_flags = [ + # Do not resolve our symlinked resource prefixes to real paths. + "-no-canonical-prefixes", + # Reproducibility + "-Wno-builtin-macro-redefined", + "-D__DATE__=\"redacted\"", + "-D__TIMESTAMP__=\"redacted\"", + "-D__TIME__=\"redacted\"", + "-fdebug-prefix-map=%{toolchain_path_prefix}=%{debug_toolchain_path_prefix}", + ] + overrides.get("extra_compile_flags", []) + + if custom_target_triple: + unfiltered_compile_flags.append("--target={}".format(custom_target_triple)) + + # Linker flags: + enable_hosted_linker_flags = not overrides.get("omit_hosted_linker_flags", False) + if host_platform == "k8": linker_flags = [ # Use the lld linker. "-fuse-ld=lld", - # The linker has no way of knowing if there are C++ objects; so we always link C++ libraries. - "-L%{toolchain_path_prefix}/lib", + ] + + # If we're compiling for the host, we want to include the lib folder in + # the library search directories so that any static libs we specify get + # picked up. + # + # If we're *not* compiling for the host we do not want to include these + # since they'll interfere with the sysroot for the target and, if these + # come first on the command line, potentially cause the linker to try + # to use the wrong version of libc++ and friends. + if not custom_target_triple: + linker_flags += [ + "-L%{toolchain_path_prefix}/lib", + ] + + # Unless the target has a compelling reason not to want to do so (i.e. + # has a special linker that doesn't support this syntax), we'd like to + # prefer _statically_ linking the C++ libraries: + prefer_static_cxx_libs_on_linux_hosts = \ + overrides.get("prefer_static_cxx_libs_on_linux_hosts", True) + linker_flags += [ "-l:libc++.a", "-l:libc++abi.a", - "-l:libunwind.a", - # Compiler runtime features. - "-rtlib=compiler-rt", - # To support libunwind. - "-lpthread", - "-ldl", - # Other linker flags. - "-Wl,--build-id=md5", - "-Wl,--hash-style=gnu", - "-Wl,-z,relro,-z,now", + ] if prefer_static_cxx_libs_on_linux_hosts else [ + "-lc++", + "-lc++abi", ] - elif ctx.attr.cpu == "darwin": + + if enable_hosted_linker_flags: + linker_flags += [ + "-l:libunwind.a", + # Compiler runtime features. + "-rtlib=compiler-rt", + # To support libunwind. + "-lpthread", + "-ldl", + ] + + # Other linker flags. + if overrides.get("linker_include_build_id_on_linux_hosts", True): + linker_flags += ["-Wl,--build-id=md5"] + if overrides.get("linker_use_gnu_hash_style_on_linux_hosts", True): + linker_flags += ["-Wl,--hash-style=gnu"] + if overrides.get("linker_use_elf_hardening_so_flags_on_linux_hosts", True): + linker_flags += ["-Wl,-z,relro,-z,now"] + elif host_platform == "darwin": linker_flags = [ - # Difficult to guess options to statically link C++ libraries with the macOS linker. + # Difficult to guess options to statically link C++ libraries with + # the macOS linker. "-lc++", "-lc++abi", - "-headerpad_max_install_names", - "-undefined", - "dynamic_lookup", ] + + if enable_hosted_linker_flags: + linker_flags += [ + "-headerpad_max_install_names", + "-undefined", "dynamic_lookup", + ] else: fail("Unreachable") - opt_feature = feature(name = "opt") - fastbuild_feature = feature(name = "fastbuild") - dbg_feature = feature(name = "dbg") - - random_seed_feature = feature(name = "random_seed", enabled = True) - supports_pic_feature = feature(name = "supports_pic", enabled = True) - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - - unfiltered_compile_flags_feature = feature( - name = "unfiltered_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - flags = [ - # Do not resolve our smylinked resource prefixes to real paths. - "-no-canonical-prefixes", - # Reproducibility - "-Wno-builtin-macro-redefined", - "-D__DATE__=\"redacted\"", - "-D__TIMESTAMP__=\"redacted\"", - "-D__TIME__=\"redacted\"", - "-fdebug-prefix-map=%{toolchain_path_prefix}=%{debug_toolchain_path_prefix}", - ], - ), - ], - ), - ], - ) + link_flags = [ + "-lm", + "-no-canonical-prefixes", + ] + linker_flags + overrides.get("extra_linker_flags", []) - default_link_flags_feature = feature( - name = "default_link_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = [ - "-lm", - "-no-canonical-prefixes", - ] + linker_flags, - ), - ], - ), - ] + ([ - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["-Wl,--gc-sections"])], - with_features = [with_feature_set(features = ["opt"])], - ), - ] if ctx.attr.cpu == "k8" else []), - ) + if custom_target_triple: + link_flags.append("--target={}".format(custom_target_triple)) - default_compile_flags_feature = feature( - name = "default_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - flags = [ - # Security - "-U_FORTIFY_SOURCE", # https://github.com/google/sanitizers/issues/247 - "-fstack-protector", - "-fno-omit-frame-pointer", - # Diagnostics - "-fcolor-diagnostics", - "-Wall", - "-Wthread-safety", - "-Wself-assign", - ], - ), - ], - ), - flag_set( - actions = all_compile_actions, - flag_groups = [flag_group(flags = ["-g", "-fstandalone-debug"])], - with_features = [with_feature_set(features = ["dbg"])], - ), - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - flags = [ - "-g0", - "-O2", - "-D_FORTIFY_SOURCE=1", - "-DNDEBUG", - "-ffunction-sections", - "-fdata-sections", - ], - ), - ], - with_features = [with_feature_set(features = ["opt"])], - ), - flag_set( - actions = all_cpp_compile_actions, - flag_groups = [flag_group(flags = ["-std=c++17", "-stdlib=libc++"])], - ), - ], - ) + opt_link_flags = ["-Wl,--gc-sections"] if host_platform == "k8" else [] - objcopy_embed_flags_feature = feature( - name = "objcopy_embed_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = ["objcopy_embed_data"], - flag_groups = [flag_group(flags = ["-I", "binary"])], - ), - ], - ) - user_compile_flags_feature = feature( - name = "user_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - expand_if_available = "user_compile_flags", - flags = ["%{user_compile_flags}"], - iterate_over = "user_compile_flags", - ), - ], - ), - ], - ) + # Default compiler flags: + compile_flags = [ + # Security + "-U_FORTIFY_SOURCE", # https://github.com/google/sanitizers/issues/247 + "-fstack-protector", + "-fno-omit-frame-pointer", + # Diagnostics + "-fcolor-diagnostics", + "-Wall", + "-Wthread-safety", + "-Wself-assign", + ] - sysroot_feature = feature( - name = "sysroot", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions + all_link_actions, - flag_groups = [ - flag_group( - expand_if_available = "sysroot", - flags = ["--sysroot=%{sysroot}"], - ), - ], - ), - ], - ) + dbg_compile_flags = ["-g", "-fstandalone-debug"] - coverage_feature = feature( - name = "coverage", - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - flags = ["-fprofile-instr-generate", "-fcoverage-mapping"], - ), - ], - ), - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["-fprofile-instr-generate"])], - ), - ], - provides = ["profile"], - ) + opt_compile_flags = [ + "-g0", + "-O2", + "-D_FORTIFY_SOURCE=1", + "-DNDEBUG", + "-ffunction-sections", + "-fdata-sections", + ] - framework_paths_feature = feature( - name = "framework_paths", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - "objc-executable", - "objc++-executable", - ], - flag_groups = [ - flag_group( - flags = ["-F%{framework_paths}"], - iterate_over = "framework_paths", - ), - ], - ), - ], - ) + # Some targets *can't* pick between `libc++` and `libstdc++` so the + # `-stdlib` is just unused and emits a warnings. + add_cxx_stdlib_flag = not overrides.get("omit_cxx_stdlib_flag", False) + cxx_flags = ["-std=c++17"] + ["-stdlib=libc++"] if add_cxx_stdlib_flag else [] - include_paths_feature = feature( - name = "include_paths", - enabled = True, - flag_sets = [ - flag_set( - actions = preprocessor_compile_actions, - flag_groups = [ - flag_group( - flags = ["/I%{quote_include_paths}"], - iterate_over = "quote_include_paths", - ), - flag_group( - flags = ["/I%{include_paths}"], - iterate_over = "include_paths", - ), - flag_group( - flags = ["/I%{system_include_paths}"], - iterate_over = "system_include_paths", - ), - ], - ), - ], - ) + # Coverage flags: + coverage_compile_flags = ["-fprofile-instr-generate", "-fcoverage-mapping"] + coverage_link_flags = ["-fprofile-instr-generate"] - dependency_file_feature = feature( - name = "dependency_file", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_header_parsing, - ], - flag_groups = [ - flag_group( - expand_if_available = "dependency_file", - flags = ["/DEPENDENCY_FILE", "%{dependency_file}"], - ), - ], - ), - ], - ) - compiler_input_flags_feature = feature( - name = "compiler_input_flags", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - expand_if_available = "source_file", - flags = ["/c", "%{source_file}"], - ), - ], - ), - ], - ) + ## NOTE: framework paths is missing here; unix_cc_toolchain_config + ## doesn't seem to have a feature for this. - compiler_output_flags_feature = feature( - name = "compiler_output_flags", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.assemble], - flag_groups = [ - flag_group( - expand_if_available = "output_file", - expand_if_not_available = "output_assembly_file", - flags = ["/Fo%{output_file}", "/Zi"], - ), - ], - ), - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - expand_if_available = "output_file", - expand_if_not_available = "output_assembly_file", - flags = ["/Fo%{output_file}"], - ), - flag_group( - expand_if_available = "output_file", - flags = ["/Fa%{output_file}"], - ), - flag_group( - expand_if_available = "output_file", - flags = ["/P", "/Fi%{output_file}"], - ), - ], - ), - ], - ) - - features = [ - opt_feature, - fastbuild_feature, - dbg_feature, - random_seed_feature, - supports_pic_feature, - supports_dynamic_linker_feature, - unfiltered_compile_flags_feature, - default_link_flags_feature, - default_compile_flags_feature, - objcopy_embed_flags_feature, - user_compile_flags_feature, - sysroot_feature, - coverage_feature, - # Windows only features. - # input_paths_feature - # dependency_file_feature - # compiler_input_flags_feature - # compiler_output_flags_feature - ] - if (ctx.attr.cpu == "darwin"): - features.extend([framework_paths_feature]) + # C++ built-in include directories: cxx_builtin_include_directories = [ "%{toolchain_path_prefix}include/c++/v1", "%{toolchain_path_prefix}lib/clang/%{llvm_version}/include", "%{toolchain_path_prefix}lib64/clang/%{llvm_version}/include", ] - if (ctx.attr.cpu == "k8"): + + # If `builtin_sysroot` is supported, use the `sysroot_prefix` here. + # + # If not, we'll just substitute in the `builtin_sysroot` path directly. + sysroot_for_include_dirs = "%{sysroot_prefix}" if sysroot_prefix_supported else builtin_sysroot + + if (host_platform == "k8"): cxx_builtin_include_directories += [ - "%{sysroot_prefix}/include", - "%{sysroot_prefix}/usr/include", - "%{sysroot_prefix}/usr/local/include", + "{}/include".format(sysroot_for_include_dirs), + "{}/usr/include".format(sysroot_for_include_dirs), + "{}/usr/local/include".format(sysroot_for_include_dirs), ] + [ %{k8_additional_cxx_builtin_include_directories} ] - elif (ctx.attr.cpu == "darwin"): + elif (host_platform == "darwin"): cxx_builtin_include_directories += [ - "%{sysroot_prefix}/usr/include", - "%{sysroot_prefix}/System/Library/Frameworks", + "{}/usr/include".format(sysroot_for_include_dirs), + "{}/System/Library/Frameworks".format(sysroot_for_include_dirs), "/Library/Frameworks", ] + [ %{darwin_additional_cxx_builtin_include_directories} @@ -522,135 +267,93 @@ def _impl(ctx): else: fail("Unreachable") - artifact_name_patterns = [] - - if (ctx.attr.cpu == "darwin"): - make_variables = [ - make_variable( - name = "STACK_FRAME_UNLIMITED", - value = "-Wframe-larger-than=100000000 -Wno-vla", - ), - ] - elif (ctx.attr.cpu == "k8"): - make_variables = [] - else: - fail("Unreachable") - if (ctx.attr.cpu == "k8"): - tool_paths = [ - tool_path( - name = "ld", - path = "%{tools_path_prefix}bin/ld.lld", - ), - tool_path( - name = "cpp", - path = "%{tools_path_prefix}bin/clang-cpp", - ), - tool_path( - name = "dwp", - path = "%{tools_path_prefix}bin/llvm-dwp", - ), - tool_path( - name = "gcov", - path = "%{tools_path_prefix}bin/llvm-profdata", - ), - tool_path( - name = "nm", - path = "%{tools_path_prefix}bin/llvm-nm", - ), - tool_path( - name = "objcopy", - path = "%{tools_path_prefix}bin/llvm-objcopy", - ), - tool_path( - name = "objdump", - path = "%{tools_path_prefix}bin/llvm-objdump", - ), - tool_path(name = "strip", path = "/usr/bin/strip"), - tool_path( - name = "gcc", - path = "%{tools_path_prefix}bin/clang", - ), - tool_path( - name = "ar", - path = "%{tools_path_prefix}bin/llvm-ar", - ), - ] - elif (ctx.attr.cpu == "darwin"): - tool_paths = [ - tool_path(name = "ld", path = "%{tools_path_prefix}bin/ld"), - tool_path( - name = "cpp", - path = "%{tools_path_prefix}bin/clang-cpp", - ), - tool_path( - name = "dwp", - path = "%{tools_path_prefix}bin/llvm-dwp", - ), - tool_path( - name = "gcov", - path = "%{tools_path_prefix}bin/llvm-profdata", - ), - tool_path( - name = "nm", - path = "%{tools_path_prefix}bin/llvm-nm", - ), - tool_path( - name = "objcopy", - path = "%{tools_path_prefix}bin/llvm-objcopy", - ), - tool_path( - name = "objdump", - path = "%{tools_path_prefix}bin/llvm-objdump", - ), - tool_path(name = "strip", path = "/usr/bin/strip"), - tool_path( - name = "gcc", - path = "%{tools_path_prefix}bin/cc_wrapper.sh", - ), - tool_path(name = "ar", path = "/usr/bin/libtool"), - ] - else: - fail("Unreachable") - - out = ctx.actions.declare_file(ctx.label.name) - ctx.actions.write(out, "Fake executable") - return [ - cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = toolchain_identifier, - host_system_name = host_system_name, - target_system_name = target_system_name, - target_cpu = target_cpu, - target_libc = target_libc, - compiler = compiler, - abi_version = abi_version, - abi_libc_version = abi_libc_version, - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = builtin_sysroot, - cc_target_os = cc_target_os, - ), - DefaultInfo( - executable = out, - ), - ] - -cc_toolchain_config = rule( - attrs = { - "cpu": attr.string( - mandatory = True, - values = [ - "darwin", - "k8", - ], - ), - }, - executable = True, - provides = [CcToolchainConfigInfo], - implementation = _impl, -) + ## NOTE: make variables are missing here; unix_cc_toolchain_config doesn't + ## pass these to `create_cc_toolchain_config_info`. + + + # Tool paths: + # `llvm-strip` was introduced in V7 (https://reviews.llvm.org/D46407): + LLVM_MAJOR_VER = "%{llvm_version}".split(".") + LLVM_MAJOR_VER = int(LLVM_MAJOR_VER[0]) if len(LLVM_MAJOR_VER) else 0 + strip_binary = \ + "%{tools_path_prefix}bin/llvm-strip" if LLVM_MAJOR_VER >= 7 else "/usr/bin/strip" + + tool_paths = { + "cpp": "%{tools_path_prefix}bin/clang-cpp", + "dwp": "%{tools_path_prefix}bin/llvm-dwp", + "gcov": "%{tools_path_prefix}bin/llvm-profdata", + "llvm-cov": "%{tools_path_prefix}bin/llvm-cov", + "nm": "%{tools_path_prefix}bin/llvm-nm", + "objcopy": "%{tools_path_prefix}bin/llvm-objcopy", + "objdump": "%{tools_path_prefix}bin/llvm-objdump", + "strip": strip_binary, + } + linker_tool = overrides.get("custom_linker_tool", { + "k8": "ld.lld", + # ld.lld Mach-O support is still experimental: + "darwin": "ld", + }) + tool_paths.update({ + "k8": { + "ld": "%{tools_path_prefix}bin/{}".format(linker_tool["k8"]), + "gcc": "%{tools_path_prefix}bin/clang", + "ar": "%{tools_path_prefix}bin/llvm-ar", + }, + "darwin": { + "ld": "%{tools_path_prefix}bin/{}".format(linker_tool["darwin"]), + # See `cc_wrapper.sh.tpl` for details: + "gcc": "%{tools_path_prefix}bin/cc_wrapper.sh", + # No idea why we use `libtool` instead of `llvm-ar` on macOS: + "ar": "/usr/bin/libtool", + }, + }[host_platform]) + + if overrides.get("use_llvm_ar_instead_of_libtool_on_macos", False) and host_platform == "darwin": + tool_paths["ar"] = "%{tools_path_prefix}bin/llvm-ar" + + # Start-end group linker support: + # This was added to `lld` in this patch: http://reviews.llvm.org/D18814 + # + # The oldest version of LLVM that we support is 6.0.0 which was released + # after the above patch was merged, so we just set this to `True` when `lld` + # is being used as the linker, which is always... except on macOS since + # `lld` Mach-O support is still experimental (and potentially for extra + # targets). + supports_start_end_lib = tool_paths["ld"].endswith("ld.lld") + + + # Source: https://cs.opensource.google/bazel/bazel/+/master:tools/cpp/unix_cc_toolchain_config.bzl + bazel_cc_toolchain_config( + name = name, + cpu = target_cpu, + compiler = compiler, + toolchain_identifier = toolchain_identifier, + host_system_name = host_system_name, + target_system_name = target_system_name, + target_libc = target_libc, + abi_version = abi_version, + abi_libc_version = abi_libc_version, + cxx_builtin_include_directories = cxx_builtin_include_directories, + tool_paths = tool_paths, + compile_flags = compile_flags, + dbg_compile_flags = dbg_compile_flags, + opt_compile_flags = opt_compile_flags, + cxx_flags = cxx_flags, + link_flags = link_flags, + # link_libs = _, + opt_link_flags = opt_link_flags, + unfiltered_compile_flags = unfiltered_compile_flags, + coverage_compile_flags = coverage_compile_flags, + coverage_link_flags = coverage_link_flags, + supports_start_end_lib = supports_start_end_lib, + + # This was only added in bazel v4.0.0. + # + # See: https://github.com/bazelbuild/bazel/commit/da345f1f249ebf28bec88c6e0d63260dfaef14e9 + **( + {"builtin_sysroot": builtin_sysroot or "/"} + if sysroot_prefix_supported + else {} + ) + ) diff --git a/toolchain/cc_wrapper.sh.tpl b/toolchain/cc_wrapper.sh.tpl index b2afed71..e3e7ea0d 100755 --- a/toolchain/cc_wrapper.sh.tpl +++ b/toolchain/cc_wrapper.sh.tpl @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# OS X relpath is not really working. This is a wrapper script around gcc +# OS X realpath is not really working. This is a wrapper script around gcc # to simulate relpath behavior. # # This wrapper uses install_name_tool to replace all paths in the binary @@ -56,8 +56,19 @@ done if [[ "${PATH}:" == *"%{toolchain_path_prefix}bin:"* ]]; then # GoCompile sets the PATH to the directory containing the linker, and changes CWD. clang "$@" -else +elif [[ -f %{toolchain_path_prefix}bin/clang ]]; then %{toolchain_path_prefix}bin/clang "$@" +else + # Some consumers of `CcToolchainConfigInfo`s will use a PWD that *isn't* + # the execroot (i.e. `cmake` from `rules_foreign_cc`). For cases like this, + # we'll try to find `clang` by assuming it's next to this script. + potential_clang_path="$(dirname "${0}")/clang" + if [[ -f ${potential_clang_path} ]]; then + "${potential_clang_path}" "${@}" + else + echo "ERROR: could not find clang; PWD is: $(pwd)." + exit 5 + fi fi function get_library_path() { diff --git a/toolchain/deps.bzl b/toolchain/deps.bzl index bb6f8264..8224c5f5 100644 --- a/toolchain/deps.bzl +++ b/toolchain/deps.bzl @@ -23,3 +23,13 @@ def bazel_toolchain_dependencies(): strip_prefix = "rules_cc-726dd8157557f1456b3656e26ab21a1646653405", urls = ["https://github.com/bazelbuild/rules_cc/archive/726dd8157557f1456b3656e26ab21a1646653405.tar.gz"], ) + + if not native.existing_rule("platforms"): + http_archive( + name = "platforms", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.4/platforms-0.0.4.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.4/platforms-0.0.4.tar.gz", + ], + sha256 = "079945598e4b6cc075846f7fd6a9d0857c33a7afc0de868c2ccb96405225135d", + ) diff --git a/toolchain/internal/configure.bzl b/toolchain/internal/configure.bzl index 9522db80..8e2cd8a6 100644 --- a/toolchain/internal/configure.bzl +++ b/toolchain/internal/configure.bzl @@ -12,6 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +load( + "@com_grail_bazel_toolchain//toolchain/internal:extra_targets.bzl", + _cpu_names = "cpu_names", + _extra_target_setup = "extra_target_setup", + _overrides_for_target = "overrides_for_target", + _split_target_triple = "split_target_triple", + _target_triple_to_constraints = "target_triple_to_constraints", +) load( "@com_grail_bazel_toolchain//toolchain/internal:llvm_distributions.bzl", _download_llvm = "download_llvm", @@ -19,7 +27,8 @@ load( ) load( "@com_grail_bazel_toolchain//toolchain/internal:sysroot.bzl", - _sysroot_path = "sysroot_path", + _host_sysroot_path = "host_sysroot_path", + _target_sysroot_path = "target_sysroot_path", ) load("@rules_cc//cc:defs.bzl", _cc_toolchain = "cc_toolchain") @@ -30,12 +39,194 @@ def _makevars_ld_flags(rctx): # lld, as of LLVM 7, is experimental for Mach-O, so we use it only on linux. return "-fuse-ld=lld" -def _include_dirs_str(rctx, cpu): - dirs = rctx.attr.cxx_builtin_include_directories.get(cpu) +def _include_dirs_str(rctx, host_platform): + dirs = rctx.attr.cxx_builtin_include_directories.get(host_platform) if not dirs: return "" return ("\n" + 12 * " ").join(["\"%s\"," % d for d in dirs]) +def _host_os(rctx): + os_name = rctx.os.name + + if os_name == "mac os x": return "darwin" + elif os_name == "linux": return "linux" + else: + fail("Unsupported OS: " + os_name) + +def _extra_toolchains_for_toolchain_suite(target_triple, cpus_in_toolchain_suite, host_os): + arch, _vendor, _os, _env = _split_target_triple(target_triple) + cpus = _cpu_names(arch) + + # This is more than a little broken. + # + # `cc_toolchain_suite` only allows us to associate a toolchain with a + # particular `--cpu` and `--compiler` value and does not provide any way + # to "select" a toolchain based on the host platform, just on the target + # cpu. The `toolchains` param on `cc_toolchain_suite` is also + # non-configurable (and I don't know of a good way to select on _host_ + # platform anyways) so to get around this, we only emit the toolchain + # entries for the current host platform here (i.e. we fix on the host OS + # at repo rule evaluation time rather than letting Bazel take care of it + # later like we do most everywhere else). Ultimately, this is fine. + # + # See: https://docs.bazel.build/versions/main/be/c-cpp.html#cc_toolchain_suite.toolchains + # + # Worse, absent documentation/consensus for what values of `--cpu` are + # commonly used, we just use the CPU values for the `@platforms//cpu` + # constraint; these may or may not match what is commonly used and what + # users are putting in their platform mappings. + # + # See: https://docs.bazel.build/versions/main/platforms-intro.html#platform-mappings + # + # Since toolchain resolution is what will be supported in the future (for + # which we do the right thing), this isn't the end of the world. It's + # unclear if it's possible for us to do better here though. (TODO) + + # Bazel complains if we have multiple entries in a `cc_toolchain_suite` for + # a particular "cpu" so we only take the first target triple's toolchain + # that provides a toolchain for a particular "cpu". + # + # i.e. if given `wasm32-unknown-unknown` and `wasm32-unknown-wasi` as + # `extra_targets` – both of which are suitable toolchains for + # `cpu = "wasm32"` – whichever hits this function first will be used as + # the "wasm32" entry in the generated `cc_toolchain_suite`. + # + # This is definitely not great but as mentioned, toolchain resolution is + # really what users should be using anyways. + cpus = [ c for c in cpus if c not in cpus_in_toolchain_suite ] + cpus_in_toolchain_suite.update(**{ c: None for c in cpus }) + + if len(cpus) > 0: + return "# For `{}`:".format(target_triple) + ''.join([ + """ + "{cpu}": ":cc-clang-{host_os}_{target}", + "{cpu}|clang": ":cc-clang-{host_os}_{target}",\n""".format( + cpu = cpu, + host_os = host_os, + target = target_triple, + ) + for cpu in cpus + ]) + else: + return "# No `cc_toolchain_suite` entries for `{}`.".format( + target_triple + ) + + +def _extra_cc_toolchain_config(rctx, target_triple, extra_sysroots): + target_constraints = _target_triple_to_constraints(target_triple) + arch, _vendor, os, _env = _split_target_triple(target_triple) + + extra_overrides = _overrides_for_target(rctx, target_triple) + + # Use the architecture as the CPU name if there is no @platforms//cpu value + # for the architecture. + cpu = (_cpu_names(arch) + [arch])[0] + + sysroot_override = "" + sysroot_definition = "" + if target_triple in extra_sysroots: + path, label = extra_sysroots[target_triple] + sysroot_override = """"sysroot_path": "{path}",""".format(path = path) + + if label: + sysroot_definition = """ +filegroup( + name = "sysroot_components_{target_triple}", + srcs = ["{label}"], +) + +""".format( + target_triple = target_triple, + label = label, +) + + overrides_var_name = target_triple.replace("-", "_").upper() + "_OVERRIDES" + return """ +# For `{target}`: + +{sysroot_definition} +{overrides_var_name} = {{ + "target_system_name": "{target}", + "target_cpu": "{cpu}", + "target_libc": "{os}", + "abi_libc_version": "{os}", + {sysroot_override} +}} +{overrides_var_name}.update({extra_overrides}) + +cc_toolchain_config( + name = "local_linux_{target}", + host_platform = "k8", + custom_target_triple = "{target}", + overrides = {overrides_var_name}, +) + +cc_toolchain_config( + name = "local_darwin_{target}", + host_platform = "darwin", + custom_target_triple = "{target}", + overrides = {overrides_var_name}, +) + +toolchain( + name = "cc-toolchain-linux_{target}", + exec_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + target_compatible_with = [ + {target_constraints} + ], + toolchain = ":cc-clang-linux_{target}", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +toolchain( + name = "cc-toolchain-darwin_{target}", + exec_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:osx", + ], + target_compatible_with = [ + {target_constraints} + ], + toolchain = ":cc-clang-darwin_{target}", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) +""".format( + target = target_triple, + cpu = cpu, + os = os, + sysroot_override = sysroot_override, + extra_overrides = extra_overrides, + sysroot_definition = sysroot_definition, + target_constraints = '\n '.join([ + '"{}",'.format(c) for c in target_constraints + ]), + overrides_var_name = overrides_var_name, +) + +def _extra_conditional_cc_toolchain_config(rctx, target_triple, extra_sysroots): + absolute_paths = rctx.attr.absolute_paths + + sysroot_label = ":host_sysroot_components" # this is the fallback + if target_triple in extra_sysroots: + _path, label = extra_sysroots[target_triple] + if label: + # defined by `_extra_cc_toolchain_config` + sysroot_label = ":sysroot_components_{}".format(target_triple) + + return """ +# For `{target}`: +conditional_cc_toolchain("cc-clang-linux_{target}", ":local_linux_{target}", False, "{sysroot_label}", {abs_paths}) +conditional_cc_toolchain("cc-clang-darwin_{target}", ":local_darwin_{target}", True, "{sysroot_label}", {abs_paths}) +""".format( + target = target_triple, + abs_paths = absolute_paths, + sysroot_label = sysroot_label, + ) + def llvm_toolchain_impl(rctx): if rctx.os.name.startswith("windows"): rctx.file("BUILD") @@ -52,20 +243,51 @@ def llvm_register_toolchains(): else: toolchain_path_prefix = relative_path_prefix - sysroot_path, sysroot = _sysroot_path(rctx) + extra_sysroots = { + target: _target_sysroot_path(rctx, target) for target in rctx.attr.extra_targets + } + + cpus_already_in_toolchain_suite = {} + host_os = _host_os(rctx) + + host_sysroot_path, host_sysroot = _host_sysroot_path(rctx) + host_sysroot_label = "\"%s\"" % str(host_sysroot) if host_sysroot else "" substitutions = { "%{repo_name}": rctx.name, "%{llvm_version}": rctx.attr.llvm_version, + "%{bazel_version}": native.bazel_version, "%{toolchain_path_prefix}": toolchain_path_prefix, "%{tools_path_prefix}": (repo_path + "/") if rctx.attr.absolute_paths else "", "%{debug_toolchain_path_prefix}": relative_path_prefix, - "%{sysroot_path}": sysroot_path, - "%{sysroot_prefix}": "%sysroot%" if sysroot_path else "", - "%{sysroot_label}": "\"%s\"" % str(sysroot) if sysroot else "", + "%{host_sysroot_path}": host_sysroot_path, + "%{sysroot_prefix}": "%sysroot%", + "%{host_sysroot_label}": host_sysroot_label, "%{absolute_paths}": "True" if rctx.attr.absolute_paths else "False", "%{makevars_ld_flags}": _makevars_ld_flags(rctx), "%{k8_additional_cxx_builtin_include_directories}": _include_dirs_str(rctx, "k8"), "%{darwin_additional_cxx_builtin_include_directories}": _include_dirs_str(rctx, "darwin"), + + "%{extra_cc_toolchain_config}": '\n'.join([ + _extra_cc_toolchain_config(rctx, t, extra_sysroots) for t in rctx.attr.extra_targets + ]), + "%{extra_conditional_cc_toolchain_config}": '\n'.join([ + _extra_conditional_cc_toolchain_config(rctx, t, extra_sysroots) for t in rctx.attr.extra_targets + ]), + "%{extra_toolchains_for_toolchain_suite}": '\n '.join([ + _extra_toolchains_for_toolchain_suite(t, cpus_already_in_toolchain_suite, host_os) + for t in rctx.attr.extra_targets + ]), + "%{extra_toolchains_for_registration}": ' \n'.join([ + "# For `{target}`:\n".format(target = target) + ''.join([ + """ "@{repo}//:cc-toolchain-{host}_{target}",\n""".format( + repo = rctx.name, + host = host, + target = target, + ) + for host in ["linux", "darwin"] + ]) + for target in rctx.attr.extra_targets + ]), } rctx.template( @@ -109,41 +331,71 @@ def llvm_register_toolchains(): if not _download_llvm(rctx): _download_llvm_preconfigured(rctx) -def conditional_cc_toolchain(name, darwin, absolute_paths = False): - # Toolchain macro for BUILD file to use conditional logic. + # Finally, do additional set up for the extra targets: + for target in rctx.attr.extra_targets: + _extra_target_setup(rctx, target) - toolchain_config = "local_darwin" if darwin else "local_linux" - toolchain_identifier = "clang-darwin" if darwin else "clang-linux" +def conditional_cc_toolchain( + name, + toolchain_config, + host_is_darwin, + sysroot_label, + absolute_paths = False, + llvm_repo_label_prefix = None, +): + # If not specified, use the repo of the callee. + if not llvm_repo_label_prefix: + llvm_repo_label_prefix = "" + # Toolchain macro for BUILD file to use conditional logic. if absolute_paths: + empty = "{}:empty".format(llvm_repo_label_prefix) + _cc_toolchain( name = name, - all_files = ":empty", - compiler_files = ":empty", - dwp_files = ":empty", - linker_files = ":empty", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 0 if darwin else 1, + all_files = empty, + compiler_files = empty, + dwp_files = empty, + linker_files = empty, + objcopy_files = empty, + strip_files = empty, + supports_param_files = 0 if host_is_darwin else 1, toolchain_config = toolchain_config, ) else: - extra_files = [":cc_wrapper"] if darwin else [] - native.filegroup(name = name + "-all-files", srcs = [":all_components"] + extra_files) - native.filegroup(name = name + "-archiver-files", srcs = [":ar"] + extra_files) - native.filegroup(name = name + "-assembler-files", srcs = [":as"] + extra_files) - native.filegroup(name = name + "-compiler-files", srcs = [":compiler_components"] + extra_files) - native.filegroup(name = name + "-linker-files", srcs = [":linker_components"] + extra_files) + r = llvm_repo_label_prefix + + cc_wrapper = "{}:cc_wrapper".format(r) + clang = "{}:clang".format(r) + include = "{}:include".format(r) + ld = "{}:ld".format(r) + ar = "{}:ar".format(r) + lib = "{}:lib".format(r) + binutils_components = "{}:binutils_components".format(r) + as_ = "{}:as".format(r) + empty = "{}:empty".format(r) + objcopy = "{}:objcopy".format(r) + + extra_files = [cc_wrapper] if host_is_darwin else [] + native.filegroup(name = name + "-compiler_components", srcs = [clang, include, sysroot_label]) + native.filegroup(name = name + "-linker_components", srcs = [clang, ld, ar, lib, sysroot_label]) + native.filegroup(name = name + "-all_components", srcs = [binutils_components, name + "-compiler_components", name + "-linker_components"]) + + native.filegroup(name = name + "-all-files", srcs = [name + "-all_components"] + extra_files) + native.filegroup(name = name + "-archiver-files", srcs = [ar] + extra_files) + native.filegroup(name = name + "-assembler-files", srcs = [as_] + extra_files) + native.filegroup(name = name + "-compiler-files", srcs = [name + "-compiler_components"] + extra_files) + native.filegroup(name = name + "-linker-files", srcs = [name + "-linker_components"] + extra_files) _cc_toolchain( name = name, all_files = name + "-all-files", ar_files = name + "-archiver-files", as_files = name + "-assembler-files", compiler_files = name + "-compiler-files", - dwp_files = ":empty", + dwp_files = empty, linker_files = name + "-linker-files", - objcopy_files = ":objcopy", - strip_files = ":empty", - supports_param_files = 0 if darwin else 1, + objcopy_files = objcopy, + strip_files = empty, + supports_param_files = 0 if host_is_darwin else 1, toolchain_config = toolchain_config, ) diff --git a/toolchain/internal/extra_targets.bzl b/toolchain/internal/extra_targets.bzl new file mode 100644 index 00000000..c7b9e73d --- /dev/null +++ b/toolchain/internal/extra_targets.bzl @@ -0,0 +1,448 @@ +""" +Helpers for configuring toolchains that target platforms different than the +host platform. +""" + +load("//toolchain/internal/extra_targets:wasi.bzl", "get_wasi_sysroot", "install_wasi_compiler_rt") + +# A mapping from [LLVM target triple][tt] [architectures][a] to Bazel platform +# [CPU][c]s. +# +# [tt]: https://llvm.org/doxygen/Triple_8h_source.html +# [a]: https://llvm.org/doxygen/classllvm_1_1Triple.html#a547abd13f7a3c063aa72c8192a868154 +# [c]: https://github.com/bazelbuild/platforms/blob/main/cpu/BUILD +LLVM_ARCH_TO_BAZEL_PLATFORM_CPU = { + # ARM (little endian): arm, armv.*, xscale + # -> ARM subarch table + + # ARM (big endian): armeb + "armeb": None, + # AArch64 (little endian): aarch64 + "aarch64": "aarch64", + # AArch64 (big endian): aarch64_be + "aarch64_be": None, + # AArch64 (little endian) ILP32: aarch64_32 + "aarch64_32": "arm64_32", + # ARC: Synopsys ARC + "arc": None, + # AVR: Atmel AVR microcontroller + "avr": None, + # eBPF or extended BPF or 64-bit BPF (little endian) + "bpfel": None, + # eBPF or extended BPF or 64-bit BPF (big endian) + "bpfeb": None, + # CSKY: csky + "csky": None, + # Hexagon: hexagon + "hexagon": None, + # M68k: Motorola 680x0 family + "m68k": None, + # MIPS: mips, mipsallegrex, mipsr6 + "mips": None, + "mipsallegrex": None, + "mipsr6": None, + # MIPSEL: mipsel, mipsallegrexe, mipsr6el + "mipsel": None, + "mipsallegrexe": None, + "mipsr6el": None, + # MIPS64: mips64, mips64r6, mipsn32, mipsn32r6 + "mips64": "mips64", + "mips64r6": "mips64", + "mipsn32": "mips64", + "mipsn32r6": "mips64", + # MIPS64EL: mips64el, mips64r6el, mipsn32el, mipsn32r6el + "mips64el": None, + "mips64r6el": None, + "mipsn32el": None, + "mipsn32r6el": None, + # MSP430: msp430 + "msp430": None, + # PPC: powerpc + "ppc": "ppc", + # PPCLE: powerpc (little endian) + "ppcle": None, + # PPC64: powerpc64, ppu + "ppc64": None, + # PPC64LE: powerpc64le + "ppc64le": None, + # R600: AMD GPUs HD2XXX - HD6XXX + "r600": None, + # AMDGCN: AMD GCN GPUs + "amdgcn": None, + # RISC-V (32-bit): riscv32 + "riscv32": None, + # RISC-V (64-bit): riscv64 + "riscv64": "riscv64", + # Sparc: sparc + "sparc": None, + # Sparcv9: Sparcv9 + "sparcv9": None, + # Sparc: (endianness = little). NB: 'Sparcle' is a CPU variant + "sparcel": None, + # SystemZ: s390x + "systemz": "s390x", + "s390x": "s390x", + # TCE (http://tce.cs.tut.fi/): tce + "tce": None, + # TCE little endian (http://tce.cs.tut.fi/): tcele + "tcele": None, + # Thumb (little endian): thumb, thumbv.* + # -> ARM subarch table + + # Thumb (big endian): thumbeb + "thumbeb": "arm", + # X86: i[3-9]86 + "x86": ["i386", "x86_32"], + "i386": ["i386", "x86_32"], + "i486": ["i386", "x86_32"], + "i586": ["i386", "x86_32"], + "i686": ["i386", "x86_32"], + "i786": ["i386", "x86_32"], + "i886": ["i386", "x86_32"], + "i986": ["i386", "x86_32"], + # X86-64: amd64, x86_64 + "x86_64": "x86_64", + "amd64": "x86_64", + # XCore: xcore + "xcore": None, + # NVPTX: 32-bit + "nvptx": None, + # NVPTX: 64-bit + "nvptx64": None, + # le32: generic little-endian 32-bit CPU (PNaCl) + "le32": None, + # le64: generic little-endian 64-bit CPU (PNaCl) + "le64": None, + # AMDIL + "amdil": None, + # AMDIL with 64-bit pointers + "amdil64": None, + # AMD HSAIL + "hsail": None, + # AMD HSAIL with 64-bit pointers + "hsail64": None, + # SPIR: standard portable IR for OpenCL 32-bit version + "spir": None, + # SPIR: standard portable IR for OpenCL 64-bit version + "spir64": None, + # Kalimba: generic kalimba + "kalimba": None, + # SHAVE: Movidius vector VLIW processors + "shave": None, + # Lanai: Lanai 32-bit + "lanai": None, + # WebAssembly with 32-bit pointers + "wasm32": "wasm32", + # WebAssembly with 64-bit pointers + "wasm64": "wasm64", + # 32-bit RenderScript + "renderscript32": None, + # 64-bit RenderScript + "renderscript64": None, + # NEC SX-Aurora Vector Engine + "ve": None, +} + +# For `armv.*` and `thumbv.*`. +# +# Not all of these sub-architectures are allowed for both `arm` and `thumb` +# target triples but this is a good enough for what we're doing. +# +# See this file for some context: https://github.com/llvm/llvm-project/blob/main/llvm/lib/Support/ARMTargetParser.cpp +# And this: https://llvm.org/doxygen/classllvm_1_1Triple.html#a9ffca842bbaefcf99484f59a83b618d4 +# +# TODO: verify that these are right +LLVM_ARCH_TO_BAZEL_PLATFORM_CPU_ARM_SUBARCHS = { + "v8.7a": "aarch64", + "v8.6a": "aarch64", + "v8.5a": "aarch64", + "v8.4a": "aarch64", + "v8.3a": "aarch64", + "v8.2a": "aarch64", + "v8.1a": "aarch64", + "v8": "aarch64", + "v8r": None, + "v8m.base": "armv8-m", + "v8m.main": "armv8-m", + "v8.1m.main": "armv8-m", + "v7": "armv7", + # This is questionable; `hf` is typically in the env part of the triple so + # we could check for that and then pick between `armv7e-mf` and `armv7e-m`. + # + # Or we could just say the compiler supports both (which *is* true). + # + # Eventually we should probably do the former though (TODO). + # + # Ideally `arm-fpu` would be it's own constraint. + "v7em": ["armv7e-m", "armv7e-mf"], + "v7m": "armv7m", + "v7s": None, + "v7k": "armv7k", + "v7ve": None, + "v6": None, + "v6m": "armv6-m", + "v6k": None, + "v6t2": None, + "v5": None, + "v5te": None, + "v4t": None, + + # For arm64e: + "64e": "arm64e" +} + +# A mapping from [LLVM target triple][tt] [operating systems][o] to Bazel +# platform [operating systems][baz-o] constraints. +# +# [tt]: https://llvm.org/doxygen/Triple_8h_source.html +# [o]: https://llvm.org/doxygen/classllvm_1_1Triple.html#a3cfefc755ab656000934f91193afb1cd +# [baz-o]: https://github.com/bazelbuild/platforms/blob/main/os/BUILD +# +# See: https://github.com/llvm/llvm-project/blob/944dfa4975e8d55ca9d97f6eb7222ff1d0f7291a/llvm/lib/Support/Triple.cpp#L505-L541 +LLVM_OS_TO_BAZEL_PLATFORM_OS = { + "ananas": None, + "cloudabi": None, + "darwin": None, + "dragonfly": None, + "freebsd": "freebsd", + "fuchsia": None, + "ios": "ios", + "kfreebsd": None, + "linux": "linux", + "lv2": None, + "macos": "macos", + "netbsd": None, + "openbsd": "openbsd", + "solaris": None, + "win32": None, + "windows": "windows", + "zos": None, + "haiku": None, + "minix": None, + "rtems": None, + "nacl": None, + "aix": None, + "cuda": None, + "nvcl": None, + "amdhsa": None, + "ps4": None, + "elfiamcu": None, + "tvos": "tvos", + "watchos": "watchos", + "mesa3d": None, + "contiki": None, + "amdpal": None, + "hermit": None, + "hurd": None, + "wasi": "wasi", + "emscripten": None, + + # No OS; bare metal. + "none": "none", +} + +# A mapping from [LLVM target triple][tt] [environments][e] to Bazel +# platform constraints. +# +# [tt]: https://llvm.org/doxygen/Triple_8h_source.html +# [e]: https://llvm.org/doxygen/classllvm_1_1Triple.html#a1778f5c464f88710033f7e11e84a9324 +# +# See: https://github.com/llvm/llvm-project/blob/944dfa4975e8d55ca9d97f6eb7222ff1d0f7291a/llvm/lib/Support/Triple.cpp#L544-L568 +LLVM_ENV_TO_BAZEL_PLATFORM_CONSTRAINTS = { + "eabihf": None, + "eabi": None, + "gnuabin32": None, + "gnuabi64": None, + "gnueabihf": None, + "gnueabi": None, + "gnux32": None, + "gnu_ilp32": None, + "code16": None, + "gnu": None, + "android": "os:android", + "musleabihf": None, + "musleabi": None, + "muslx32": None, + "musl": None, + "msvc": None, + "itanium": None, + "cygnus": None, + "coreclr": None, + "simulator": None, + "macabi": None, +} + +def prefix_list_or_single(constraint_base, constraints): + if type(constraints) == "list": + return ["{}{}".format(constraint_base, c) for c in constraints] + elif constraints: + return ["{}{}".format(constraint_base, constraints)] + else: + return [] + +def cpu_names(arch): + constraints = None + + if arch.startswith("arm"): + constraints = LLVM_ARCH_TO_BAZEL_PLATFORM_CPU_ARM_SUBARCHS.get(arch[len("arm"):]) + + # If a more specific constraint isn't available, fall back to "arm": + if not constraints: constraints = "arm" + elif arch.startswith("thumb"): + constraints = LLVM_ARCH_TO_BAZEL_PLATFORM_CPU_ARM_SUBARCHS.get(arch[len("thumb"):]) + else: + if arch not in LLVM_ARCH_TO_BAZEL_PLATFORM_CPU: + fail("Unrecognized architecture: `{}`.".format(arch)) + constraints = LLVM_ARCH_TO_BAZEL_PLATFORM_CPU.get(arch) + + return prefix_list_or_single("", constraints) + +def cpu_constraints(arch): + return prefix_list_or_single("@platforms//cpu:", cpu_names(arch)) + +def os_constraints(os): + # NOTE: we do not error if the given OS name is not in our table. + constraints = LLVM_OS_TO_BAZEL_PLATFORM_OS.get(os) + + return prefix_list_or_single("@platforms//os:", constraints) + +def env_constraints(env): + # `env` is optional: + if not env: return [] + + if env in LLVM_ENV_TO_BAZEL_PLATFORM_CONSTRAINTS: + constraints = LLVM_ENV_TO_BAZEL_PLATFORM_CONSTRAINTS[env] + + return prefix_list_or_single("@platforms//", constraints) + else: + fail("Unrecognized environment in target triple: `{}`.".format(env)) + +def split_target_triple(triple): + """Splits a target triple into its parts. + + Args: + triple: the triple to split + + Returns: + the triple's constituent architecture, vendor, operating system, and env + parts + """ + + # [As per LLVM](https://llvm.org/doxygen/Triple_8h_source.html), a triple + # consists of: `arch-vendor-operating_system(-environment)?`. + parts = triple.split("-") + + if len(parts) == 3: + parts.append(None) + if len(parts) != 4: + fail("`{}` is not a valid target triple.".format(triple)) + + return parts + +def target_triple_to_constraints(triple): + arch, _vendor, os, env = split_target_triple(triple) + + # NOTE: we don't generate constraints from the vendor part. + return cpu_constraints(arch) + os_constraints(os) + env_constraints(env) + +# TODO: this kind of logic is what needs to be filled in for other targets: + +README = "\n\nSee https://github.com/grailbio/bazel-toolchain/blob/master/README.md#setting-up-toolchains-for-other-targets" + \ + " for more information." + +def overrides_for_target(rctx, triple): + arch, _vendor, os, _env = split_target_triple(triple) + llvm_version = rctx.attr.llvm_version + llvm_major_version = int(llvm_version.split(".")[0]) + + if arch == "wasm32" or arch == "wasm64": + overrides = { + "omit_hosted_linker_flags": True, + "omit_cxx_stdlib_flag": True, + + # libtool doesn't seem to be able to handle wasm + "use_llvm_ar_instead_of_libtool_on_macos": True, + + # lld ultimately shells out to `wasm-ld` which does *not* support + # start end groups for libraries which is why this override is + # important + "custom_linker_tool": { + "darwin": "wasm-ld", + "k8": "wasm-ld", + }, + + # wasm-ld doesn't understand `-l:libfoo.a` style syntax unfortunately + "prefer_static_cxx_libs_on_linux_hosts": False, + + # not yet supported on wasm; see: https://github.com/WebAssembly/tool-conventions/issues/133 + "linker_include_build_id_on_linux_hosts": False, + + # not support by `wasm-ld`: + "linker_use_gnu_hash_style_on_linux_hosts": False, + + # not applicable for wasm (we're not dynamically linking): + "linker_use_elf_hardening_so_flags_on_linux_hosts": False, + } + + # `clang-12` specifically uses `-mthread-model posix` by default + # which causes `libc++` to define `__STDCPP_THREADS__` which doesn't + # play nice with the WASI libc defines. + # + # This is "fixed" in newer versions of LLVM: + # https://reviews.llvm.org/D96091 + # + # But for `clang-12` we need to pass in `-mthread-model single`. + # + # See: https://github.com/WebAssembly/wasi-sdk/issues/173 + if llvm_major_version == 12: + overrides.update({ + "extra_compile_flags": ["-mthread-model", "single"], + }) + + return overrides + else: + print( + ("`{}` support has not been added to bazel-toolchain; you may " + + "need to manually adjust compiler flags and toolchain " + + "configurations yourself!" + README).format(triple) + ) + + return {} + +def sysroot_for_target(rctx, triple): + arch, _vendor, os, _env = split_target_triple(triple) + + if not rctx.path("sysroots").exists: + rctx.file( + "sysroots/BUILD", + content = "", + executable = False, + ) + + # NOTE: I think this sysroot can be used on `wasm32-unknown-unknown` too; + # it seems to gate all wasi functionality correctly. + if arch == "wasm32" and (os == "wasi" or os == "unknown" or os == "none"): + return get_wasi_sysroot(rctx, for_non_wasi = os != "wasi") + else: + print( + ("`{}` support has not been added to bazel-toolchain; you may " + + "need to find a sysroot and manually configure a toolchain " + + "yourself!" + README).format(triple) + ) + + return None + +# Runs *after* the toolchain has been fetched and extracted. +def extra_target_setup(rctx, triple): + arch, _vendor, os, _env = split_target_triple(triple) + + # TODO: I think compiler_rt for wasi can be used on + # `wasm32-unknown-unknown` too. + if arch == "wasm32" and (os == "wasi" or os == "unknown" or os == "none"): + install_wasi_compiler_rt(rctx, for_non_wasi = os != "wasi") + else: + print( + ("`{}` support has not been added to bazel-toolchain; you may " + + "need to grab compiler_rt or do additional toolchain setup " + + "yourself!" + README).format(triple) + ) diff --git a/toolchain/internal/extra_targets/BUILD b/toolchain/internal/extra_targets/BUILD new file mode 100644 index 00000000..e69de29b diff --git a/toolchain/internal/extra_targets/wasi.bzl b/toolchain/internal/extra_targets/wasi.bzl new file mode 100644 index 00000000..4f8104fb --- /dev/null +++ b/toolchain/internal/extra_targets/wasi.bzl @@ -0,0 +1,179 @@ +WASI_SYSROOT_LINKS = { + 8: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-8/wasi-sysroot-8.0.tar.gz", + "57fbc9b41f1daf99fa197ed026f105e38cbba0828333ffb3c24c835d660c5499", + ), + 9: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-9/wasi-sysroot-9.0.tar.gz", + "7aeaed38d3f4d02350460a6e8f2b73db8d732d30f659095fe58440d24d6dbdd7", + ), + 10: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-10/wasi-sysroot-10.0.tar.gz", + "d87ddfa3c460faa6960d2440b51370f626603635cff310138a9c14757d1307d9", + ), + 11: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/wasi-sysroot-11.0.tar.gz", + "7523bc938efa491108b519101208a8e1dec794041377eb05a6102620ce43220a", + ), + 12: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sysroot-12.0.tar.gz", + "eed85df38110578a0366478c696cb755a6a01167a23ac1de70138b748401a2b4", + ), +} + +# A fallback (so that users can use the newer versions without us *having* to update this list): +def wasi_sysroot_url(llvm_major_version): + return "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{v}/wasi-sysroot-{v}.0.tar.gz".format(v = llvm_major_version) + +WASI_SYSROOT_EXTRACT_PATH = "sysroots/wasi" +WASM_SYSROOT_PATH = "sysroots/wasm" + +_sysroot_build_file_contents = """ +filegroup( + name = "sysroot", + srcs = glob(["*/**"]), + visibility = ["//visibility:public"], +) +""" + +# Returns a `Label` in the current repository pointing to the sysroot that was fetched. +def get_wasi_sysroot(rctx, for_non_wasi = False): + if not rctx.path(WASI_SYSROOT_EXTRACT_PATH).exists: + llvm_version = rctx.attr.llvm_version + llvm_major_version = int(llvm_version.split(".")[0]) + common_download_params = { + "output": WASI_SYSROOT_EXTRACT_PATH, + "stripPrefix": "wasi-sysroot", + "canonical_id": str(llvm_major_version), + } + + if llvm_major_version in WASI_SYSROOT_LINKS: + url, sha = WASI_SYSROOT_LINKS[llvm_major_version] + rctx.download_and_extract( + url = url, + sha256 = sha, + **common_download_params + ) + else: + url = wasi_sysroot_url(llvm_major_version) + print("We don't have a WASI sysroot URL for LLVM {}; we'll try to use `{}`..".format(llvm_major_version, url)) + + res = rctx.download_and_extract( + url = url, + **common_download_params + ) + + print( + "\n\nIt worked! Feel free to make a PR adding `{}` as the WASI sysroot URL for LLVM {} with sha256 = `{}`.\n\n".format( + url, + llvm_major_version, + res.sha256 + ) + ) + + rctx.file( + WASI_SYSROOT_EXTRACT_PATH + "/BUILD", + executable = False, + content = _sysroot_build_file_contents, + ) + + # Because the WASI libc headers gate everything on WASI, we should be + # able to safely use the same WASI sysroot for non-WASI wasm32 targets. + if for_non_wasi: + if not rctx.path(WASM_SYSROOT_PATH).exists: + rctx.file( + WASM_SYSROOT_PATH + "/BUILD", + executable = False, + content = _sysroot_build_file_contents, + ) + + rctx.symlink( + WASI_SYSROOT_EXTRACT_PATH + "/lib/wasm32-wasi", + WASM_SYSROOT_PATH + "/lib", + ) + + rctx.symlink( + WASI_SYSROOT_EXTRACT_PATH + "/include", + WASM_SYSROOT_PATH + "/include", + ) + + return "@" + rctx.attr.name + "//" + WASM_SYSROOT_PATH + ":sysroot" + else: + return "@" + rctx.attr.name + "//" + WASI_SYSROOT_EXTRACT_PATH + ":sysroot" + + +WASI_COMPILER_RT_LINKS = { + 8: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-8/libclang_rt.builtins-wasm32-wasi-8.0.tar.gz", + "4b81bacf931820db80a41011320fc266117a3672eb5a4f4082caf533235a60f5", + ), + 9: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-9/libclang_rt.builtins-wasm32-wasi-9.0.tar.gz", + "b7d18202009f1528a4eb18c9a8551bb165d42475de27f6023156318f711b5abe", + ), + 10: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-10/libclang_rt.builtins-wasm32-wasi-10.0.tar.gz", + "4901bcf47107d6e696cf3900284b3fd5813caeb1f9d6f9f6b1960325b941429e", + ), + 11: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/libclang_rt.builtins-wasm32-wasi-11.0.tar.gz", + "6bf6998e5d9a4eacde44da4276c7c5be021c795e22243c689da905772a4be442", + ), + 12: ( + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/libclang_rt.builtins-wasm32-wasi-12.0.tar.gz", + "5a0d8b8ce56be1615dc87acefaaa01573760d03e6f59de0e45207f775eea963b", + ), +} + +# A fallback (so that users can use the newer versions without us *having* to update this list): +def wasi_compiler_rt_url(llvm_major_version): + return "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-{v}/libclang_rt.builtins-wasm32-wasi-{v}.0.tar.gz".format(v = llvm_major_version) + +COMPILER_RT_FILE_NAME = "libclang_rt.builtins-wasm32.a" + +def install_wasi_compiler_rt(rctx, for_non_wasi = False): + llvm_version = rctx.attr.llvm_version + llvm_major_version = int(llvm_version.split(".")[0]) + + output_file_non_wasi = "lib/clang/{}/lib/{}".format(llvm_version, COMPILER_RT_FILE_NAME) + output_dir_wasi = "lib/clang/{}/lib/wasi".format(llvm_version) + output_file_wasi = output_dir_wasi + "/" + COMPILER_RT_FILE_NAME + + common_download_params = { + "output": output_dir_wasi, + "stripPrefix": "lib/wasi", + "canonical_id": str(llvm_major_version), + } + + # Don't download it again if we've already grabbed it. + if not rctx.path(output_file_wasi).exists: + if llvm_major_version in WASI_COMPILER_RT_LINKS: + url, sha = WASI_COMPILER_RT_LINKS[llvm_major_version] + rctx.download_and_extract( + url = url, + sha256 = sha, + **common_download_params + ) + else: + url = wasi_compiler_rt_url(llvm_major_version) + print("We don't have a WASI compiler_rt URL for LLVM {}; we'll try to use `{}`..".format(llvm_major_version, url)) + + res = rctx.download_and_extract( + url = url, + **common_download_params + ) + + print( + "\n\nIt worked! Feel free to make a PR adding `{}` as the WASI compiler_rt URL for LLVM {} with sha256 = `{}`.\n\n".format( + url, + llvm_major_version, + res.sha256 + ) + ) + + # We should be able to reuse the WASI compiler-rt for non-WASI wasm32 targets: + if for_non_wasi and not rctx.path(output_file_non_wasi).exists: + rctx.symlink( + output_file_wasi, + output_file_non_wasi, + ) diff --git a/toolchain/internal/sysroot.bzl b/toolchain/internal/sysroot.bzl index 45880f3e..78c67b64 100644 --- a/toolchain/internal/sysroot.bzl +++ b/toolchain/internal/sysroot.bzl @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load(":extra_targets.bzl", "sysroot_for_target") + def _darwin_sdk_path(rctx): if rctx.os.name != "mac os x": return "" @@ -23,24 +25,16 @@ def _darwin_sdk_path(rctx): print(exec_result.stderr) return exec_result.stdout.strip() -def _default_sysroot(rctx): +def _default_host_sysroot(rctx): if rctx.os.name == "mac os x": return _darwin_sdk_path(rctx) else: return "" -# Return the sysroot path and the label to the files, if sysroot is not a system path. -def sysroot_path(rctx): - if rctx.os.name == "linux": - sysroot = rctx.attr.sysroot.get("linux", default = "") - elif rctx.os.name == "mac os x": - sysroot = rctx.attr.sysroot.get("darwin", default = "") - else: - fail("Unsupported OS: " + rctx.os.name) - - if not sysroot: - return (_default_sysroot(rctx), None) - +# Takes a sysroot absolute path or label and returns a (path, label) pair. +# +# If given an absolute path, the label will be `None`. +def process_sysroot(sysroot): # If the sysroot is an absolute path, use it as-is. Check for things that # start with "/" and not "//" to identify absolute paths, but also support # passing the sysroot as "/" to indicate the root directory. @@ -52,3 +46,44 @@ def sysroot_path(rctx): return (sysroot.workspace_root + "/" + sysroot.package, sysroot) else: return (sysroot.package, sysroot) + +# Checks if the user has explicitly specified a sysroot for the given target; if they +# have then it uses that. +# +# If not, this function consults `extra_targets.bzl%sysroot_for_target`. +# +# If that comes up empty, this function falls back to the default sysroot for the host. +def target_sysroot_path(rctx, target_triple): + sysroot = None + + # Try for an explicitly specified sysroot first: + if rctx.os.name == "linux": sysroot = rctx.attr.sysroot.get("linux_{}".format(target_triple)) + elif rctx.os.name == "mac os x": sysroot = rctx.attr.sysroot.get("darwin_{}".format(target_triple)) + else: + fail("Unsupported OS: " + rctx.os.name) + + if sysroot: return process_sysroot(sysroot) + + # If that didn't work, consult `sysroot_for_target`: + sysroot = sysroot_for_target(rctx, target_triple) + + # `sysroot_for_target` always returns a Label but it's fine; we can still + # call `process_sysroot_` + if sysroot: return process_sysroot(sysroot) + + # Finally, as a last resort just use the default sysroot (an absolute path): + return (_default_host_sysroot(rctx), None) + +# Return the sysroot path and the label to the files, if sysroot is not a system path. +def host_sysroot_path(rctx): + if rctx.os.name == "linux": + sysroot = rctx.attr.sysroot.get("linux", default = "") + elif rctx.os.name == "mac os x": + sysroot = rctx.attr.sysroot.get("darwin", default = "") + else: + fail("Unsupported OS: " + rctx.os.name) + + if not sysroot: + return (_default_host_sysroot(rctx), None) + else: + return process_sysroot(sysroot) diff --git a/toolchain/rules.bzl b/toolchain/rules.bzl index 0cb40a8c..23eb4fb2 100644 --- a/toolchain/rules.bzl +++ b/toolchain/rules.bzl @@ -40,7 +40,13 @@ llvm_toolchain = repository_rule( "the set of files that form the sysroot for the compiler. If the value begins " + "with exactly one forward slash '/', then the value is assumed to be a system " + "path. Else, the value will be assumed to be a label containing the files and " + - "the sysroot path will be taken as the path to the package of this label."), + "the sysroot path will be taken as the path to the package of this label." + + "" + + "Note that this can also be used to specify sysroots for additional targets " + + "configured with `extra_targets`. For example, to specify sysroots for the " + + "toolchain for `wasm32-unknown-unknown`, use " + + "`{ 'linux_wasm32-unknown-unknown': '...', 'darwin_wasm32-unknown-unknown': " + + "'...' }`."), ), "cxx_builtin_include_directories": attr.string_list_dict( mandatory = False, @@ -55,6 +61,10 @@ llvm_toolchain = repository_rule( default = False, doc = "Use absolute paths in the toolchain. Avoids sandbox overhead.", ), + "extra_targets": attr.string_list( + default = [], + doc = "Extra target triples to configure toolchains for.", + ), "_llvm_release_name": attr.label( default = "@com_grail_bazel_toolchain//toolchain/tools:llvm_release_name.py", allow_single_file = True, diff --git a/toolchain/toolchains.bzl.tpl b/toolchain/toolchains.bzl.tpl index 41db9ea2..0a238445 100644 --- a/toolchain/toolchains.bzl.tpl +++ b/toolchain/toolchains.bzl.tpl @@ -16,4 +16,8 @@ def llvm_register_toolchains(): native.register_toolchains( "@%{repo_name}//:cc-toolchain-linux", "@%{repo_name}//:cc-toolchain-darwin", + + %{extra_toolchains_for_registration} ) + +def register_toolchain(t): native.register_toolchains(t) diff --git a/toolchain/tools/llvm_release_name.py b/toolchain/tools/llvm_release_name.py index 8ee544ff..b04fb3f1 100755 --- a/toolchain/tools/llvm_release_name.py +++ b/toolchain/tools/llvm_release_name.py @@ -77,7 +77,9 @@ def _linux(llvm_version): os_name = "linux-sles%s" % version elif distname == "ubuntu" and version.startswith("14.04"): os_name = "linux-gnu-ubuntu-14.04" - elif (distname == "ubuntu" and version.startswith("20.04")) or (distname == "linuxmint" and version.startswith("20")): + elif ((distname == "ubuntu" and version.startswith("20.04")) or + (distname == "linuxmint" and version.startswith("20")) or + (distname == "pop" and version.startswith("20"))): if major_llvm_version < 11: # There is no binary packages specifically for 20.04, but those for 18.04 works on # 20.04 @@ -85,15 +87,17 @@ def _linux(llvm_version): else: # release 11.0.0 started providing packaging for ubuntu 20 os_name = "linux-gnu-ubuntu-20.04" - elif (distname == "ubuntu" and version.startswith("18.04")) or (distname == "linuxmint" and version.startswith("19")): - os_name = "linux-gnu-ubuntu-18.04" - elif (distname == "ubuntu" and version.startswith("20")) or (distname == "pop" and version.startswith("20")): - # use ubuntu 18.04 clang LLVM release for ubuntu 20.04 - os_name = "linux-gnu-ubuntu-18.04" + elif ((distname == "ubuntu" and version.startswith("18.04")) or + (distname == "linuxmint" and version.startswith("19")) or + (distname == "fedora" and major_llvm_version >= 7) or + (distname == "debian" and (version is None or int(version) == 10 or int(version) == 11))): + if major_llvm_version > 10: + # LLVM seems to have stopped providing binary packages for ubuntu 18.04. + os_name = "linux-gnu-ubuntu-16.04" + else: + os_name = "linux-gnu-ubuntu-18.04" elif distname in ["ubuntu", "manjaro"] or (distname == "linuxmint" and version.startswith("18")): os_name = "linux-gnu-ubuntu-16.04" - elif distname == "debian" and (version is None or int(version) == 10): - os_name = "linux-gnu-ubuntu-18.04" elif distname == "debian" and int(version) == 9 and major_llvm_version >= 7: os_name = "linux-gnu-ubuntu-16.04" elif distname == "debian" and int(version) == 8 and major_llvm_version < 7: @@ -103,12 +107,14 @@ def _linux(llvm_version): os_name = "linux-gnu-Fedora27" elif distname == "centos" and major_llvm_version >= 7: os_name = "linux-sles11.3" - elif distname == "fedora" and major_llvm_version >= 7: - os_name = "linux-gnu-ubuntu-18.04" elif distname == "arch" and major_llvm_version >= 11: os_name = "linux-gnu-ubuntu-20.04" elif distname == "arch" and major_llvm_version >= 10: - os_name = "linux-gnu-ubuntu-18.04" + if major_llvm_version > 10: + # LLVM seems to have stopped providing binary packages for ubuntu 18.04. + os_name = "linux-gnu-ubuntu-16.04" + else: + os_name = "linux-gnu-ubuntu-18.04" elif distname == "arch" and major_llvm_version >= 7: os_name = "linux-gnu-ubuntu-16.04" elif distname == "amzn":