From 4819ab0c991b6ce9958839a7f72f43ef518bb37e Mon Sep 17 00:00:00 2001 From: max Date: Sat, 9 Nov 2024 08:07:51 -0600 Subject: [PATCH] Add LLVM as submodule (and add GHA build scripts) --- .../ci_manylinux_x86_64_clang_llvm.yml | 118 ++++++++++++++++ .gitignore | 34 +++++ .gitmodules | 5 + build_tools/cmake/build_llvm.sh | 57 ++++++++ build_tools/cmake/llvm_cache.cmake | 127 ++++++++++++++++++ third_party/llvm-project | 1 + 6 files changed, 342 insertions(+) create mode 100644 .github/workflows/ci_manylinux_x86_64_clang_llvm.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100755 build_tools/cmake/build_llvm.sh create mode 100644 build_tools/cmake/llvm_cache.cmake create mode 160000 third_party/llvm-project diff --git a/.github/workflows/ci_manylinux_x86_64_clang_llvm.yml b/.github/workflows/ci_manylinux_x86_64_clang_llvm.yml new file mode 100644 index 00000000..42981c56 --- /dev/null +++ b/.github/workflows/ci_manylinux_x86_64_clang_llvm.yml @@ -0,0 +1,118 @@ +name: CI - manylinux x86_64 clang LLVM + +on: + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/ci_manylinux_x86_64_clang_llvm.yml" + - "third_party/llvm-project" + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + manylinux_x64_clang_llvm: + name: Build LLVM and MLIR + + runs-on: ubuntu-22.04 + + env: + CACHE_DIR: /.container-cache + # either the PR number or `branch-N` where N always increments + CACHE_KEY: manylinux_x64_clang_llvm_${{ format('{0}-{1}', github.ref_name, github.run_number) }} + + container: + image: quay.io/pypa/manylinux_2_28_x86_64 + + steps: + + - name: "Set unified TZ" + uses: szenius/set-timezone@v2.0 + with: + # this is an arbitrary choice + timezoneLinux: "Asia/Singapore" + timezoneMacos: "Asia/Singapore" + timezoneWindows: "Singapore Standard Time" + + - name: Restore cache + uses: actions/cache/restore@v3 + with: + path: ${{ env.CACHE_DIR }} + key: ${{ env.CACHE_KEY }} + restore-keys: manylinux_x64_clang_llvm_ + + - name: "Check out repository" + uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: "Install OS deps" + run: | + dnf install -y epel-release + dnf install -y sudo ncurses-compat-libs tmate python3-pip + + - name: "Setup compiler/toolchain" + uses: aminya/setup-cpp@v1 + with: + compiler: llvm-18 + cmake: true + ninja: true + ccache: true + + - name: "Python deps" + run: | + python3.12 -m pip install -r third_party/llvm-project/mlir/python/requirements.txt + + - name: "Build LLVM and MLIR" + run: | + export CCACHE_DIR="${{ env.CACHE_DIR }}" + export CCACHE_MAXSIZE=700M + export CCACHE_COMPILERCHECK="string:$(clang --version)" + export CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros + + export CMAKE_C_COMPILER_LAUNCHER=ccache + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export CMAKE_C_COMPILER=clang + export CMAKE_CXX_COMPILER=clang++ + export CMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" + export CMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" + export CMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" + export Python3_EXECUTABLE="$(which python3.12)" + + export LLVM_SOURCE_DIR="$PWD/third_party/llvm-project" + export LLVM_BUILD_DIR="$PWD/llvm-build" + export LLVM_INSTALL_DIR="$PWD/llvm-install" + + pushd $LLVM_SOURCE_DIR && llvm_sha_short=$(git rev-parse --short HEAD) && popd + build_tools/cmake/build_llvm.sh + + rm -rf $LLVM_BUILD_DIR $LLVM_SOURCE_DIR + tar cf llvm-dist-linux-$llvm_sha_short.tar $LLVM_INSTALL_DIR + rm -rf $LLVM_INSTALL_DIR + + - name: "Upload dist" + uses: actions/upload-artifact@v4 + with: + name: linux_x86_64_llvm_packages + path: llvm-dist-*.tar + if-no-files-found: error + + - name: "Save cache" + uses: actions/cache/save@v3 + # if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }} + if: ${{ !cancelled() }} + with: + path: ${{ env.CACHE_DIR }} + key: ${{ env.CACHE_KEY }} + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3.18 + with: + limit-access-to-actor: true + install-dependencies: false \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..4d4754ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +.idea \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..1cc3edb9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,5 @@ +[submodule "third_party/llvm-project"] + path = third_party/llvm-project + url = https://github.com/llvm/llvm-project.git + ignore = dirty + shallow = true diff --git a/build_tools/cmake/build_llvm.sh b/build_tools/cmake/build_llvm.sh new file mode 100755 index 00000000..721ed58f --- /dev/null +++ b/build_tools/cmake/build_llvm.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +TD="$(cd $(dirname $0) && pwd)" +REPO_ROOT="$(cd $TD/../.. && pwd)" + +LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-${REPO_ROOT}/third_party/llvm-project}" +LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${REPO_ROOT}/llvm-build}" +LLVM_INSTALL_DIR="${LLVM_INSTALL_DIR:-${REPO_ROOT}/llvm-install}" + +mkdir -p $LLVM_BUILD_DIR +mkdir -p $LLVM_INSTALL_DIR + +LLVM_SOURCE_DIR="$(realpath "${LLVM_SOURCE_DIR}")" +LLVM_BUILD_DIR="$(realpath "${LLVM_BUILD_DIR}")" +LLVM_INSTALL_DIR="$(realpath "${LLVM_INSTALL_DIR}")" + +echo "Paths canonicalized as:" +echo "LLVM_SOURCE_DIR=${LLVM_SOURCE_DIR}" +echo "LLVM_BUILD_DIR=${LLVM_BUILD_DIR}" +echo "LLVM_INSTALL_DIR=${LLVM_INSTALL_DIR}" + +command="$1" +shift + +python3_command="" +if (command -v python3 &> /dev/null); then + python3_command="python3" +elif (command -v python &> /dev/null); then + python3_command="python" +fi + +Python3_EXECUTABLE="${Python3_EXECUTABLE:-$(which $python3_command)}" + +set -euo pipefail + +echo "*********************** BUILDING LLVM *********************************" + +cmake_options=( + -GNinja + -S "${LLVM_SOURCE_DIR}/llvm" + -B "${LLVM_BUILD_DIR}" + -DLLVM_TARGETS_TO_BUILD="${LLVM_TARGETS_TO_BUILD:-host}" + -DCMAKE_BUILD_TYPE=Release + -DPython3_EXECUTABLE="$Python3_EXECUTABLE" + -C "$TD/llvm_cache.cmake" + -DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL_DIR}" +) + +echo "Source Directory: ${LLVM_SOURCE_DIR}" +echo "Build Directory: ${LLVM_BUILD_DIR}" +echo "CMake Options: ${cmake_options[*]}" + +cmake "${cmake_options[@]}" +cmake --build "${LLVM_BUILD_DIR}" \ + --target install-toolchain-distribution \ + --target install-development-distribution \ + --target install-mlirdevelopment-distribution diff --git a/build_tools/cmake/llvm_cache.cmake b/build_tools/cmake/llvm_cache.cmake new file mode 100644 index 00000000..d0c260de --- /dev/null +++ b/build_tools/cmake/llvm_cache.cmake @@ -0,0 +1,127 @@ +set(LLVM_ENABLE_PROJECTS "llvm;mlir" CACHE STRING "") + +# All the tools will use libllvm shared library +set(LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "") +set(LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "") +# When exceptions are disabled, unwind tables are large and useless +set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "") + +# useful things +set(LLVM_BUILD_TOOLS ON CACHE BOOL "") +set(LLVM_BUILD_UTILS ON CACHE BOOL "") +set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "") +set(LLVM_ENABLE_WARNINGS ON CACHE BOOL "") +set(LLVM_FORCE_ENABLE_STATS ON CACHE BOOL "") +set(LLVM_INCLUDE_TOOLS ON CACHE BOOL "") +set(LLVM_INSTALL_UTILS ON CACHE BOOL "") + +# MLIR + +set(MLIR_ENABLE_BINDINGS_PYTHON ON CACHE BOOL "") +set(MLIR_ENABLE_EXECUTION_ENGINE ON CACHE BOOL "") +set(MLIR_ENABLE_SPIRV_CPU_RUNNER ON CACHE BOOL "") + +# space savings + +set(LLVM_BUILD_BENCHMARKS OFF CACHE BOOL "") +set(LLVM_BUILD_EXAMPLES OFF CACHE BOOL "") +set(LLVM_ENABLE_LIBCXX OFF CACHE BOOL "") +set(LLVM_ENABLE_LIBCXX OFF CACHE BOOL "") +set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "") +set(LLVM_ENABLE_LIBXML2 OFF CACHE BOOL "") +set(LLVM_ENABLE_RTTI ON CACHE BOOL "") +set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "") +set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "") +set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "") +set(LLVM_ENABLE_ZSTD OFF CACHE BOOL "") +set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "") +set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "") +set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "") +set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "") + +# tests + +option(RUN_TESTS "" OFF) +set(LLVM_INCLUDE_TESTS ${RUN_TESTS} CACHE BOOL "") +set(LLVM_BUILD_TESTS ${RUN_TESTS} CACHE BOOL "") +set(MLIR_INCLUDE_INTEGRATION_TESTS ${RUN_TESTS} CACHE BOOL "") +set(MLIR_INCLUDE_TESTS ${RUN_TESTS} CACHE BOOL "") + +### Distributions ### + +set(LLVM_INSTALL_TOOLCHAIN_ONLY OFF CACHE BOOL "") +set(LLVM_DISTRIBUTIONS + Development + MlirDevelopment + Toolchain + CACHE STRING "") + +set(LLVM_TOOLCHAIN_TOOLS + llvm-addr2line + llvm-ar + llvm-cxxfilt + llvm-dis + llvm-dwarfdump + llvm-lib + llvm-link + llvm-mc + llvm-nm + llvm-objcopy + llvm-objdump + llvm-ranlib + llvm-rc + llvm-readelf + llvm-readobj + llvm-size + llvm-strip + llvm-symbolizer + llvm-xray + CACHE STRING "") + +set(LLVM_BUILD_UTILS ON CACHE BOOL "") +set(LLVM_INSTALL_UTILS ON CACHE BOOL "") +set(LLVM_TOOLCHAIN_UTILITIES + FileCheck + count + not + CACHE STRING "") + +set(LLVM_Toolchain_DISTRIBUTION_COMPONENTS + LLVM + LTO + ${LLVM_TOOLCHAIN_TOOLS} + ${LLVM_TOOLCHAIN_UTILITIES} + CACHE STRING "") + +set(LLVM_Development_DISTRIBUTION_COMPONENTS + Remarks + cmake-exports + development-cmake-exports + llc + llvm-config + llvm-headers + llvm-libraries + opt + toolchain-cmake-exports + CACHE STRING "") + +set(LLVM_MLIR_TOOLS + mlir-opt + mlir-reduce + mlir-tblgen + mlir-translate + CACHE STRING "") + +set(LLVM_MLIR_Python_COMPONENTS + MLIRPythonModules + mlir-python-sources + CACHE STRING "") + +set(LLVM_MlirDevelopment_DISTRIBUTION_COMPONENTS + MLIRPythonModules + mlir-cmake-exports + mlir-headers + mlir-libraries + ${LLVM_MLIR_TOOLS} + ${LLVM_MLIR_Python_COMPONENTS} + CACHE STRING "") diff --git a/third_party/llvm-project b/third_party/llvm-project new file mode 160000 index 00000000..1aff96b3 --- /dev/null +++ b/third_party/llvm-project @@ -0,0 +1 @@ +Subproject commit 1aff96b3dfcc58d62fda5b1452a8029f1a737cc2