Add LLVM as submodule (and add GHA build scripts)

.github/workflows/ci_manylinux_x86_64_clang_llvm.yml

@@ -0,0 +1,118 @@
+name: CI - manylinux x86_64 clang LLVM
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - ".github/workflows/ci_manylinux_x86_64_clang_llvm.yml"
+      - "third_party/llvm-project"
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  manylinux_x64_clang_llvm:
+    name: Build LLVM and MLIR
+
+    runs-on: ubuntu-22.04
+
+    env:
+      CACHE_DIR: /.container-cache
+      # either the PR number or `branch-N` where N always increments
+      CACHE_KEY: manylinux_x64_clang_llvm_${{ format('{0}-{1}', github.ref_name, github.run_number) }}
+
+    container:
+      image: quay.io/pypa/manylinux_2_28_x86_64
+
+    steps:
+
+      - name: "Set unified TZ"
+        uses: szenius/[email protected]
+        with:
+          # this is an arbitrary choice
+          timezoneLinux: "Asia/Singapore"
+          timezoneMacos: "Asia/Singapore"
+          timezoneWindows: "Singapore Standard Time"
+
+      - name: Restore cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ env.CACHE_DIR }}
+          key:  ${{ env.CACHE_KEY }}
+          restore-keys: manylinux_x64_clang_llvm_
+
+      - name: "Check out repository"
+        uses: actions/[email protected]
+        with:
+          submodules: true
+
+      - name: "Install OS deps"
+        run: |
+          dnf install -y epel-release 
+          dnf install -y sudo ncurses-compat-libs tmate python3-pip
+
+      - name: "Setup compiler/toolchain"
+        uses: aminya/setup-cpp@v1
+        with:
+          compiler: llvm-18
+          cmake: true
+          ninja: true
+          ccache: true
+
+      - name: "Python deps"
+        run: |
+          python3.12 -m pip install -r third_party/llvm-project/mlir/python/requirements.txt
+
+      - name: "Build LLVM and MLIR"
+        run: |
+          export CCACHE_DIR="${{ env.CACHE_DIR }}"
+          export CCACHE_MAXSIZE=700M
+          export CCACHE_COMPILERCHECK="string:$(clang --version)"
+          export CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros
+
+          export CMAKE_C_COMPILER_LAUNCHER=ccache
+          export CMAKE_CXX_COMPILER_LAUNCHER=ccache
+          export CMAKE_C_COMPILER=clang 
+          export CMAKE_CXX_COMPILER=clang++
+          export CMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld"
+          export CMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld"
+          export CMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld"
+          export Python3_EXECUTABLE="$(which python3.12)"
+
+          export LLVM_SOURCE_DIR="$PWD/third_party/llvm-project"
+          export LLVM_BUILD_DIR="$PWD/llvm-build"
+          export LLVM_INSTALL_DIR="$PWD/llvm-install"
+
+          pushd $LLVM_SOURCE_DIR && llvm_sha_short=$(git rev-parse --short HEAD) && popd
+          build_tools/cmake/build_llvm.sh
+
+          rm -rf $LLVM_BUILD_DIR $LLVM_SOURCE_DIR
+          tar cf llvm-dist-linux-$llvm_sha_short.tar $LLVM_INSTALL_DIR
+          rm -rf $LLVM_INSTALL_DIR
+
+      - name: "Upload dist"
+        uses: actions/upload-artifact@v4
+        with:
+          name: linux_x86_64_llvm_packages
+          path: llvm-dist-*.tar
+          if-no-files-found: error
+
+      - name: "Save cache"
+        uses: actions/cache/save@v3
+        # if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }}
+        if: ${{ !cancelled() }}
+        with:
+          path: ${{ env.CACHE_DIR }}
+          key: ${{ env.CACHE_KEY }}
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/[email protected]
+        with:
+          limit-access-to-actor: true
+          install-dependencies: false

.gitignore

@@ -0,0 +1,34 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+.idea

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "third_party/llvm-project"]
+	path = third_party/llvm-project
+	url = https://github.com/llvm/llvm-project.git
+	ignore = dirty
+	shallow = true

build_tools/cmake/build_llvm.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+TD="$(cd $(dirname $0) && pwd)"
+REPO_ROOT="$(cd $TD/../.. && pwd)"
+
+LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-${REPO_ROOT}/third_party/llvm-project}"
+LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${REPO_ROOT}/llvm-build}"
+LLVM_INSTALL_DIR="${LLVM_INSTALL_DIR:-${REPO_ROOT}/llvm-install}"
+
+mkdir -p $LLVM_BUILD_DIR
+mkdir -p $LLVM_INSTALL_DIR
+
+LLVM_SOURCE_DIR="$(realpath "${LLVM_SOURCE_DIR}")"
+LLVM_BUILD_DIR="$(realpath "${LLVM_BUILD_DIR}")"
+LLVM_INSTALL_DIR="$(realpath "${LLVM_INSTALL_DIR}")"
+
+echo "Paths canonicalized as:"
+echo "LLVM_SOURCE_DIR=${LLVM_SOURCE_DIR}"
+echo "LLVM_BUILD_DIR=${LLVM_BUILD_DIR}"
+echo "LLVM_INSTALL_DIR=${LLVM_INSTALL_DIR}"
+
+command="$1"
+shift
+
+python3_command=""
+if (command -v python3 &> /dev/null); then
+  python3_command="python3"
+elif (command -v python &> /dev/null); then
+  python3_command="python"
+fi
+
+Python3_EXECUTABLE="${Python3_EXECUTABLE:-$(which $python3_command)}"
+
+set -euo pipefail
+
+echo "*********************** BUILDING LLVM *********************************"
+
+cmake_options=(
+  -GNinja
+  -S "${LLVM_SOURCE_DIR}/llvm"
+  -B "${LLVM_BUILD_DIR}"
+  -DLLVM_TARGETS_TO_BUILD="${LLVM_TARGETS_TO_BUILD:-host}"
+  -DCMAKE_BUILD_TYPE=Release
+  -DPython3_EXECUTABLE="$Python3_EXECUTABLE"
+  -C "$TD/llvm_cache.cmake"
+  -DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL_DIR}"
+)
+
+echo "Source Directory: ${LLVM_SOURCE_DIR}"
+echo "Build Directory: ${LLVM_BUILD_DIR}"
+echo "CMake Options: ${cmake_options[*]}"
+
+cmake "${cmake_options[@]}"
+cmake --build "${LLVM_BUILD_DIR}" \
+  --target install-toolchain-distribution \
+  --target install-development-distribution \
+  --target install-mlirdevelopment-distribution

build_tools/cmake/llvm_cache.cmake

@@ -0,0 +1,127 @@
+set(LLVM_ENABLE_PROJECTS "llvm;mlir" CACHE STRING "")
+
+# All the tools will use libllvm shared library
+set(LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "")
+set(LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "")
+# When exceptions are disabled, unwind tables are large and useless
+set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "")
+
+# useful things
+set(LLVM_BUILD_TOOLS ON CACHE BOOL "")
+set(LLVM_BUILD_UTILS ON CACHE BOOL "")
+set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "")
+set(LLVM_ENABLE_WARNINGS ON CACHE BOOL "")
+set(LLVM_FORCE_ENABLE_STATS ON CACHE BOOL "")
+set(LLVM_INCLUDE_TOOLS ON CACHE BOOL "")
+set(LLVM_INSTALL_UTILS ON CACHE BOOL "")
+
+# MLIR
+
+set(MLIR_ENABLE_BINDINGS_PYTHON ON CACHE BOOL "")
+set(MLIR_ENABLE_EXECUTION_ENGINE ON CACHE BOOL "")
+set(MLIR_ENABLE_SPIRV_CPU_RUNNER ON CACHE BOOL "")
+
+# space savings
+
+set(LLVM_BUILD_BENCHMARKS OFF CACHE BOOL "")
+set(LLVM_BUILD_EXAMPLES OFF CACHE BOOL "")
+set(LLVM_ENABLE_LIBCXX OFF CACHE BOOL "")
+set(LLVM_ENABLE_LIBCXX OFF CACHE BOOL "")
+set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
+set(LLVM_ENABLE_LIBXML2 OFF CACHE BOOL "")
+set(LLVM_ENABLE_RTTI ON CACHE BOOL "")
+set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "")
+set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "")
+set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "")
+set(LLVM_ENABLE_ZSTD OFF CACHE BOOL "")
+set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")
+set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "")
+set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
+set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "")
+
+# tests
+
+option(RUN_TESTS "" OFF)
+set(LLVM_INCLUDE_TESTS ${RUN_TESTS} CACHE BOOL "")
+set(LLVM_BUILD_TESTS ${RUN_TESTS} CACHE BOOL "")
+set(MLIR_INCLUDE_INTEGRATION_TESTS ${RUN_TESTS} CACHE BOOL "")
+set(MLIR_INCLUDE_TESTS ${RUN_TESTS} CACHE BOOL "")
+
+### Distributions ###
+
+set(LLVM_INSTALL_TOOLCHAIN_ONLY OFF CACHE BOOL "")
+set(LLVM_DISTRIBUTIONS
+      Development
+      MlirDevelopment
+      Toolchain
+    CACHE STRING "")
+
+set(LLVM_TOOLCHAIN_TOOLS
+  llvm-addr2line
+  llvm-ar
+  llvm-cxxfilt
+  llvm-dis
+  llvm-dwarfdump
+  llvm-lib
+  llvm-link
+  llvm-mc
+  llvm-nm
+  llvm-objcopy
+  llvm-objdump
+  llvm-ranlib
+  llvm-rc
+  llvm-readelf
+  llvm-readobj
+  llvm-size
+  llvm-strip
+  llvm-symbolizer
+  llvm-xray
+  CACHE STRING "")
+
+set(LLVM_BUILD_UTILS ON CACHE BOOL "")
+set(LLVM_INSTALL_UTILS ON CACHE BOOL "")
+set(LLVM_TOOLCHAIN_UTILITIES
+    FileCheck
+    count
+    not
+    CACHE STRING "")
+
+set(LLVM_Toolchain_DISTRIBUTION_COMPONENTS
+      LLVM
+      LTO
+      ${LLVM_TOOLCHAIN_TOOLS}
+      ${LLVM_TOOLCHAIN_UTILITIES}
+    CACHE STRING "")
+
+set(LLVM_Development_DISTRIBUTION_COMPONENTS
+      Remarks
+      cmake-exports
+      development-cmake-exports
+      llc
+      llvm-config
+      llvm-headers
+      llvm-libraries
+      opt
+      toolchain-cmake-exports
+    CACHE STRING "")
+
+set(LLVM_MLIR_TOOLS
+      mlir-opt
+      mlir-reduce
+      mlir-tblgen
+      mlir-translate
+    CACHE STRING "")
+
+set(LLVM_MLIR_Python_COMPONENTS
+      MLIRPythonModules
+      mlir-python-sources
+    CACHE STRING "")
+
+set(LLVM_MlirDevelopment_DISTRIBUTION_COMPONENTS
+      MLIRPythonModules
+      mlir-cmake-exports
+      mlir-headers
+      mlir-libraries
+      ${LLVM_MLIR_TOOLS}
+      ${LLVM_MLIR_Python_COMPONENTS}
+    CACHE STRING "")