diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 000000000..14b8decbc --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,83 @@ +--- +name: "Nix Build" +"on": [push, pull_request] +jobs: + build: + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + nixpkgs: + - pinned + - release-21.11 + - release-21.05 + - nixpkgs-unstable + fail-fast: false + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2.4.0 + + - name: Install nix + uses: cachix/install-nix-action@v16 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + + - name: Link to bifrost binary cache on cachix.org + uses: cachix/cachix-action@v10 + with: + name: bifrost + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + + - name: Lock the version of nixpkgs specified by the matrix + run: | + if "${{ matrix.nixpkgs }}" != "pinned"; then + nix flake lock --override-input nixpkgs \ + "nixpkgs/${{ matrix.nixpkgs }}" + fi + nix flake metadata + + - name: Check for well-formed flake + run: | + nix flake check + + - name: Build for default python3 + run: | + nix build --print-build-logs .#bifrost-py3 + + - name: Build with enable-debug for python3, if pinned nixpkgs + if: matrix.nixpkgs == 'pinned' + run: | + nix build --print-build-logs .#bifrost-py3-debug + + - name: Build for python38, unless nixpkgs-unstable + # For unstable, 3.8 packages are no longer in the build cache + if: matrix.nixpkgs != 'nixpkgs-unstable' + run: | + nix build --print-build-logs .#bifrost-py38 + + - name: Run Python test suite + # Skipped during nix build because it needs network data. + run: | + cd test + bash download_test_data.sh + nix run ..#python3-bifrost -- -m bifrost.telemetry --disable + nix run ..#python3-bifrost -- -m unittest discover + + - name: Build documentation + run: | + nix build --print-build-logs --out-link result-doc .#bifrost-doc + + - name: Deploy documentation to gh-pages + uses: JamesIves/github-pages-deploy-action@v4.2.5 + # Maybe only deploy if changes in doc/ folder? (But some doc + # pages generated from code, so should reflect code changes?) + if: | + matrix.os == 'ubuntu-latest' + && matrix.nixpkgs == 'pinned' + && github.event_name == 'push' + && github.ref == 'refs/heads/master' + with: + branch: gh-pages + folder: result-doc/share/doc/bifrost/html diff --git a/CHANGELOG b/CHANGELOG index 4db4aa6c7..e2a6ae44f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,6 @@ 0.10.1 * Cleaned up the Makefile outputs + * Added support for configurable, reproducible building with nix 0.10.0 * Switched over to an autotools-based build system diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..51f0349be --- /dev/null +++ b/flake.lock @@ -0,0 +1,80 @@ +{ + "nodes": { + "ctypesgen": { + "flake": false, + "locked": { + "lastModified": 1575761084, + "narHash": "sha256-0VuIufvB1TYK8EXSFr8EegNxxRxnoHsxEKQX9y7LOdY=", + "owner": "ctypesgen", + "repo": "ctypesgen", + "rev": "b7ccd0764ef7d74e9ad5816924950d05b47ecc8c", + "type": "github" + }, + "original": { + "owner": "ctypesgen", + "ref": "ctypesgen-1.0.2", + "repo": "ctypesgen", + "type": "github" + } + }, + "flake-utils": { + "locked": { + "lastModified": 1619345332, + "narHash": "sha256-qHnQkEp1uklKTpx3MvKtY6xzgcqXDsz5nLilbbuL+3A=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "2ebf2558e5bf978c7fb8ea927dfaed8fefab2e28", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1641947035, + "narHash": "sha256-l4dnwWvh2Yte2I9sAwGhMG/qkvLkfV0v/ssgkZ5KNY4=", + "path": "/nix/store/3ck36g7rh9v9fa924h149pq7h6cx9nzx-source", + "rev": "9acedfd7ef32253237311dc3c78286773dbcb0d6", + "type": "path" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1639823344, + "narHash": "sha256-jlsQb2y6A5dB1R0wVPLOfDGM0wLyfYqEJNzMtXuzCXw=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "ff9c0b459ddc4b79c06e19d44251daa8e9cd1746", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "ff9c0b459ddc4b79c06e19d44251daa8e9cd1746", + "type": "github" + } + }, + "root": { + "inputs": { + "ctypesgen": "ctypesgen", + "nixpkgs": "nixpkgs", + "pre-commit-hooks": "pre-commit-hooks" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..147c5b5c6 --- /dev/null +++ b/flake.nix @@ -0,0 +1,317 @@ +{ + description = + "A stream processing framework for high-throughput applications."; + + inputs.ctypesgen = { + url = "github:ctypesgen/ctypesgen/ctypesgen-1.0.2"; + flake = false; + }; + + inputs.pre-commit-hooks = { + url = + "github:cachix/pre-commit-hooks.nix/ff9c0b459ddc4b79c06e19d44251daa8e9cd1746"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + outputs = inputs@{ self, nixpkgs, pre-commit-hooks, ... }: + let + inherit (nixpkgs) lib; + + # Parse the version info in the AC_INIT declaration. + acVersion = lib.head + (builtins.match "AC_INIT\\(\\[bifrost], *\\[([.0-9]+)].*" (lib.head + (lib.filter (lib.strings.hasPrefix "AC_INIT") + (lib.splitString "\n" (lib.readFile ./configure.ac))))); + + # Add a git hash if available; but if repo isn't clean then flake won’t + # provide shortRev and version ends in ".dev". + version = "${acVersion}.dev" + + lib.optionalString (self ? shortRev) "+g${self.shortRev}"; + + # Can inspect the cuda version to guess at what architectures would be + # most useful. Take care not to instatiate the cuda package though, which + # would happen if you start inspecting header files or trying to run nvcc. + + defaultGpuArchs = _cudatoolkit: [ "70" "75" ]; + + # At time of writing (2022-03-24): + # PACKAGE VERSION ARCHS + # cudatoolkit │ │ + # ≡ ~_10 │ │ + # ≡ ~_10_2 10.2.89 30 32 35 37 50 52 53 60 61 62 70 72 75 + # cudatoolkit_11 │ │ (deprecated) + # ≡ ~_11_4 11.4.2 │ (35 37 50)52 53 60 61 62 70 72 75 80 86 87 + # cudatoolkit_11_5 11.5.0 │ (35 37 50)52 53 60 61 62 70 72 75 80 86 87 + # │ │*Experimented w/using all supported archs, but + # │ │ had to eliminate 87 because not in cufft lib. + + bifrost = { stdenv, ctags, ncurses, file, enableDebug ? false + , enablePython ? true, python3, enableCuda ? false, cudatoolkit + , util-linuxMinimal, gpuArchs ? defaultGpuArchs cudatoolkit }: + stdenv.mkDerivation { + name = lib.optionalString (!enablePython) "lib" + "bifrost" + + lib.optionalString enablePython + "-py${lib.versions.majorMinor python3.version}" + + lib.optionalString enableCuda + "-cuda${lib.versions.majorMinor cudatoolkit.version}" + + lib.optionalString enableDebug "-debug" + "-${version}"; + inherit version; + src = ./.; + buildInputs = [ ctags ncurses ] ++ lib.optionals enablePython [ + python3 + python3.pkgs.ctypesgen + python3.pkgs.setuptools + python3.pkgs.pip + python3.pkgs.wheel + ] ++ lib.optionals enableCuda [ cudatoolkit util-linuxMinimal ]; + propagatedBuildInputs = lib.optionals enablePython [ + python3.pkgs.contextlib2 + python3.pkgs.graphviz + python3.pkgs.matplotlib + python3.pkgs.numpy + python3.pkgs.pint + python3.pkgs.scipy + python3.pkgs.simplejson + ]; + patchPhase = + # libtool wants file command, and refers to it in /usr/bin + '' + sed -i 's:/usr/bin/file:${file}/bin/file:' configure + '' + + # Use pinned ctypesgen, not one from pypi. + '' + sed -i 's/ctypesgen==1.0.2/ctypesgen/' python/setup.py + '' + + # Mimic the process of buildPythonPackage, which explicitly + # creates wheel, then installs with pip. + '' + sed -i -e "s:build @PYBUILDFLAGS@:bdist_wheel:" \ + -e "s:@PYINSTALLFLAGS@ .:${ + lib.concatStringsSep " " [ + "--prefix=${placeholder "out"}" + "--no-index" + "--no-warn-script-location" + "--no-cache" + ] + } dist/*.whl:" \ + python/Makefile.in + ''; + # Had difficulty specifying this with configureFlags, because it + # wants to quote the args and that fails with spaces in gpuArchs. + configurePhase = lib.concatStringsSep " " + ([ "./configure" "--disable-static" ''--prefix="$out"'' ] + ++ lib.optionals enableDebug [ "--enable-debug" ] + ++ lib.optionals enableCuda [ + "--with-cuda-home=${cudatoolkit}" + ''--with-gpu-archs="${lib.concatStringsSep " " gpuArchs}"'' + "--with-nvcc-flags='-Wno-deprecated-gpu-targets'" + "LDFLAGS=-L${cudatoolkit}/lib/stubs" + ]); + preBuild = lib.optionalString enablePython '' + make -C python bifrost/libbifrost_generated.py + sed -e "s:^add_library_search_dirs(\[:&'$out/lib':" \ + -e 's:name_formats = \["%s":&,"lib%s","lib%s.so":' \ + -i python/bifrost/libbifrost_generated.py + ''; + # This can be a helpful addition to above sed; prints each path + # tried when loading library: + # -e "s:return self\.Lookup(path):print(path); &:" \ + makeFlags = + lib.optionals enableCuda [ "CUDA_LIBDIR64=$(CUDA_HOME)/lib" ]; + preInstall = '' + mkdir -p "$out/lib" + ''; + }; + + ctypesgen = + { buildPythonPackage, setuptools-scm, toml, glibc, stdenv, gcc }: + buildPythonPackage rec { + pname = "ctypesgen"; + # Setup tools won’t be able to run git describe to generate the + # version, but we can include the shortRev. + version = "1.0.2.dev+g${inputs.ctypesgen.shortRev}"; + SETUPTOOLS_SCM_PRETEND_VERSION = version; + src = inputs.ctypesgen; + buildInputs = [ setuptools-scm toml ]; + postPatch = + # Version detection in the absence of ‘git describe’ is broken, + # even with an explicit VERSION file. + '' + sed -e 's/\(VERSION = \).*$/\1"${pname}-${version}"/' \ + -e 's/\(VERSION_NUMBER = \).*$/\1"${version}"/' \ + -i ctypesgen/version.py + '' + + # Test suite invokes ‘run.py’, replace that with actual script. + '' + sed -e "s:\(script = \).*:\1'${ + placeholder "out" + }/bin/ctypesgen':" \ + -e "s:run\.py:ctypesgen:" \ + -i ctypesgen/test/testsuite.py + '' + + # At runtime, ctypesgen invokes ‘gcc -E’. It won’t be available in + # the darwin stdenv so let's explicitly patch full path to gcc in + # nix store, making gcc a true prerequisite, which it is. There + # are also runs of gcc specified in test suite. + '' + sed -i 's:gcc -E:${gcc}/bin/gcc -E:' ctypesgen/options.py + '' + + # Some tests explicitly load ‘libm’ and ‘libc’. They won’t be + # found on NixOS unless we patch in the ‘glibc’ path. + lib.optionalString stdenv.isLinux '' + sed -e 's:libm.so.6:${glibc}/lib/&:' \ + -e 's:libc.so.6:${glibc}/lib/&:' \ + -i ctypesgen/test/testsuite.py + ''; + checkPhase = "python ctypesgen/test/testsuite.py"; + }; + + pyOverlay = self: _: { + ctypesgen = self.callPackage ctypesgen { }; + bifrost = self.toPythonModule (self.callPackage bifrost { + enablePython = true; + python3 = self.python; + }); + }; + + bifrost-doc = + { stdenv, python3, ctags, doxygen, docDir ? "/share/doc/bifrost" }: + stdenv.mkDerivation { + name = "bifrost-doc-${version}"; + inherit version; + src = ./.; + buildInputs = [ + ctags + doxygen + python3 + python3.pkgs.bifrost + python3.pkgs.sphinx + python3.pkgs.breathe + ]; + buildPhase = '' + make doc + make -C docs html + cd docs/build/html + mv _static static + mv _sources sources + find . -type f -exec sed -i \ + -e '/\(href\|src\)=\"\(\.\.\/\)\?_static/ s/_static/static/' \ + -e '/\(href\|src\)=\"\(\.\.\/\)\?_modules/ s/_modules/modules/' \ + -e '/\(href\|src\)=\"\(\.\.\/\)\?_sources/ s/_sources/sources/' \ + -e '/\$\.ajax\(.*\)_sources/ s/_sources/sources/' \ + {} \; + cd ../../.. + ''; + installPhase = '' + mkdir -p "$out${docDir}" + cp -r docs/build/html "$out${docDir}" + ''; + }; + + # Enable pre-configured packages for these systems. + eachSystem = do: + lib.genAttrs [ "x86_64-linux" "x86_64-darwin" ] (system: + do (import nixpkgs { + inherit system; + config.allowUnfree = true; + overlays = lib.attrValues self.overlays; + })); + + # Which python3 packages should be modified by the overlay? + isPython = name: builtins.match "python3[0-9]*" name != null; + pythonAttrs = lib.filterAttrs (name: _: isPython name); + + in { + overlays.default = final: prev: + { + bifrost = final.callPackage bifrost { }; + bifrost-doc = final.callPackage bifrost-doc { }; + } + # Apply the python overlay to every python package set we find. + // lib.mapAttrs (_: py: py.override { packageOverrides = pyOverlay; }) + (pythonAttrs prev); + + packages = eachSystem (pkgs: + let + shortenPy = lib.replaceStrings [ "thon" ] [ "" ]; + + # Which cuda versions should be target by the packages? Let's just do + # the default 10 and 11. It's easy to generate other point releases + # from the overlay. (Versions prior to 10 are not supported anymore by + # nixpkgs.) + isCuda = name: builtins.match "cudatoolkit(_1[01])" name != null; + shortenCuda = lib.replaceStrings [ "toolkit" "_" ] [ "" "" ]; + cudaAttrs = lib.filterAttrs + (name: pkg: isCuda name && lib.elem pkgs.system pkg.meta.platforms) + pkgs; + + eachBool = f: lib.concatMap f [ true false ]; + eachCuda = f: lib.concatMap f ([ null ] ++ lib.attrNames cudaAttrs); + eachConfig = f: + eachBool (enableDebug: + eachCuda (cuda: + f (lib.optionalString (cuda != null) "-${shortenCuda cuda}" + + lib.optionalString enableDebug "-debug") { + inherit enableDebug; + enableCuda = cuda != null; + cudatoolkit = pkgs.${cuda}; + })); + + # Runnable ctypesgen per python. Though it's just the executable we + # need, it's possible something about ctypes library could change + # between releases. + cgens = lib.mapAttrs' (name: py: { + name = "ctypesgen-${shortenPy name}"; + value = py.pkgs.ctypesgen; + }) (pythonAttrs pkgs); + + # The whole set of bifrost packages, with or without python (and each + # python version), and for each configuration. + bfs = lib.listToAttrs (eachConfig (suffix: config: + [{ + name = "libbifrost${suffix}"; + value = + pkgs.bifrost.override (config // { enablePython = false; }); + }] ++ lib.mapAttrsToList (name: py: { + name = "bifrost-${shortenPy name}${suffix}"; + value = py.pkgs.bifrost.override config; + }) (pythonAttrs pkgs))); + + # Now generate pythons with bifrost packaged. + pys = lib.listToAttrs (eachConfig (suffix: config: + lib.mapAttrsToList (name: py: { + name = "${name}-bifrost${suffix}"; + value = py.withPackages (p: [ (p.bifrost.override config) ]); + }) (pythonAttrs pkgs))); + + in { inherit (pkgs) bifrost-doc; } // cgens // bfs // pys); + + devShells = eachSystem (pkgs: { + default = let + pre-commit = pre-commit-hooks.lib.${pkgs.system}.run { + src = ./.; + hooks.nixfmt.enable = true; + hooks.nix-linter.enable = true; + hooks.yamllint.enable = true; + }; + + in pkgs.mkShell { + inherit (pre-commit) shellHook; + + # Tempting to include bifrost-doc.buildInputs here, but that requires + # bifrost to already be built. + buildInputs = pkgs.bifrost.buildInputs + ++ pkgs.bifrost.propagatedBuildInputs ++ [ + pkgs.black + pkgs.ctags + pkgs.doxygen + pkgs.nixfmt + pkgs.nix-linter + pkgs.python3.pkgs.breathe + pkgs.python3.pkgs.sphinx + pkgs.yamllint + ]; + }; + }); + }; +}