From 02ceae28106748f2da5b6012df6b53f30d3ccfac Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Feb 2023 16:00:49 -0600 Subject: [PATCH 1/2] Pass out nodes in a specific order --- .pre-commit-config.yaml | 16 +++++----------- Cargo.lock | 14 +++++++------- Cargo.toml | 7 ++----- pyproject.toml | 15 +++++++++++++-- src/lib.rs | 3 +++ test_graphlib.py | 19 +++++++++++++++++++ 6 files changed, 49 insertions(+), 25 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bd123d..beca684 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ files: ^python/.*|^tests/.*|^src/.* repos: - repo: https://github.com/ambv/black - rev: 22.3.0 + rev: 23.1.0 hooks: - id: black - repo: local @@ -18,13 +18,8 @@ repos: language: system types: [rust] pass_filenames: false - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - args: ["--max-line-length=88"] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.950 + rev: v1.0.0 hooks: - id: mypy - repo: https://github.com/pre-commit/pre-commit-hooks @@ -32,8 +27,7 @@ repos: hooks: - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/pycqa/isort - rev: 5.10.1 + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: 'v0.0.247' hooks: - - id: isort - name: isort (python) + - id: ruff diff --git a/Cargo.lock b/Cargo.lock index ec75db1..9ff4ff2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,13 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "_graphlib2" +version = "0.4.8" +dependencies = [ + "pyo3", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -20,13 +27,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "graphlib2" -version = "0.4.7" -dependencies = [ - "pyo3", -] - [[package]] name = "indoc" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index 550f6d3..76978c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,7 @@ [package] -name = "graphlib2" -version = "0.4.7" +name = "_graphlib2" +version = "0.4.8" edition = "2021" -description = "Rust port of the Python stdlib graphlib modules" readme = "README.md" license-file = "LICENSE.txt" @@ -15,6 +14,4 @@ version = "^0.17.2" features = ["extension-module", "abi3-py37"] [package.metadata.maturin] -python-source = "python" -description-content-type = "text/markdown; charset=UTF-8; variant=GFM" name = "graphlib2._graphlib2" diff --git a/pyproject.toml b/pyproject.toml index 87be5db..99b6e24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,17 @@ build-backend = "maturin" [tool.maturin] sdist-include = ["Cargo.lock"] strip = true +python-source = "python" +profile = "release" -[tool.isort] -profile = "black" + +[tool.ruff] +extend-select = [ + "RUF100", # dissallow unused # noqa comments + "I001", # isort +] +ignore = [ + "E501", # leave line lengths up to black + "E402", # leave imports up to isort + "E731", # assigning lambdas is convenient +] diff --git a/src/lib.rs b/src/lib.rs index abbeeb5..9b4a1f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -138,6 +138,9 @@ impl PreparedState { fn get_ready<'py>(&mut self, py: Python<'py>) -> &'py PyTuple { let id2node = &self.dag.id2node; self.n_passed_out += self.ready_nodes.len(); + // Pass out nodes ordered by most to least + self.ready_nodes + .sort_unstable_by_key(|node| -(self.dag.parents[*node].len() as i64)); PyTuple::new( py, self.ready_nodes diff --git a/test_graphlib.py b/test_graphlib.py index 09ab257..9aa5d12 100644 --- a/test_graphlib.py +++ b/test_graphlib.py @@ -354,5 +354,24 @@ def run(ts: graphlib.TopologicalSorter[int]) -> None: assert not ts.is_active() +def test_ready_ordered_by_number_of_upstream() -> None: + """Ready nodes should be passed out such that if + they are processed in order and done() is called on nodes individually + we optimize unblocking more tasks so that we can saturate the + threads / tasks / etc. that we have. + """ + graphs = [ + {0: [], 1: [], 2: [1], 3: [1]}, + { 1: [], 0: [], 2: [1], 3: [1]}, + ] + for graph in graphs: + ts: graphlib.TopologicalSorter[int] = graphlib.TopologicalSorter() + for node in graph: + ts.add(node, *graph[node]) + ts.prepare() + assert list(ts.get_ready()) == [1, 0] + + + if __name__ == "__main__": pytest.main([__file__,]) From c89268e0c3f626ce5d78c4525d2f6e4a8b5f50b1 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Feb 2023 16:06:27 -0600 Subject: [PATCH 2/2] update maturin --- .github/workflows/python.yaml | 16 ++++++++-------- pyproject.toml | 2 +- requirements-dev.txt | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index a3a79c0..ce104b8 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -34,7 +34,7 @@ jobs: with: target: x86_64 args: --release --out dist --sdist - maturin-version: "v0.13.0" + maturin-version: "v0.14.13" - name: Install built wheel - x86_64 run: | pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall @@ -44,7 +44,7 @@ jobs: uses: messense/maturin-action@v1 with: args: --release --universal2 --out dist - maturin-version: "v0.13.0" + maturin-version: "v0.14.13" - name: Install built wheel - universal2 run: | pip install dist/${{ env.PACKAGE_NAME }}-*universal2.whl --force-reinstall @@ -78,7 +78,7 @@ jobs: with: target: ${{ matrix.target }} args: --release --out dist - maturin-version: "v0.13.0" + maturin-version: "v0.14.13" - name: Install built wheel shell: bash run: | @@ -108,7 +108,7 @@ jobs: target: ${{ matrix.target }} manylinux: auto args: --release --out dist - maturin-version: "v0.13.0" + maturin-version: "v0.14.13" - name: Install built wheel if: matrix.target == 'x86_64' run: | @@ -137,7 +137,7 @@ jobs: target: ${{ matrix.target }} manylinux: auto args: --release --out dist - maturin-version: "v0.13.0" + maturin-version: "v0.14.13" - uses: uraimo/run-on-arch-action@v2.0.5 if: matrix.target != 'ppc64' name: Install built wheel @@ -178,7 +178,7 @@ jobs: target: ${{ matrix.target }} manylinux: musllinux_1_2 args: --release --out dist - maturin-version: "v0.13.0" + maturin-version: "v0.14.13" - name: Install built wheel if: matrix.target == 'x86_64-unknown-linux-musl' uses: addnab/docker-run-action@v3 @@ -216,7 +216,7 @@ jobs: target: ${{ matrix.platform.target }} manylinux: musllinux_1_2 args: --release --out dist - maturin-version: "v0.13.0" + maturin-version: "v0.14.13" - uses: uraimo/run-on-arch-action@master name: Install built wheel with: @@ -256,7 +256,7 @@ jobs: - name: Build wheels uses: messense/maturin-action@v1 with: - maturin-version: "v0.13.0" + maturin-version: "v0.14.13" target: ${{ matrix.target }} manylinux: auto args: --release --out dist -i pypy${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index 99b6e24..920e702 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ documentation = "https://github.com/adriangb/graphlib2/README.md" repository = "https://github.com/adriangb/graphlib2" [build-system] -requires = ["maturin>=0.13.0<14"] +requires = ["maturin>=0.14.0,<0.15.0"] build-backend = "maturin" [tool.maturin] diff --git a/requirements-dev.txt b/requirements-dev.txt index ee15951..3b4a1f0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ pytest==6.2.5 -maturin>=0.13.0<14 +maturin>=0.14.0,<0.15.0 pre-commit==2.16.0