From 83bed19f360b69dad26b7d9b00ffd837c8075b7a Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Sun, 31 Mar 2024 14:57:52 -0400
Subject: [PATCH 001/103] Fix wording of comment about the /cygdrive prefix
---
git/repo/fun.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/repo/fun.py b/git/repo/fun.py
index e44d9c644..182cf82ed 100644
--- a/git/repo/fun.py
+++ b/git/repo/fun.py
@@ -112,7 +112,7 @@ def find_submodule_git_dir(d: PathLike) -> Optional[PathLike]:
path = content[8:]
if Git.is_cygwin():
- # Cygwin creates submodules prefixed with `/cygdrive/...` suffixes.
+ # Cygwin creates submodules prefixed with `/cygdrive/...`.
# Cygwin git understands Cygwin paths much better than Windows ones.
# Also the Cygwin tests are assuming Cygwin paths.
path = cygpath(path)
From 988d97bf12c5b15ff4693a5893134271dd8d8a28 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Sun, 31 Mar 2024 15:40:47 -0400
Subject: [PATCH 002/103] Fix typo in _get_exe_extensions PATHEXT fallback
PATHEXT lists file extensions with the ".". In the fallback given
in _get_exe_extensions, the other extensions had this, but ".COM"
was listed without the ".". This fixes that.
This is very minor because _get_exe_extensions is nonpublic and not
currently used on native Windows, which is the platform where the
PATHEXT fallback code would be used.
Specifically, _get_exe_extensions is called only in py_where, which
while named with no leading underscore is nonpublic do not being
(and never having been) listed in __all__. As its docstring states,
it is an implementation detail of is_cygwin_git and not intended
for any other use. More specifically, is_cygwin_git currently
immediately returns False on *native* Windows (even if the git
executable GitPython is using is a Cygwin git executable). Only on
Cygwin, or other systems that are not native Windows, does it try
to check the git executable (by calling its _is_cygwin_git helper,
which uses py_where).
---
git/util.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/util.py b/git/util.py
index 8c1c26012..11f963e02 100644
--- a/git/util.py
+++ b/git/util.py
@@ -339,7 +339,7 @@ def _get_exe_extensions() -> Sequence[str]:
if PATHEXT:
return tuple(p.upper() for p in PATHEXT.split(os.pathsep))
elif sys.platform == "win32":
- return (".BAT", "COM", ".EXE")
+ return (".BAT", ".COM", ".EXE")
else:
return ()
From f18df8edcf5de5971e4dd01f15ad32411e38244e Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Sun, 31 Mar 2024 18:07:32 -0400
Subject: [PATCH 003/103] Don't pass --disable-warnings to pytest
---
pyproject.toml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 6cb05f96e..ee54edb78 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,7 +3,7 @@ requires = ["setuptools"]
build-backend = "setuptools.build_meta"
[tool.pytest.ini_options]
-addopts = "--cov=git --cov-report=term --disable-warnings -ra"
+addopts = "--cov=git --cov-report=term -ra"
filterwarnings = "ignore::DeprecationWarning"
python_files = "test_*.py"
tmp_path_retention_policy = "failed"
@@ -13,7 +13,6 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc
# --cov-report term-missing # to terminal with line numbers
# --cov-report html:path # html file at path
# --maxfail # number of errors before giving up
-# -disable-warnings # Disable pytest warnings (not codebase warnings)
# -rfE # default test summary: list fail and error
# -ra # test summary: list all non-passing (fail, error, skip, xfail, xpass)
# --ignore-glob=**/gitdb/* # ignore glob paths
From 0152b528a035566364fc8a0e1a80829c6d495301 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Mon, 1 Apr 2024 14:53:43 -0400
Subject: [PATCH 004/103] Update the comment about `--mixed` and paths
This updates the comment in HEAD.reset about why `--mixed` is
omitted from the git command constructed to perform a reset where
paths are being passed, adding specific information about the git
versions where this is deprecated, and changing the more-info link
from an old GitPython issue that is no longer retrievable to #1876.
---
git/refs/head.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/git/refs/head.py b/git/refs/head.py
index 7e6fc3377..683634451 100644
--- a/git/refs/head.py
+++ b/git/refs/head.py
@@ -99,8 +99,8 @@ def reset(
if index:
mode = "--mixed"
- # It appears some git versions declare mixed and paths deprecated.
- # See http://github.com/Byron/GitPython/issues#issue/2.
+ # Explicit "--mixed" when passing paths is deprecated since git 1.5.4.
+ # See https://github.com/gitpython-developers/GitPython/discussions/1876.
if paths:
mode = None
# END special case
From a9593c7c56e76bdef35245be00bf23abdc3ba4c0 Mon Sep 17 00:00:00 2001
From: Eduard Talanov <89387701+EduardTalanov@users.noreply.github.com>
Date: Fri, 5 Apr 2024 10:10:53 +0200
Subject: [PATCH 005/103] Update remote.py
Fixed the error of updating shallow submodules
---
git/remote.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/remote.py b/git/remote.py
index f2ecd0f36..37c991d27 100644
--- a/git/remote.py
+++ b/git/remote.py
@@ -316,7 +316,7 @@ class FetchInfo(IterableObj):
ERROR,
) = [1 << x for x in range(8)]
- _re_fetch_result = re.compile(r"^ *(.) (\[[\w \.$@]+\]|[\w\.$@]+) +(.+) -> ([^ ]+)( \(.*\)?$)?")
+ _re_fetch_result = re.compile(r"^ *(?:.{0,3})(.) (\[[\w \.$@]+\]|[\w\.$@]+) +(.+) -> ([^ ]+)( \(.*\)?$)?")
_flag_map: Dict[flagKeyLiteral, int] = {
"!": ERROR,
From 55c30a34185e13c31ab14c39f7a7dd0db3a494e4 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 11 Apr 2024 19:55:10 -0400
Subject: [PATCH 006/103] OSS-Fuzz test initial migration
Migrates the OSS-Fuzz tests and setup scripts from the OSS-Fuzz
repository to GitPython's repo as discussed here:
https://github.com/gitpython-developers/GitPython/issues/1887#issuecomment-2028599381
These files include the changes that were originally proposed in:
https://github.com/google/oss-fuzz/pull/11763
Additional changes include:
- A first pass at documenting the contents of the fuzzing set up in a
dedicated README.md
- Adding the dictionary files to this repo for improved visibility. Seed
corpra zips are still located in an external repo pending further
discussion regarding where those should live in the long term.
---
fuzzing/README.md | 172 ++++++++++++++++++
fuzzing/dictionaries/fuzz_config.dict | 56 ++++++
fuzzing/dictionaries/fuzz_tree.dict | 13 ++
fuzzing/fuzz-targets/fuzz_config.py | 51 ++++++
fuzzing/fuzz-targets/fuzz_tree.py | 59 ++++++
fuzzing/oss-fuzz-scripts/build.sh | 37 ++++
.../container-environment-bootstrap.sh | 56 ++++++
7 files changed, 444 insertions(+)
create mode 100644 fuzzing/README.md
create mode 100644 fuzzing/dictionaries/fuzz_config.dict
create mode 100644 fuzzing/dictionaries/fuzz_tree.dict
create mode 100644 fuzzing/fuzz-targets/fuzz_config.py
create mode 100644 fuzzing/fuzz-targets/fuzz_tree.py
create mode 100644 fuzzing/oss-fuzz-scripts/build.sh
create mode 100644 fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
diff --git a/fuzzing/README.md b/fuzzing/README.md
new file mode 100644
index 000000000..6853c5002
--- /dev/null
+++ b/fuzzing/README.md
@@ -0,0 +1,172 @@
+# Fuzzing GitPython
+
+[][oss-fuzz-issue-tracker]
+
+This directory contains files related to GitPython's suite of fuzz tests that are executed daily on automated
+infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working
+with fuzzing in GitPython.
+
+The details about the latest OSS-Fuzz test status, including build logs and coverage reports, is made available
+at [this link](https://introspector.oss-fuzz.com/project-profile?project=gitpython).
+
+## How to Contribute
+
+There are many ways to contribute to GitPython's fuzzing efforts! Contributions are welcomed through issues,
+discussions, or pull requests on this repository.
+
+Areas that are particularly appreciated include:
+
+- **Tackling the existing backlog of open issues**. While fuzzing is an effective way to identify bugs, that information
+ isn't useful unless they are fixed. If you are not sure where to start, the issues tab is a great place to get ideas!
+- **Improvements to this (or other) documentation** make it easier for new contributors to get involved, so even small
+ improvements can have a large impact over time. If you see something that could be made easier by a documentation
+ update of any size, please consider suggesting it!
+
+For everything else, such as expanding test coverage, optimizing test performance, or enhancing error detection
+capabilities, jump in to the "Getting Started" section below.
+
+## Getting Started with Fuzzing GitPython
+
+> [!TIP]
+> **New to fuzzing or unfamiliar with OSS-Fuzz?**
+>
+> These resources are an excellent place to start:
+>
+> - [OSS-Fuzz documentation][oss-fuzz-docs] - Continuous fuzzing service for open source software.
+> - [Google/fuzzing][google-fuzzing-repo] - Tutorials, examples, discussions, research proposals, and other resources
+ related to fuzzing.
+> - [CNCF Fuzzing Handbook](https://github.com/cncf/tag-security/blob/main/security-fuzzing-handbook/handbook-fuzzing.pdf) -
+ A comprehensive guide for fuzzing open source software.
+> - [Efficient Fuzzing Guide by The Chromium Project](https://chromium.googlesource.com/chromium/src/+/main/testing/libfuzzer/efficient_fuzzing.md) -
+ Explores strategies to enhance the effectiveness of your fuzz tests, recommended for those looking to optimize their
+ testing efforts.
+
+### Setting Up Your Local Environment
+
+Before contributing to fuzzing efforts, ensure Python and Docker are installed on your machine. Docker is required for
+running fuzzers in containers provided by OSS-Fuzz. [Install Docker](https://docs.docker.com/get-docker/) following the
+official guide if you do not already have it.
+
+### Understanding Existing Fuzz Targets
+
+Review the `fuzz-targets/` directory to familiarize yourself with how existing tests are implemented. See
+the [Files & Directories Overview](#files--directories-overview) for more details on the directory structure.
+
+### Contributing to Fuzz Tests
+
+Start by reviewing the [Atheris documentation][atheris-repo] and the section
+on [Running Fuzzers Locally](#running-fuzzers-locally) to begin writing or improving fuzz tests.
+
+## Files & Directories Overview
+
+The `fuzzing/` directory is organized into three key areas:
+
+### Fuzz Targets (`fuzz-targets/`)
+
+Contains Python files for each fuzz test, targeting specific functionalities of GitPython.
+
+**Things to Know**:
+
+- Each fuzz test targets a specific part of GitPython's functionality.
+- Test files adhere to the naming convention: `fuzz_<API Under Test>.py`, where `<API Under Test>` indicates the
+ functionality targeted by the test.
+- Any functionality that involves performing operations on input data is a possible candidate for fuzz testing, but
+ features that involve processing untrusted user input or parsing operations are typically going to be the most
+ interesting.
+- The goal of these tests is to identify previously unknown or unexpected error cases caused by a given input. For that
+ reason, fuzz tests should gracefully handle anticipated exception cases with a `try`/`except` block to avoid false
+ positives that halt the fuzzing engine.
+
+### Dictionaries (`dictionaries/`)
+
+Provides hints to the fuzzing engine about inputs that might trigger unique code paths. Each fuzz target may have a
+corresponding `.dict` file. For details on how these are used, refer
+to [LibFuzzer documentation](https://llvm.org/docs/LibFuzzer.html#dictionaries).
+
+**Things to Know**:
+
+- OSS-Fuzz loads dictionary files per fuzz target if one exists with the same name, all others are ignored.
+- Most entries in the dictionary files found here are escaped hex or Unicode values that were recommended by the fuzzing
+ engine after previous runs.
+- A default set of dictionary entries are created for all fuzz targets as part of the build process, regardless of an
+ existing file here.
+- Development or updates to dictionaries should reflect the varied formats and edge cases relevant to the
+ functionalities under test.
+- Example dictionaries (some of which are used to build the default dictionaries mentioned above) are can be found here:
+ - [AFL++ dictionary repository](https://github.com/AFLplusplus/AFLplusplus/tree/stable/dictionaries#readme)
+ - [Google/fuzzing dictionary repository](https://github.com/google/fuzzing/tree/master/dictionaries)
+
+### OSS-Fuzz Scripts (`oss-fuzz-scripts/`)
+
+Includes scripts for building and integrating fuzz targets with OSS-Fuzz:
+
+- **`container-environment-bootstrap.sh`** - Sets up the execution environment. It is responsible for fetching default
+ dictionary entries and ensuring all required build dependencies are installed and up-to-date.
+- **`build.sh`** - Executed within the Docker container, this script builds fuzz targets with necessary instrumentation
+ and prepares seed corpora and dictionaries for use.
+
+## Running Fuzzers Locally
+
+### Direct Execution of Fuzz Targets
+
+For quick testing of changes, [Atheris][atheris-repo] makes it possible to execute a fuzz target directly:
+
+1. Install Atheris following the [installation guide][atheris-repo] for your operating system.
+2. Execute a fuzz target, for example:
+
+```shell
+python fuzzing/fuzz-targets/fuzz_config.py
+```
+
+### Running OSS-Fuzz Locally
+
+This approach uses Docker images provided by OSS-Fuzz for building and running fuzz tests locally. It offers
+comprehensive features but requires a local clone of the OSS-Fuzz repository and sufficient disk space for Docker
+containers.
+
+#### Preparation
+
+Set environment variables to simplify command usage:
+
+```shell
+export SANITIZER=address # Can be either 'address' or 'undefined'.
+export FUZZ_TARGET=fuzz_config # specify the fuzz target without the .py extension.
+```
+
+#### Build and Run
+
+Clone the OSS-Fuzz repository and prepare the Docker environment:
+
+```shell
+git clone --depth 1 https://github.com/google/oss-fuzz.git oss-fuzz
+cd oss-fuzz
+python infra/helper.py build_image gitpython
+python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython
+```
+
+Verify the build of your fuzzers with the optional `check_build` command:
+
+```shell
+python infra/helper.py check_build gitpython
+```
+
+Execute the desired fuzz target:
+
+```shell
+python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET
+```
+
+#### Next Steps
+
+For detailed instructions on advanced features like reproducing OSS-Fuzz issues or using the Fuzz Introspector, refer
+to [the official OSS-Fuzz documentation][oss-fuzz-docs].
+
+[oss-fuzz-repo]: https://github.com/google/oss-fuzz
+
+[oss-fuzz-docs]: https://google.github.io/oss-fuzz
+
+[oss-fuzz-issue-tracker]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:gitpython
+
+[google-fuzzing-repo]: https://github.com/google/fuzzing
+
+[atheris-repo]: https://github.com/google/atheris
diff --git a/fuzzing/dictionaries/fuzz_config.dict b/fuzzing/dictionaries/fuzz_config.dict
new file mode 100644
index 000000000..b545ddfc8
--- /dev/null
+++ b/fuzzing/dictionaries/fuzz_config.dict
@@ -0,0 +1,56 @@
+"\\004\\000\\000\\000\\000\\000\\000\\000"
+"\\006\\000\\000\\000\\000\\000\\000\\000"
+"_validate_value_"
+"\\000\\000\\000\\000\\000\\000\\000\\000"
+"rem"
+"__eq__"
+"\\001\\000\\000\\000"
+"__abstrac"
+"_mutating_methods_"
+"items"
+"\\0021\\""
+"\\001\\000"
+"\\000\\000\\000\\000"
+"DEFAULT"
+"getfloat"
+"\\004\\000\\000\\000\\000\\000\\000\\000"
+"news"
+"\\037\\000\\000\\000\\000\\000\\000\\000"
+"\\001\\000\\000\\000\\000\\000\\000\\037"
+"\\000\\000\\000\\000\\000\\000\\000\\014"
+"list"
+"\\376\\377\\377\\377\\377\\377\\377\\377"
+"items_all"
+"\\004\\000\\000\\000\\000\\000\\000\\000"
+"\\377\\377\\377\\377\\377\\377\\377\\014"
+"\\001\\000\\000\\000"
+"_acqui"
+"\\000\\000\\000\\000\\000\\000\\000\\000"
+"__ne__"
+"__exit__"
+"__modu"
+"uucp"
+"__str__"
+"\\001\\000\\000\\000"
+"\\017\\000\\000\\000\\000\\000\\000\\000"
+"_has_incl"
+"update"
+"\\377\\377\\377\\377\\377\\377\\377\\023"
+"setdef"
+"setdefaul"
+"\\000\\000\\000\\000"
+"\\001\\000\\000\\000"
+"\\001\\000"
+"\\022\\000\\000\\000\\000\\000\\000\\000"
+"_value_to_string"
+"__abstr"
+"\\001\\000\\000\\000\\000\\000\\000\\000"
+"\\000\\000\\000\\000\\000\\000\\000\\022"
+"\\377\\377\\377\\377"
+"\\004\\000\\000\\000\\000\\000\\000\\000"
+"\\000\\000\\000\\000\\000\\000\\000\\000"
+"\\000\\000\\000\\000\\000\\000\\000\\037"
+"\\001\\000\\000\\000\\000\\000\\000\\013"
+"_OPT_TM"
+"__name__"
+"_get_conv"
diff --git a/fuzzing/dictionaries/fuzz_tree.dict b/fuzzing/dictionaries/fuzz_tree.dict
new file mode 100644
index 000000000..3ebe52b7f
--- /dev/null
+++ b/fuzzing/dictionaries/fuzz_tree.dict
@@ -0,0 +1,13 @@
+"\\001\\000\\000\\000"
+"_join_multiline_va"
+"setdef"
+"1\\000\\000\\000\\000\\000\\000\\000"
+"\\000\\000\\000\\000\\000\\000\\000\\020"
+"\\377\\377\\377\\377\\377\\377\\377r"
+"\\001\\000\\000\\000\\000\\000\\000\\001"
+"\\000\\000\\000\\000\\000\\000\\000\\014"
+"\\000\\000\\000\\000\\000\\000\\000\\003"
+"\\001\\000"
+"\\032\\000\\000\\000\\000\\000\\000\\000"
+"-\\000\\000\\000\\000\\000\\000\\000"
+"__format"
diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py
new file mode 100644
index 000000000..1403c96e4
--- /dev/null
+++ b/fuzzing/fuzz-targets/fuzz_config.py
@@ -0,0 +1,51 @@
+#!/usr/bin/python3
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import atheris
+import sys
+import io
+from configparser import MissingSectionHeaderError, ParsingError
+
+with atheris.instrument_imports():
+ from git import GitConfigParser
+
+
+def TestOneInput(data):
+ sio = io.BytesIO(data)
+ sio.name = "/tmp/fuzzconfig.config"
+ git_config = GitConfigParser(sio)
+ try:
+ git_config.read()
+ except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError):
+ return -1 # Reject inputs raising expected exceptions
+ except (IndexError, ValueError) as e:
+ if isinstance(e, IndexError) and "string index out of range" in str(e):
+ # Known possibility that might be patched
+ # See: https://github.com/gitpython-developers/GitPython/issues/1887
+ pass
+ elif isinstance(e, ValueError) and "embedded null byte" in str(e):
+ # The `os.path.expanduser` function, which does not accept strings
+ # containing null bytes might raise this.
+ return -1
+ else:
+ raise e # Raise unanticipated exceptions as they might be bugs
+
+
+def main():
+ atheris.Setup(sys.argv, TestOneInput)
+ atheris.Fuzz()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py
new file mode 100644
index 000000000..53258fb1e
--- /dev/null
+++ b/fuzzing/fuzz-targets/fuzz_tree.py
@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import atheris
+import io
+import sys
+import os
+import shutil
+
+with atheris.instrument_imports():
+ from git.objects import Tree
+ from git.repo import Repo
+
+
+def TestOneInput(data):
+ fdp = atheris.FuzzedDataProvider(data)
+ git_dir = "/tmp/.git"
+ head_file = os.path.join(git_dir, "HEAD")
+ refs_dir = os.path.join(git_dir, "refs")
+ common_dir = os.path.join(git_dir, "commondir")
+ objects_dir = os.path.join(git_dir, "objects")
+
+ if os.path.isdir(git_dir):
+ shutil.rmtree(git_dir)
+
+ os.mkdir(git_dir)
+ with open(head_file, "w") as f:
+ f.write(fdp.ConsumeUnicodeNoSurrogates(1024))
+ os.mkdir(refs_dir)
+ os.mkdir(common_dir)
+ os.mkdir(objects_dir)
+
+ _repo = Repo("/tmp/")
+
+ fuzz_tree = Tree(_repo, Tree.NULL_BIN_SHA, 0, "")
+ try:
+ fuzz_tree._deserialize(io.BytesIO(data))
+ except IndexError:
+ return -1
+
+
+def main():
+ atheris.Setup(sys.argv, TestOneInput)
+ atheris.Fuzz()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
new file mode 100644
index 000000000..fdab7a1e0
--- /dev/null
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+python3 -m pip install .
+
+# Directory to look in for dictionaries, options files, and seed corpa:
+SEED_DATA_DIR="$SRC/seed_data"
+
+find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \
+ ! \( -name '__base.*' \) -exec printf 'Copying: %s\n' {} \; \
+ -exec chmod a-x {} \; \
+ -exec cp {} "$OUT" \;
+
+# Build fuzzers in $OUT.
+find "$SRC" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do
+ compile_python_fuzzer "$fuzz_harness"
+
+ common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict"
+ if [[ -r "$common_base_dictionary_filename" ]]; then
+ # Strip the `.py` extension from the filename and replace it with `.dict`.
+ fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict"
+ output_file="$OUT/$fuzz_harness_dictionary_filename"
+
+ printf 'Appending %s to %s\n' "$common_base_dictionary_filename" "$output_file"
+ if [[ -s "$output_file" ]]; then
+ # If a dictionary file for this fuzzer already exists and is not empty,
+ # we append a new line to the end of it before appending any new entries.
+ #
+ # libfuzzer will happily ignore multiple empty lines in a dictionary but crash
+ # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.)
+ # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32
+ echo >>"$output_file"
+ fi
+ cat "$common_base_dictionary_filename" >>"$output_file"
+ fi
+done
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
new file mode 100644
index 000000000..43c21a8da
--- /dev/null
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+#################
+# Prerequisites #
+#################
+
+for cmd in python3 git wget rsync; do
+ command -v "$cmd" >/dev/null 2>&1 || {
+ printf '[%s] Required command %s not found, exiting.\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$cmd" >&2
+ exit 1
+ }
+done
+
+SEED_DATA_DIR="$SRC/seed_data"
+mkdir -p "$SEED_DATA_DIR"
+
+#############
+# Functions #
+#############
+
+download_and_concatenate_common_dictionaries() {
+ # Assign the first argument as the target file where all contents will be concatenated
+ target_file="$1"
+
+ # Shift the arguments so the first argument (target_file path) is removed
+ # and only URLs are left for the loop below.
+ shift
+
+ for url in "$@"; do
+ wget -qO- "$url" >>"$target_file"
+ # Ensure there's a newline between each file's content
+ echo >>"$target_file"
+ done
+}
+
+fetch_seed_corpra() {
+ # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo.
+ git clone --depth 1 https://github.com/DaveLak/oss-fuzz-inputs.git oss-fuzz-inputs &&
+ rsync -avc oss-fuzz-inputs/gitpython/corpra/ "$SEED_DATA_DIR/" &&
+ rm -rf oss-fuzz-inputs; # Clean up the cloned repo to keep the Docker image as slim as possible.
+}
+
+########################
+# Main execution logic #
+########################
+
+fetch_seed_corpra;
+
+download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \
+ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \
+ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict";
+
+# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below.
+python3 -m pip install --upgrade pip;
+python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0'; # Uses the latest versions know to work at the time of this commit.
From d0c6ee62ad64a074ca885c0c2edbbc5074542b6e Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 11 Apr 2024 20:11:34 -0400
Subject: [PATCH 007/103] Update documentation to include fuzzing specific info
As per discussion in https://github.com/gitpython-developers/GitPython/discussions/1889
---
CONTRIBUTING.md | 5 +++++
README.md | 12 ++++++++++++
2 files changed, 17 insertions(+)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index e108f1b80..8536d7f73 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -8,3 +8,8 @@ The following is a short step-by-step rundown of what one typically would do to
- Try to avoid massive commits and prefer to take small steps, with one commit for each.
- Feel free to add yourself to AUTHORS file.
- Create a pull request.
+
+## Fuzzing Test Specific Documentation
+
+For details related to contributing to the fuzzing test suite and OSS-Fuzz integration, please
+refer to the dedicated [fuzzing README](./fuzzing/README.md).
diff --git a/README.md b/README.md
index 9bedaaae7..39f5496dc 100644
--- a/README.md
+++ b/README.md
@@ -240,5 +240,17 @@ Please have a look at the [contributions file][contributing].
[3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license].
+> [!NOTE]
+> There are two special case files located in the `fuzzzing/` directory that are licensed differently:
+>
+> `fuzz_config.py` and `fuzz_tree.py` were migrated here from the OSS-Fuzz project repository where they were initially
+> created and retain the original licence and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google
+> LLC respectively.)
+>
+> - **These files do not impact the licence under which GitPython releases or source code are distributed.**
+> - The files located in the `fuzzzing/` directory are part of the project test suite and neither packaged nor distributed as
+ part of any release.
+
+
[contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md
[license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE
From 1bc9a1a8250aa7291255cf389be2fa871c9049db Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 11 Apr 2024 21:19:24 -0400
Subject: [PATCH 008/103] Improve fuzzing README
Adds additional documentation links and fixes some typos.
---
fuzzing/README.md | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/fuzzing/README.md b/fuzzing/README.md
index 6853c5002..97a62724a 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -6,7 +6,7 @@ This directory contains files related to GitPython's suite of fuzz tests that ar
infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working
with fuzzing in GitPython.
-The details about the latest OSS-Fuzz test status, including build logs and coverage reports, is made available
+The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is made available
at [this link](https://introspector.oss-fuzz.com/project-profile?project=gitpython).
## How to Contribute
@@ -23,7 +23,7 @@ Areas that are particularly appreciated include:
update of any size, please consider suggesting it!
For everything else, such as expanding test coverage, optimizing test performance, or enhancing error detection
-capabilities, jump in to the "Getting Started" section below.
+capabilities, jump into the "Getting Started" section below.
## Getting Started with Fuzzing GitPython
@@ -63,7 +63,7 @@ The `fuzzing/` directory is organized into three key areas:
### Fuzz Targets (`fuzz-targets/`)
-Contains Python files for each fuzz test, targeting specific functionalities of GitPython.
+Contains Python files for each fuzz test.
**Things to Know**:
@@ -81,7 +81,7 @@ Contains Python files for each fuzz test, targeting specific functionalities of
Provides hints to the fuzzing engine about inputs that might trigger unique code paths. Each fuzz target may have a
corresponding `.dict` file. For details on how these are used, refer
-to [LibFuzzer documentation](https://llvm.org/docs/LibFuzzer.html#dictionaries).
+to the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.html#dictionaries).
**Things to Know**:
@@ -105,6 +105,11 @@ Includes scripts for building and integrating fuzz targets with OSS-Fuzz:
- **`build.sh`** - Executed within the Docker container, this script builds fuzz targets with necessary instrumentation
and prepares seed corpora and dictionaries for use.
+**Where to learn more:**
+
+- [OSS-Fuzz documentation on the build.sh](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#buildsh)
+- [See GitPython's build.sh and Dockerfile in the OSS-Fuzz repository](https://github.com/google/oss-fuzz/tree/master/projects/gitpython)
+
## Running Fuzzers Locally
### Direct Execution of Fuzz Targets
@@ -153,9 +158,21 @@ python infra/helper.py check_build gitpython
Execute the desired fuzz target:
```shell
-python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET
+python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET -- -max_total_time=60 -print_final_stats=1
```
+> [!TIP]
+> In the example above, the "`-- -max_total_time=60 -print_final_stats=1`" portion of the command is optional but quite
+> useful.
+>
+> Every argument provided after "`--`" in the above command is passed to the fuzzing engine directly. In this case:
+> - `-max_total_time=60` tells the LibFuzzer to stop execution after 60 seconds have elapsed.
+> - `-print_final_stats=1` tells the LibFuzzer to print a summary of useful metrics about the target run upon
+ completion.
+>
+> But almost any [LibFuzzer option listed in the documentation](https://llvm.org/docs/LibFuzzer.html#options) should
+> work as well.
+
#### Next Steps
For detailed instructions on advanced features like reproducing OSS-Fuzz issues or using the Fuzz Introspector, refer
From 576a858b7298bd14b1e87b118150df963af447dd Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 11 Apr 2024 22:06:05 -0400
Subject: [PATCH 009/103] Updates to support easily running OSS-Fuzz using
local repo sources
- Updates the fuzzing documentation to include steps for working with
locally modified versions of the gitpython repository.
- Updates the build.sh script to make the fuzz target search path more
specific, reducing the risk of local OSS-Fuzz builds picking up
files located outside of where we expect them (for example, in a .venv
directory.)
- add artifacts produced by local OSS-Fuzz runs to gitignore
---
.gitignore | 3 +++
fuzzing/README.md | 19 +++++++++++++++++--
fuzzing/oss-fuzz-scripts/build.sh | 2 +-
3 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/.gitignore b/.gitignore
index 7765293d8..d85569405 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,3 +47,6 @@ output.txt
# Finder metadata
.DS_Store
+
+# Files created by OSS-Fuzz when running locally
+fuzz_*.pkg.spec
diff --git a/fuzzing/README.md b/fuzzing/README.md
index 97a62724a..acaf17d06 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -134,8 +134,10 @@ containers.
Set environment variables to simplify command usage:
```shell
-export SANITIZER=address # Can be either 'address' or 'undefined'.
-export FUZZ_TARGET=fuzz_config # specify the fuzz target without the .py extension.
+# $SANITIZER can be either 'address' or 'undefined':
+export SANITIZER=address
+# specify the fuzz target without the .py extension:
+export FUZZ_TARGET=fuzz_config
```
#### Build and Run
@@ -149,6 +151,19 @@ python infra/helper.py build_image gitpython
python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython
```
+> [!TIP]
+> The `build_fuzzers` command above accepts a local file path pointing to your gitpython repository clone as the last
+> argument.
+> This makes it easy to build fuzz targets you are developing locally in this repository without changing anything in
+> the OSS-Fuzz repo!
+> For example, if you have cloned this repository (or a fork of it) into: `~/code/GitPython`
+> Then running this command would build new or modified fuzz targets using the `~/code/GitPython/fuzzing/fuzz-targets`
+> directory:
+> ```shell
+> python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython ~/code/GitPython
+> ```
+
+
Verify the build of your fuzzers with the optional `check_build` command:
```shell
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
index fdab7a1e0..aff1c4347 100644
--- a/fuzzing/oss-fuzz-scripts/build.sh
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -13,7 +13,7 @@ find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name
-exec cp {} "$OUT" \;
# Build fuzzers in $OUT.
-find "$SRC" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do
+find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do
compile_python_fuzzer "$fuzz_harness"
common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict"
From 5e56e96821878bd2808c640b8b39f84738ed8cf8 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 11 Apr 2024 23:37:54 -0400
Subject: [PATCH 010/103] Clarify documentation
- Fix typos in the documentation on dictionaries
- Link to the fuzzing directory in the main README where it is
referenced.
---
README.md | 2 +-
fuzzing/README.md | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index 39f5496dc..b4cdc0a97 100644
--- a/README.md
+++ b/README.md
@@ -241,7 +241,7 @@ Please have a look at the [contributions file][contributing].
[3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license].
> [!NOTE]
-> There are two special case files located in the `fuzzzing/` directory that are licensed differently:
+> There are two special case files located in the [`fuzzzing/` directory](./fuzzing) that are licensed differently:
>
> `fuzz_config.py` and `fuzz_tree.py` were migrated here from the OSS-Fuzz project repository where they were initially
> created and retain the original licence and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google
diff --git a/fuzzing/README.md b/fuzzing/README.md
index acaf17d06..c57e31d86 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -80,8 +80,8 @@ Contains Python files for each fuzz test.
### Dictionaries (`dictionaries/`)
Provides hints to the fuzzing engine about inputs that might trigger unique code paths. Each fuzz target may have a
-corresponding `.dict` file. For details on how these are used, refer
-to the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.html#dictionaries).
+corresponding `.dict` file. For information about dictionary syntax, refer to
+the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.html#dictionaries).
**Things to Know**:
@@ -92,7 +92,7 @@ to the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.
existing file here.
- Development or updates to dictionaries should reflect the varied formats and edge cases relevant to the
functionalities under test.
-- Example dictionaries (some of which are used to build the default dictionaries mentioned above) are can be found here:
+- Example dictionaries (some of which are used to build the default dictionaries mentioned above) can be found here:
- [AFL++ dictionary repository](https://github.com/AFLplusplus/AFLplusplus/tree/stable/dictionaries#readme)
- [Google/fuzzing dictionary repository](https://github.com/google/fuzzing/tree/master/dictionaries)
From 2041ba9972e7720f05bf570e2304fc0a5a2463d7 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Sun, 14 Apr 2024 22:59:01 -0400
Subject: [PATCH 011/103] Use gitpython-developers org ownd repository for seed
corpra
This repo was created after discussion in PR #1901.
---
fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
index 43c21a8da..881161fae 100644
--- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -36,9 +36,9 @@ download_and_concatenate_common_dictionaries() {
fetch_seed_corpra() {
# Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo.
- git clone --depth 1 https://github.com/DaveLak/oss-fuzz-inputs.git oss-fuzz-inputs &&
- rsync -avc oss-fuzz-inputs/gitpython/corpra/ "$SEED_DATA_DIR/" &&
- rm -rf oss-fuzz-inputs; # Clean up the cloned repo to keep the Docker image as slim as possible.
+ git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets &&
+ rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" &&
+ rm -rf qa-assets; # Clean up the cloned repo to keep the Docker image as slim as possible.
}
########################
From 945a767ccd13c84946b2a49fbde4227fdfc84a26 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 16 Apr 2024 02:06:50 -0400
Subject: [PATCH 012/103] Updates to comply with the terms of the Apache
License
Addresses feedback and encorperates suggestions from PR #1901 to ensure
that the Apache License requirements are met for the two files that they
apply to, and the documentation pertaining to licensing of the files in
this repository is clear and concise.
The contects of LICENSE-APACHE were coppied from the LICENSE file of the
OSS-Fuzz repository that the two fuzz harnesses came from as of commit:
https://github.com/google/oss-fuzz/blob/c2c0632831767ff05c568e7b552cef2801d739ff/LICENSE
---
README.md | 13 +-
fuzzing/LICENSE-APACHE | 201 ++++++++++++++++++++++++++++
fuzzing/README.md | 12 ++
fuzzing/fuzz-targets/fuzz_config.py | 6 +
fuzzing/fuzz-targets/fuzz_tree.py | 6 +
5 files changed, 227 insertions(+), 11 deletions(-)
create mode 100644 fuzzing/LICENSE-APACHE
diff --git a/README.md b/README.md
index b4cdc0a97..987e40e6c 100644
--- a/README.md
+++ b/README.md
@@ -240,17 +240,8 @@ Please have a look at the [contributions file][contributing].
[3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license].
-> [!NOTE]
-> There are two special case files located in the [`fuzzzing/` directory](./fuzzing) that are licensed differently:
->
-> `fuzz_config.py` and `fuzz_tree.py` were migrated here from the OSS-Fuzz project repository where they were initially
-> created and retain the original licence and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google
-> LLC respectively.)
->
-> - **These files do not impact the licence under which GitPython releases or source code are distributed.**
-> - The files located in the `fuzzzing/` directory are part of the project test suite and neither packaged nor distributed as
- part of any release.
-
+Two files exclusively used for fuzz testing are subject to [a separate license, detailed here](./fuzzing/README.md#license).
+These files are not included in the wheel or sdist packages published by the maintainers of GitPython.
[contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md
[license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE
diff --git a/fuzzing/LICENSE-APACHE b/fuzzing/LICENSE-APACHE
new file mode 100644
index 000000000..8dada3eda
--- /dev/null
+++ b/fuzzing/LICENSE-APACHE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/fuzzing/README.md b/fuzzing/README.md
index c57e31d86..09d6fc003 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -193,6 +193,18 @@ python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET -- -max_total_time=60 -
For detailed instructions on advanced features like reproducing OSS-Fuzz issues or using the Fuzz Introspector, refer
to [the official OSS-Fuzz documentation][oss-fuzz-docs].
+## LICENSE
+
+All files located within the `fuzzing/` directory are subject to [the same license](../LICENSE)
+as [the other files in this repository](../README.md#license) with two exceptions:
+
+Two files located in this directory, [`fuzz_config.py`](./fuzz-targets/fuzz_config.py)
+and [`fuzz_tree.py`](./fuzz-targets/fuzz_tree.py), have been migrated here from the OSS-Fuzz project repository where
+they were originally created. As such, these two files retain their original license and copyright notice (Apache
+License, Version 2.0 and Copyright 2023 Google LLC respectively.) Each file includes a notice in their respective header
+comments stating that they have been modified. [LICENSE-APACHE](./LICENSE-APACHE) contains the original license used by
+the OSS-Fuzz project repository at the time they were migrated.
+
[oss-fuzz-repo]: https://github.com/google/oss-fuzz
[oss-fuzz-docs]: https://google.github.io/oss-fuzz
diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py
index 1403c96e4..fc2f0960a 100644
--- a/fuzzing/fuzz-targets/fuzz_config.py
+++ b/fuzzing/fuzz-targets/fuzz_config.py
@@ -12,6 +12,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
+###############################################################################
+# Note: This file has been modified by contributors to GitPython.
+# The original state of this file may be referenced here:
+# https://github.com/google/oss-fuzz/commit/f26f254558fc48f3c9bc130b10507386b94522da
+###############################################################################
import atheris
import sys
import io
diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py
index 53258fb1e..b4e0e6b55 100644
--- a/fuzzing/fuzz-targets/fuzz_tree.py
+++ b/fuzzing/fuzz-targets/fuzz_tree.py
@@ -12,6 +12,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
+###############################################################################
+# Note: This file has been modified by contributors to GitPython.
+# The original state of this file may be referenced here:
+# https://github.com/google/oss-fuzz/commit/f26f254558fc48f3c9bc130b10507386b94522da
+###############################################################################
import atheris
import io
import sys
From 68194a913fa0d9f601a55fcd08ff13b7ac35be75 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 16 Apr 2024 14:41:36 -0400
Subject: [PATCH 013/103] Remove shebangs from fuzz harnesses
Prefer executing these files using the OSS-Fuzz or `python` command
methods outlined in the `fuzzing/README`.
Based on feedback and discussion on:
https://github.com/gitpython-developers/GitPython/pull/1901
---
fuzzing/fuzz-targets/fuzz_config.py | 1 -
fuzzing/fuzz-targets/fuzz_tree.py | 1 -
2 files changed, 2 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py
index fc2f0960a..0a06956c8 100644
--- a/fuzzing/fuzz-targets/fuzz_config.py
+++ b/fuzzing/fuzz-targets/fuzz_config.py
@@ -1,4 +1,3 @@
-#!/usr/bin/python3
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py
index b4e0e6b55..464235098 100644
--- a/fuzzing/fuzz-targets/fuzz_tree.py
+++ b/fuzzing/fuzz-targets/fuzz_tree.py
@@ -1,4 +1,3 @@
-#!/usr/bin/python3
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
From 8954c7151e098a0b12d4d2dec277fe6c63980579 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 16 Apr 2024 14:45:49 -0400
Subject: [PATCH 014/103] Replace shebang in `build.sh` with ShellCheck
directive
This script is meant to be sourced by the OSS-Fuzz file of the same
name, rather than executed directly. The shebang may lead to the
incorrect assumption that the script is meant for direct execution.
Replacing it with this directive instructs ShellCheck to treat
the script as a Bash script, regardless of how it is executed.
Based @EliahKagan's suggestion and feedback on:
https://github.com/gitpython-developers/GitPython/pull/1901
---
fuzzing/oss-fuzz-scripts/build.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
index aff1c4347..4c7de8799 100644
--- a/fuzzing/oss-fuzz-scripts/build.sh
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+# shellcheck shell=bash
set -euo pipefail
From b0a5b8e66c4da3d603d8e27a71c70aaad53542b8 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 16 Apr 2024 14:59:05 -0400
Subject: [PATCH 015/103] Set executable bit on
`container-environment-bootstrap.sh`
This script is executed directly, not sourced as is the case with
`build.sh`, so it should have an executable bit set to avoid ambiguity.
Based @EliahKagan's suggestion and feedback on:
https://github.com/gitpython-developers/GitPython/pull/1901
---
fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 0
1 file changed, 0 insertions(+), 0 deletions(-)
mode change 100644 => 100755 fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
old mode 100644
new mode 100755
From 25f360090cb6a7fd0f01bc127f2a2280659757a2 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 16 Apr 2024 15:31:05 -0400
Subject: [PATCH 016/103] Minor clarity improvements in `fuzzing/README.md`
- Make the link text for the OSS-Fuzz test status URL more descriptive
- Fix capitalization of GitPython repository name
Based @EliahKagan's suggestion and feedback on:
https://github.com/gitpython-developers/GitPython/pull/1901
---
fuzzing/README.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/fuzzing/README.md b/fuzzing/README.md
index 09d6fc003..65e311d4a 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -6,8 +6,8 @@ This directory contains files related to GitPython's suite of fuzz tests that ar
infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working
with fuzzing in GitPython.
-The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is made available
-at [this link](https://introspector.oss-fuzz.com/project-profile?project=gitpython).
+The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is available
+on [the Open Source Fuzzing Introspection website](https://introspector.oss-fuzz.com/project-profile?project=gitpython).
## How to Contribute
@@ -152,7 +152,7 @@ python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython
```
> [!TIP]
-> The `build_fuzzers` command above accepts a local file path pointing to your gitpython repository clone as the last
+> The `build_fuzzers` command above accepts a local file path pointing to your GitPython repository clone as the last
> argument.
> This makes it easy to build fuzz targets you are developing locally in this repository without changing anything in
> the OSS-Fuzz repo!
From d79c176384f1a5b6cb615f500037dbcecd9ee7d9 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 16 Apr 2024 16:10:08 -0400
Subject: [PATCH 017/103] Simplify read delimiter to use empty string in fuzz
harness loop
Replaces the null character delimiter `-d $'\0'` with the simpler
empty string `-d ''` in the fuzzing harness build loop.
This changes leverages the Bash `read` builtin behavior to avoid
unnecessary complexity and improving script readability.
Based @EliahKagan's suggestion and feedback on:
https://github.com/gitpython-developers/GitPython/pull/1901
---
fuzzing/oss-fuzz-scripts/build.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
index 4c7de8799..a412a1d15 100644
--- a/fuzzing/oss-fuzz-scripts/build.sh
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -13,7 +13,7 @@ find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name
-exec cp {} "$OUT" \;
# Build fuzzers in $OUT.
-find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do
+find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do
compile_python_fuzzer "$fuzz_harness"
common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict"
From e038526b846f4bc5e75a91c736f3384616800aa1 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 16 Apr 2024 16:27:43 -0400
Subject: [PATCH 018/103] Remove unnecessary semicolon for consistent script
formatting
Based @EliahKagan's suggestion and feedback on:
https://github.com/gitpython-developers/GitPython/pull/1901
---
.../container-environment-bootstrap.sh | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
index 881161fae..87f817993 100755
--- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -38,19 +38,19 @@ fetch_seed_corpra() {
# Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo.
git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets &&
rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" &&
- rm -rf qa-assets; # Clean up the cloned repo to keep the Docker image as slim as possible.
+ rm -rf qa-assets # Clean up the cloned repo to keep the Docker image as slim as possible.
}
########################
# Main execution logic #
########################
-fetch_seed_corpra;
+fetch_seed_corpra
download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \
"https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \
- "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict";
+ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict"
# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below.
-python3 -m pip install --upgrade pip;
-python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0'; # Uses the latest versions know to work at the time of this commit.
+python3 -m pip install --upgrade pip
+python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' # Uses the latest versions know to work at the time of this commit.
From d25ae2def1f995afcb7fad69250b12f5bf07b3bb Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 16 Apr 2024 16:38:21 -0400
Subject: [PATCH 019/103] Fix various misspellings of "corpora" & improve
script comments
A misspelling in the https://github.com/gitpython-developers/qa-assets
repository is still present here. It will need to be fixed in that
repository first.
"corpora" is a difficult word to spell consistently I guess. This made
for a good opportunity to improve the phrasing of two other comments at
at least.
Based @EliahKagan's suggestion and feedback on:
https://github.com/gitpython-developers/GitPython/pull/1901
---
fuzzing/oss-fuzz-scripts/build.sh | 4 ++--
.../oss-fuzz-scripts/container-environment-bootstrap.sh | 7 ++++---
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
index a412a1d15..ab46ec7a2 100644
--- a/fuzzing/oss-fuzz-scripts/build.sh
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -4,7 +4,7 @@ set -euo pipefail
python3 -m pip install .
-# Directory to look in for dictionaries, options files, and seed corpa:
+# Directory to look in for dictionaries, options files, and seed corpora:
SEED_DATA_DIR="$SRC/seed_data"
find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \
@@ -27,7 +27,7 @@ find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d
# If a dictionary file for this fuzzer already exists and is not empty,
# we append a new line to the end of it before appending any new entries.
#
- # libfuzzer will happily ignore multiple empty lines in a dictionary but crash
+ # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error
# if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.)
# See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32
echo >>"$output_file"
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
index 87f817993..0be012ccd 100755
--- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -34,7 +34,7 @@ download_and_concatenate_common_dictionaries() {
done
}
-fetch_seed_corpra() {
+fetch_seed_corpora() {
# Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo.
git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets &&
rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" &&
@@ -45,7 +45,7 @@ fetch_seed_corpra() {
# Main execution logic #
########################
-fetch_seed_corpra
+fetch_seed_corpora
download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \
"https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \
@@ -53,4 +53,5 @@ download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \
# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below.
python3 -m pip install --upgrade pip
-python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' # Uses the latest versions know to work at the time of this commit.
+ # Upgrade to the latest versions known to work at the time the below changes were introduced:
+python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0'
From 23a505f3ef51c4c26998fed924f4edad2438c757 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Wed, 17 Apr 2024 19:40:44 -0400
Subject: [PATCH 020/103] Remove comment suggesting the `undefined` sanitizer
is a valid option
Also makes come structural improvements to how the local instructions
for running OSS-Fuzz are presented now that only the single `address`
sanitizer is a valid option.
The `undefined` sanitizer was removed from GitPython's `project.yaml`
OSS-Fuzz configuration file at the request of OSS-Fuzz project reviewers
in https://github.com/google/oss-fuzz/pull/11803.
The `undefined` sanitizer is only useful in Python projects that use
native exstensions (such as C, C++, Rust, ect.), which GitPython does
not currently do. This commit updates the `fuzzing/README` reference to
that sanitizer accoirdingly.
See:
- https://github.com/google/oss-fuzz/pull/11803/commits/b210fb21427f1f994c91f07e95ca0cc977f61f66
- https://github.com/google/oss-fuzz/pull/11803#discussion_r1569160945
---
fuzzing/README.md | 28 +++++++++++++---------------
1 file changed, 13 insertions(+), 15 deletions(-)
diff --git a/fuzzing/README.md b/fuzzing/README.md
index 65e311d4a..ab9f6a63f 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -129,18 +129,7 @@ This approach uses Docker images provided by OSS-Fuzz for building and running f
comprehensive features but requires a local clone of the OSS-Fuzz repository and sufficient disk space for Docker
containers.
-#### Preparation
-
-Set environment variables to simplify command usage:
-
-```shell
-# $SANITIZER can be either 'address' or 'undefined':
-export SANITIZER=address
-# specify the fuzz target without the .py extension:
-export FUZZ_TARGET=fuzz_config
-```
-
-#### Build and Run
+#### Build the Execution Environment
Clone the OSS-Fuzz repository and prepare the Docker environment:
@@ -148,7 +137,7 @@ Clone the OSS-Fuzz repository and prepare the Docker environment:
git clone --depth 1 https://github.com/google/oss-fuzz.git oss-fuzz
cd oss-fuzz
python infra/helper.py build_image gitpython
-python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython
+python infra/helper.py build_fuzzers --sanitizer address gitpython
```
> [!TIP]
@@ -160,16 +149,25 @@ python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython
> Then running this command would build new or modified fuzz targets using the `~/code/GitPython/fuzzing/fuzz-targets`
> directory:
> ```shell
-> python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython ~/code/GitPython
+> python infra/helper.py build_fuzzers --sanitizer address gitpython ~/code/GitPython
> ```
-
Verify the build of your fuzzers with the optional `check_build` command:
```shell
python infra/helper.py check_build gitpython
```
+#### Run a Fuzz Target
+
+Setting an environment variable for the fuzz target argument of the execution command makes it easier to quickly select
+a different target between runs:
+
+```shell
+# specify the fuzz target without the .py extension:
+export FUZZ_TARGET=fuzz_config
+```
+
Execute the desired fuzz target:
```shell
From 1d54d4b0b2bdba60cc742f73a4b8d5a88cce8f64 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Wed, 17 Apr 2024 22:11:00 -0400
Subject: [PATCH 021/103] Remove unintentional leading space from comment
---
fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
index 0be012ccd..662808e27 100755
--- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -53,5 +53,5 @@ download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \
# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below.
python3 -m pip install --upgrade pip
- # Upgrade to the latest versions known to work at the time the below changes were introduced:
+# Upgrade to the latest versions known to work at the time the below changes were introduced:
python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0'
From fdce8375c80abd02b1dc08bd218f09849fea8233 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Sat, 20 Apr 2024 16:48:00 -0400
Subject: [PATCH 022/103] Dockerized "Direct Execution of Fuzz Targets"
Adds a Dockerfile to enable easily executing the fuzz targets directly
inside a container environment instead of directly on a host machine.
This addresses concerns raised in PR #1901 related to how `fuzz_tree.py`
writes to the real `/tmp` directory of the file system it is executed on
as part of setting up its own test fixtures, but also makes for an
easier to use development workflow.
See this related comment on PR #1901 for additional context:
https://github.com/gitpython-developers/GitPython/pull/1901#issuecomment-2063818998
---
fuzzing/README.md | 43 ++++++++++++++++++++++++----
fuzzing/local-dev-helpers/Dockerfile | 22 ++++++++++++++
2 files changed, 59 insertions(+), 6 deletions(-)
create mode 100644 fuzzing/local-dev-helpers/Dockerfile
diff --git a/fuzzing/README.md b/fuzzing/README.md
index ab9f6a63f..0a62b4c85 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -44,8 +44,7 @@ capabilities, jump into the "Getting Started" section below.
### Setting Up Your Local Environment
Before contributing to fuzzing efforts, ensure Python and Docker are installed on your machine. Docker is required for
-running fuzzers in containers provided by OSS-Fuzz. [Install Docker](https://docs.docker.com/get-docker/) following the
-official guide if you do not already have it.
+running fuzzers in containers provided by OSS-Fuzz and for safely executing test files directly. [Install Docker](https://docs.docker.com/get-docker/) following the official guide if you do not already have it.
### Understanding Existing Fuzz Targets
@@ -110,19 +109,51 @@ Includes scripts for building and integrating fuzz targets with OSS-Fuzz:
- [OSS-Fuzz documentation on the build.sh](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#buildsh)
- [See GitPython's build.sh and Dockerfile in the OSS-Fuzz repository](https://github.com/google/oss-fuzz/tree/master/projects/gitpython)
+### Local Development Helpers (`local-dev-helpers/`)
+
+Contains tools to make local development tasks easier.
+See [the "Running Fuzzers Locally" section below](#running-fuzzers-locally) for further documentation and use cases related to files found here.
+
## Running Fuzzers Locally
+> [!WARNING]
+> **Some fuzz targets in this repository write to the filesystem** during execution.
+> For that reason, it is strongly recommended to **always use Docker when executing fuzz targets**, even when it may be
+> possible to do so without it.
+>
+> Although [I/O operations such as writing to disk are not considered best practice](https://github.com/google/fuzzing/blob/master/docs/good-fuzz-target.md#io), the current implementation of at least one test requires it.
+> See [the "Setting Up Your Local Environment" section above](#setting-up-your-local-environment) if you do not already have Docker installed on your machine.
+>
+> PRs that replace disk I/O with in-memory alternatives are very much welcomed!
+
### Direct Execution of Fuzz Targets
-For quick testing of changes, [Atheris][atheris-repo] makes it possible to execute a fuzz target directly:
+Directly executing fuzz targets allows for quick iteration and testing of changes which can be helpful during early
+development of new fuzz targets or for validating changes made to an existing test.
+The [Dockerfile](./local-dev-helpers/Dockerfile) located in the `local-dev-helpers/` subdirectory provides a lightweight
+container environment preconfigured with [Atheris][atheris-repo] that makes it easy to execute a fuzz target directly.
+
+**From the root directory of your GitPython repository clone**:
-1. Install Atheris following the [installation guide][atheris-repo] for your operating system.
-2. Execute a fuzz target, for example:
+1. Build the local development helper image:
```shell
-python fuzzing/fuzz-targets/fuzz_config.py
+docker build -f fuzzing/local-dev-helpers/Dockerfile -t gitpython-fuzzdev .
```
+2. Then execute a fuzz target inside the image, for example:
+
+```shell
+ docker run -it -v "$PWD":/src gitpython-fuzzdev python fuzzing/fuzz-targets/fuzz_config.py -atheris_runs=10000
+```
+
+The above command executes [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) and exits after `10000` runs, or earlier if
+the fuzzer finds an error.
+
+Docker CLI's `-v` flag specifies a volume mount in Docker that maps the directory in which the command is run (which
+should be the root directory of your local GitPython clone) to a directory inside the container, so any modifications
+made between invocations will be reflected immediately without the need to rebuild the image each time.
+
### Running OSS-Fuzz Locally
This approach uses Docker images provided by OSS-Fuzz for building and running fuzz tests locally. It offers
diff --git a/fuzzing/local-dev-helpers/Dockerfile b/fuzzing/local-dev-helpers/Dockerfile
new file mode 100644
index 000000000..77808ed1d
--- /dev/null
+++ b/fuzzing/local-dev-helpers/Dockerfile
@@ -0,0 +1,22 @@
+# syntax=docker/dockerfile:1
+
+# Use the same Python version as OSS-Fuzz to accidental incompatibilities in test code
+FROM python:3.8-slim
+
+LABEL project="GitPython Fuzzing Local Dev Helper"
+
+WORKDIR /src
+
+COPY . .
+
+# Update package managers, install necessary packages, and cleanup unnecessary files in a single RUN to keep the image smaller.
+RUN apt-get update && \
+ apt-get install --no-install-recommends -y git && \
+ python -m pip install --upgrade pip && \
+ python -m pip install atheris && \
+ python -m pip install -e . && \
+ apt-get clean && \
+ apt-get autoremove -y && \
+ rm -rf /var/lib/apt/lists/* /root/.cache
+
+CMD ["bash"]
From f1451219c5a3a221615d3d38ac251bf5bbe46119 Mon Sep 17 00:00:00 2001
From: DaveLak <github@themoderndev.com>
Date: Sun, 21 Apr 2024 12:19:57 -0400
Subject: [PATCH 023/103] Fix Atheris install in local dev helper Docker image
The Atheris package bundles a binary that supplies libFuzzer on some
host machines, but in some cases (such as ARM based mac hosts) Atheris
seems to require building libFuzzer at install time while pip builds the
wheel. In the latter case, clang and related dependencies must be
present and available for the build, which itself requires using a non
"slim" version of the Python base image and not passing the
`--no-install-recommends` flag to `apt-get install` as both prevent the
required related libraries from being automatically installed.
It is also worth noting that at the time of this commit, the default
version of LLVM & Clang installed when `clang` is installed from `apt`
is version 14, while the latest stable version is 17 and OSS-Fuzz uses
15. The decision to install the default version (14) available via the
debian repos was intentional because a) it appears to work fine for our
needs and Atheris version b) specifying a different version requires
more complexity depending on install method, but the goal of this
Dockerfile is simplicity and low maintenance.
If it becomes neccissary to upgrade Clang/LLVM in the future, one option
to consider besides installing from source is the apt repository
maintained by the LLVM project: https://apt.llvm.org/
See the discussion in this issue for additional context to this change:
https://github.com/gitpython-developers/GitPython/pull/1904
---
fuzzing/local-dev-helpers/Dockerfile | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/fuzzing/local-dev-helpers/Dockerfile b/fuzzing/local-dev-helpers/Dockerfile
index 77808ed1d..426de05dd 100644
--- a/fuzzing/local-dev-helpers/Dockerfile
+++ b/fuzzing/local-dev-helpers/Dockerfile
@@ -1,7 +1,7 @@
# syntax=docker/dockerfile:1
# Use the same Python version as OSS-Fuzz to accidental incompatibilities in test code
-FROM python:3.8-slim
+FROM python:3.8-bookworm
LABEL project="GitPython Fuzzing Local Dev Helper"
@@ -11,12 +11,12 @@ COPY . .
# Update package managers, install necessary packages, and cleanup unnecessary files in a single RUN to keep the image smaller.
RUN apt-get update && \
- apt-get install --no-install-recommends -y git && \
+ apt-get install -y git clang && \
python -m pip install --upgrade pip && \
python -m pip install atheris && \
python -m pip install -e . && \
apt-get clean && \
apt-get autoremove -y && \
- rm -rf /var/lib/apt/lists/* /root/.cache
+ rm -rf /var/lib/apt/lists/*
CMD ["bash"]
From f4b95cf089706a29396b744b53a4ecdcc924d31c Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Mon, 22 Apr 2024 16:07:54 -0400
Subject: [PATCH 024/103] Fix Fuzzer Crash in ClusterFuzz Due to Missing Git
Executable
A Git executable is not globally available in the ClusterFuzz container
environment where OSS-Fuzz executes fuzz tests, causing an error in the fuzz
harnesses when GitPython attempts to initialize, crashing the tests before they
can run.
To avoid this issue, we bundle the `git` binary that is available in the OSS-Fuzz
build container with the fuzz harness via Pyinstaller's `--add-binary` flag in
`build.sh` and use GitPython's `git.refresh(<full-path-to-git-executable>)`
method inside a Pyinstaller runtime check to initialize GitPython with the
bundled Git executable when running from the bundled application.
In all other execution environments, we assume a `git` executable is available
globally.
Fixes:
- https://github.com/gitpython-developers/GitPython/issues/1905
- https://github.com/google/oss-fuzz/issues/10600
---
fuzzing/fuzz-targets/fuzz_config.py | 9 +++++++--
fuzzing/fuzz-targets/fuzz_tree.py | 11 +++++++----
fuzzing/oss-fuzz-scripts/build.sh | 2 +-
3 files changed, 15 insertions(+), 7 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py
index 0a06956c8..7623ab98f 100644
--- a/fuzzing/fuzz-targets/fuzz_config.py
+++ b/fuzzing/fuzz-targets/fuzz_config.py
@@ -20,16 +20,21 @@
import atheris
import sys
import io
+import os
from configparser import MissingSectionHeaderError, ParsingError
with atheris.instrument_imports():
- from git import GitConfigParser
+ import git
def TestOneInput(data):
+ if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+ path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
+ git.refresh(path_to_bundled_git_binary)
+
sio = io.BytesIO(data)
sio.name = "/tmp/fuzzconfig.config"
- git_config = GitConfigParser(sio)
+ git_config = git.GitConfigParser(sio)
try:
git_config.read()
except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError):
diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py
index 464235098..7187c4a6f 100644
--- a/fuzzing/fuzz-targets/fuzz_tree.py
+++ b/fuzzing/fuzz-targets/fuzz_tree.py
@@ -24,11 +24,14 @@
import shutil
with atheris.instrument_imports():
- from git.objects import Tree
- from git.repo import Repo
+ import git
def TestOneInput(data):
+ if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+ path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
+ git.refresh(path_to_bundled_git_binary)
+
fdp = atheris.FuzzedDataProvider(data)
git_dir = "/tmp/.git"
head_file = os.path.join(git_dir, "HEAD")
@@ -46,9 +49,9 @@ def TestOneInput(data):
os.mkdir(common_dir)
os.mkdir(objects_dir)
- _repo = Repo("/tmp/")
+ _repo = git.Repo("/tmp/")
- fuzz_tree = Tree(_repo, Tree.NULL_BIN_SHA, 0, "")
+ fuzz_tree = git.Tree(_repo, git.Tree.NULL_BIN_SHA, 0, "")
try:
fuzz_tree._deserialize(io.BytesIO(data))
except IndexError:
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
index ab46ec7a2..be31ac32a 100644
--- a/fuzzing/oss-fuzz-scripts/build.sh
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -14,7 +14,7 @@ find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name
# Build fuzzers in $OUT.
find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do
- compile_python_fuzzer "$fuzz_harness"
+ compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):."
common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict"
if [[ -r "$common_base_dictionary_filename" ]]; then
From 2b0a9693ea98ab7ff025a8ab1235b6f8ea0da676 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Mon, 22 Apr 2024 16:36:17 -0400
Subject: [PATCH 025/103] Add GitPython's standard license header comments to
oss-fuzz scripts
These files are already BSD-3-Clause even without the headers, but
adding these comments and the `LICENSE-BSD` symlink to the root level
`LICENSE` file are helpful to reinforce that there are only two
particular files in the `fuzzing/` that are not under BSD-3-Clause.
See:
https://github.com/gitpython-developers/GitPython/pull/1901#discussion_r1567849271
---
fuzzing/LICENSE-BSD | 1 +
fuzzing/oss-fuzz-scripts/build.sh | 3 +++
fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 4 ++++
3 files changed, 8 insertions(+)
create mode 120000 fuzzing/LICENSE-BSD
diff --git a/fuzzing/LICENSE-BSD b/fuzzing/LICENSE-BSD
new file mode 120000
index 000000000..ea5b60640
--- /dev/null
+++ b/fuzzing/LICENSE-BSD
@@ -0,0 +1 @@
+../LICENSE
\ No newline at end of file
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
index ab46ec7a2..a79cbe895 100644
--- a/fuzzing/oss-fuzz-scripts/build.sh
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -1,4 +1,7 @@
# shellcheck shell=bash
+#
+# This file is part of GitPython and is released under the
+# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/
set -euo pipefail
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
index 662808e27..76ec97c7f 100755
--- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -1,4 +1,8 @@
#!/usr/bin/env bash
+#
+# This file is part of GitPython and is released under the
+# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/
+
set -euo pipefail
#################
From b021a76354bb2779fc8f43c647a3a85b67f3d01a Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Mon, 22 Apr 2024 16:43:03 -0400
Subject: [PATCH 026/103] Add GitPython's standard license header comments to
top level scripts
While discussing adding similar license comments to the shell scripts
introduced in PR #1901, it was noticed that the shell scripts in the
repository root directory did not have such comments and suggested that
we could add them when the scripts in the `fuzzing/` directory were
updated, so this commit does just that.
See:
https://github.com/gitpython-developers/GitPython/pull/1901#discussion_r1567849271
---
build-release.sh | 3 +++
check-version.sh | 3 +++
init-tests-after-clone.sh | 3 +++
3 files changed, 9 insertions(+)
diff --git a/build-release.sh b/build-release.sh
index 49c13b93a..1a8dce2c2 100755
--- a/build-release.sh
+++ b/build-release.sh
@@ -1,5 +1,8 @@
#!/bin/bash
#
+# This file is part of GitPython and is released under the
+# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/
+#
# This script builds a release. If run in a venv, it auto-installs its tools.
# You may want to run "make release" instead of running this script directly.
diff --git a/check-version.sh b/check-version.sh
index dac386e46..579cf789f 100755
--- a/check-version.sh
+++ b/check-version.sh
@@ -1,5 +1,8 @@
#!/bin/bash
#
+# This file is part of GitPython and is released under the
+# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/
+#
# This script checks if we are in a consistent state to build a new release.
# See the release instructions in README.md for the steps to make this pass.
# You may want to run "make release" instead of running this script directly.
diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh
index 118e1de22..bfada01b0 100755
--- a/init-tests-after-clone.sh
+++ b/init-tests-after-clone.sh
@@ -1,4 +1,7 @@
#!/bin/sh
+#
+# This file is part of GitPython and is released under the
+# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/
set -eu
From 47e5738074e7f9acfb64d164206770bbd41685a0 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Mon, 22 Apr 2024 18:48:26 -0400
Subject: [PATCH 027/103] Fix IndexError in GitConfigParser when config value
ends in new line
Improve the guarding `if` check in `GitConfigParser`'s `string_decode`
function to safely handle empty strings and prevent `IndexError`s when
accessing string elements.
This resolves an IndexError in the `GitConfigParser`'s `.read()`
method when the config file contains a quoted value containing a
trailing new line.
Fixes:
https://github.com/gitpython-developers/GitPython/issues/1887
---
fuzzing/fuzz-targets/fuzz_config.py | 8 ++------
git/config.py | 2 +-
test/test_config.py | 8 ++++++++
3 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py
index 0a06956c8..81dcf9a88 100644
--- a/fuzzing/fuzz-targets/fuzz_config.py
+++ b/fuzzing/fuzz-targets/fuzz_config.py
@@ -34,12 +34,8 @@ def TestOneInput(data):
git_config.read()
except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError):
return -1 # Reject inputs raising expected exceptions
- except (IndexError, ValueError) as e:
- if isinstance(e, IndexError) and "string index out of range" in str(e):
- # Known possibility that might be patched
- # See: https://github.com/gitpython-developers/GitPython/issues/1887
- pass
- elif isinstance(e, ValueError) and "embedded null byte" in str(e):
+ except ValueError as e:
+ if isinstance(e, ValueError) and "embedded null byte" in str(e):
# The `os.path.expanduser` function, which does not accept strings
# containing null bytes might raise this.
return -1
diff --git a/git/config.py b/git/config.py
index 3ce9b123f..c9b49684c 100644
--- a/git/config.py
+++ b/git/config.py
@@ -452,7 +452,7 @@ def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None:
e = None # None, or an exception.
def string_decode(v: str) -> str:
- if v[-1] == "\\":
+ if v and v[-1] == "\\":
v = v[:-1]
# END cut trailing escapes to prevent decode error
diff --git a/test/test_config.py b/test/test_config.py
index 0911d0262..92997422d 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -142,6 +142,14 @@ def test_multi_line_config(self):
)
self.assertEqual(len(config.sections()), 23)
+ def test_config_value_with_trailing_new_line(self):
+ config_content = b'[section-header]\nkey:"value\n"'
+ config_file = io.BytesIO(config_content)
+ config_file.name = "multiline_value.config"
+
+ git_config = GitConfigParser(config_file)
+ git_config.read() # This should not throw an exception
+
def test_base(self):
path_repo = fixture_path("git_config")
path_global = fixture_path("git_config_global")
From c2283f6e3566606300f64c44a12197f0b65f0d71 Mon Sep 17 00:00:00 2001
From: Sebastian Thiel <sebastian.thiel@icloud.com>
Date: Tue, 23 Apr 2024 08:21:58 +0200
Subject: [PATCH 028/103] Avoid unnecessary isinstance check
---
fuzzing/fuzz-targets/fuzz_config.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py
index 81dcf9a88..80ab7b08d 100644
--- a/fuzzing/fuzz-targets/fuzz_config.py
+++ b/fuzzing/fuzz-targets/fuzz_config.py
@@ -35,7 +35,7 @@ def TestOneInput(data):
except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError):
return -1 # Reject inputs raising expected exceptions
except ValueError as e:
- if isinstance(e, ValueError) and "embedded null byte" in str(e):
+ if "embedded null byte" in str(e):
# The `os.path.expanduser` function, which does not accept strings
# containing null bytes might raise this.
return -1
From 1a0ab5bbdaa9df5d04d1b6946af419492b650fce Mon Sep 17 00:00:00 2001
From: Sebastian Thiel <sebastian.thiel@icloud.com>
Date: Fri, 26 Apr 2024 07:15:13 +0200
Subject: [PATCH 029/103] Use endswith() for more clarity
---
git/config.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/config.py b/git/config.py
index c9b49684c..de3508360 100644
--- a/git/config.py
+++ b/git/config.py
@@ -452,7 +452,7 @@ def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None:
e = None # None, or an exception.
def string_decode(v: str) -> str:
- if v and v[-1] == "\\":
+ if v and v.endswith("\\"):
v = v[:-1]
# END cut trailing escapes to prevent decode error
From dac3535d3dc4aaff9bd98a6ea70f46b132537694 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Fri, 26 Apr 2024 18:13:11 -0400
Subject: [PATCH 030/103] Attempt 2 - Fix Missing Git Executable Causing
ClusterFuzz Crash
This is a second attempt at #1906 and should resolve:
- https://github.com/gitpython-developers/GitPython/issues/1905
- https://github.com/google/oss-fuzz/issues/10600
PR #1906 had the right idea but wrong implementation, and the differences between
the ClusterFuzz image that it was supposed to fix and the OSS-Fuzz image where
the fix was tested led to the issue not being fully resolved.
The root cause of the issue is the same: A Git executable is not globally
available in the ClusterFuzz container environment where OSS-Fuzz executes
fuzz tests.
#1906 attempted to fix the issue by bundling the Git binary and using
GitPython's `git.refresh(<full-path-to-git-executable>)` method to set it
inside the `TestOneInput` function of the test harness.
However, GitPython attempts to set the binary at import time via its `__init__`
hook, and crashes the test if no executable is found during the import.
This issue is fixed here by setting the environment variable that GitPython
looks in before importing it, so it's available for the import. This was tested
by setting the `$PATH` to an empty string inside the test files, which
reproduced the crash, then adding the changes introduced here with `$PATH` still
empty, which avoided the crash indicating that the bundled Git executable is
working as expected.
---
fuzzing/fuzz-targets/fuzz_config.py | 8 ++++----
fuzzing/fuzz-targets/fuzz_tree.py | 8 ++++----
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py
index 6f2caad4b..4eddc32ff 100644
--- a/fuzzing/fuzz-targets/fuzz_config.py
+++ b/fuzzing/fuzz-targets/fuzz_config.py
@@ -23,15 +23,15 @@
import os
from configparser import MissingSectionHeaderError, ParsingError
+if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+ path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
+ os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+
with atheris.instrument_imports():
import git
def TestOneInput(data):
- if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
- path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
- git.refresh(path_to_bundled_git_binary)
-
sio = io.BytesIO(data)
sio.name = "/tmp/fuzzconfig.config"
git_config = git.GitConfigParser(sio)
diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py
index 7187c4a6f..4e2038add 100644
--- a/fuzzing/fuzz-targets/fuzz_tree.py
+++ b/fuzzing/fuzz-targets/fuzz_tree.py
@@ -23,15 +23,15 @@
import os
import shutil
+if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+ path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
+ os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+
with atheris.instrument_imports():
import git
def TestOneInput(data):
- if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
- path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
- git.refresh(path_to_bundled_git_binary)
-
fdp = atheris.FuzzedDataProvider(data)
git_dir = "/tmp/.git"
head_file = os.path.join(git_dir, "HEAD")
From c84e643c6aa177f364ebe28e4c7bab1e37fb0242 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Sun, 28 Apr 2024 22:41:11 -0400
Subject: [PATCH 031/103] Replace the suboptimal fuzz_tree harness with a
better alternative
As discussed in the initial fuzzing integration PR[^1], `fuzz_tree.py`'s
implementation was not ideal in terms of coverage and its reading/writing to
hard-coded paths inside `/tmp` was problematic as (among other concerns), it
causes intermittent crashes on ClusterFuzz[^2] when multiple workers execute
the test at the same time on the same machine.
The changes here replace `fuzz_tree.py` completely with a completely new
`fuzz_repo.py` fuzz target which:
- Uses `tempfile.TemporaryDirectory()` to safely manage tmpdir creation and
tear down, including during multi-worker execution runs.
- Retains the same feature coverage as `fuzz_tree.py`, but it also adds
considerably more from much smaller data inputs and with less memory consumed
(and it doesn't even have a seed corpus or target specific dictionary yet.)
- Can likely be improved further in the future by exercising additional features
of `Repo` to the harness.
Because `fuzz_tree.py` was removed and `fuzz_repo.py` was not derived from it,
the Apache License call outs in the docs were also updated as they only apply to
the singe `fuzz_config.py` file now.
[^1]: https://github.com/gitpython-developers/GitPython/pull/1901#discussion_r1565001609
[^2]: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=68355
---
README.md | 4 +-
fuzzing/README.md | 16 +++----
fuzzing/dictionaries/fuzz_tree.dict | 13 ------
fuzzing/fuzz-targets/fuzz_repo.py | 47 ++++++++++++++++++++
fuzzing/fuzz-targets/fuzz_tree.py | 67 -----------------------------
5 files changed, 57 insertions(+), 90 deletions(-)
delete mode 100644 fuzzing/dictionaries/fuzz_tree.dict
create mode 100644 fuzzing/fuzz-targets/fuzz_repo.py
delete mode 100644 fuzzing/fuzz-targets/fuzz_tree.py
diff --git a/README.md b/README.md
index 987e40e6c..d365a6584 100644
--- a/README.md
+++ b/README.md
@@ -240,8 +240,8 @@ Please have a look at the [contributions file][contributing].
[3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license].
-Two files exclusively used for fuzz testing are subject to [a separate license, detailed here](./fuzzing/README.md#license).
-These files are not included in the wheel or sdist packages published by the maintainers of GitPython.
+One file exclusively used for fuzz testing is subject to [a separate license, detailed here](./fuzzing/README.md#license).
+This file is not included in the wheel or sdist packages published by the maintainers of GitPython.
[contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md
[license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE
diff --git a/fuzzing/README.md b/fuzzing/README.md
index 0a62b4c85..9d02bf72f 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -225,14 +225,14 @@ to [the official OSS-Fuzz documentation][oss-fuzz-docs].
## LICENSE
All files located within the `fuzzing/` directory are subject to [the same license](../LICENSE)
-as [the other files in this repository](../README.md#license) with two exceptions:
-
-Two files located in this directory, [`fuzz_config.py`](./fuzz-targets/fuzz_config.py)
-and [`fuzz_tree.py`](./fuzz-targets/fuzz_tree.py), have been migrated here from the OSS-Fuzz project repository where
-they were originally created. As such, these two files retain their original license and copyright notice (Apache
-License, Version 2.0 and Copyright 2023 Google LLC respectively.) Each file includes a notice in their respective header
-comments stating that they have been modified. [LICENSE-APACHE](./LICENSE-APACHE) contains the original license used by
-the OSS-Fuzz project repository at the time they were migrated.
+as [the other files in this repository](../README.md#license) with one exception:
+
+[`fuzz_config.py`](./fuzz-targets/fuzz_config.py) was migrated to this repository from the OSS-Fuzz project's repository
+where it was originally created. As such, [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) retains its original license
+and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google LLC respectively) as in a header
+comment, followed by a notice stating that it has have been modified contributors to GitPython.
+[LICENSE-APACHE](./LICENSE-APACHE) contains the original license used by the OSS-Fuzz project repository at the time the
+file was migrated.
[oss-fuzz-repo]: https://github.com/google/oss-fuzz
diff --git a/fuzzing/dictionaries/fuzz_tree.dict b/fuzzing/dictionaries/fuzz_tree.dict
deleted file mode 100644
index 3ebe52b7f..000000000
--- a/fuzzing/dictionaries/fuzz_tree.dict
+++ /dev/null
@@ -1,13 +0,0 @@
-"\\001\\000\\000\\000"
-"_join_multiline_va"
-"setdef"
-"1\\000\\000\\000\\000\\000\\000\\000"
-"\\000\\000\\000\\000\\000\\000\\000\\020"
-"\\377\\377\\377\\377\\377\\377\\377r"
-"\\001\\000\\000\\000\\000\\000\\000\\001"
-"\\000\\000\\000\\000\\000\\000\\000\\014"
-"\\000\\000\\000\\000\\000\\000\\000\\003"
-"\\001\\000"
-"\\032\\000\\000\\000\\000\\000\\000\\000"
-"-\\000\\000\\000\\000\\000\\000\\000"
-"__format"
diff --git a/fuzzing/fuzz-targets/fuzz_repo.py b/fuzzing/fuzz-targets/fuzz_repo.py
new file mode 100644
index 000000000..7bd82c120
--- /dev/null
+++ b/fuzzing/fuzz-targets/fuzz_repo.py
@@ -0,0 +1,47 @@
+import atheris
+import io
+import sys
+import os
+import tempfile
+
+if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+ path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
+ os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+
+with atheris.instrument_imports():
+ import git
+
+
+def TestOneInput(data):
+ fdp = atheris.FuzzedDataProvider(data)
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ repo = git.Repo.init(path=temp_dir)
+
+ # Generate a minimal set of files based on fuzz data to minimize I/O operations.
+ file_paths = [os.path.join(temp_dir, f"File{i}") for i in range(min(3, fdp.ConsumeIntInRange(1, 3)))]
+ for file_path in file_paths:
+ with open(file_path, "wb") as f:
+ # The chosen upperbound for count of bytes we consume by writing to these
+ # files is somewhat arbitrary and may be worth experimenting with if the
+ # fuzzer coverage plateaus.
+ f.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512)))
+
+ repo.index.add(file_paths)
+ repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 80)))
+
+ fuzz_tree = git.Tree(repo, git.Tree.NULL_BIN_SHA, 0, "")
+
+ try:
+ fuzz_tree._deserialize(io.BytesIO(data))
+ except IndexError:
+ return -1
+
+
+def main():
+ atheris.Setup(sys.argv, TestOneInput)
+ atheris.Fuzz()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py
deleted file mode 100644
index 4e2038add..000000000
--- a/fuzzing/fuzz-targets/fuzz_tree.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# Copyright 2023 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-###############################################################################
-# Note: This file has been modified by contributors to GitPython.
-# The original state of this file may be referenced here:
-# https://github.com/google/oss-fuzz/commit/f26f254558fc48f3c9bc130b10507386b94522da
-###############################################################################
-import atheris
-import io
-import sys
-import os
-import shutil
-
-if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
- path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
- os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
-
-with atheris.instrument_imports():
- import git
-
-
-def TestOneInput(data):
- fdp = atheris.FuzzedDataProvider(data)
- git_dir = "/tmp/.git"
- head_file = os.path.join(git_dir, "HEAD")
- refs_dir = os.path.join(git_dir, "refs")
- common_dir = os.path.join(git_dir, "commondir")
- objects_dir = os.path.join(git_dir, "objects")
-
- if os.path.isdir(git_dir):
- shutil.rmtree(git_dir)
-
- os.mkdir(git_dir)
- with open(head_file, "w") as f:
- f.write(fdp.ConsumeUnicodeNoSurrogates(1024))
- os.mkdir(refs_dir)
- os.mkdir(common_dir)
- os.mkdir(objects_dir)
-
- _repo = git.Repo("/tmp/")
-
- fuzz_tree = git.Tree(_repo, git.Tree.NULL_BIN_SHA, 0, "")
- try:
- fuzz_tree._deserialize(io.BytesIO(data))
- except IndexError:
- return -1
-
-
-def main():
- atheris.Setup(sys.argv, TestOneInput)
- atheris.Fuzz()
-
-
-if __name__ == "__main__":
- main()
From 48abb1cbc138cd9c013369ea4608dd2fe5ca7a62 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Sat, 4 May 2024 14:40:26 -0400
Subject: [PATCH 032/103] Add git.Blob fuzz target
Based on the `test_blob.py` unit test.
---
fuzzing/dictionaries/fuzz_blob.dict | 1 +
fuzzing/fuzz-targets/fuzz_blob.py | 36 +++++++++++++++++++++++++++++
2 files changed, 37 insertions(+)
create mode 100644 fuzzing/dictionaries/fuzz_blob.dict
create mode 100644 fuzzing/fuzz-targets/fuzz_blob.py
diff --git a/fuzzing/dictionaries/fuzz_blob.dict b/fuzzing/dictionaries/fuzz_blob.dict
new file mode 100644
index 000000000..7f123f830
--- /dev/null
+++ b/fuzzing/dictionaries/fuzz_blob.dict
@@ -0,0 +1 @@
+"\\377\\377\\377\\377\\377\\377\\377\\377"
diff --git a/fuzzing/fuzz-targets/fuzz_blob.py b/fuzzing/fuzz-targets/fuzz_blob.py
new file mode 100644
index 000000000..9d296de40
--- /dev/null
+++ b/fuzzing/fuzz-targets/fuzz_blob.py
@@ -0,0 +1,36 @@
+import atheris
+import sys
+import os
+import tempfile
+
+if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+ path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
+ os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+
+with atheris.instrument_imports():
+ import git
+
+
+def TestOneInput(data):
+ fdp = atheris.FuzzedDataProvider(data)
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ repo = git.Repo.init(path=temp_dir)
+ blob = git.Blob(
+ repo,
+ **{
+ "binsha": git.Blob.NULL_BIN_SHA,
+ "path": fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()),
+ },
+ )
+
+ _ = blob.mime_type
+
+
+def main():
+ atheris.Setup(sys.argv, TestOneInput)
+ atheris.Fuzz()
+
+
+if __name__ == "__main__":
+ main()
From 6823e4543f33eb623df14a5a27c9731199de7a4f Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Sat, 4 May 2024 15:44:23 -0400
Subject: [PATCH 033/103] Use fuzzed data for all git.Blob arguments
This increases the edges reached by the fuzzer, making for a more
effective test with higher coverage.
---
fuzzing/fuzz-targets/fuzz_blob.py | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_blob.py b/fuzzing/fuzz-targets/fuzz_blob.py
index 9d296de40..ce888e85f 100644
--- a/fuzzing/fuzz-targets/fuzz_blob.py
+++ b/fuzzing/fuzz-targets/fuzz_blob.py
@@ -16,13 +16,17 @@ def TestOneInput(data):
with tempfile.TemporaryDirectory() as temp_dir:
repo = git.Repo.init(path=temp_dir)
- blob = git.Blob(
- repo,
- **{
- "binsha": git.Blob.NULL_BIN_SHA,
- "path": fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()),
- },
- )
+ binsha = fdp.ConsumeBytes(20)
+ mode = fdp.ConsumeInt(fdp.ConsumeIntInRange(0, fdp.remaining_bytes()))
+ path = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes())
+
+ try:
+ blob = git.Blob(repo, binsha, mode, path)
+ except AssertionError as e:
+ if "Require 20 byte binary sha, got" in str(e):
+ return -1
+ else:
+ raise e
_ = blob.mime_type
From e15caab8e70adc44b796bd3d972e1d34d30ad7ee Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 7 May 2024 19:26:54 +0200
Subject: [PATCH 034/103] lint: switch order Ruff's hooks `fix` -> `format`
---
.pre-commit-config.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 585b4f04d..987d86cd9 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,12 +1,12 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.3.2
+ rev: v0.4.3
hooks:
- - id: ruff-format
- exclude: ^git/ext/
- id: ruff
args: ["--fix"]
exclude: ^git/ext/
+ - id: ruff-format
+ exclude: ^git/ext/
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.9.0.6
From 2cfd2007b4a73bb061506e7c521570e9a0ec3f96 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Wed, 8 May 2024 03:20:18 -0400
Subject: [PATCH 035/103] Update OSS-Fuzz Scripts to Use New QA-Assets Repo
Structure
This change is required to support the changes to the seed data repo
structure introduced in:
https://github.com/gitpython-developers/qa-assets/pull/2
This moves most of the seed data related build steps into the OSS-Fuzz
Docker image build via `container-environment-bootstrap.sh`. This
includes moveing the dictionaries into that repo.
The fuzzing/README.md here should be updated in a follow-up with a link
to the qa-assets repo (and probably some context setting about corpora
in general) but I have opted to defer that as I think the functionality
added by the seed data improvements is valuable as is and shouldn't be
blocked by documentation writers block.
---
fuzzing/README.md | 19 ------
fuzzing/dictionaries/fuzz_blob.dict | 1 -
fuzzing/dictionaries/fuzz_config.dict | 56 ----------------
fuzzing/oss-fuzz-scripts/build.sh | 27 +-------
.../container-environment-bootstrap.sh | 64 +++++++++++++++----
5 files changed, 53 insertions(+), 114 deletions(-)
delete mode 100644 fuzzing/dictionaries/fuzz_blob.dict
delete mode 100644 fuzzing/dictionaries/fuzz_config.dict
diff --git a/fuzzing/README.md b/fuzzing/README.md
index 9d02bf72f..286f529eb 100644
--- a/fuzzing/README.md
+++ b/fuzzing/README.md
@@ -76,25 +76,6 @@ Contains Python files for each fuzz test.
reason, fuzz tests should gracefully handle anticipated exception cases with a `try`/`except` block to avoid false
positives that halt the fuzzing engine.
-### Dictionaries (`dictionaries/`)
-
-Provides hints to the fuzzing engine about inputs that might trigger unique code paths. Each fuzz target may have a
-corresponding `.dict` file. For information about dictionary syntax, refer to
-the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.html#dictionaries).
-
-**Things to Know**:
-
-- OSS-Fuzz loads dictionary files per fuzz target if one exists with the same name, all others are ignored.
-- Most entries in the dictionary files found here are escaped hex or Unicode values that were recommended by the fuzzing
- engine after previous runs.
-- A default set of dictionary entries are created for all fuzz targets as part of the build process, regardless of an
- existing file here.
-- Development or updates to dictionaries should reflect the varied formats and edge cases relevant to the
- functionalities under test.
-- Example dictionaries (some of which are used to build the default dictionaries mentioned above) can be found here:
- - [AFL++ dictionary repository](https://github.com/AFLplusplus/AFLplusplus/tree/stable/dictionaries#readme)
- - [Google/fuzzing dictionary repository](https://github.com/google/fuzzing/tree/master/dictionaries)
-
### OSS-Fuzz Scripts (`oss-fuzz-scripts/`)
Includes scripts for building and integrating fuzz targets with OSS-Fuzz:
diff --git a/fuzzing/dictionaries/fuzz_blob.dict b/fuzzing/dictionaries/fuzz_blob.dict
deleted file mode 100644
index 7f123f830..000000000
--- a/fuzzing/dictionaries/fuzz_blob.dict
+++ /dev/null
@@ -1 +0,0 @@
-"\\377\\377\\377\\377\\377\\377\\377\\377"
diff --git a/fuzzing/dictionaries/fuzz_config.dict b/fuzzing/dictionaries/fuzz_config.dict
deleted file mode 100644
index b545ddfc8..000000000
--- a/fuzzing/dictionaries/fuzz_config.dict
+++ /dev/null
@@ -1,56 +0,0 @@
-"\\004\\000\\000\\000\\000\\000\\000\\000"
-"\\006\\000\\000\\000\\000\\000\\000\\000"
-"_validate_value_"
-"\\000\\000\\000\\000\\000\\000\\000\\000"
-"rem"
-"__eq__"
-"\\001\\000\\000\\000"
-"__abstrac"
-"_mutating_methods_"
-"items"
-"\\0021\\""
-"\\001\\000"
-"\\000\\000\\000\\000"
-"DEFAULT"
-"getfloat"
-"\\004\\000\\000\\000\\000\\000\\000\\000"
-"news"
-"\\037\\000\\000\\000\\000\\000\\000\\000"
-"\\001\\000\\000\\000\\000\\000\\000\\037"
-"\\000\\000\\000\\000\\000\\000\\000\\014"
-"list"
-"\\376\\377\\377\\377\\377\\377\\377\\377"
-"items_all"
-"\\004\\000\\000\\000\\000\\000\\000\\000"
-"\\377\\377\\377\\377\\377\\377\\377\\014"
-"\\001\\000\\000\\000"
-"_acqui"
-"\\000\\000\\000\\000\\000\\000\\000\\000"
-"__ne__"
-"__exit__"
-"__modu"
-"uucp"
-"__str__"
-"\\001\\000\\000\\000"
-"\\017\\000\\000\\000\\000\\000\\000\\000"
-"_has_incl"
-"update"
-"\\377\\377\\377\\377\\377\\377\\377\\023"
-"setdef"
-"setdefaul"
-"\\000\\000\\000\\000"
-"\\001\\000\\000\\000"
-"\\001\\000"
-"\\022\\000\\000\\000\\000\\000\\000\\000"
-"_value_to_string"
-"__abstr"
-"\\001\\000\\000\\000\\000\\000\\000\\000"
-"\\000\\000\\000\\000\\000\\000\\000\\022"
-"\\377\\377\\377\\377"
-"\\004\\000\\000\\000\\000\\000\\000\\000"
-"\\000\\000\\000\\000\\000\\000\\000\\000"
-"\\000\\000\\000\\000\\000\\000\\000\\037"
-"\\001\\000\\000\\000\\000\\000\\000\\013"
-"_OPT_TM"
-"__name__"
-"_get_conv"
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
index 58c9adb5a..e0b3a50ab 100644
--- a/fuzzing/oss-fuzz-scripts/build.sh
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -7,34 +7,13 @@ set -euo pipefail
python3 -m pip install .
-# Directory to look in for dictionaries, options files, and seed corpora:
-SEED_DATA_DIR="$SRC/seed_data"
-
-find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \
- ! \( -name '__base.*' \) -exec printf 'Copying: %s\n' {} \; \
+find "$SRC" -maxdepth 1 \
+ \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \
+ -exec printf '[%s] Copying: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" {} \; \
-exec chmod a-x {} \; \
-exec cp {} "$OUT" \;
# Build fuzzers in $OUT.
find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do
compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):."
-
- common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict"
- if [[ -r "$common_base_dictionary_filename" ]]; then
- # Strip the `.py` extension from the filename and replace it with `.dict`.
- fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict"
- output_file="$OUT/$fuzz_harness_dictionary_filename"
-
- printf 'Appending %s to %s\n' "$common_base_dictionary_filename" "$output_file"
- if [[ -s "$output_file" ]]; then
- # If a dictionary file for this fuzzer already exists and is not empty,
- # we append a new line to the end of it before appending any new entries.
- #
- # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error
- # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.)
- # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32
- echo >>"$output_file"
- fi
- cat "$common_base_dictionary_filename" >>"$output_file"
- fi
done
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
index 76ec97c7f..bbdcf5357 100755
--- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -9,23 +9,20 @@ set -euo pipefail
# Prerequisites #
#################
-for cmd in python3 git wget rsync; do
+for cmd in python3 git wget zip; do
command -v "$cmd" >/dev/null 2>&1 || {
printf '[%s] Required command %s not found, exiting.\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$cmd" >&2
exit 1
}
done
-SEED_DATA_DIR="$SRC/seed_data"
-mkdir -p "$SEED_DATA_DIR"
-
#############
# Functions #
#############
download_and_concatenate_common_dictionaries() {
# Assign the first argument as the target file where all contents will be concatenated
- target_file="$1"
+ local target_file="$1"
# Shift the arguments so the first argument (target_file path) is removed
# and only URLs are left for the loop below.
@@ -38,22 +35,61 @@ download_and_concatenate_common_dictionaries() {
done
}
-fetch_seed_corpora() {
- # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo.
- git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets &&
- rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" &&
- rm -rf qa-assets # Clean up the cloned repo to keep the Docker image as slim as possible.
+create_seed_corpora_zips() {
+ local seed_corpora_dir="$1"
+ local output_zip
+ for dir in "$seed_corpora_dir"/*; do
+ if [ -d "$dir" ] && [ -n "$dir" ]; then
+ output_zip="$SRC/$(basename "$dir")_seed_corpus.zip"
+ printf '[%s] Zipping the contents of %s into %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dir" "$output_zip"
+ zip -jur "$output_zip" "$dir"/*
+ fi
+ done
+}
+
+prepare_dictionaries_for_fuzz_targets() {
+ local dictionaries_dir="$1"
+ local fuzz_targets_dir="$2"
+ local common_base_dictionary_filename="$WORK/__base.dict"
+
+ printf '[%s] Copying .dict files from %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dictionaries_dir" "$SRC/"
+ cp -v "$dictionaries_dir"/*.dict "$SRC/"
+
+ download_and_concatenate_common_dictionaries "$common_base_dictionary_filename" \
+ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \
+ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict"
+
+ find "$fuzz_targets_dir" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do
+ if [[ -r "$common_base_dictionary_filename" ]]; then
+ # Strip the `.py` extension from the filename and replace it with `.dict`.
+ fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict"
+ local output_file="$SRC/$fuzz_harness_dictionary_filename"
+
+ printf '[%s] Appending %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$common_base_dictionary_filename" "$output_file"
+ if [[ -s "$output_file" ]]; then
+ # If a dictionary file for this fuzzer already exists and is not empty,
+ # we append a new line to the end of it before appending any new entries.
+ #
+ # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error
+ # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.)
+ # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32
+ echo >>"$output_file"
+ fi
+ cat "$common_base_dictionary_filename" >>"$output_file"
+ fi
+ done
}
########################
# Main execution logic #
########################
+# Seed corpora and dictionaries are hosted in a separate repository to avoid additional bloat in this repo.
+# We clone into the $WORK directory because OSS-Fuzz cleans it up after building the image, keeping the image small.
+git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets"
-fetch_seed_corpora
+create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora"
-download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \
- "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \
- "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict"
+prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing"
# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below.
python3 -m pip install --upgrade pip
From a915adf08e570d8989bb070f647e2a3ee941871d Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Wed, 8 May 2024 17:06:19 -0400
Subject: [PATCH 036/103] Add `Diff` Fuzz Target
Adds a new `fuzz_diff.py` fuzz target that covers `Diff` class
initialization using fuzzed data.
---
fuzzing/fuzz-targets/fuzz_diff.py | 54 +++++++++++++++++++++++++++++++
1 file changed, 54 insertions(+)
create mode 100644 fuzzing/fuzz-targets/fuzz_diff.py
diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py
new file mode 100644
index 000000000..cf01e7ffa
--- /dev/null
+++ b/fuzzing/fuzz-targets/fuzz_diff.py
@@ -0,0 +1,54 @@
+import sys
+import os
+import tempfile
+from binascii import Error as BinasciiError
+
+import atheris
+
+if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+ path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
+ os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+
+with atheris.instrument_imports():
+ from git import Repo, Diff
+
+
+def TestOneInput(data):
+ fdp = atheris.FuzzedDataProvider(data)
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ repo = Repo.init(path=temp_dir)
+ try:
+ Diff(
+ repo,
+ a_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
+ b_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
+ a_blob_id=fdp.ConsumeBytes(20),
+ b_blob_id=fdp.ConsumeBytes(20),
+ a_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
+ b_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
+ new_file=fdp.ConsumeBool(),
+ deleted_file=fdp.ConsumeBool(),
+ copied_file=fdp.ConsumeBool(),
+ raw_rename_from=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
+ raw_rename_to=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
+ diff=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
+ change_type=fdp.PickValueInList(["A", "D", "C", "M", "R", "T", "U"]),
+ score=fdp.ConsumeIntInRange(0, fdp.remaining_bytes()),
+ )
+ except BinasciiError:
+ return -1
+ except AssertionError as e:
+ if "Require 20 byte binary sha, got" in str(e):
+ return -1
+ else:
+ raise e
+
+
+def main():
+ atheris.Setup(sys.argv, TestOneInput)
+ atheris.Fuzz()
+
+
+if __name__ == "__main__":
+ main()
From 989ae1ac03e25a5ce51d4c615128dcf75b9e24f5 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Wed, 8 May 2024 19:28:29 -0400
Subject: [PATCH 037/103] Read class properties & call methods to cover more
features
Property access and private methods on the `Diff` class are complex and
involve encoding and decoding operations that warrant being tested.
This test borrows its design from the `test_diff.py` unit test file.
---
fuzzing/fuzz-targets/fuzz_diff.py | 31 ++++++++++++++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py
index cf01e7ffa..ba44995f2 100644
--- a/fuzzing/fuzz-targets/fuzz_diff.py
+++ b/fuzzing/fuzz-targets/fuzz_diff.py
@@ -1,5 +1,6 @@
import sys
import os
+import io
import tempfile
from binascii import Error as BinasciiError
@@ -13,13 +14,26 @@
from git import Repo, Diff
+class BytesProcessAdapter:
+ """Allows bytes to be used as process objects returned by subprocess.Popen."""
+
+ def __init__(self, input_string):
+ self.stdout = io.BytesIO(input_string)
+ self.stderr = io.BytesIO()
+
+ def wait(self):
+ return 0
+
+ poll = wait
+
+
def TestOneInput(data):
fdp = atheris.FuzzedDataProvider(data)
with tempfile.TemporaryDirectory() as temp_dir:
repo = Repo.init(path=temp_dir)
try:
- Diff(
+ diff = Diff(
repo,
a_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
b_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())),
@@ -44,6 +58,21 @@ def TestOneInput(data):
else:
raise e
+ _ = diff.__str__()
+ _ = diff.a_path
+ _ = diff.b_path
+ _ = diff.rename_from
+ _ = diff.rename_to
+ _ = diff.renamed_file
+
+ diff_index = diff._index_from_patch_format(
+ repo, proc=BytesProcessAdapter(fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())))
+ )
+
+ diff._handle_diff_line(
+ lines_bytes=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), repo=repo, index=diff_index
+ )
+
def main():
atheris.Setup(sys.argv, TestOneInput)
From 315a2fd03c94c93d4a7089d23d734e4aaccbe066 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Wed, 15 May 2024 13:36:29 -0400
Subject: [PATCH 038/103] Instrument test utility functions to increase fuzzer
efficiency
Fuzz Introspector was reporting a high percentage of fuzz blockers
in the `fuzz_diff` test. This means the fuzzing engine was unable
to gain visibility into functions lower in the call stack than the
blocking functions, making it less effective at producing interesting
input data.
This clears a large percentage of the fuzz blockers by adding fuzzer
instrumentation to them via the `@atheris.instrument_func` decorator.
---
fuzzing/fuzz-targets/fuzz_diff.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py
index ba44995f2..d4bd68b57 100644
--- a/fuzzing/fuzz-targets/fuzz_diff.py
+++ b/fuzzing/fuzz-targets/fuzz_diff.py
@@ -17,16 +17,19 @@
class BytesProcessAdapter:
"""Allows bytes to be used as process objects returned by subprocess.Popen."""
+ @atheris.instrument_func
def __init__(self, input_string):
self.stdout = io.BytesIO(input_string)
self.stderr = io.BytesIO()
+ @atheris.instrument_func
def wait(self):
return 0
poll = wait
+@atheris.instrument_func
def TestOneInput(data):
fdp = atheris.FuzzedDataProvider(data)
From cf81c6c98155c24d69af6f1a8eca368ad1a5d962 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Sun, 26 May 2024 16:40:23 -0400
Subject: [PATCH 039/103] Momentarily downgrade Git on Cygwin to investigate
failures
Using this older version is not in general secure, since the new
version is a security update. It is sometimes acceptable to run
software with security bugs in CI workflows, but the intent of this
change is just to check if the version of the Cygwin `git` package
is the cause of the failures. If so, they can probably be fixed or
worked around in a better way than downgrading. (Furthermore, the
lower version of the `git` package will not always be avaialable
from Cygwin's repositories.)
---
.github/workflows/cygwin-test.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml
index 61e6a3089..a2fe588ad 100644
--- a/.github/workflows/cygwin-test.yml
+++ b/.github/workflows/cygwin-test.yml
@@ -30,7 +30,7 @@ jobs:
- name: Set up Cygwin
uses: egor-tensin/setup-cygwin@v4
with:
- packages: python39=3.9.16-1 python39-pip python39-virtualenv git
+ packages: python39=3.9.16-1 python39-pip python39-virtualenv git=2.43.0-1
- name: Arrange for verbose output
run: |
From eb06a18d83eda0ae04e2a00b2d656da147e9188a Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Sun, 26 May 2024 16:45:57 -0400
Subject: [PATCH 040/103] Unpin Cygwin `git`; add our `.git` as a
`safe.directory`
This undoes the change of pinning Git to an earlier version (before
the recent security update) on Cygwin, and instead adds the `.git`
subdirectory of the `GitPython` directory as an additional value of
the multi-valued `safe.directory` Git configuration variable.
---
.github/workflows/cygwin-test.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml
index a2fe588ad..bde4ea659 100644
--- a/.github/workflows/cygwin-test.yml
+++ b/.github/workflows/cygwin-test.yml
@@ -30,7 +30,7 @@ jobs:
- name: Set up Cygwin
uses: egor-tensin/setup-cygwin@v4
with:
- packages: python39=3.9.16-1 python39-pip python39-virtualenv git=2.43.0-1
+ packages: python39=3.9.16-1 python39-pip python39-virtualenv git
- name: Arrange for verbose output
run: |
@@ -40,6 +40,7 @@ jobs:
- name: Special configuration for Cygwin git
run: |
git config --global --add safe.directory "$(pwd)"
+ git config --global --add safe.directory "$(pwd)/.git"
git config --global core.autocrlf false
- name: Prepare this repo for tests
From d3b181d54a8da6b8561474dba1333682b47b7ba7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 13:22:29 +0000
Subject: [PATCH 041/103] Bump Vampire/setup-wsl from 3.0.0 to 3.1.0
Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 3.0.0 to 3.1.0.
- [Release notes](https://github.com/vampire/setup-wsl/releases)
- [Commits](https://github.com/vampire/setup-wsl/compare/v3.0.0...v3.1.0)
---
updated-dependencies:
- dependency-name: Vampire/setup-wsl
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
---
.github/workflows/pythonpackage.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 4c918a92d..574048620 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -44,7 +44,7 @@ jobs:
- name: Set up WSL (Windows)
if: startsWith(matrix.os, 'windows')
- uses: Vampire/setup-wsl@v3.0.0
+ uses: Vampire/setup-wsl@v3.1.0
with:
distribution: Debian
From 7bdcfa556ad476d89a3643137e97ee5749e3c7df Mon Sep 17 00:00:00 2001
From: Sebastian Thiel <sebastian.thiel@icloud.com>
Date: Mon, 27 May 2024 21:27:07 +0200
Subject: [PATCH 042/103] Update to the fixed version of `Vampire`
---
.github/workflows/pythonpackage.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 574048620..031b0e6b2 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -44,7 +44,7 @@ jobs:
- name: Set up WSL (Windows)
if: startsWith(matrix.os, 'windows')
- uses: Vampire/setup-wsl@v3.1.0
+ uses: Vampire/setup-wsl@v3.1.1
with:
distribution: Debian
From 6d52bdbe6a546ecb76e28f7dde45b44fe8577010 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Wed, 29 May 2024 22:06:40 -0400
Subject: [PATCH 043/103] Add Submodules Fuzz Target
Fuzz Introspector heuristics suggest the Submodule API code represent
"optimal analysis targets" that should yield a meaningful increase in
code coverage. The changes here introduce a first pass at implementing a
fuzz harness that cover the primary APIs/methods related to Submodules.
Of particular interest to me is the `Submodule.config_writer()`
coverage.
Please note however, there is likely plenty of room for improvement in
this harness in terms of both code coverage as well as performance; the
latter of which will see significant benefit from a well curated seed
corpus of `.gitmodules` file like inputs. The `ParsingError` raised by
the fuzzer without a good seed corpus hinders test efficacy
significantly.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 93 ++++++++++++++++++++++++++
fuzzing/fuzz-targets/utils.py | 22 ++++++
2 files changed, 115 insertions(+)
create mode 100644 fuzzing/fuzz-targets/fuzz_submodule.py
create mode 100644 fuzzing/fuzz-targets/utils.py
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
new file mode 100644
index 000000000..ddcbaa00f
--- /dev/null
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -0,0 +1,93 @@
+import atheris
+import sys
+import os
+import tempfile
+from configparser import ParsingError
+from utils import is_expected_exception_message
+
+if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+ path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
+ os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+
+with atheris.instrument_imports():
+ from git import Repo, GitCommandError, InvalidGitRepositoryError
+
+
+def TestOneInput(data):
+ fdp = atheris.FuzzedDataProvider(data)
+
+ with tempfile.TemporaryDirectory() as repo_temp_dir:
+ repo = Repo.init(path=repo_temp_dir)
+ repo.index.commit("Initial commit")
+
+ try:
+ with tempfile.TemporaryDirectory() as submodule_temp_dir:
+ sub_repo = Repo.init(submodule_temp_dir, bare=fdp.ConsumeBool())
+ sub_repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512)))
+
+ submodule_name = f"submodule_{fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))}"
+ submodule_path = os.path.join(repo.working_tree_dir, submodule_name)
+ submodule_url = sub_repo.git_dir
+
+ submodule = repo.create_submodule(submodule_name, submodule_path, url=submodule_url)
+ repo.index.commit(f"Added submodule {submodule_name}")
+
+ with submodule.config_writer() as writer:
+ key_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes()))
+ value_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes()))
+
+ writer.set_value(
+ fdp.ConsumeUnicodeNoSurrogates(key_length), fdp.ConsumeUnicodeNoSurrogates(value_length)
+ )
+ writer.release()
+
+ submodule.update(init=fdp.ConsumeBool(), dry_run=fdp.ConsumeBool(), force=fdp.ConsumeBool())
+
+ submodule_repo = submodule.module()
+ new_file_path = os.path.join(
+ submodule_repo.working_tree_dir,
+ f"new_file_{fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))}",
+ )
+ with open(new_file_path, "wb") as new_file:
+ new_file.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512)))
+ submodule_repo.index.add([new_file_path])
+ submodule_repo.index.commit("Added new file to submodule")
+
+ repo.submodule_update(recursive=fdp.ConsumeBool())
+ submodule_repo.head.reset(commit="HEAD~1", working_tree=fdp.ConsumeBool(), head=fdp.ConsumeBool())
+ # Use fdp.PickValueInList to ensure at least one of 'module' or 'configuration' is True
+ module_option_value, configuration_option_value = fdp.PickValueInList(
+ [(True, False), (False, True), (True, True)]
+ )
+ submodule.remove(
+ module=module_option_value,
+ configuration=configuration_option_value,
+ dry_run=fdp.ConsumeBool(),
+ force=fdp.ConsumeBool(),
+ )
+ repo.index.commit(f"Removed submodule {submodule_name}")
+
+ except (ParsingError, GitCommandError, InvalidGitRepositoryError, FileNotFoundError, BrokenPipeError):
+ return -1
+ except (ValueError, OSError) as e:
+ expected_messages = [
+ "SHA is empty",
+ "Reference at",
+ "embedded null byte",
+ "This submodule instance does not exist anymore",
+ "cmd stdin was empty",
+ "File name too long",
+ ]
+ if is_expected_exception_message(e, expected_messages):
+ return -1
+ else:
+ raise e
+
+
+def main():
+ atheris.Setup(sys.argv, TestOneInput)
+ atheris.Fuzz()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py
new file mode 100644
index 000000000..42faa8eb0
--- /dev/null
+++ b/fuzzing/fuzz-targets/utils.py
@@ -0,0 +1,22 @@
+import atheris # pragma: no cover
+from typing import List # pragma: no cover
+
+
+@atheris.instrument_func
+def is_expected_exception_message(exception: Exception, error_message_list: List[str]) -> bool: # pragma: no cover
+ """
+ Checks if the message of a given exception matches any of the expected error messages, case-insensitively.
+
+ Args:
+ exception (Exception): The exception object raised during execution.
+ error_message_list (List[str]): A list of error message substrings to check against the exception's message.
+
+ Returns:
+ bool: True if the exception's message contains any of the substrings from the error_message_list,
+ case-insensitively, otherwise False.
+ """
+ exception_message = str(exception).lower()
+ for error in error_message_list:
+ if error.lower() in exception_message:
+ return True
+ return False
From 9e67138819c7e081fee89a5b855c89b538a8f604 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Thu, 30 May 2024 12:22:45 +0200
Subject: [PATCH 044/103] precommit: enable `end-of-file-fixer`
---
.pre-commit-config.yaml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 585b4f04d..50f430084 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,6 +18,7 @@ repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
+ - id: end-of-file-fixer
- id: check-toml
- id: check-yaml
- id: check-merge-conflict
From 96e21f0055060b01d03b705cdb582edd3551aa43 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Thu, 30 May 2024 12:24:19 +0200
Subject: [PATCH 045/103] apply
---
doc/source/index.rst | 1 -
doc/source/intro.rst | 1 -
doc/source/roadmap.rst | 1 -
test/fixtures/.gitconfig | 2 +-
test/fixtures/blame | 2 +-
test/fixtures/cat_file_blob | 2 +-
test/fixtures/git_config | 1 -
test/fixtures/git_config_with_empty_value | 2 +-
test/fixtures/rev_list_bisect_all | 1 -
test/fixtures/rev_list_commit_diffs | 1 -
test/fixtures/rev_list_commit_idabbrev | 1 -
test/fixtures/rev_list_commit_stats | 1 -
12 files changed, 4 insertions(+), 12 deletions(-)
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 72db8ee5a..ca5229ac3 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -21,4 +21,3 @@ Indices and tables
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
-
diff --git a/doc/source/intro.rst b/doc/source/intro.rst
index 4f22a0942..d053bd117 100644
--- a/doc/source/intro.rst
+++ b/doc/source/intro.rst
@@ -122,4 +122,3 @@ License Information
===================
GitPython is licensed under the New BSD License. See the LICENSE file for
more information.
-
diff --git a/doc/source/roadmap.rst b/doc/source/roadmap.rst
index a573df33a..34c953626 100644
--- a/doc/source/roadmap.rst
+++ b/doc/source/roadmap.rst
@@ -6,4 +6,3 @@ The full list of milestones including associated tasks can be found on GitHub:
https://github.com/gitpython-developers/GitPython/issues
Select the respective milestone to filter the list of issues accordingly.
-
diff --git a/test/fixtures/.gitconfig b/test/fixtures/.gitconfig
index 6a0459f6b..f6c25c15a 100644
--- a/test/fixtures/.gitconfig
+++ b/test/fixtures/.gitconfig
@@ -1,3 +1,3 @@
[alias]
rbi = "!g() { git rebase -i origin/${1:-master} ; } ; g"
- expush = "!f() { git branch -f tmp ; { git rbi $1 && git push ; } ; git reset --hard tmp ; git rebase origin/${1:-master}; } ; f"
\ No newline at end of file
+ expush = "!f() { git branch -f tmp ; { git rbi $1 && git push ; } ; git reset --hard tmp ; git rebase origin/${1:-master}; } ; f"
diff --git a/test/fixtures/blame b/test/fixtures/blame
index 10c141dda..949976c5d 100644
--- a/test/fixtures/blame
+++ b/test/fixtures/blame
@@ -128,4 +128,4 @@ b6e1b765e0c15586a2c5b9832854f95defd71e1f 23 23
634396b2f541a9f2d58b00be1a07f0c358b999b3 11 24 2
VERSION = '1.0.0'
634396b2f541a9f2d58b00be1a07f0c358b999b3 12 25
- end
\ No newline at end of file
+ end
diff --git a/test/fixtures/cat_file_blob b/test/fixtures/cat_file_blob
index 70c379b63..802992c42 100644
--- a/test/fixtures/cat_file_blob
+++ b/test/fixtures/cat_file_blob
@@ -1 +1 @@
-Hello world
\ No newline at end of file
+Hello world
diff --git a/test/fixtures/git_config b/test/fixtures/git_config
index a8cad56e8..d3066d86e 100644
--- a/test/fixtures/git_config
+++ b/test/fixtures/git_config
@@ -43,4 +43,3 @@
# inclusions should be processed immediately
[sec]
var1 = value1_main
-
diff --git a/test/fixtures/git_config_with_empty_value b/test/fixtures/git_config_with_empty_value
index 0427caea5..83de84c8b 100644
--- a/test/fixtures/git_config_with_empty_value
+++ b/test/fixtures/git_config_with_empty_value
@@ -1,4 +1,4 @@
[color]
ui
[core]
- filemode = true
\ No newline at end of file
+ filemode = true
diff --git a/test/fixtures/rev_list_bisect_all b/test/fixtures/rev_list_bisect_all
index 342ea94ae..60d382d01 100644
--- a/test/fixtures/rev_list_bisect_all
+++ b/test/fixtures/rev_list_bisect_all
@@ -48,4 +48,3 @@ committer David Aguilar <davvid@gmail.com> 1220418344 -0700
This resolves the issue mentioned in that thread.
Signed-off-by: David Aguilar <davvid@gmail.com>
-
diff --git a/test/fixtures/rev_list_commit_diffs b/test/fixtures/rev_list_commit_diffs
index 20397e2e4..c39df2061 100644
--- a/test/fixtures/rev_list_commit_diffs
+++ b/test/fixtures/rev_list_commit_diffs
@@ -5,4 +5,3 @@ author Tom Preston-Werner <tom@mojombo.com> 1193200199 -0700
committer Tom Preston-Werner <tom@mojombo.com> 1193200199 -0700
fix some initialization warnings
-
diff --git a/test/fixtures/rev_list_commit_idabbrev b/test/fixtures/rev_list_commit_idabbrev
index 9385ba713..6266df93e 100644
--- a/test/fixtures/rev_list_commit_idabbrev
+++ b/test/fixtures/rev_list_commit_idabbrev
@@ -5,4 +5,3 @@ author tom <tom@taco.(none)> 1195608462 -0800
committer tom <tom@taco.(none)> 1195608462 -0800
fix tests on other machines
-
diff --git a/test/fixtures/rev_list_commit_stats b/test/fixtures/rev_list_commit_stats
index 60aa8cf58..c78aadeb5 100644
--- a/test/fixtures/rev_list_commit_stats
+++ b/test/fixtures/rev_list_commit_stats
@@ -4,4 +4,3 @@ author Tom Preston-Werner <tom@mojombo.com> 1191997100 -0700
committer Tom Preston-Werner <tom@mojombo.com> 1191997100 -0700
initial grit setup
-
From 2ce9675b7238fb1a498f1ea4f5dae8a26d4b89ec Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Thu, 30 May 2024 12:25:33 +0200
Subject: [PATCH 046/103] precommit: enable `validate-pyproject`
---
.pre-commit-config.yaml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 585b4f04d..02950db8c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -21,3 +21,8 @@ repos:
- id: check-toml
- id: check-yaml
- id: check-merge-conflict
+
+- repo: https://github.com/abravalheri/validate-pyproject
+ rev: v0.16
+ hooks:
+ - id: validate-pyproject
\ No newline at end of file
From 7b684cd43cf0f9c54adb8a8def54fa07f5cfd145 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 30 May 2024 10:34:49 -0400
Subject: [PATCH 047/103] Add graceful handling of expected exceptions in
`fuzz_submodule.py`
Fixes: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=69350
**`IsADirectoryError`**
Fuzzer provided input data can sometimes produce filenames that look
like directories and raise `IsADirectoryError` exceptions which crash
the fuzzer. This commit catches those cases and returns -1 to instruct
libfuzzer that the inputs are not valuable to add to the corpus.
**`FileExistsError`**
Similar to the above, this is a possible exception case produced by the
fuzzed data and not a bug so its handled the same.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index ddcbaa00f..9406fc68b 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -67,7 +67,15 @@ def TestOneInput(data):
)
repo.index.commit(f"Removed submodule {submodule_name}")
- except (ParsingError, GitCommandError, InvalidGitRepositoryError, FileNotFoundError, BrokenPipeError):
+ except (
+ ParsingError,
+ GitCommandError,
+ InvalidGitRepositoryError,
+ FileNotFoundError,
+ FileExistsError,
+ IsADirectoryError,
+ BrokenPipeError,
+ ):
return -1
except (ValueError, OSError) as e:
expected_messages = [
From 6c00ce602eb19eda342e827a25d005610ce92fa8 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 30 May 2024 13:53:42 -0400
Subject: [PATCH 048/103] Improve file name generation to prevent "File name
too long" `OSError`'s
Adds a utility function to limit the maximum file name legnth produced
by the fuzzer to a max size dictated by the host its run on.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 13 ++++++-------
fuzzing/fuzz-targets/utils.py | 15 +++++++++++++++
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 9406fc68b..817ce8f98 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -3,7 +3,7 @@
import os
import tempfile
from configparser import ParsingError
-from utils import is_expected_exception_message
+from utils import is_expected_exception_message, get_max_filename_length
if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
@@ -42,12 +42,12 @@ def TestOneInput(data):
writer.release()
submodule.update(init=fdp.ConsumeBool(), dry_run=fdp.ConsumeBool(), force=fdp.ConsumeBool())
-
submodule_repo = submodule.module()
- new_file_path = os.path.join(
- submodule_repo.working_tree_dir,
- f"new_file_{fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))}",
+
+ new_file_name = fdp.ConsumeUnicodeNoSurrogates(
+ fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(submodule_repo.working_tree_dir)))
)
+ new_file_path = os.path.join(submodule_repo.working_tree_dir, new_file_name)
with open(new_file_path, "wb") as new_file:
new_file.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512)))
submodule_repo.index.add([new_file_path])
@@ -77,14 +77,13 @@ def TestOneInput(data):
BrokenPipeError,
):
return -1
- except (ValueError, OSError) as e:
+ except ValueError as e:
expected_messages = [
"SHA is empty",
"Reference at",
"embedded null byte",
"This submodule instance does not exist anymore",
"cmd stdin was empty",
- "File name too long",
]
if is_expected_exception_message(e, expected_messages):
return -1
diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py
index 42faa8eb0..86f049341 100644
--- a/fuzzing/fuzz-targets/utils.py
+++ b/fuzzing/fuzz-targets/utils.py
@@ -1,4 +1,5 @@
import atheris # pragma: no cover
+import os
from typing import List # pragma: no cover
@@ -20,3 +21,17 @@ def is_expected_exception_message(exception: Exception, error_message_list: List
if error.lower() in exception_message:
return True
return False
+
+
+@atheris.instrument_func
+def get_max_filename_length(path: str) -> int:
+ """
+ Get the maximum filename length for the filesystem containing the given path.
+
+ Args:
+ path (str): The path to check the filesystem for.
+
+ Returns:
+ int: The maximum filename length.
+ """
+ return os.pathconf(path, "PC_NAME_MAX")
From 2a2294f9d1e46d9bbe11cd2031d62e5441fe19c4 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 30 May 2024 14:02:27 -0400
Subject: [PATCH 049/103] Improve `fuzz_submodule.py` coverage & efficacy
The fuzzer was having trouble analyzing `fuzz_submodule.py` when using
the `atheris.instrument_imports()` context manager. Switching to
`atheris.instrument_all()` instead slightly increases the startup time
for the fuzzer, but significantly improves the fuzzing engines ability
to identify new coverage.
The changes here also disable warnings that are logged to `stdout` from
the SUT. These warnings are expected to happen with some inputs and
clutter the fuzzer output logs. They can be optionally re-enabled for
debugging by passing a flag o the Python interpreter command line or
setting the `PYTHONWARNINGS` environment variable.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 16 +++++++++++++---
fuzzing/fuzz-targets/utils.py | 4 ++--
2 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 817ce8f98..53f5a7884 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -4,13 +4,22 @@
import tempfile
from configparser import ParsingError
from utils import is_expected_exception_message, get_max_filename_length
+from git import Repo, GitCommandError, InvalidGitRepositoryError
-if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
+if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover
path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
-with atheris.instrument_imports():
- from git import Repo, GitCommandError, InvalidGitRepositoryError
+if not sys.warnoptions: # pragma: no cover
+ # The warnings filter below can be overridden by passing the -W option
+ # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable.
+ import warnings
+ import logging
+
+ # Fuzzing data causes some plugins to generate a large number of warnings
+ # which are not usually interesting and make the test output hard to read, so we ignore them.
+ warnings.simplefilter("ignore")
+ logging.getLogger().setLevel(logging.ERROR)
def TestOneInput(data):
@@ -92,6 +101,7 @@ def TestOneInput(data):
def main():
+ atheris.instrument_all()
atheris.Setup(sys.argv, TestOneInput)
atheris.Fuzz()
diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py
index 86f049341..f522d2959 100644
--- a/fuzzing/fuzz-targets/utils.py
+++ b/fuzzing/fuzz-targets/utils.py
@@ -1,5 +1,5 @@
import atheris # pragma: no cover
-import os
+import os # pragma: no cover
from typing import List # pragma: no cover
@@ -24,7 +24,7 @@ def is_expected_exception_message(exception: Exception, error_message_list: List
@atheris.instrument_func
-def get_max_filename_length(path: str) -> int:
+def get_max_filename_length(path: str) -> int: # pragma: no cover
"""
Get the maximum filename length for the filesystem containing the given path.
From 57a56a8a2874d2ab76f4034b9d3c98e09ed7fa35 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 30 May 2024 14:12:02 -0400
Subject: [PATCH 050/103] Add graceful handling for `NotADirectoryError`s
---
fuzzing/fuzz-targets/fuzz_submodule.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 53f5a7884..cfd1a6d3f 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -83,6 +83,7 @@ def TestOneInput(data):
FileNotFoundError,
FileExistsError,
IsADirectoryError,
+ NotADirectoryError,
BrokenPipeError,
):
return -1
From 2b64dee466ed72523684f90a037d604355121df0 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 30 May 2024 14:17:20 -0400
Subject: [PATCH 051/103] Improve comment wording
---
fuzzing/fuzz-targets/fuzz_submodule.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index cfd1a6d3f..92b569949 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -16,7 +16,7 @@
import warnings
import logging
- # Fuzzing data causes some plugins to generate a large number of warnings
+ # Fuzzing data causes some modules to generate a large number of warnings
# which are not usually interesting and make the test output hard to read, so we ignore them.
warnings.simplefilter("ignore")
logging.getLogger().setLevel(logging.ERROR)
From 882425ded5ae210c7092b87f4ea6bc871784ae89 Mon Sep 17 00:00:00 2001
From: Sebastian Thiel <sebastian.thiel@icloud.com>
Date: Fri, 31 May 2024 07:24:47 +0200
Subject: [PATCH 052/103] Add missing newline in `prec-commit-config.yaml`
Just to be sure the coming linting won't be disturbed by that.
---
.pre-commit-config.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 02950db8c..fe966adad 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -25,4 +25,4 @@ repos:
- repo: https://github.com/abravalheri/validate-pyproject
rev: v0.16
hooks:
- - id: validate-pyproject
\ No newline at end of file
+ - id: validate-pyproject
From 59a0c88a08de4b35608d82b107844915a787f192 Mon Sep 17 00:00:00 2001
From: Andrej730 <azhilenkov@gmail.com>
Date: Mon, 3 Jun 2024 00:26:11 +0500
Subject: [PATCH 053/103] Fix IndexFile items argument type
Error before commit:
path: os.PathLike = ...
repo = git.Repo(path_dir)
repo.index.add(path)
---
git/index/base.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/git/index/base.py b/git/index/base.py
index b8161ea52..fc4474cac 100644
--- a/git/index/base.py
+++ b/git/index/base.py
@@ -658,7 +658,7 @@ def _to_relative_path(self, path: PathLike) -> PathLike:
return os.path.relpath(path, self.repo.working_tree_dir)
def _preprocess_add_items(
- self, items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]
+ self, items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]]
) -> Tuple[List[PathLike], List[BaseIndexEntry]]:
"""Split the items into two lists of path strings and BaseEntries."""
paths = []
@@ -749,7 +749,7 @@ def _entries_for_paths(
def add(
self,
- items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]],
+ items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]],
force: bool = True,
fprogress: Callable = lambda *args: None,
path_rewriter: Union[Callable[..., PathLike], None] = None,
@@ -976,7 +976,7 @@ def _items_to_rela_paths(
@default_index
def remove(
self,
- items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]],
+ items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]],
working_tree: bool = False,
**kwargs: Any,
) -> List[str]:
@@ -1036,7 +1036,7 @@ def remove(
@default_index
def move(
self,
- items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]],
+ items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]],
skip_errors: bool = False,
**kwargs: Any,
) -> List[Tuple[str, str]]:
From 77fb5f06bd86a02f481a1d34ca0938bb5b7f5219 Mon Sep 17 00:00:00 2001
From: Andrej730 <azhilenkov@gmail.com>
Date: Mon, 3 Jun 2024 00:28:45 +0500
Subject: [PATCH 054/103] Specify DiffIndex generic type
Example before this commit:
repo = git.Repo(path_dir)
diff = repo.index.diff(None)
modified_files = [d for d in repo.index.diff(None)]
reveal_type(modified_files) # list[Unknown] instead of list[Diff]
---
git/diff.py | 8 ++++----
git/index/base.py | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/git/diff.py b/git/diff.py
index f89b12d98..e9f7e209f 100644
--- a/git/diff.py
+++ b/git/diff.py
@@ -187,7 +187,7 @@ def diff(
paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None,
create_patch: bool = False,
**kwargs: Any,
- ) -> "DiffIndex":
+ ) -> "DiffIndex[Diff]":
"""Create diffs between two items being trees, trees and index or an index and
the working tree. Detects renames automatically.
@@ -581,7 +581,7 @@ def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_m
return None
@classmethod
- def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex:
+ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex["Diff"]:
"""Create a new :class:`DiffIndex` from the given process output which must be
in patch format.
@@ -674,7 +674,7 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn
return index
@staticmethod
- def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> None:
+ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]) -> None:
lines = lines_bytes.decode(defenc)
# Discard everything before the first colon, and the colon itself.
@@ -747,7 +747,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non
index.append(diff)
@classmethod
- def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex":
+ def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex[Diff]":
"""Create a new :class:`DiffIndex` from the given process output which must be
in raw format.
diff --git a/git/index/base.py b/git/index/base.py
index fc4474cac..28b60a880 100644
--- a/git/index/base.py
+++ b/git/index/base.py
@@ -1478,7 +1478,7 @@ def diff(
paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None,
create_patch: bool = False,
**kwargs: Any,
- ) -> git_diff.DiffIndex:
+ ) -> git_diff.DiffIndex[git_diff.Diff]:
"""Diff this index against the working copy or a :class:`~git.objects.tree.Tree`
or :class:`~git.objects.commit.Commit` object.
From 491e134d2a930d12cc4250951e9e986dbab2be2d Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 4 Jun 2024 06:58:07 -0400
Subject: [PATCH 055/103] Fix Improper Import Order Breaking `fuzz_submodule`
Fuzzer
ClusterFuzz runs of the `fuzz_submodule` target have been failing
because the `git` import was placed before the condition that sets the
Git executable path.
The order in which `git` is imported matters because it attempts to find
a Git executable as the import is loaded (via `refresh()` in
`git/__init__.py`.) As per #1909, we configure the ClusterFuzz
environment to use a bundled Git executable via the env variable
condition in all fuzz targets.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 92b569949..ca47690ea 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -4,12 +4,13 @@
import tempfile
from configparser import ParsingError
from utils import is_expected_exception_message, get_max_filename_length
-from git import Repo, GitCommandError, InvalidGitRepositoryError
if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover
path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+from git import Repo, GitCommandError, InvalidGitRepositoryError
+
if not sys.warnoptions: # pragma: no cover
# The warnings filter below can be overridden by passing the -W option
# to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable.
From d59708812f50362f526d4c6aa67b7218d12024f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Krzy=C5=9Bk=C3=B3w?= <kamilzary@gmail.com>
Date: Sat, 8 Jun 2024 01:23:21 +0200
Subject: [PATCH 056/103] Add deprecation test for DiffIndex.iter_change_type
---
test/deprecation/test_basic.py | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/test/deprecation/test_basic.py b/test/deprecation/test_basic.py
index 6235a836c..3bf0287c7 100644
--- a/test/deprecation/test_basic.py
+++ b/test/deprecation/test_basic.py
@@ -31,7 +31,7 @@
if TYPE_CHECKING:
from pathlib import Path
- from git.diff import Diff
+ from git.diff import Diff, DiffIndex
from git.objects.commit import Commit
# ------------------------------------------------------------------------
@@ -54,6 +54,12 @@ def diff(commit: "Commit") -> Generator["Diff", None, None]:
yield diff
+@pytest.fixture
+def diffs(commit: "Commit") -> Generator["DiffIndex", None, None]:
+ """Fixture to supply a DiffIndex."""
+ yield commit.diff(NULL_TREE)
+
+
def test_diff_renamed_warns(diff: "Diff") -> None:
"""The deprecated Diff.renamed property issues a deprecation warning."""
with pytest.deprecated_call():
@@ -122,3 +128,10 @@ def test_iterable_obj_inheriting_does_not_warn() -> None:
class Derived(IterableObj):
pass
+
+
+def test_diff_iter_change_type(diffs: "DiffIndex") -> None:
+ """The internal DiffIndex.iter_change_type function issues no deprecation warning."""
+ with assert_no_deprecation_warning():
+ for change_type in diffs.change_type:
+ [*diffs.iter_change_type(change_type=change_type)]
From e1c660d224a1d27469c9275940c9db841570b8d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Krzy=C5=9Bk=C3=B3w?=
<34622465+kamilkrzyskow@users.noreply.github.com>
Date: Tue, 28 May 2024 05:53:44 +0200
Subject: [PATCH 057/103] Fix iter_change_type diff renamed property to prevent
warning
---
git/diff.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/diff.py b/git/diff.py
index e9f7e209f..8d2646f99 100644
--- a/git/diff.py
+++ b/git/diff.py
@@ -325,7 +325,7 @@ def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]:
yield diffidx
elif change_type == "C" and diffidx.copied_file:
yield diffidx
- elif change_type == "R" and diffidx.renamed:
+ elif change_type == "R" and diffidx.renamed_file:
yield diffidx
elif change_type == "M" and diffidx.a_blob and diffidx.b_blob and diffidx.a_blob != diffidx.b_blob:
yield diffidx
From f1ec1f15ec13e369bb5a4d758e94d7877e481ed3 Mon Sep 17 00:00:00 2001
From: Nick Papior <nickpapior@gmail.com>
Date: Thu, 13 Jun 2024 14:35:03 +0200
Subject: [PATCH 058/103] fixed doc to not faulty do #1924
Signed-off-by: Nick Papior <nickpapior@gmail.com>
---
test/test_docs.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/test/test_docs.py b/test/test_docs.py
index b3547c1de..cc0bbf26a 100644
--- a/test/test_docs.py
+++ b/test/test_docs.py
@@ -469,11 +469,11 @@ def test_references_and_objects(self, rw_dir):
# ![30-test_references_and_objects]
# [31-test_references_and_objects]
- git = repo.git
- git.checkout("HEAD", b="my_new_branch") # Create a new branch.
- git.branch("another-new-one")
- git.branch("-D", "another-new-one") # Pass strings for full control over argument order.
- git.for_each_ref() # '-' becomes '_' when calling it.
+ git_cmd = repo.git
+ git_cmd.checkout("HEAD", b="my_new_branch") # Create a new branch.
+ git_cmd.branch("another-new-one")
+ git_cmd.branch("-D", "another-new-one") # Pass strings for full control over argument order.
+ git_cmd.for_each_ref() # '-' becomes '_' when calling it.
# ![31-test_references_and_objects]
repo.git.clear_cache()
From d35998f5f420db780a30d73b703296498e3aa531 Mon Sep 17 00:00:00 2001
From: Guillaume Cardoen <G.Cardoen@outlook.com>
Date: Mon, 17 Jun 2024 09:28:32 +0200
Subject: [PATCH 059/103] fix: fix beginning whitespace error
---
git/diff.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/diff.py b/git/diff.py
index 8d2646f99..9c6ae59e0 100644
--- a/git/diff.py
+++ b/git/diff.py
@@ -695,7 +695,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]
change_type: Lit_change_type = cast(Lit_change_type, _change_type[0])
score_str = "".join(_change_type[1:])
score = int(score_str) if score_str.isdigit() else None
- path = path.strip()
+ path = path.strip("\n")
a_path = path.encode(defenc)
b_path = path.encode(defenc)
deleted_file = False
From 9910a886ddeb05a39b774e9f3520837fd9a76dca Mon Sep 17 00:00:00 2001
From: Guillaume Cardoen <G.Cardoen@outlook.com>
Date: Mon, 17 Jun 2024 09:31:51 +0200
Subject: [PATCH 060/103] test: add test for diff with beginning whitespace
---
test/test_diff.py | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/test/test_diff.py b/test/test_diff.py
index 928a9f428..6cae3fbf2 100644
--- a/test/test_diff.py
+++ b/test/test_diff.py
@@ -529,3 +529,23 @@ def test_diff_patch_with_external_engine(self, rw_dir):
self.assertEqual(len(index_against_head), 1)
index_against_working_tree = repo.index.diff(None, create_patch=True)
self.assertEqual(len(index_against_working_tree), 1)
+
+ @with_rw_directory
+ def test_beginning_space(self, rw_dir):
+ # Create a file beginning by a whitespace
+ repo = Repo.init(rw_dir)
+ file = osp.join(rw_dir, " file.txt")
+ with open(file, "w") as f:
+ f.write("hello world")
+ repo.git.add(Git.polish_url(file))
+ repo.index.commit("first commit")
+
+ # Diff the commit with an empty tree
+ # and check the paths
+ diff_index = repo.head.commit.diff(NULL_TREE)
+ d = diff_index[0]
+ a_path = d.a_path
+ b_path = d.b_path
+ self.assertEqual(a_path, " file.txt")
+ self.assertEqual(b_path, " file.txt")
+
\ No newline at end of file
From 97fad9cb8322e510647cf58cc023702a7b7e077f Mon Sep 17 00:00:00 2001
From: Guillaume Cardoen <G.Cardoen@outlook.com>
Date: Tue, 18 Jun 2024 08:51:00 +0200
Subject: [PATCH 061/103] style: ruff
---
test/test_diff.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/test/test_diff.py b/test/test_diff.py
index 6cae3fbf2..612fbd9e0 100644
--- a/test/test_diff.py
+++ b/test/test_diff.py
@@ -539,7 +539,7 @@ def test_beginning_space(self, rw_dir):
f.write("hello world")
repo.git.add(Git.polish_url(file))
repo.index.commit("first commit")
-
+
# Diff the commit with an empty tree
# and check the paths
diff_index = repo.head.commit.diff(NULL_TREE)
@@ -548,4 +548,3 @@ def test_beginning_space(self, rw_dir):
b_path = d.b_path
self.assertEqual(a_path, " file.txt")
self.assertEqual(b_path, " file.txt")
-
\ No newline at end of file
From f96eb0cdaeb6e33bf7725e1fb0385509f6030969 Mon Sep 17 00:00:00 2001
From: Patrick Massot <patrickmassot@free.fr>
Date: Mon, 24 Jun 2024 14:02:53 -0400
Subject: [PATCH 062/103] Change aliases to work around mypy issue.
Fixes #1934
Note this should also gives better LSP support to these property aliases.
---
git/remote.py | 11 +++++++++--
git/repo/base.py | 24 ++++++++++++++++++++----
2 files changed, 29 insertions(+), 6 deletions(-)
diff --git a/git/remote.py b/git/remote.py
index 37c991d27..15e360064 100644
--- a/git/remote.py
+++ b/git/remote.py
@@ -828,8 +828,15 @@ def remove(cls, repo: "Repo", name: str) -> str:
name._clear_cache()
return name
- # `rm` is an alias.
- rm = remove
+ @classmethod
+ def rm(cls, repo: "Repo", name: str) -> str:
+ """Alias of remove.
+ Remove the remote with the given name.
+
+ :return:
+ The passed remote name to remove
+ """
+ return cls.remove(repo, name)
def rename(self, new_name: str) -> "Remote":
"""Rename self to the given `new_name`.
diff --git a/git/repo/base.py b/git/repo/base.py
index 51ea76901..346248ddb 100644
--- a/git/repo/base.py
+++ b/git/repo/base.py
@@ -402,6 +402,17 @@ def heads(self) -> "IterableList[Head]":
"""
return Head.list_items(self)
+ @property
+ def branches(self) -> "IterableList[Head]":
+ """Alias for heads.
+ A list of :class:`~git.refs.head.Head` objects representing the branch heads
+ in this repo.
+
+ :return:
+ ``git.IterableList(Head, ...)``
+ """
+ return self.heads
+
@property
def references(self) -> "IterableList[Reference]":
"""A list of :class:`~git.refs.reference.Reference` objects representing tags,
@@ -412,11 +423,16 @@ def references(self) -> "IterableList[Reference]":
"""
return Reference.list_items(self)
- # Alias for references.
- refs = references
+ @property
+ def refs(self) -> "IterableList[Reference]":
+ """Alias for references.
+ A list of :class:`~git.refs.reference.Reference` objects representing tags,
+ heads and remote references.
- # Alias for heads.
- branches = heads
+ :return:
+ ``git.IterableList(Reference, ...)``
+ """
+ return self.references
@property
def index(self) -> "IndexFile":
From 366a60760cea066b40ed33815fa8256b25afdfcc Mon Sep 17 00:00:00 2001
From: jirka <jirka.borovec@seznam.cz>
Date: Tue, 16 Jul 2024 12:35:36 +0200
Subject: [PATCH 063/103] exclude: test/fixtures/
---
.pre-commit-config.yaml | 1 +
test/fixtures/.gitconfig | 2 +-
test/fixtures/blame | 2 +-
test/fixtures/cat_file_blob | 2 +-
test/fixtures/git_config | 1 +
test/fixtures/git_config_with_empty_value | 2 +-
test/fixtures/rev_list_bisect_all | 1 +
test/fixtures/rev_list_commit_diffs | 1 +
test/fixtures/rev_list_commit_idabbrev | 1 +
test/fixtures/rev_list_commit_stats | 1 +
10 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 50f430084..5491c4297 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -19,6 +19,7 @@ repos:
rev: v4.5.0
hooks:
- id: end-of-file-fixer
+ exclude: test/fixtures/
- id: check-toml
- id: check-yaml
- id: check-merge-conflict
diff --git a/test/fixtures/.gitconfig b/test/fixtures/.gitconfig
index f6c25c15a..6a0459f6b 100644
--- a/test/fixtures/.gitconfig
+++ b/test/fixtures/.gitconfig
@@ -1,3 +1,3 @@
[alias]
rbi = "!g() { git rebase -i origin/${1:-master} ; } ; g"
- expush = "!f() { git branch -f tmp ; { git rbi $1 && git push ; } ; git reset --hard tmp ; git rebase origin/${1:-master}; } ; f"
+ expush = "!f() { git branch -f tmp ; { git rbi $1 && git push ; } ; git reset --hard tmp ; git rebase origin/${1:-master}; } ; f"
\ No newline at end of file
diff --git a/test/fixtures/blame b/test/fixtures/blame
index 949976c5d..10c141dda 100644
--- a/test/fixtures/blame
+++ b/test/fixtures/blame
@@ -128,4 +128,4 @@ b6e1b765e0c15586a2c5b9832854f95defd71e1f 23 23
634396b2f541a9f2d58b00be1a07f0c358b999b3 11 24 2
VERSION = '1.0.0'
634396b2f541a9f2d58b00be1a07f0c358b999b3 12 25
- end
+ end
\ No newline at end of file
diff --git a/test/fixtures/cat_file_blob b/test/fixtures/cat_file_blob
index 802992c42..70c379b63 100644
--- a/test/fixtures/cat_file_blob
+++ b/test/fixtures/cat_file_blob
@@ -1 +1 @@
-Hello world
+Hello world
\ No newline at end of file
diff --git a/test/fixtures/git_config b/test/fixtures/git_config
index d3066d86e..a8cad56e8 100644
--- a/test/fixtures/git_config
+++ b/test/fixtures/git_config
@@ -43,3 +43,4 @@
# inclusions should be processed immediately
[sec]
var1 = value1_main
+
diff --git a/test/fixtures/git_config_with_empty_value b/test/fixtures/git_config_with_empty_value
index 83de84c8b..0427caea5 100644
--- a/test/fixtures/git_config_with_empty_value
+++ b/test/fixtures/git_config_with_empty_value
@@ -1,4 +1,4 @@
[color]
ui
[core]
- filemode = true
+ filemode = true
\ No newline at end of file
diff --git a/test/fixtures/rev_list_bisect_all b/test/fixtures/rev_list_bisect_all
index 60d382d01..342ea94ae 100644
--- a/test/fixtures/rev_list_bisect_all
+++ b/test/fixtures/rev_list_bisect_all
@@ -48,3 +48,4 @@ committer David Aguilar <davvid@gmail.com> 1220418344 -0700
This resolves the issue mentioned in that thread.
Signed-off-by: David Aguilar <davvid@gmail.com>
+
diff --git a/test/fixtures/rev_list_commit_diffs b/test/fixtures/rev_list_commit_diffs
index c39df2061..20397e2e4 100644
--- a/test/fixtures/rev_list_commit_diffs
+++ b/test/fixtures/rev_list_commit_diffs
@@ -5,3 +5,4 @@ author Tom Preston-Werner <tom@mojombo.com> 1193200199 -0700
committer Tom Preston-Werner <tom@mojombo.com> 1193200199 -0700
fix some initialization warnings
+
diff --git a/test/fixtures/rev_list_commit_idabbrev b/test/fixtures/rev_list_commit_idabbrev
index 6266df93e..9385ba713 100644
--- a/test/fixtures/rev_list_commit_idabbrev
+++ b/test/fixtures/rev_list_commit_idabbrev
@@ -5,3 +5,4 @@ author tom <tom@taco.(none)> 1195608462 -0800
committer tom <tom@taco.(none)> 1195608462 -0800
fix tests on other machines
+
diff --git a/test/fixtures/rev_list_commit_stats b/test/fixtures/rev_list_commit_stats
index c78aadeb5..60aa8cf58 100644
--- a/test/fixtures/rev_list_commit_stats
+++ b/test/fixtures/rev_list_commit_stats
@@ -4,3 +4,4 @@ author Tom Preston-Werner <tom@mojombo.com> 1191997100 -0700
committer Tom Preston-Werner <tom@mojombo.com> 1191997100 -0700
initial grit setup
+
From 1c88b0a734142cfc05114ad2ca0794c565294fb9 Mon Sep 17 00:00:00 2001
From: jirka <jirka.borovec@seznam.cz>
Date: Tue, 7 May 2024 19:32:10 +0200
Subject: [PATCH 064/103] use codespell
---
.pre-commit-config.yaml | 6 ++++++
pyproject.toml | 5 +++++
2 files changed, 11 insertions(+)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 551d8be34..23272bc25 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,4 +1,10 @@
repos:
+- repo: https://github.com/codespell-project/codespell
+ rev: v2.2.4
+ hooks:
+ - id: codespell
+ additional_dependencies: [tomli]
+
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.4.3
hooks:
diff --git a/pyproject.toml b/pyproject.toml
index ee54edb78..7fc809a6d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -78,3 +78,8 @@ lint.unfixable = [
"test/**" = [
"B018", # useless-expression
]
+
+[tool.codespell]
+#skip = '*.po,*.ts,./src/3rdParty,./src/Test'
+#count = true
+quiet-level = 3
\ No newline at end of file
From 2ce013cc0043f7968f126ea38482a32077efa991 Mon Sep 17 00:00:00 2001
From: Jirka <jirka.borovec@seznam.cz>
Date: Tue, 7 May 2024 19:38:44 +0200
Subject: [PATCH 065/103] fix & skip
---
.pre-commit-config.yaml | 1 +
git/index/base.py | 2 +-
git/remote.py | 2 +-
pyproject.toml | 3 ++-
4 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 23272bc25..03730febd 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,6 +4,7 @@ repos:
hooks:
- id: codespell
additional_dependencies: [tomli]
+ args: ["--write-changes"]
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.4.3
diff --git a/git/index/base.py b/git/index/base.py
index 28b60a880..a317e71c0 100644
--- a/git/index/base.py
+++ b/git/index/base.py
@@ -1443,7 +1443,7 @@ def reset(
key = entry_key(path, 0)
self.entries[key] = nie[key]
except KeyError:
- # If key is not in theirs, it musn't be in ours.
+ # If key is not in theirs, it mustn't be in ours.
try:
del self.entries[key]
except KeyError:
diff --git a/git/remote.py b/git/remote.py
index 15e360064..1e09e210e 100644
--- a/git/remote.py
+++ b/git/remote.py
@@ -250,7 +250,7 @@ def _from_line(cls, remote: "Remote", line: str) -> "PushInfo":
flags |= cls.NEW_TAG
elif "[new branch]" in summary:
flags |= cls.NEW_HEAD
- # uptodate encoded in control character
+ # up-to-date encoded in control character
else:
# Fast-forward or forced update - was encoded in control character,
# but we parse the old and new commit.
diff --git a/pyproject.toml b/pyproject.toml
index 7fc809a6d..8b4522824 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -80,6 +80,7 @@ lint.unfixable = [
]
[tool.codespell]
-#skip = '*.po,*.ts,./src/3rdParty,./src/Test'
+skip = 'test/fixtures/reflog_*'
+ignore-words-list="gud,doesnt"
#count = true
quiet-level = 3
\ No newline at end of file
From 93993b201458fd18059e97fa25a08a14fae2af1f Mon Sep 17 00:00:00 2001
From: jirka <jirka.borovec@seznam.cz>
Date: Wed, 17 Jul 2024 12:31:02 +0200
Subject: [PATCH 066/103] fixing
---
.pre-commit-config.yaml | 3 ++-
README.md | 2 +-
doc/source/changes.rst | 2 +-
git/objects/util.py | 6 +++---
pyproject.toml | 3 +--
test/test_exc.py | 2 +-
test/test_index.py | 6 +++---
7 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 03730febd..692c7fa2a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,7 +4,8 @@ repos:
hooks:
- id: codespell
additional_dependencies: [tomli]
- args: ["--write-changes"]
+ # args: ["--write-changes"] # consider enabling for auto-fif
+ exclude: "test/fixtures/"
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.4.3
diff --git a/README.md b/README.md
index d365a6584..59c6f995b 100644
--- a/README.md
+++ b/README.md
@@ -101,7 +101,7 @@ In the less common case that you do not want to install test dependencies, `pip
#### With editable *dependencies* (not preferred, and rarely needed)
-In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediatley reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython.
+In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediately reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython.
If you want to do that *and* you want the versions in GitPython's git submodules to be used, then pass `-e git/ext/gitdb` and/or `-e git/ext/gitdb/gitdb/ext/smmap` to `pip install`. This can be done in any order, and in separate `pip install` commands or the same one, so long as `-e` appears before *each* path. For example, you can install GitPython, gitdb, and smmap editably in the currently active virtual environment this way:
diff --git a/doc/source/changes.rst b/doc/source/changes.rst
index 0bc757134..3c903423c 100644
--- a/doc/source/changes.rst
+++ b/doc/source/changes.rst
@@ -20,7 +20,7 @@ https://github.com/gitpython-developers/GitPython/releases/tag/3.1.42
3.1.41
======
-This release is relevant for security as it fixes a possible arbitary
+This release is relevant for security as it fixes a possible arbitrary
code execution on Windows.
See this PR for details: https://github.com/gitpython-developers/GitPython/pull/1792
diff --git a/git/objects/util.py b/git/objects/util.py
index 5c56e6134..a68d701f5 100644
--- a/git/objects/util.py
+++ b/git/objects/util.py
@@ -568,11 +568,11 @@ def addToStack(
yield rval
# Only continue to next level if this is appropriate!
- nd = d + 1
- if depth > -1 and nd > depth:
+ next_d = d + 1
+ if depth > -1 and next_d > depth:
continue
- addToStack(stack, item, branch_first, nd)
+ addToStack(stack, item, branch_first, next_d)
# END for each item on work stack
diff --git a/pyproject.toml b/pyproject.toml
index 8b4522824..603e2597c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -80,7 +80,6 @@ lint.unfixable = [
]
[tool.codespell]
-skip = 'test/fixtures/reflog_*'
ignore-words-list="gud,doesnt"
#count = true
-quiet-level = 3
\ No newline at end of file
+quiet-level = 3
diff --git a/test/test_exc.py b/test/test_exc.py
index c1eae7240..2e979f5a1 100644
--- a/test/test_exc.py
+++ b/test/test_exc.py
@@ -52,7 +52,7 @@
_streams_n_substrings = (
None,
- "steram",
+ "stream",
"ομορφο stream",
)
diff --git a/test/test_index.py b/test/test_index.py
index b92258c92..2684cfd81 100644
--- a/test/test_index.py
+++ b/test/test_index.py
@@ -1018,7 +1018,7 @@ class Mocked:
@pytest.mark.xfail(
type(_win_bash_status) is WinBashStatus.Absent,
reason="Can't run a hook on Windows without bash.exe.",
- rasies=HookExecutionError,
+ raises=HookExecutionError,
)
@pytest.mark.xfail(
type(_win_bash_status) is WinBashStatus.WslNoDistro,
@@ -1077,7 +1077,7 @@ def test_hook_uses_shell_not_from_cwd(self, rw_dir, case):
@pytest.mark.xfail(
type(_win_bash_status) is WinBashStatus.Absent,
reason="Can't run a hook on Windows without bash.exe.",
- rasies=HookExecutionError,
+ raises=HookExecutionError,
)
@pytest.mark.xfail(
type(_win_bash_status) is WinBashStatus.WslNoDistro,
@@ -1120,7 +1120,7 @@ def test_pre_commit_hook_fail(self, rw_repo):
@pytest.mark.xfail(
type(_win_bash_status) is WinBashStatus.Absent,
reason="Can't run a hook on Windows without bash.exe.",
- rasies=HookExecutionError,
+ raises=HookExecutionError,
)
@pytest.mark.xfail(
type(_win_bash_status) is WinBashStatus.Wsl,
From 813520c123d44a8cf87a219fc710faeb1f1559ca Mon Sep 17 00:00:00 2001
From: Jirka Borovec <6035284+Borda@users.noreply.github.com>
Date: Wed, 17 Jul 2024 12:34:14 +0200
Subject: [PATCH 067/103] Apply suggestions from code review
---
git/remote.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/remote.py b/git/remote.py
index 1e09e210e..9de3dace4 100644
--- a/git/remote.py
+++ b/git/remote.py
@@ -250,7 +250,7 @@ def _from_line(cls, remote: "Remote", line: str) -> "PushInfo":
flags |= cls.NEW_TAG
elif "[new branch]" in summary:
flags |= cls.NEW_HEAD
- # up-to-date encoded in control character
+ # `uptodate` encoded in control character
else:
# Fast-forward or forced update - was encoded in control character,
# but we parse the old and new commit.
From ce8a69a4141d2149bac2cbf56ea7d4b1f2ed7257 Mon Sep 17 00:00:00 2001
From: Jonas Scharpf <jonas.scharpf@checkmk.com>
Date: Wed, 17 Jul 2024 11:01:09 +0200
Subject: [PATCH 068/103] Add type of change to files_dict of a commit
This allows to not only get the total, inserted or deleted number of
lines being changed but also the type of change like
Added (A), Copied (C), Deleted (D), Modified (M), Renamed (R),
type changed (T), Unmerged (U), Unknown (X), or pairing Broken (B)
---
AUTHORS | 1 +
git/objects/commit.py | 24 +++++++++++++++++-------
git/types.py | 1 +
git/util.py | 4 +++-
test/fixtures/diff_numstat | 5 +++--
test/test_commit.py | 9 ++++++---
test/test_stats.py | 12 +++++++++---
7 files changed, 40 insertions(+), 16 deletions(-)
diff --git a/AUTHORS b/AUTHORS
index 9311b3962..45b14c961 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -54,5 +54,6 @@ Contributors are:
-Wenhan Zhu <wzhu.cosmos _at_ gmail.com>
-Eliah Kagan <eliah.kagan _at_ gmail.com>
-Ethan Lin <et.repositories _at_ gmail.com>
+-Jonas Scharpf <jonas.scharpf _at_ checkmk.com>
Portions derived from other open source works and are clearly marked.
diff --git a/git/objects/commit.py b/git/objects/commit.py
index d957c9051..0ceb46609 100644
--- a/git/objects/commit.py
+++ b/git/objects/commit.py
@@ -377,15 +377,25 @@ def stats(self) -> Stats:
:return:
:class:`Stats`
"""
- if not self.parents:
- text = self.repo.git.diff_tree(self.hexsha, "--", numstat=True, no_renames=True, root=True)
- text2 = ""
- for line in text.splitlines()[1:]:
+
+ def process_lines(lines: List[str]) -> str:
+ text = ""
+ for file_info, line in zip(lines, lines[len(lines) // 2 :]):
+ change_type = file_info.split("\t")[0][-1]
(insertions, deletions, filename) = line.split("\t")
- text2 += "%s\t%s\t%s\n" % (insertions, deletions, filename)
- text = text2
+ text += "%s\t%s\t%s\t%s\n" % (change_type, insertions, deletions, filename)
+ return text
+
+ if not self.parents:
+ lines = self.repo.git.diff_tree(
+ self.hexsha, "--", numstat=True, no_renames=True, root=True, raw=True
+ ).splitlines()[1:]
+ text = process_lines(lines)
else:
- text = self.repo.git.diff(self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True)
+ lines = self.repo.git.diff(
+ self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True, raw=True
+ ).splitlines()
+ text = process_lines(lines)
return Stats._list_from_string(self.repo, text)
@property
diff --git a/git/types.py b/git/types.py
index 584450146..cce184530 100644
--- a/git/types.py
+++ b/git/types.py
@@ -248,6 +248,7 @@ class Files_TD(TypedDict):
insertions: int
deletions: int
lines: int
+ change_type: str
class Total_TD(TypedDict):
diff --git a/git/util.py b/git/util.py
index 11f963e02..9e8ac821d 100644
--- a/git/util.py
+++ b/git/util.py
@@ -910,6 +910,7 @@ class Stats:
deletions = number of deleted lines as int
insertions = number of inserted lines as int
lines = total number of lines changed as int, or deletions + insertions
+ change_type = type of change as str, A|C|D|M|R|T|U|X|B
``full-stat-dict``
@@ -938,7 +939,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats":
"files": {},
}
for line in text.splitlines():
- (raw_insertions, raw_deletions, filename) = line.split("\t")
+ (change_type, raw_insertions, raw_deletions, filename) = line.split("\t")
insertions = raw_insertions != "-" and int(raw_insertions) or 0
deletions = raw_deletions != "-" and int(raw_deletions) or 0
hsh["total"]["insertions"] += insertions
@@ -949,6 +950,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats":
"insertions": insertions,
"deletions": deletions,
"lines": insertions + deletions,
+ "change_type": change_type,
}
hsh["files"][filename.strip()] = files_dict
return Stats(hsh["total"], hsh["files"])
diff --git a/test/fixtures/diff_numstat b/test/fixtures/diff_numstat
index 44c6ca2d5..b76e467eb 100644
--- a/test/fixtures/diff_numstat
+++ b/test/fixtures/diff_numstat
@@ -1,2 +1,3 @@
-29 18 a.txt
-0 5 b.txt
+M 29 18 a.txt
+M 0 5 b.txt
+A 7 0 c.txt
\ No newline at end of file
diff --git a/test/test_commit.py b/test/test_commit.py
index 5832258de..37c66e3e7 100644
--- a/test/test_commit.py
+++ b/test/test_commit.py
@@ -135,9 +135,12 @@ def test_stats(self):
commit = self.rorepo.commit("33ebe7acec14b25c5f84f35a664803fcab2f7781")
stats = commit.stats
- def check_entries(d):
+ def check_entries(d, has_change_type=False):
assert isinstance(d, dict)
- for key in ("insertions", "deletions", "lines"):
+ keys = ("insertions", "deletions", "lines")
+ if has_change_type:
+ keys += ("change_type",)
+ for key in keys:
assert key in d
# END assertion helper
@@ -148,7 +151,7 @@ def check_entries(d):
assert "files" in stats.total
for _filepath, d in stats.files.items():
- check_entries(d)
+ check_entries(d, True)
# END for each stated file
# Check that data is parsed properly.
diff --git a/test/test_stats.py b/test/test_stats.py
index eec73c802..91d2cf6ae 100644
--- a/test/test_stats.py
+++ b/test/test_stats.py
@@ -14,13 +14,19 @@ def test_list_from_string(self):
output = fixture("diff_numstat").decode(defenc)
stats = Stats._list_from_string(self.rorepo, output)
- self.assertEqual(2, stats.total["files"])
- self.assertEqual(52, stats.total["lines"])
- self.assertEqual(29, stats.total["insertions"])
+ self.assertEqual(3, stats.total["files"])
+ self.assertEqual(59, stats.total["lines"])
+ self.assertEqual(36, stats.total["insertions"])
self.assertEqual(23, stats.total["deletions"])
self.assertEqual(29, stats.files["a.txt"]["insertions"])
self.assertEqual(18, stats.files["a.txt"]["deletions"])
+ self.assertEqual("M", stats.files["a.txt"]["change_type"])
self.assertEqual(0, stats.files["b.txt"]["insertions"])
self.assertEqual(5, stats.files["b.txt"]["deletions"])
+ self.assertEqual("M", stats.files["b.txt"]["change_type"])
+
+ self.assertEqual(7, stats.files["c.txt"]["insertions"])
+ self.assertEqual(0, stats.files["c.txt"]["deletions"])
+ self.assertEqual("A", stats.files["c.txt"]["change_type"])
From 58a9a58f58e6aae220efda8ce95bf4c2e0fd9ca0 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Wed, 24 Jul 2024 02:10:57 -0400
Subject: [PATCH 069/103] Use Alpine Linux in WSL on CI
Some of the CI tests use WSL. This switches the WSL distribution
from Debian to Alpine, which might be slightly faster. For the way
it is being used here, the main expected speed improvement would be
to how long the image would take to download, as Alpine is smaller.
(The reason for this is thus unrelated to the reason for the Alpine
docker CI test job added in #1826. There, the goal was to test on a
wider variety of systems and environments, and that runs the whole
test suite in Alpine. This just changes the WSL distro, used by a
few tests on Windows, from Debian to Alpine.)
Two things have changed that, taken together, have unblocked this:
- https://github.com/Vampire/setup-wsl/issues/50 was fixed, so the
action we are using is able to install Alpine Linux. See:
https://github.com/gitpython-developers/GitPython/pull/1917#pullrequestreview-2081550232
- #1893 was fixed in #1888. So if switching the WSL distro from
Debian to Alpine breaks any tests, including by making them fail
in an unexpected way that raises the wrong exception, we are
likely to find out.
---
.github/workflows/pythonpackage.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 031b0e6b2..61ab2206c 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -46,7 +46,7 @@ jobs:
if: startsWith(matrix.os, 'windows')
uses: Vampire/setup-wsl@v3.1.1
with:
- distribution: Debian
+ distribution: Alpine
- name: Prepare this repo for tests
run: |
From ce5eefd90b6c652083ce615583b5ee62b39ae187 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Wed, 24 Jul 2024 02:39:03 -0400
Subject: [PATCH 070/103] Enable Python 3.8 and 3.9 on M1 runners
These were excluded in 9ad28c3 (#1817) due to
https://github.com/actions/setup-python/issues/808, which was later
fixed by https://github.com/actions/python-versions/pull/259.
Because Python 3.7 has been end-of-life for a while, it is very
unlikely to have AArch64 builds added in python-versions for use
on GitHub Actions CI runners (preinstalled or via setup-python).
---
.github/workflows/pythonpackage.yml | 4 ----
1 file changed, 4 deletions(-)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 031b0e6b2..f3c837742 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -18,10 +18,6 @@ jobs:
exclude:
- os: "macos-14"
python-version: "3.7"
- - os: "macos-14"
- python-version: "3.8"
- - os: "macos-14"
- python-version: "3.9"
include:
- experimental: false
From 055394a548d19dded2ad9791a208bbcc54879b14 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Wed, 24 Jul 2024 03:25:31 -0400
Subject: [PATCH 071/103] Install bash in WSL Alpine distro
Because Alpine Linux does not ship with bash, and the tests that
use WSL use it.
---
.github/workflows/pythonpackage.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 61ab2206c..1902ecb19 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -47,6 +47,7 @@ jobs:
uses: Vampire/setup-wsl@v3.1.1
with:
distribution: Alpine
+ additional-packages: bash
- name: Prepare this repo for tests
run: |
From c2bbaf47e14dac5f0470938b3ecf67836ca1695d Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Wed, 24 Jul 2024 04:34:00 -0400
Subject: [PATCH 072/103] Remove the non-ARM macOS CI jobs
This keeps only the macos-14 jobs, which run on Apple Silicon M1,
and removes the macos-13 jobs, which ran on x86-64.
Other operating systems jobs continue to run on x86-64 machines
(and none on ARM, yet). Only the macOS jobs are removed.
This change leaves Python 3.7 without any macOS test job. That is
probably okay, since it has been end-of-life for some time, and
it remains tested on Ubuntu and Windows.
---
.github/workflows/pythonpackage.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 49f6c5254..7547aecf9 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -13,7 +13,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: ["ubuntu-latest", "macos-13", "macos-14", "windows-latest"]
+ os: ["ubuntu-latest", "macos-14", "windows-latest"]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
exclude:
- os: "macos-14"
From be6744b6e4365fc42996a1be7f026f133f992928 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Wed, 24 Jul 2024 04:38:06 -0400
Subject: [PATCH 073/103] Use the macos-latest label rather than macos-14
Currently they are the same. The macos-latest label will move to
later versions automatically in the future, like the ubuntu-latest
and windows-latest labels that we are already using.
In this repo, the macos-14 label had been used originally because
it was added before the migration of macos-latest to be macos-14
was completed. See https://github.com/github/roadmap/issues/926. It
was kept for clarity of constrast with the macos-13 jobs that were
also in use, some for the same Python versions.
Now that the macos-13 jobs have been removed in c2bbaf4, the
macos-latest label can be used here without confusion.
---
.github/workflows/pythonpackage.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 7547aecf9..0f1d17544 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -13,10 +13,10 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: ["ubuntu-latest", "macos-14", "windows-latest"]
+ os: ["ubuntu-latest", "macos-latest", "windows-latest"]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
exclude:
- - os: "macos-14"
+ - os: "macos-latest"
python-version: "3.7"
include:
- experimental: false
From af0cd933e84b9f83210c0f12f95a456606ee79e9 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 6 Jun 2024 02:17:25 -0400
Subject: [PATCH 074/103] Fix "OSError: [Errno 36] File name too long" in
fuzz_submodule
Fixes a bug in the `fuzz_submodule` harness where the fuzzed data can
produce file names that exceed the maximum size allowed byt the OS. This
issue came up previously and was fixed in #1922, but the submodule file
name fixed here was missed in that PR.
Fixes: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=69456
---
fuzzing/fuzz-targets/fuzz_submodule.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index ca47690ea..9f5828d8d 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -35,12 +35,13 @@ def TestOneInput(data):
sub_repo = Repo.init(submodule_temp_dir, bare=fdp.ConsumeBool())
sub_repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512)))
- submodule_name = f"submodule_{fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))}"
+ submodule_name = fdp.ConsumeUnicodeNoSurrogates(
+ fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(repo.working_tree_dir)))
+ )
submodule_path = os.path.join(repo.working_tree_dir, submodule_name)
- submodule_url = sub_repo.git_dir
- submodule = repo.create_submodule(submodule_name, submodule_path, url=submodule_url)
- repo.index.commit(f"Added submodule {submodule_name}")
+ submodule = repo.create_submodule(submodule_name, submodule_path, url=sub_repo.git_dir)
+ repo.index.commit("Added submodule")
with submodule.config_writer() as writer:
key_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes()))
From 7de1556d3895c718f0f0772530ff7cde5457d9d8 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 8 Aug 2024 16:54:37 -0400
Subject: [PATCH 075/103] Filter out non-bug exceptions using a pre-defined
exception list.
This reduces false positive test failures by identifying and
gracefully handling exceptions that are explicitly raised by GitPython,
thus reducing the false-positive fuzzing test failure rate.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 56 +++++++++++++++----
fuzzing/oss-fuzz-scripts/build.sh | 2 +-
.../container-environment-bootstrap.sh | 11 ++++
3 files changed, 56 insertions(+), 13 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 9f5828d8d..05c543bf8 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -1,16 +1,51 @@
+# ruff: noqa: E402
import atheris
import sys
import os
+import traceback
import tempfile
from configparser import ParsingError
-from utils import is_expected_exception_message, get_max_filename_length
+from utils import get_max_filename_length
+import re
+
+bundle_dir = os.path.dirname(os.path.abspath(__file__))
if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover
- path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
- os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+ bundled_git_binary_path = os.path.join(bundle_dir, "git")
+ os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path
from git import Repo, GitCommandError, InvalidGitRepositoryError
+
+def load_exception_list(file_path):
+ """Load and parse the exception list from a file."""
+ try:
+ with open(file_path, "r") as file:
+ lines = file.readlines()
+ exception_list = set()
+ for line in lines:
+ match = re.match(r"(.+):(\d+):", line)
+ if match:
+ file_path = match.group(1).strip()
+ line_number = int(match.group(2).strip())
+ exception_list.add((file_path, line_number))
+ return exception_list
+ except FileNotFoundError:
+ print("File not found: %s", file_path)
+ return set()
+ except Exception as e:
+ print("Error loading exception list: %s", e)
+ return set()
+
+
+def check_exception_against_list(exception_list, exc_traceback):
+ """Check if the exception traceback matches any entry in the exception list."""
+ for filename, lineno, _, _ in traceback.extract_tb(exc_traceback):
+ if (filename, lineno) in exception_list:
+ return True
+ return False
+
+
if not sys.warnoptions: # pragma: no cover
# The warnings filter below can be overridden by passing the -W option
# to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable.
@@ -89,17 +124,14 @@ def TestOneInput(data):
BrokenPipeError,
):
return -1
- except ValueError as e:
- expected_messages = [
- "SHA is empty",
- "Reference at",
- "embedded null byte",
- "This submodule instance does not exist anymore",
- "cmd stdin was empty",
- ]
- if is_expected_exception_message(e, expected_messages):
+ except Exception as e:
+ exc_traceback = e.__traceback__
+ exception_list = load_exception_list(os.path.join(bundle_dir, "explicit-exceptions-list.txt"))
+ if check_exception_against_list(exception_list, exc_traceback):
+ print("Exception matches an entry in the exception list.")
return -1
else:
+ print("Exception does not match any entry in the exception list.")
raise e
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
index e0b3a50ab..c156e872d 100644
--- a/fuzzing/oss-fuzz-scripts/build.sh
+++ b/fuzzing/oss-fuzz-scripts/build.sh
@@ -15,5 +15,5 @@ find "$SRC" -maxdepth 1 \
# Build fuzzers in $OUT.
find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do
- compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):."
+ compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." --add-data="$SRC/explicit-exceptions-list.txt:."
done
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
index bbdcf5357..af1ddf014 100755
--- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -91,6 +91,17 @@ create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora"
prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing"
+pushd "$SRC/gitpython/"
+# Search for 'raise' and 'assert' statements in Python files within GitPython's 'git/' directory and its submodules,
+# remove trailing colons, and save to 'explicit-exceptions-list.txt'. This file can then be used by fuzz harnesses to
+# check exception tracebacks:
+# If an exception found by the fuzzer originated in a file + line number in explicit-exceptions-list.txt, then it is not a bug.
+
+git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- "git/**/*.py" > "$SRC/explicit-exceptions-list.txt"
+
+popd
+
+
# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below.
python3 -m pip install --upgrade pip
# Upgrade to the latest versions known to work at the time the below changes were introduced:
From 799b9cae745f50f2c0c590e8b3e19bfea199c463 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 8 Aug 2024 18:58:28 -0400
Subject: [PATCH 076/103] Improve `check_exception_against_list` matching logic
using regex
Changes:
- `match_exception_with_traceback` uses regular expressions for more
flexible matching of file paths and line numbers. This allows for
partial matches and more complex patterns.
- Improve `check_exception_against_list` by delegating to
`match_exception_with_traceback` for checking tracebacks against
exception list entries.
- `load_exception_list`: Remains largely unchanged, as it correctly
parses the file and line number from each exception entry. However,
we ensure the set consists of regex patterns to match against
tracebacks.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 05c543bf8..37f069079 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -31,21 +31,27 @@ def load_exception_list(file_path):
exception_list.add((file_path, line_number))
return exception_list
except FileNotFoundError:
- print("File not found: %s", file_path)
+ print(f"File not found: {file_path}")
return set()
except Exception as e:
- print("Error loading exception list: %s", e)
+ print(f"Error loading exception list: {e}")
return set()
-def check_exception_against_list(exception_list, exc_traceback):
- """Check if the exception traceback matches any entry in the exception list."""
+def match_exception_with_traceback(exception_list, exc_traceback):
+ """Match exception traceback with the entries in the exception list."""
for filename, lineno, _, _ in traceback.extract_tb(exc_traceback):
- if (filename, lineno) in exception_list:
- return True
+ for file_pattern, line_pattern in exception_list:
+ if re.fullmatch(file_pattern, filename) and re.fullmatch(line_pattern, str(lineno)):
+ return True
return False
+def check_exception_against_list(exception_list, exc_traceback):
+ """Check if the exception traceback matches any entry in the exception list."""
+ return match_exception_with_traceback(exception_list, exc_traceback)
+
+
if not sys.warnoptions: # pragma: no cover
# The warnings filter below can be overridden by passing the -W option
# to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable.
@@ -128,10 +134,8 @@ def TestOneInput(data):
exc_traceback = e.__traceback__
exception_list = load_exception_list(os.path.join(bundle_dir, "explicit-exceptions-list.txt"))
if check_exception_against_list(exception_list, exc_traceback):
- print("Exception matches an entry in the exception list.")
return -1
else:
- print("Exception does not match any entry in the exception list.")
raise e
From 2e9c23995b70372a18edc4d0b143b6b522d3fb39 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 8 Aug 2024 19:38:06 -0400
Subject: [PATCH 077/103] Extract environment setup and exception checking
boilerplate logic
Changes:
- Simplify exception handling in test harnesses via `handle_exception(e)`
in the `except Exception as e:` block.
- `setup_git_environment` is a step towards centralizing environment
variable and logging configuration set up consistently across
different fuzzing scripts. **Only applying it to a single test for
now is an intentional choice in case it fails to work in the
ClusterFuzz environment!** If it proves successful, a follow-up
change set will be welcome.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 70 +++------------------
fuzzing/fuzz-targets/utils.py | 87 +++++++++++++++++++++++++-
2 files changed, 95 insertions(+), 62 deletions(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 37f069079..634572bf2 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -1,67 +1,17 @@
-# ruff: noqa: E402
import atheris
import sys
import os
-import traceback
import tempfile
from configparser import ParsingError
-from utils import get_max_filename_length
-import re
-
-bundle_dir = os.path.dirname(os.path.abspath(__file__))
-
-if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover
- bundled_git_binary_path = os.path.join(bundle_dir, "git")
- os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path
-
from git import Repo, GitCommandError, InvalidGitRepositoryError
+from utils import (
+ setup_git_environment,
+ handle_exception,
+ get_max_filename_length,
+)
-
-def load_exception_list(file_path):
- """Load and parse the exception list from a file."""
- try:
- with open(file_path, "r") as file:
- lines = file.readlines()
- exception_list = set()
- for line in lines:
- match = re.match(r"(.+):(\d+):", line)
- if match:
- file_path = match.group(1).strip()
- line_number = int(match.group(2).strip())
- exception_list.add((file_path, line_number))
- return exception_list
- except FileNotFoundError:
- print(f"File not found: {file_path}")
- return set()
- except Exception as e:
- print(f"Error loading exception list: {e}")
- return set()
-
-
-def match_exception_with_traceback(exception_list, exc_traceback):
- """Match exception traceback with the entries in the exception list."""
- for filename, lineno, _, _ in traceback.extract_tb(exc_traceback):
- for file_pattern, line_pattern in exception_list:
- if re.fullmatch(file_pattern, filename) and re.fullmatch(line_pattern, str(lineno)):
- return True
- return False
-
-
-def check_exception_against_list(exception_list, exc_traceback):
- """Check if the exception traceback matches any entry in the exception list."""
- return match_exception_with_traceback(exception_list, exc_traceback)
-
-
-if not sys.warnoptions: # pragma: no cover
- # The warnings filter below can be overridden by passing the -W option
- # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable.
- import warnings
- import logging
-
- # Fuzzing data causes some modules to generate a large number of warnings
- # which are not usually interesting and make the test output hard to read, so we ignore them.
- warnings.simplefilter("ignore")
- logging.getLogger().setLevel(logging.ERROR)
+# Setup the git environment
+setup_git_environment()
def TestOneInput(data):
@@ -131,12 +81,10 @@ def TestOneInput(data):
):
return -1
except Exception as e:
- exc_traceback = e.__traceback__
- exception_list = load_exception_list(os.path.join(bundle_dir, "explicit-exceptions-list.txt"))
- if check_exception_against_list(exception_list, exc_traceback):
+ if isinstance(e, ValueError) and "embedded null byte" in str(e):
return -1
else:
- raise e
+ return handle_exception(e)
def main():
diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py
index f522d2959..97e6eab98 100644
--- a/fuzzing/fuzz-targets/utils.py
+++ b/fuzzing/fuzz-targets/utils.py
@@ -1,6 +1,9 @@
import atheris # pragma: no cover
import os # pragma: no cover
-from typing import List # pragma: no cover
+import re # pragma: no cover
+import traceback # pragma: no cover
+import sys # pragma: no cover
+from typing import Set, Tuple, List # pragma: no cover
@atheris.instrument_func
@@ -35,3 +38,85 @@ def get_max_filename_length(path: str) -> int: # pragma: no cover
int: The maximum filename length.
"""
return os.pathconf(path, "PC_NAME_MAX")
+
+
+@atheris.instrument_func
+def read_lines_from_file(file_path: str) -> list:
+ """Read lines from a file and return them as a list."""
+ try:
+ with open(file_path, "r") as f:
+ return [line.strip() for line in f if line.strip()]
+ except FileNotFoundError:
+ print(f"File not found: {file_path}")
+ return []
+ except IOError as e:
+ print(f"Error reading file {file_path}: {e}")
+ return []
+
+
+@atheris.instrument_func
+def load_exception_list(file_path: str = "explicit-exceptions-list.txt") -> Set[Tuple[str, str]]:
+ """Load and parse the exception list from a default or specified file."""
+ try:
+ bundle_dir = os.path.dirname(os.path.abspath(__file__))
+ full_path = os.path.join(bundle_dir, file_path)
+ lines = read_lines_from_file(full_path)
+ exception_list: Set[Tuple[str, str]] = set()
+ for line in lines:
+ match = re.match(r"(.+):(\d+):", line)
+ if match:
+ file_path: str = match.group(1).strip()
+ line_number: str = str(match.group(2).strip())
+ exception_list.add((file_path, line_number))
+ return exception_list
+ except Exception as e:
+ print(f"Error loading exception list: {e}")
+ return set()
+
+
+@atheris.instrument_func
+def match_exception_with_traceback(exception_list: Set[Tuple[str, str]], exc_traceback) -> bool:
+ """Match exception traceback with the entries in the exception list."""
+ for filename, lineno, _, _ in traceback.extract_tb(exc_traceback):
+ for file_pattern, line_pattern in exception_list:
+ # Ensure filename and line_number are strings for regex matching
+ if re.fullmatch(file_pattern, filename) and re.fullmatch(line_pattern, str(lineno)):
+ return True
+ return False
+
+
+@atheris.instrument_func
+def check_exception_against_list(exc_traceback, exception_file: str = "explicit-exceptions-list.txt") -> bool:
+ """Check if the exception traceback matches any entry in the exception list."""
+ exception_list = load_exception_list(exception_file)
+ return match_exception_with_traceback(exception_list, exc_traceback)
+
+
+@atheris.instrument_func
+def handle_exception(e: Exception) -> int:
+ """Encapsulate exception handling logic for reusability."""
+ exc_traceback = e.__traceback__
+ if check_exception_against_list(exc_traceback):
+ return -1
+ else:
+ raise e
+
+
+@atheris.instrument_func
+def setup_git_environment() -> None:
+ """Set up the environment variables for Git."""
+ bundle_dir = os.path.dirname(os.path.abspath(__file__))
+ if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover
+ bundled_git_binary_path = os.path.join(bundle_dir, "git")
+ os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path
+
+ if not sys.warnoptions: # pragma: no cover
+ # The warnings filter below can be overridden by passing the -W option
+ # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable.
+ import warnings
+ import logging
+
+ # Fuzzing data causes some modules to generate a large number of warnings
+ # which are not usually interesting and make the test output hard to read, so we ignore them.
+ warnings.simplefilter("ignore")
+ logging.getLogger().setLevel(logging.ERROR)
From 27de8676c64b549038b4fdd994a20f1ce996ad5e Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Thu, 8 Aug 2024 20:35:13 -0400
Subject: [PATCH 078/103] Fix buggy `git grep` pathspec args
To ensure that all necessary files are included in the
explicit-exceptions-list.txt file and unwanted files and directories are
not.
---
.../container-environment-bootstrap.sh | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
index af1ddf014..924a3cbf3 100755
--- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
+++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -92,12 +92,12 @@ create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora"
prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing"
pushd "$SRC/gitpython/"
-# Search for 'raise' and 'assert' statements in Python files within GitPython's 'git/' directory and its submodules,
-# remove trailing colons, and save to 'explicit-exceptions-list.txt'. This file can then be used by fuzz harnesses to
-# check exception tracebacks:
-# If an exception found by the fuzzer originated in a file + line number in explicit-exceptions-list.txt, then it is not a bug.
+# Search for 'raise' and 'assert' statements in Python files within GitPython's source code and submodules, saving the
+# matched file path, line number, and line content to a file named 'explicit-exceptions-list.txt'.
+# This file can then be used by fuzz harnesses to check exception tracebacks and filter out explicitly raised or otherwise
+# anticipated exceptions to reduce false positive test failures.
-git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- "git/**/*.py" > "$SRC/explicit-exceptions-list.txt"
+git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- '*.py' -- ':!setup.py' -- ':!test/**' -- ':!fuzzing/**' > "$SRC/explicit-exceptions-list.txt"
popd
From 2ed33345667706c5755708e88c989ede06f2414f Mon Sep 17 00:00:00 2001
From: David Lakin <david.js.lakin@gmail.com>
Date: Fri, 9 Aug 2024 00:06:44 -0400
Subject: [PATCH 079/103] Fix order of environment setup and git module import
The environment setup must happen before the `git` module is imported,
otherwise GitPython won't be able to find the Git executable and raise
an exception that causes the ClusterFuzz fuzzer runs to fail.
---
fuzzing/fuzz-targets/fuzz_submodule.py | 2 +-
pyproject.toml | 4 ++++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 634572bf2..997133b70 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -3,7 +3,6 @@
import os
import tempfile
from configparser import ParsingError
-from git import Repo, GitCommandError, InvalidGitRepositoryError
from utils import (
setup_git_environment,
handle_exception,
@@ -12,6 +11,7 @@
# Setup the git environment
setup_git_environment()
+from git import Repo, GitCommandError, InvalidGitRepositoryError
def TestOneInput(data):
diff --git a/pyproject.toml b/pyproject.toml
index 603e2597c..6cf4b3f5d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -78,6 +78,10 @@ lint.unfixable = [
"test/**" = [
"B018", # useless-expression
]
+"fuzzing/fuzz-targets/**" = [
+ "E402", # environment setup must happen before the `git` module is imported, thus cannot happen at top of file
+]
+
[tool.codespell]
ignore-words-list="gud,doesnt"
From 096851b61fa99df233176b090146efb52e524f48 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Fri, 9 Aug 2024 11:01:34 -0400
Subject: [PATCH 080/103] Gracefully handle `PermissionError` exceptions that
crash fuzzer
Fuzzing inputs sometimes produce directory paths that are protected
inside the fuzzer execution environment. This is not an issue in
GitPython's code, so it should not crash the fuzzer.
Fixes OSS-Fuzz Issue 69456:
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=69870
---
fuzzing/fuzz-targets/fuzz_submodule.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index 997133b70..c2bf1e4fe 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -78,6 +78,7 @@ def TestOneInput(data):
IsADirectoryError,
NotADirectoryError,
BrokenPipeError,
+ PermissionError,
):
return -1
except Exception as e:
From 7126ce16a03e0aea5ef4d031c62596992a6d7cb5 Mon Sep 17 00:00:00 2001
From: David Lakin <github@themoderndev.com>
Date: Tue, 13 Aug 2024 01:09:37 -0400
Subject: [PATCH 081/103] Fuzzing: Gracefully Handle Uninteresting Error to Fix
OSS-Fuzz Issue 71095
Fuzzing data can generate filenames that trigger:
> OSError: [Errno 36] File name too long
The changes here add handling for these exceptions because they di not
indicate a bug and should not crash the fuzzer.
a
---
fuzzing/fuzz-targets/fuzz_submodule.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
index c2bf1e4fe..d22b0aa5b 100644
--- a/fuzzing/fuzz-targets/fuzz_submodule.py
+++ b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -84,6 +84,8 @@ def TestOneInput(data):
except Exception as e:
if isinstance(e, ValueError) and "embedded null byte" in str(e):
return -1
+ elif isinstance(e, OSError) and "File name too long" in str(e):
+ return -1
else:
return handle_exception(e)
From d1582d181bfeb5138d9cae306b40dfa2fe87fe39 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 18:08:22 -0400
Subject: [PATCH 082/103] Update versions of pre-commit hooks
---
.pre-commit-config.yaml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 692c7fa2a..7d93876ed 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/codespell-project/codespell
- rev: v2.2.4
+ rev: v2.3.0
hooks:
- id: codespell
additional_dependencies: [tomli]
@@ -8,7 +8,7 @@ repos:
exclude: "test/fixtures/"
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.4.3
+ rev: v0.6.0
hooks:
- id: ruff
args: ["--fix"]
@@ -17,14 +17,14 @@ repos:
exclude: ^git/ext/
- repo: https://github.com/shellcheck-py/shellcheck-py
- rev: v0.9.0.6
+ rev: v0.10.0.1
hooks:
- id: shellcheck
args: [--color]
exclude: ^test/fixtures/polyglot$|^git/ext/
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.5.0
+ rev: v4.6.0
hooks:
- id: end-of-file-fixer
exclude: test/fixtures/
@@ -33,6 +33,6 @@ repos:
- id: check-merge-conflict
- repo: https://github.com/abravalheri/validate-pyproject
- rev: v0.16
+ rev: v0.19
hooks:
- id: validate-pyproject
From 016fa44a64ac244de2335b00338af67e3f8585ee Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 18:09:06 -0400
Subject: [PATCH 083/103] Have codespell ignore words that cause new false
positives
---
pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyproject.toml b/pyproject.toml
index 6cf4b3f5d..090972eed 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -84,6 +84,6 @@ lint.unfixable = [
[tool.codespell]
-ignore-words-list="gud,doesnt"
+ignore-words-list="afile,assertIn,doesnt,gud,uptodate"
#count = true
quiet-level = 3
From c82bb65fd263603b374b925f61483efc47c2a264 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 18:13:28 -0400
Subject: [PATCH 084/103] Fix a spelling error that codespell didn't catch
---
.pre-commit-config.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7d93876ed..90b899f8e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,7 +4,7 @@ repos:
hooks:
- id: codespell
additional_dependencies: [tomli]
- # args: ["--write-changes"] # consider enabling for auto-fif
+ # args: ["--write-changes"] # consider enabling for auto-fix
exclude: "test/fixtures/"
- repo: https://github.com/astral-sh/ruff-pre-commit
From 9556f63a965877db19002849d7bfeec71e84a2c7 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 18:19:03 -0400
Subject: [PATCH 085/103] Drop suggestion to auto-fix spelling (many false
positives)
---
.pre-commit-config.yaml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 90b899f8e..c47d9a2c7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,7 +4,6 @@ repos:
hooks:
- id: codespell
additional_dependencies: [tomli]
- # args: ["--write-changes"] # consider enabling for auto-fix
exclude: "test/fixtures/"
- repo: https://github.com/astral-sh/ruff-pre-commit
From 7a138eea78fd922b21f6049d273aaeca5f02bfb0 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 18:55:48 -0400
Subject: [PATCH 086/103] Fix small inconsistencies in test/fixtures/
exclusions
---
.pre-commit-config.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c47d9a2c7..f5635b2a0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,7 +4,7 @@ repos:
hooks:
- id: codespell
additional_dependencies: [tomli]
- exclude: "test/fixtures/"
+ exclude: ^test/fixtures/
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.6.0
@@ -26,7 +26,7 @@ repos:
rev: v4.6.0
hooks:
- id: end-of-file-fixer
- exclude: test/fixtures/
+ exclude: ^test/fixtures/
- id: check-toml
- id: check-yaml
- id: check-merge-conflict
From 53ec790e0dbc1ec9e4451394edb5c572c807b817 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 19:01:02 -0400
Subject: [PATCH 087/103] Fix inconsistent indentation
---
.pre-commit-config.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f5635b2a0..0cbf5aa73 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -34,4 +34,4 @@ repos:
- repo: https://github.com/abravalheri/validate-pyproject
rev: v0.19
hooks:
- - id: validate-pyproject
+ - id: validate-pyproject
From bdfa280f6dd412464419dd133ad02781cd27a312 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 19:02:14 -0400
Subject: [PATCH 088/103] Temporarily let end-of-file-fixer break LICENSE-BSD
symlink
On Windows, when `core.symlinks` is `false` or unset (since it
defaults to `false` on Windows), Git checks out symbolic links as
regular files whose contents are symlinks' target paths. Modifying
those regular files and committing the changes alters the symlink
target in the repository, and when they are checked out as actual
symlinks, the targets are different.
But the `end-of-file-fixer` pre-commit hook automatically adds
newlines to the end of regular files that lack them. It doesn't do
this on actual symlinks, but it does do it to regular files that
stand in for symlinks. This causes it to carry a risk of breaking
symlinks if it is run on Windows and the changes committed, and it
is easy to miss that this will happen because `git diff` output
shows it the same way as other additions of absent newlines.
This deliberately commits the change that end-of-file-fixer makes
to the `LICENSE-BSD` symlink, in order to allow a mitigation beyond
just excluding that symlink (or replacing it with a regular file)
to be tested. This change must be undone, of course.
---
fuzzing/LICENSE-BSD | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fuzzing/LICENSE-BSD b/fuzzing/LICENSE-BSD
index ea5b60640..4f88f81bf 120000
--- a/fuzzing/LICENSE-BSD
+++ b/fuzzing/LICENSE-BSD
@@ -1 +1 @@
-../LICENSE
\ No newline at end of file
+../LICENSE
From 965ea8bebcd768f6cadbc6cae6b7fe65868f1fb6 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 20:17:24 -0400
Subject: [PATCH 089/103] Enable check-symlinks pre-commit hook
Rationale:
- Small but likely benefit in general, since there are no currently
foreseen intentional use cases of committing of broken/dangling
symlinks in this project. So such symlinks that arise are likely
unintentional.
- If the end-of-file-fixer hook has run on a Windows system where
`core.symlinks` has *not* been set to `true`, and symlinks' paths
have not been excluded, then a newline character is added to the
end of the path held in the regular file Git checks out to stand
in for the symlink. Because it is not actually a symlink, this
will not detect the problem at that time (regardless of the order
in which this and that hook run relative to each other). But when
it is then run on CI on a system where symlinks are checked out,
it will detect the problem.
---
.pre-commit-config.yaml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0cbf5aa73..3f6892687 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,6 +27,7 @@ repos:
hooks:
- id: end-of-file-fixer
exclude: ^test/fixtures/
+ - id: check-symlinks
- id: check-toml
- id: check-yaml
- id: check-merge-conflict
From e9782487b8119147aa0c456c708f61ca7e3139e1 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 20:24:36 -0400
Subject: [PATCH 090/103] Revert "Temporarily let end-of-file-fixer break
LICENSE-BSD symlink"
This reverts commit bdfa280f6dd412464419dd133ad02781cd27a312.
---
fuzzing/LICENSE-BSD | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fuzzing/LICENSE-BSD b/fuzzing/LICENSE-BSD
index 4f88f81bf..ea5b60640 120000
--- a/fuzzing/LICENSE-BSD
+++ b/fuzzing/LICENSE-BSD
@@ -1 +1 @@
-../LICENSE
+../LICENSE
\ No newline at end of file
From cae0d8743a31fb0eda3c224a45c14de8cabd0d90 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Thu, 15 Aug 2024 20:28:46 -0400
Subject: [PATCH 091/103] Don't fix end-of-file in files named like licenses
The unanchored `LICENSE` and `COPYING` alternatives match the
pattern anywhere, and therefore exclude the currently used path
`fuzzing/LICENSE-BSD`.
License files are more likely than other files in this project to
be introduced as symlinks, and less likely to be noticed
immediately if they break. Symlinks can be checked out as regular
files when `core.symlinks` is set to `false`, which is rare outside
of Windows but is the default behavior when unset on Windows.
This exclusion fixes the current problem that end-of-file-fixer
breaks those links by adding a newline character to the end (the
symlinks are checked out broken if that is committed). It also
guards against most future cases involving licenses, though
possibly not all, and not other unrelated cases where symlinks may
be used for other purposes.
Although the pre-commit-hooks repository also provides a
destroyed-symlinks hook that detects the situation of a symlink
that has been replaced by a regular file, this does not add that
hook, because this situation is not inherently a problem. The code
here does not require symlinks to be checked out to work, and
adding that would break significant uses of the repository on
Windows.
Note that this leaves the situation where a license file may be a
symlink to another license file and may thus be checked out as a
regular file containing that file's path. However, it is easy to
understand that situation and manually follow the path. That
differs from the scenario where a symlink is created but broken,
because attempting to open it gives an error, and the error message
is often non-obvious, reporting that a file is not found but giving
the name of the symlink rather than its target.
---
.pre-commit-config.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3f6892687..424cc5f37 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -26,7 +26,7 @@ repos:
rev: v4.6.0
hooks:
- id: end-of-file-fixer
- exclude: ^test/fixtures/
+ exclude: ^test/fixtures/|COPYING|LICENSE
- id: check-symlinks
- id: check-toml
- id: check-yaml
From 16fc99fee45412c3dae44bdd7f59d921a11c00b3 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Sat, 17 Aug 2024 12:51:13 -0400
Subject: [PATCH 092/103] Upgrade sphinx to ~7.1.2
The old pinned version and its explicitly constrained dependencies
are retained for now for Python 3.7 so that documentation can be
built even with 3.7. (This could maybe be removed soon as a
preliminary step toward evenutally dropping 3.7 support.)
For Python 3.8 and higher, version 7.1.2 is used, allowing later
patch versions but constrained to remain 7.1.*. This is so the same
versions are likely to be selected on all Python version from 3.8
and higher, to minimize small differences in generated documentation
that different versions could give, and also to simplify debugging.
The reason to upgrade Sphinx now is to suppport Python 3.13, which
shall be (and, in the pre-releases available, is) incompatible with
versions of Sphinx below 6.2. This is because those earlier Sphinx
versions use the deprecated `imghdr` module, which 3.13 removes:
- https://docs.python.org/3.13/whatsnew/3.13.html#whatsnew313-pep594
- https://github.com/python/cpython/issues/104818
On old versions of Sphinx, that gives the error:
Extension error:
Could not import extension sphinx.builders.epub3 (exception: No module named 'imghdr')
Using Sphinx 6.2 is sufficient to avoid this, but there do not seem
to be any disadvantages for GitPython to remain below 7.1.2.
The reason we did not upgrade Sphinx before is that, last time we
considered doing so, we ran into a problem of new warnings (that we
treat as errors). This is detailed in the "Can we upgrade Sphinx?"
section of #1802, especially the "What Sphinx 5 is talking about"
subsection. The problem is warnings about `Actor` when it comes
in through type annotations:
WARNING: more than one target found for cross-reference 'Actor': git.objects.util.Actor, git.util.Actor
So this includes other changes to fix that problem as well. The
solution used here is to import `Actor` even when `TYPE_CHECKING`
is `false`, and write it unquoted in annotations, as `Actor` rather
than `"Actor"`. This allows Sphinx to discern where it should
consider it to be located, for the purpose of linking to its
documentation.
This does not incur overhead, because:
- The affected modules already have imports from `git.util`, so also
importing `Actor` from `git.util` does not cause any modules to
be imported that were not imported otherwise, nor even to be
imported at a different time.
- Even if that that had not been the case, most modules in GitPython
including `git.util` have members imported them into the top-level
`git` module in `git.__init__` when `git` is imported (and thus
when any Python submodule of `git` is imported).
The only disadvantage is the presence of the additional name in
those modules at runtime, which a user might inadvertently use and
thus write brittle code that could break if it is later removed.
But:
- The affected modules define `__all__` and do not include
`"Actor"` in `__all__`, so it is non-public.
- There are many places in GitPython (and most Python libraries)
where the onus is already on the author of code that uses the
library to avoid doing this.
The reasons for this approach, rather than any of several others,
were:
1. I did not write out the annotations as `git.util.Actor` to
resolve the ambiguity because annotations should, and for some
uses must, also be interpretable as expressions. But while
`from git.util import Actor` works and makes `Actor` available,
`git.util.Actor` cannot be used as an expression even after
`import git.util`. This is because, even after such an import,
`git.util` actually refers to `git.index.util`. This is as
detailed in the warnings issued when it is accessed, originally
from an overly broad `*` import but retained because removing it
could be a breaking change. See `git/__init__.py` for details.
2. The reason I did not write out the annotations as
`git.objects.util.Actor` to resolve the ambiguity is that not
all occurrences where Sphinx needed to be told which module to
document it as being from were within the `git.objects` Python
submodule. Two of the warnings were in `git/objects/tag.py`,
where annotating it that way would not be confusing. But the
other two were in `git/index/base.py`.
3. Although removing `Actor` from `git.objects.util.__all__` would
resolve the ambiguity, this should not be done, because:
- This would be a breaking change.
- It seems to be there deliberately, since `git.objects.util`
contains other members that relate to it directly.
4. The reasons I did not take this opportunity to move the contents
of `git/util.py` to a new file in that directory and make
`git/util.py` re-export the contents, even though this would
allow a solution analogous to (1) but for the new module to be
used, while also simplifying importing elsewhere, were:
- That seems like a change that should be done separately, based
either on the primary benefit to users or on a greater need
for it.
- If and when that is done, it may make sense to change the
interface as well. For example, `git/util.py` has a number of
members that it makes available for use throughout GitPython
but that are deliberately omitted from `__all__` and are meant
as non-public outside the project. One approach would be to make
a module with a leading `_` for these "internal" members, and
another public ones with everything else. But that also cannot
be decided based on the considerations that motivate the
changes here.
- The treatment of `HIDE_WINDOWS_KNOWN_ERRORS`, which is public
in `git/util.py` and which currently *does* have an effect,
will need to be considered. Although it cannot be re-bound by
assigning to `git.util.HIDE_WINDOWS_KNOWN_ERRORS` because the
`git.util` subexpression would evaluate to `git.index.util`,
there may be code that re-binds it in another way, such as by
accessing the module through `sys.modules`. Unlike functions
and classes that should not be monkey-patched from code
outside GitPython's test suite anyway, this attribute may
reasonably be assigned to, so it matters what module it is
actually in, unless the action of assigning to it elsewhere
is customized dynamically to carry over to the "real" place.
5. An alternative solution that may be reasonable in the near
future is to modify `reference.rst` so duplicate documentation
is no longer emitted for functions and classes that are defined
in one place but imported and "re-exported" elsewhere. I suspect
this may solve the problem, allowing the `Actor` imports to go
back under `if TYPE_CHECKING:` and to annotate with `"Actor"`
again while still running `make -C doc html` with no warnings.
However, this would entail design decisions about how to still
document those members. They should probably have links to where
they are fully documented. So an entry for `Actor` in the
section of `reference.rst` for `git.objects.util` would still
exist, but be very short and refer to the full autodoc item for
`Actor` the section for `git.util`. Since a `:class:` reference
to `git.objects.util.Actor` should still go to the stub that
links to `git.util.Actor`, it is not obvious that solving the
duplication in documentation generated for `reference.rst` ought
to be done in a way that would address the ambiguity Sphinx
warns about, even if it *can* be done in such a way.
Therefore, that should also be a separate consideration and, if
warranted, a separate change.
---
doc/requirements.txt | 13 +++++++------
git/index/base.py | 6 +++---
git/objects/tag.py | 5 ++---
3 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/doc/requirements.txt b/doc/requirements.txt
index 7769af5ae..a90a7a496 100644
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@@ -1,8 +1,9 @@
-sphinx == 4.3.2
+sphinx >= 7.1.2, < 7.2 ; python_version >= "3.8"
+sphinx == 4.3.2 ; python_version < "3.8"
+sphinxcontrib-applehelp >= 1.0.2, <= 1.0.4 ; python_version < "3.8"
+sphinxcontrib-devhelp == 1.0.2 ; python_version < "3.8"
+sphinxcontrib-htmlhelp >= 2.0.0, <= 2.0.1 ; python_version < "3.8"
+sphinxcontrib-qthelp == 1.0.3 ; python_version < "3.8"
+sphinxcontrib-serializinghtml == 1.1.5 ; python_version < "3.8"
sphinx_rtd_theme
-sphinxcontrib-applehelp >= 1.0.2, <= 1.0.4
-sphinxcontrib-devhelp == 1.0.2
-sphinxcontrib-htmlhelp >= 2.0.0, <= 2.0.1
-sphinxcontrib-qthelp == 1.0.3
-sphinxcontrib-serializinghtml == 1.1.5
sphinx-autodoc-typehints
diff --git a/git/index/base.py b/git/index/base.py
index a317e71c0..47925ad1c 100644
--- a/git/index/base.py
+++ b/git/index/base.py
@@ -28,6 +28,7 @@
from git.objects import Blob, Commit, Object, Submodule, Tree
from git.objects.util import Serializable
from git.util import (
+ Actor,
LazyMixin,
LockedFD,
join_path_native,
@@ -76,7 +77,6 @@
from git.refs.reference import Reference
from git.repo import Repo
- from git.util import Actor
Treeish = Union[Tree, Commit, str, bytes]
@@ -1117,8 +1117,8 @@ def commit(
message: str,
parent_commits: Union[List[Commit], None] = None,
head: bool = True,
- author: Union[None, "Actor"] = None,
- committer: Union[None, "Actor"] = None,
+ author: Union[None, Actor] = None,
+ committer: Union[None, Actor] = None,
author_date: Union[datetime.datetime, str, None] = None,
commit_date: Union[datetime.datetime, str, None] = None,
skip_hooks: bool = False,
diff --git a/git/objects/tag.py b/git/objects/tag.py
index a3bb0b882..88671d316 100644
--- a/git/objects/tag.py
+++ b/git/objects/tag.py
@@ -14,7 +14,7 @@
import sys
from git.compat import defenc
-from git.util import hex_to_bin
+from git.util import Actor, hex_to_bin
from . import base
from .util import get_object_type_by_name, parse_actor_and_date
@@ -30,7 +30,6 @@
if TYPE_CHECKING:
from git.repo import Repo
- from git.util import Actor
from .blob import Blob
from .commit import Commit
@@ -64,7 +63,7 @@ def __init__(
binsha: bytes,
object: Union[None, base.Object] = None,
tag: Union[None, str] = None,
- tagger: Union[None, "Actor"] = None,
+ tagger: Union[None, Actor] = None,
tagged_date: Union[int, None] = None,
tagger_tz_offset: Union[int, None] = None,
message: Union[str, None] = None,
From 44f7a738b85f75d86516a3ee1f128f4098fbcda6 Mon Sep 17 00:00:00 2001
From: Eliah Kagan <degeneracypressure@gmail.com>
Date: Sun, 18 Aug 2024 14:02:40 -0400
Subject: [PATCH 093/103] Don't support building documentation on Python 3.7
This removes the specially cased alternative lower versions of
`sphinx` and its dependencies that, since #1954, were only for
Python 3.7. As discussed in comments there, this simplifies the
documentation dependencies and avoids a situation where the version
of Python used to build the documentation has a noticeable effect
on the generated result.
This also conditions running the "Documentation" step in the main
CI test workflow (`pythonpackage.yml`) on the Python version not
being 3.7 (otherwise the job would always fail).
The only change this makes to the support status of GitPython on
Python 3.7 is to no longer support building documentation on 3.7.
GitPython can still be installed and used on 3.7 (though usually
this would not be a good idea, outside of testing, since Python 3.7
itself has not been supported by the Python Software Foundation for
quite some time). In addition, the documentation, which can be
built on any version >= 3.8 (including 3.13 starting in #1954) is
no less relevant to usage on Python 3.7 than it was before.
---
.github/workflows/pythonpackage.yml | 1 +
doc/requirements.txt | 8 +-------
2 files changed, 2 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 0f1d17544..292c9fc86 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -99,6 +99,7 @@ jobs:
continue-on-error: false
- name: Documentation
+ if: matrix.python-version != '3.7'
run: |
pip install ".[doc]"
make -C doc html
diff --git a/doc/requirements.txt b/doc/requirements.txt
index a90a7a496..81140d898 100644
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@@ -1,9 +1,3 @@
-sphinx >= 7.1.2, < 7.2 ; python_version >= "3.8"
-sphinx == 4.3.2 ; python_version < "3.8"
-sphinxcontrib-applehelp >= 1.0.2, <= 1.0.4 ; python_version < "3.8"
-sphinxcontrib-devhelp == 1.0.2 ; python_version < "3.8"
-sphinxcontrib-htmlhelp >= 2.0.0, <= 2.0.1 ; python_version < "3.8"
-sphinxcontrib-qthelp == 1.0.3 ; python_version < "3.8"
-sphinxcontrib-serializinghtml == 1.1.5 ; python_version < "3.8"
+sphinx >= 7.1.2, < 7.2
sphinx_rtd_theme
sphinx-autodoc-typehints
From f2254af5d3fd183ec150740d517bd0f8070fc67d Mon Sep 17 00:00:00 2001
From: Andrej730 <azhilenkov@gmail.com>
Date: Sat, 14 Sep 2024 10:45:53 +0500
Subject: [PATCH 094/103] _to_relative_path to support mixing slashes and
backslashes
Working on Windows you sometime end up having some paths with backslashes (windows native) and some with slashes - this PR will resolve the issue using gitpython for those kind of cases (see example below). It will also fix the issues if paths contain redundant separators or "..".
```
import git
repo = git.Repo(r"C:\gittest")
repo.index.add(r"C:\gittest\1.txt")
# Traceback (most recent call last):
# File "c:\second_test.py", line 5, in <module>
# repo.index.add(r"C:/gittest/2.txt")
# File "Python311\Lib\site-packages\git\index\base.py", line 879, in add
# paths, entries = self._preprocess_add_items(items)
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# File "Python311\Lib\site-packages\git\index\base.py", line 672, in _preprocess_add_items
# paths.append(self._to_relative_path(item))
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# File "Python311\Lib\site-packages\git\index\base.py", line 657, in _to_relative_path
# raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir))
# ValueError: Absolute path 'C:/gittest/2.txt' is not in git repository at 'C:\\gittest'
repo.index.add(r"C:/gittest/2.txt")
repo.index.commit("test")
```
---
git/index/base.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/index/base.py b/git/index/base.py
index 47925ad1c..7f53e614a 100644
--- a/git/index/base.py
+++ b/git/index/base.py
@@ -653,7 +653,7 @@ def _to_relative_path(self, path: PathLike) -> PathLike:
return path
if self.repo.bare:
raise InvalidGitRepositoryError("require non-bare repository")
- if not str(path).startswith(str(self.repo.working_tree_dir)):
+ if not osp.normpath(str(path)).startswith(osp.normpath(str(self.repo.working_tree_dir))):
raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir))
return os.path.relpath(path, self.repo.working_tree_dir)
From ca06b11efde845080354dac71e9062ea6d63ab84 Mon Sep 17 00:00:00 2001
From: Andrej730 <azhilenkov@gmail.com>
Date: Sat, 14 Sep 2024 16:51:41 +0500
Subject: [PATCH 095/103] test adding a file using non-normalized path
---
test/test_index.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/test/test_index.py b/test/test_index.py
index 2684cfd81..efd5b83a6 100644
--- a/test/test_index.py
+++ b/test/test_index.py
@@ -1181,6 +1181,18 @@ def test_index_add_pathlike(self, rw_repo):
rw_repo.index.add(file)
+ @with_rw_repo("HEAD")
+ def test_index_add_non_normalized_path(self, rw_repo):
+ git_dir = Path(rw_repo.git_dir)
+
+ file = git_dir / "file.txt"
+ file.touch()
+ non_normalized_path = file.as_posix()
+ if os.name != "nt":
+ non_normalized_path = non_normalized_path.replace("/", "\\")
+
+ rw_repo.index.add(non_normalized_path)
+
class TestIndexUtils:
@pytest.mark.parametrize("file_path_type", [str, Path])
From 46740590f7918fd5b789c95db7e41fbda06fb46f Mon Sep 17 00:00:00 2001
From: Andrej730 <azhilenkov@gmail.com>
Date: Sat, 14 Sep 2024 16:52:56 +0500
Subject: [PATCH 096/103] Remove redundant path normalization for
working_tree_dir
---
git/index/base.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/index/base.py b/git/index/base.py
index 7f53e614a..39cc9143c 100644
--- a/git/index/base.py
+++ b/git/index/base.py
@@ -653,7 +653,7 @@ def _to_relative_path(self, path: PathLike) -> PathLike:
return path
if self.repo.bare:
raise InvalidGitRepositoryError("require non-bare repository")
- if not osp.normpath(str(path)).startswith(osp.normpath(str(self.repo.working_tree_dir))):
+ if not osp.normpath(str(path)).startswith(str(self.repo.working_tree_dir)):
raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir))
return os.path.relpath(path, self.repo.working_tree_dir)
From 8327b82a1079f667006f649cb3f1bbdcc8792955 Mon Sep 17 00:00:00 2001
From: Andrej730 <azhilenkov@gmail.com>
Date: Sat, 14 Sep 2024 21:18:18 +0500
Subject: [PATCH 097/103] Fix test failing on unix
---
test/test_index.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/test_index.py b/test/test_index.py
index efd5b83a6..c586a0b5a 100644
--- a/test/test_index.py
+++ b/test/test_index.py
@@ -1189,7 +1189,7 @@ def test_index_add_non_normalized_path(self, rw_repo):
file.touch()
non_normalized_path = file.as_posix()
if os.name != "nt":
- non_normalized_path = non_normalized_path.replace("/", "\\")
+ non_normalized_path = "/" + non_normalized_path[1:].replace("/", "//")
rw_repo.index.add(non_normalized_path)
From 49ca9099dc75d0d686ec6737da36637cbee1c000 Mon Sep 17 00:00:00 2001
From: No big deal <69958306+alex20230721@users.noreply.github.com>
Date: Sat, 5 Oct 2024 17:23:19 +0800
Subject: [PATCH 098/103] Update base.py (#1965)
Improve documentation around opening repositories.
Co-authored-by: Sebastian Thiel <sebastian.thiel@icloud.com>
---
git/repo/base.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/repo/base.py b/git/repo/base.py
index 346248ddb..db89cdf41 100644
--- a/git/repo/base.py
+++ b/git/repo/base.py
@@ -179,7 +179,7 @@ def __init__(
R"""Create a new :class:`Repo` instance.
:param path:
- The path to either the root git directory or the bare git repo::
+ The path to either the worktree directory or the .git directory itself::
repo = Repo("/Users/mtrier/Development/git-python")
repo = Repo("/Users/mtrier/Development/git-python.git")
From 1bb465122f9673c9834b094c49d815148e84b8eb Mon Sep 17 00:00:00 2001
From: Florent Valette <florent.valette@ledger.fr>
Date: Mon, 14 Oct 2024 21:39:37 +0200
Subject: [PATCH 099/103] git,remote: use universal new lines for fetch/pull
stderr capture
See https://github.com/gitpython-developers/GitPython/issues/1969
stderr parser call RemoteProgress update on each line received.
With universal_newlines set to False, there is a mixup between
line feed and carriage return.
In the `handle_process_output` thread, this is thus seen as a single
line for the whole output on each steps.
---
git/remote.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/git/remote.py b/git/remote.py
index 9de3dace4..20e42b412 100644
--- a/git/remote.py
+++ b/git/remote.py
@@ -894,7 +894,7 @@ def _get_fetch_info_from_stderr(
None,
progress_handler,
finalizer=None,
- decode_streams=True,
+ decode_streams=False,
kill_after_timeout=kill_after_timeout,
)
@@ -1071,7 +1071,7 @@ def fetch(
Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_fetch_options)
proc = self.repo.git.fetch(
- "--", self, *args, as_process=True, with_stdout=False, universal_newlines=False, v=verbose, **kwargs
+ "--", self, *args, as_process=True, with_stdout=False, universal_newlines=True, v=verbose, **kwargs
)
res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout)
if hasattr(self.repo.odb, "update_cache"):
@@ -1125,7 +1125,7 @@ def pull(
Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_pull_options)
proc = self.repo.git.pull(
- "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=False, v=True, **kwargs
+ "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=True, v=True, **kwargs
)
res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout)
if hasattr(self.repo.odb, "update_cache"):
From 52cceaf2663422a79a0f1d21f905eb132e46b556 Mon Sep 17 00:00:00 2001
From: Florent Valette <florent.valette@ledger.fr>
Date: Tue, 15 Oct 2024 18:04:44 +0200
Subject: [PATCH 100/103] git,cmd: add encoding arg to popen if universal
newlines is True
---
git/cmd.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/git/cmd.py b/git/cmd.py
index 90fc39cd6..2048a43fa 100644
--- a/git/cmd.py
+++ b/git/cmd.py
@@ -1269,6 +1269,7 @@ def execute(
stdout=stdout_sink,
shell=shell,
universal_newlines=universal_newlines,
+ encoding=defenc if universal_newlines else None,
**subprocess_kwargs,
)
except cmd_not_found_exception as err:
From d6cdb67bcaa2cf606bfc0a9295aacb54677ea86d Mon Sep 17 00:00:00 2001
From: Sebastian Thiel <sebastian.thiel@icloud.com>
Date: Tue, 15 Oct 2024 20:35:29 +0200
Subject: [PATCH 101/103] See if python 3.7 still works when using an older
Ubuntu version.
This should be undone once python 3.7 is EOL.
---
.github/workflows/pythonpackage.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
index 292c9fc86..747db62f0 100644
--- a/.github/workflows/pythonpackage.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -13,7 +13,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+ os: ["ubuntu-22.04", "macos-latest", "windows-latest"]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
exclude:
- os: "macos-latest"
From e51bf80ad576256f2fbeead41ea3f0b667c77055 Mon Sep 17 00:00:00 2001
From: Sebastian Thiel <sebastian.thiel@icloud.com>
Date: Thu, 2 Jan 2025 08:24:01 +0100
Subject: [PATCH 102/103] update GitDB submodule to latest pubslished version
---
git/ext/gitdb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/git/ext/gitdb b/git/ext/gitdb
index 3d3e9572d..775cfe829 160000
--- a/git/ext/gitdb
+++ b/git/ext/gitdb
@@ -1 +1 @@
-Subproject commit 3d3e9572dc452fea53d328c101b3d1440bbefe40
+Subproject commit 775cfe8299ea5474f605935469359a9d1cdb49dc
From fb1b05124f1070ed56231a782daee0ffce9e1372 Mon Sep 17 00:00:00 2001
From: Sebastian Thiel <sebastian.thiel@icloud.com>
Date: Thu, 2 Jan 2025 08:27:54 +0100
Subject: [PATCH 103/103] bump patch level to prepare new version
---
VERSION | 2 +-
doc/source/changes.rst | 6 ++++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/VERSION b/VERSION
index d1bf6638d..e6af1c454 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.1.43
+3.1.44
diff --git a/doc/source/changes.rst b/doc/source/changes.rst
index 3c903423c..00a3c660e 100644
--- a/doc/source/changes.rst
+++ b/doc/source/changes.rst
@@ -2,6 +2,12 @@
Changelog
=========
+3.1.44
+======
+
+See the following for all changes.
+https://github.com/gitpython-developers/GitPython/releases/tag/3.1.44
+
3.1.43
======