Filter out non-bug exceptions using a pre-defined exception list.

DaveLak · DaveLak · commit 7de1556d3895 · 2024-08-08T17:09:13.000-04:00
This reduces false positive test failures by identifying and
gracefully handling exceptions that are explicitly raised by GitPython,
thus reducing the false-positive fuzzing test failure rate.
diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py
@@ -1,16 +1,51 @@
+# ruff: noqa: E402
 import atheris
 import sys
 import os
+import traceback
 import tempfile
 from configparser import ParsingError
-from utils import is_expected_exception_message, get_max_filename_length
+from utils import get_max_filename_length
+import re
+
+bundle_dir = os.path.dirname(os.path.abspath(__file__))
 
 if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):  # pragma: no cover
-    path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git"))
-    os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary
+    bundled_git_binary_path = os.path.join(bundle_dir, "git")
+    os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path
 
 from git import Repo, GitCommandError, InvalidGitRepositoryError
 
+
+def load_exception_list(file_path):
+    """Load and parse the exception list from a file."""
+    try:
+        with open(file_path, "r") as file:
+            lines = file.readlines()
+        exception_list = set()
+        for line in lines:
+            match = re.match(r"(.+):(\d+):", line)
+            if match:
+                file_path = match.group(1).strip()
+                line_number = int(match.group(2).strip())
+                exception_list.add((file_path, line_number))
+        return exception_list
+    except FileNotFoundError:
+        print("File not found: %s", file_path)
+        return set()
+    except Exception as e:
+        print("Error loading exception list: %s", e)
+        return set()
+
+
+def check_exception_against_list(exception_list, exc_traceback):
+    """Check if the exception traceback matches any entry in the exception list."""
+    for filename, lineno, _, _ in traceback.extract_tb(exc_traceback):
+        if (filename, lineno) in exception_list:
+            return True
+    return False
+
+
 if not sys.warnoptions:  # pragma: no cover
     # The warnings filter below can be overridden by passing the -W option
     # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable.
@@ -89,17 +124,14 @@ def TestOneInput(data):
             BrokenPipeError,
         ):
             return -1
-        except ValueError as e:
-            expected_messages = [
-                "SHA is empty",
-                "Reference at",
-                "embedded null byte",
-                "This submodule instance does not exist anymore",
-                "cmd stdin was empty",
-            ]
-            if is_expected_exception_message(e, expected_messages):
+        except Exception as e:
+            exc_traceback = e.__traceback__
+            exception_list = load_exception_list(os.path.join(bundle_dir, "explicit-exceptions-list.txt"))
+            if check_exception_against_list(exception_list, exc_traceback):
+                print("Exception matches an entry in the exception list.")
                 return -1
             else:
+                print("Exception does not match any entry in the exception list.")
                 raise e
 
 
diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh
@@ -15,5 +15,5 @@ find "$SRC" -maxdepth 1 \
 
 # Build fuzzers in $OUT.
 find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do
-  compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):."
+  compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." --add-data="$SRC/explicit-exceptions-list.txt:."
 done
diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh
@@ -91,6 +91,17 @@ create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora"
 
 prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing"
 
+pushd "$SRC/gitpython/"
+# Search for 'raise' and 'assert' statements in Python files within GitPython's 'git/' directory and its submodules,
+# remove trailing colons, and save to 'explicit-exceptions-list.txt'. This file can then be used by fuzz harnesses to
+# check exception tracebacks:
+# If an exception found by the fuzzer originated in a file + line number in explicit-exceptions-list.txt, then it is not a bug.
+
+git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- "git/**/*.py" > "$SRC/explicit-exceptions-list.txt"
+
+popd
+
+
 # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below.
 python3 -m pip install --upgrade pip
 # Upgrade to the latest versions known to work at the time the below changes were introduced:
