Updates pygit2 (#911)

* Updates pygit2 to pygit 1.15 and enables type checking.

* Fixes isort config.

Author: boehmseb

.isort.cfg

@@ -1,5 +1,5 @@
 [settings]
-known_third_party = PyQt5,_pytest,argparse_utils,benchbuild,click,cryptography,distro,git,github,graphviz,jinja2,matplotlib,networkx,numpy,packaging,pandas,pkg_resources,plotly,plumbum,pydriller,pygit2,pygtrie,pylatex,pytest,requests,rich,scipy,seaborn,setuptools,sklearn,tabulate,yaml
+known_firstparty = varats
 multi_line_output=3
 use_parentheses = True
 include_trailing_comma: True

docs/source/conf.py

@@ -96,6 +96,7 @@
 import click  # isort:skip
 import git  # isort:skip
 import github  # isort:skip
+import pygit2  # isort:skip
 import urllib3.exceptions  # isort:skip
 
 # Some packages use new syntax for type checking that isn't available to us

pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
   "pandas>=1.5.3",
   "plotly>=5.13.1",
   "plumbum>=1.6",
-  "pygit2>=1.10,<1.14.0",
+  "pygit2>=1.15.0,<1.16.0", # 1.16 drops support for python 3.9
   "pygraphviz>=1.7",
   "pygtrie>=2.3",
   "pylatex>=1.4.1",
@@ -98,7 +98,7 @@ module = [
   "pandas",
   "plotly.*",
   "pydriller",
-  "pygit2",
+  "pygtrie",
   "scipy.*",
   "sklearn.*",
   "PyQt5.*",

requirements.txt

@@ -16,7 +16,7 @@ plotly>=5.13.1
 plumbum>=1.6
 pre-commit>=3.2.0
 PyDriller>=2.4.1
-pygit2>=1.10
+pygit2>=1.15.0,<1.16.0
 PyGithub>=1.58
 pygraphviz>=1.7
 pygtrie>=2.3
@@ -31,6 +31,7 @@ rich>=12.6
 scikit-learn>=1.2.2
 seaborn>=0.13.0
 tabulate>=0.9
+types-pygit2  # remove when updating to pygit2 >=1.16
 types-PyYAML
 types-requests
 types-tabulate

tests/experiment/test_workload_util.py

@@ -2,9 +2,11 @@
 import unittest
 from pathlib import Path
 
-import varats.experiment.workload_util as wu
 from benchbuild.command import Command, PathToken, RootRenderer
 from benchbuild.source.base import Revision, Variant
+
+import varats.experiment.workload_util as wu
+from tests.helper_utils import run_in_test_environment, UnitTestFixtures
 from varats.paper.paper_config import load_paper_config
 from varats.projects.c_projects.xz import Xz
 from varats.projects.perf_tests.feature_perf_cs_collection import (
@@ -14,8 +16,6 @@
 from varats.utils.git_util import ShortCommitHash
 from varats.utils.settings import vara_cfg
 
-from tests.helper_utils import run_in_test_environment, UnitTestFixtures
-
 TT = PathToken.make_token(RootRenderer())
 
 

tests/provider/test_bug_provider.py

@@ -160,7 +160,7 @@ def setUp(self) -> None:
 
         def get(commit_id: str) -> pygit2.Commit:
             """Method that creates simple pygit2 Commit mocks for given ID."""
-            mock_commit = mock.create_autospec(pygit2.Commit)
+            mock_commit: pygit2.Commit = mock.create_autospec(pygit2.Commit)
             mock_commit.id = pygit2.Oid(hex=commit_id)
             return mock_commit
 
@@ -169,6 +169,7 @@ def get(commit_id: str) -> pygit2.Commit:
         self.mock_pygit.get = get
         self.mock_repo_handle = mock.create_autospec(RepositoryHandle)
         self.mock_repo_handle.pygit_repo = self.mock_pygit
+        self.mock_repo_handle.pygit_commit = get
 
     def test_issue_events_closing_bug(self) -> None:
         """Test identifying issue events that close a bug related issue, with
@@ -260,8 +261,8 @@ def test_pygit_bug_creation(self, mock_pydriller_git) -> None:
         mock_pydriller_git.return_value = DummyPydrillerRepo("")
 
         pybug = _create_corresponding_bug(
-            self.mock_pygit.get(issue_event.commit_id), self.mock_pygit,
-            issue_event.issue.number
+            self.mock_repo_handle.pygit_commit(issue_event.commit_id),
+            self.mock_repo_handle, issue_event.issue.number
         )
 
         self.assertEqual(issue_event.commit_id, str(pybug.fixing_commit.id))
@@ -357,20 +358,20 @@ def test_filter_commit_message_bugs(
         of bugs is created correctly."""
 
         first_fixing_commit = mock.create_autospec(pygit2.Commit)
-        first_fixing_commit.hex = "1241"
+        first_fixing_commit.id = "1241"
         first_fixing_commit.message = "Fixed first issue"
 
         first_non_fixing_commit = mock.create_autospec(pygit2.Commit)
-        first_non_fixing_commit.hex = "1242"
+        first_non_fixing_commit.id = "1242"
         first_non_fixing_commit.message = "Added documentation\n" + \
                                           "Grammar Errors need to be fixed"
 
         second_non_fixing_commit = mock.create_autospec(pygit2.Commit)
-        second_non_fixing_commit.hex = "1243"
+        second_non_fixing_commit.id = "1243"
         second_non_fixing_commit.message = "Added feature X"
 
         second_fixing_commit = mock.create_autospec(pygit2.Commit)
-        second_fixing_commit.hex = "1244"
+        second_fixing_commit.id = "1244"
         second_fixing_commit.message = "fixes second problem"
 
         def mock_walk(_start_id: str, _sort_mode: int):
@@ -388,14 +389,14 @@ def mock_walk(_start_id: str, _sort_mode: int):
         mock_pydriller_git.return_value = DummyPydrillerRepo("")
 
         # commit filter method for pygit bugs
-        def accept_pybugs(repo: pygit2.Repository,
+        def accept_pybugs(repo: RepositoryHandle,
                           commit: pygit2.Commit) -> tp.Optional[PygitBug]:
             if _is_closing_message(commit.message):
-                return _create_corresponding_bug(commit, self.mock_pygit)
+                return _create_corresponding_bug(commit, self.mock_repo_handle)
             return None
 
         pybug_ids = set(
-            pybug.fixing_commit.hex
+            pybug.fixing_commit.id
             for pybug in _filter_commit_message_bugs("", accept_pybugs)
         )
         expected_ids = {"1241", "1244"}
@@ -451,31 +452,31 @@ def test_basic_repo_pygit_bugs(self) -> None:
             fixing_commit="ddf0ba95408dc5508504c84e6616c49128410389"
         )
         pybug_first_intro_ids = set(
-            intro_commit.hex
+            intro_commit.id
             for intro_commit in next(iter(pybug_first)).introducing_commits
         )
 
         pybug_second = provider.find_pygit_bugs(
             fixing_commit="d846bdbe45e4d64a34115f5285079e1b5f84007f"
         )
         pybug_second_intro_ids = set(
-            intro_commit.hex
+            intro_commit.id
             for intro_commit in next(iter(pybug_second)).introducing_commits
         )
 
         pybug_third = provider.find_pygit_bugs(
             fixing_commit="2da78b2820370f6759e9086fad74155d6655e93b"
         )
         pybug_third_intro_ids = set(
-            intro_commit.hex
+            intro_commit.id
             for intro_commit in next(iter(pybug_third)).introducing_commits
         )
 
         pybug_fourth = provider.find_pygit_bugs(
             fixing_commit="3b76c8d295385358375fefdb0cf045d97ad2d193"
         )
         pybug_fourth_intro_ids = set(
-            intro_commit.hex
+            intro_commit.id
             for intro_commit in next(iter(pybug_fourth)).introducing_commits
         )
 

tests/test_containers.py

@@ -13,6 +13,8 @@
     RunLayer,
     EntryPoint,
 )
+
+from tests.helper_utils import run_in_test_environment
 from varats.containers.containers import (
     ImageBase,
     StageBuilder,
@@ -23,8 +25,6 @@
 from varats.tools.research_tools.research_tool import Distro
 from varats.utils.settings import vara_cfg, bb_cfg
 
-from tests.helper_utils import run_in_test_environment
-
 
 class TestImageBase(unittest.TestCase):
     """Test ImageBase class."""

tests/utils/test_github_util.py

@@ -5,15 +5,16 @@
 from github import Github, PaginatedList
 from github.GithubObject import GithubObject, NonCompletableGithubObject
 from github.PaginatedList import PaginatedListBase
+from github.Requester import Requester
+
+from tests.helper_utils import run_in_test_environment
 from varats.utils.github_util import (
     get_cached_github_object,
     _get_cached_pygithub_object,
     _get_cached_pygithub_object_list,
     get_cached_github_object_list,
 )
 
-from tests.helper_utils import run_in_test_environment
-
 
 class DummyGithubObject(NonCompletableGithubObject):
     """Dummy GithubObject class."""
@@ -27,8 +28,24 @@ def _useAttributes(self, attributes: tp.Any) -> None:
         pass
 
 
+class DummyRequester(Requester):
+    """Dummy Requester class."""
+
+    def __init__(self) -> None:
+        super().__init__(
+            auth=None,
+            base_url="https://base.url",
+            timeout=1,
+            user_agent="user agent",
+            per_page=10,
+            verify=False,
+            retry=None,
+            pool_size=None
+        )
+
+
 def create_dummy_github_object() -> GithubObject:
-    return DummyGithubObject(None, {}, None)
+    return DummyGithubObject(DummyRequester(), {}, {})
 
 
 class DummyPaginatedList(PaginatedListBase):

varats-core/varats/project/project_util.py

@@ -161,7 +161,7 @@ def get_commit(crp: CommitRepoPair) -> pygit2.Commit:
         Returns:
             the commit corresponding to the given CommitRepoPair
         """
-        commit = repos[crp.repository_name].pygit_repo.get(crp.commit_hash.hash)
+        commit = repos[crp.repository_name].maybe_pygit_commit(crp.commit_hash)
         if not commit:
             raise LookupError(
                 f"Could not find commit {crp} for project {project_name}."

varats-core/varats/provider/bug/bug.py

@@ -12,7 +12,7 @@
     get_project_cls_by_name,
     get_local_project_repo,
 )
-from varats.utils.git_util import FullCommitHash
+from varats.utils.git_util import FullCommitHash, RepositoryHandle
 from varats.utils.github_util import (
     get_cached_github_object_list,
     get_github_repo_name_for_project,
@@ -101,14 +101,14 @@ def as_raw_bug(pygit_bug: PygitBug) -> RawBug:
     )
 
 
-def as_pygit_bug(raw_bug: RawBug, repo: pygit2.Repository) -> PygitBug:
+def as_pygit_bug(raw_bug: RawBug, repo: RepositoryHandle) -> PygitBug:
     """Converts a ``RawBug`` to a ``PygitBug``."""
     introducing_commits: tp.Set[pygit2.Commit] = set()
     for intro_commit in raw_bug.introducing_commits:
-        introducing_commits.add(repo.get(intro_commit.hash))
+        introducing_commits.add(repo.pygit_commit(intro_commit))
     return PygitBug(
-        repo.get(raw_bug.fixing_commit.hash), introducing_commits,
-        raw_bug.issue_id, raw_bug.creation_date, raw_bug.resolution_date
+        repo[raw_bug.fixing_commit.hash], introducing_commits, raw_bug.issue_id,
+        raw_bug.creation_date, raw_bug.resolution_date
     )
 
 
@@ -239,7 +239,7 @@ def load_issue_events(github: Github) -> 'PaginatedList[IssueEvent]':
 
 def _create_corresponding_bug(
     closing_commit: pygit2.Commit,
-    project_repo: pygit2.Repository,
+    project_repo: RepositoryHandle,
     issue_id: tp.Optional[int] = None,
     creation_date: tp.Optional[datetime] = None,
     resolution_date: tp.Optional[datetime] = None
@@ -258,7 +258,7 @@ def _create_corresponding_bug(
     Returns:
         the specified bug
     """
-    pydrill_repo = pydriller.Git(project_repo.path)
+    pydrill_repo = pydriller.Git(str(project_repo.repo_path))
 
     introducing_commits: tp.Set[pygit2.Commit] = set()
     blame_dict = pydrill_repo.get_commits_last_modified_lines(
@@ -267,7 +267,7 @@ def _create_corresponding_bug(
 
     for _, introducing_set in blame_dict.items():
         for introducing_id in introducing_set:
-            introducing_commits.add(project_repo.get(introducing_id))
+            introducing_commits.add(project_repo.pygit_commit(introducing_id))
 
     return PygitBug(
         closing_commit, introducing_commits, issue_id, creation_date,
@@ -294,12 +294,12 @@ def _find_corresponding_pygit_suspect_tuple(
         A PygitSuspectTuple if the issue event represents the closing of a bug,
         None otherwise
     """
-    pygit_repo = get_local_project_repo(project_name).pygit_repo
-    pydrill_repo = pydriller.Git(pygit_repo.path)
+    repo = get_local_project_repo(project_name)
+    pydrill_repo = pydriller.Git(str(repo.repo_path))
 
     if _has_closed_a_bug(issue_event) and issue_event.commit_id:
         issue_date = issue_event.issue.created_at
-        fixing_commit = pygit_repo.get(issue_event.commit_id)
+        fixing_commit = repo.pygit_commit(issue_event.commit_id)
         pydrill_fixing_commit = pydrill_repo.get_commit(issue_event.commit_id)
         blame_dict = pydrill_repo.get_commits_last_modified_lines(
             pydrill_fixing_commit
@@ -317,9 +317,9 @@ def _find_corresponding_pygit_suspect_tuple(
                 ).committer_date.astimezone(timezone.utc)
 
                 if introduction_date > issue_date:  # commit is a suspect
-                    suspect_commits.add(pygit_repo.get(introducing_id))
+                    suspect_commits.add(repo.pygit_commit(introducing_id))
                 else:
-                    non_suspect_commits.add(pygit_repo.get(introducing_id))
+                    non_suspect_commits.add(repo.pygit_commit(introducing_id))
 
         return PygitSuspectTuple(
             fixing_commit, non_suspect_commits, suspect_commits,
@@ -386,7 +386,7 @@ def _filter_issue_bugs(
 
 def _filter_commit_message_bugs(
     project_name: str,
-    commit_filter_function: tp.Callable[[pygit2.Repository, pygit2.Commit],
+    commit_filter_function: tp.Callable[[RepositoryHandle, pygit2.Commit],
                                         tp.Optional[PygitBug]]
 ) -> tp.FrozenSet[PygitBug]:
     """
@@ -400,10 +400,11 @@ def _filter_commit_message_bugs(
         the set of bugs created by the given filter
     """
     filtered_bugs = set()
-    project_repo = get_local_project_repo(project_name).pygit_repo
+    project_repo = get_local_project_repo(project_name)
+    pygit_repo = project_repo.pygit_repo
 
-    for commit in project_repo.walk(
-        project_repo.head.target, pygit2.GIT_SORT_TIME
+    for commit in pygit_repo.walk(
+        pygit_repo.head.target, pygit2.enums.SortMode.TIME
     ):
         pybug = commit_filter_function(project_repo, commit)
         if pybug:
@@ -435,7 +436,7 @@ def accept_suspect_with_certain_introduction(
     ) -> tp.Optional[PygitBug]:
         bug = suspect.create_corresponding_bug()
 
-        if fixing_commit and bug.fixing_commit.hex != fixing_commit:
+        if fixing_commit and bug.fixing_commit.id != fixing_commit:
             return None
 
         if introducing_commit and introducing_commit not in [
@@ -470,12 +471,12 @@ def find_commit_message_bugs(
     """
 
     def accept_commit_message_pybug(
-        repo: pygit2.Repository, commit: pygit2.Commit
+        repo: RepositoryHandle, commit: pygit2.Commit
     ) -> tp.Optional[PygitBug]:
         if _is_closing_message(commit.message):
             bug = _create_corresponding_bug(commit, repo)
 
-            if fixing_commit and bug.fixing_commit.hex != fixing_commit:
+            if fixing_commit and bug.fixing_commit.id != fixing_commit:
                 return None
 
             if introducing_commit and introducing_commit not in [