Merge pull request #409 from tardyp/gitpython

Switch data_source dependency to GitPython

Author: sbs2001

etc/nix/flake.nix

@@ -74,7 +74,7 @@
             name = "vulnerablecode-${version}";
             src = vulnerablecode-src;
             dontConfigure = true; # do not use ./configure
-            propagatedBuildInputs = [ pythonEnv postgresql ];
+            propagatedBuildInputs = [ pythonEnv postgresql gitMinimal];
 
             postPatch = ''
               # Make sure the pycodestyle binary in $PATH is used.

requirements.txt

@@ -3,6 +3,7 @@ asgiref==3.2.7
 attrs==20.3.0
 backcall==0.1.0
 beautifulsoup4==4.7.1
+binaryornot==0.4.4
 cached-property==1.5.1
 cffi==1.14.0
 contextlib2==0.5.5
@@ -33,7 +34,7 @@ psycopg2==2.8.4
 ptyprocess==0.6.0
 py==1.10.0
 pycparser==2.20
-pygit2==1.5.0
+gitpython==3.1.14
 Pygments==2.7.4
 pyparsing==2.4.5
 pytest==6.2.3

vulnerabilities/data_source.py

@@ -26,6 +26,8 @@
 import shutil
 import tempfile
 import traceback
+import xml.etree.ElementTree as ET
+from binaryornot.helpers import is_binary_string
 from datetime import datetime
 from pathlib import Path
 from typing import Any
@@ -36,9 +38,7 @@
 from typing import Optional
 from typing import Set
 from typing import Tuple
-import xml.etree.ElementTree as ET
-
-import pygit2
+from git import Repo, DiffIndex
 from packageurl import PackageURL
 from univers.version_specifier import VersionSpecifier
 from univers.versions import version_class_by_package_type
@@ -318,35 +318,35 @@ def _collect_file_changes(
         file_ext: Optional[str],
     ) -> Tuple[Set[str], Set[str]]:
 
-        previous_commit = None
         added_files, updated_files = set(), set()
 
-        for commit in self._repo.walk(self._repo.head.target, pygit2.GIT_SORT_TIME):
-            commit_time = commit.commit_time + commit.commit_time_offset  # convert to UTC
-
-            if commit_time < self.cutoff_timestamp:
+        # find the most ancient commit we need to diff with
+        cutoff_commit = None
+        for commit in self._repo.iter_commits(self._repo.head):
+            if commit.committed_date < self.cutoff_timestamp:
                 break
+            cutoff_commit = commit
 
-            if previous_commit is None:
-                previous_commit = commit
-                continue
+        if cutoff_commit is None:
+            return added_files, updated_files
 
-            for d in commit.tree.diff_to_tree(previous_commit.tree).deltas:
-                if not _include_file(d.new_file.path, subdir, recursive, file_ext) or d.is_binary:
-                    continue
+        def _is_binary(d: DiffIndex):
+            return is_binary_string(d.b_blob.data_stream.read(1024))
 
-                abspath = os.path.join(self.config.working_directory, d.new_file.path)
-                # TODO
-                # Just filtering on the two status values for "added" and "modified" is too
-                # simplistic. This does not cover file renames, copies &
-                # deletions.
-                if d.status == pygit2.GIT_DELTA_ADDED:
+        for d in cutoff_commit.diff(self._repo.head.commit):
+            if not _include_file(d.b_path, subdir, recursive, file_ext) or _is_binary(d):
+                continue
+
+            abspath = os.path.join(self.config.working_directory, d.b_path)
+            if d.new_file:
+                added_files.add(abspath)
+            elif d.a_blob and d.b_blob:
+                if d.a_path != d.b_path:
+                    # consider moved files as added
                     added_files.add(abspath)
-                elif d.status == pygit2.GIT_DELTA_MODIFIED:
+                elif d.a_blob != d.b_blob:
                     updated_files.add(abspath)
 
-            previous_commit = commit
-
         # Any file that has been added and then updated inside the window of the git history we
         # looked at, should be considered "added", not "updated", since it does not exist in the
         # database yet.
@@ -363,29 +363,26 @@ def _ensure_working_directory(self) -> None:
             os.mkdir(self.config.working_directory)
 
     def _ensure_repository(self) -> None:
-        repodir = pygit2.discover_repository(self.config.working_directory)
-        if repodir is None:
+        if not os.path.exists(os.path.join(self.config.working_directory, ".git")):
             self._clone_repository()
             return
-
-        self._repo = pygit2.Repository(repodir)
+        self._repo = Repo(self.config.working_directory)
 
         if self.config.branch is None:
-            self.config.branch = self._repo.head.shorthand
-        branch = self._repo.branches[self.config.branch]
-
-        if not branch.is_checked_out():
-            self._repo.checkout(branch)
+            self.config.branch = str(self._repo.active_branch)
+        branch = self.config.branch
+        self._repo.head.reference = self._repo.heads[branch]
+        self._repo.head.reset(index=True, working_tree=True)
 
         remote = self._find_or_add_remote()
         self._update_from_remote(remote, branch)
 
     def _clone_repository(self) -> None:
         kwargs = {}
         if self.config.branch:
-            kwargs["checkout_branch"] = self.config.branch
+            kwargs["branch"] = self.config.branch
 
-        self._repo = pygit2.clone_repository(
+        self._repo = Repo.clone_from(
             self.config.repository_url, self.config.working_directory, **kwargs
         )
 
@@ -397,20 +394,19 @@ def _find_or_add_remote(self):
                 break
 
         if remote is None:
-            remote = self._repo.remotes.create(
-                "added_by_vulnerablecode", self.config.repository_url
+            remote = self._repo.create_remote(
+                "added_by_vulnerablecode", url=self.config.repository_url
             )
 
         return remote
 
     def _update_from_remote(self, remote, branch) -> None:
-        progress = remote.fetch()
-        if progress.received_objects == 0:
+        fetch_info = remote.fetch()
+        if len(fetch_info) == 0:
             return
-
-        remote_branch = self._repo.branches[f"{remote.name}/{self.config.branch}"]
-        branch.set_target(remote_branch.target)
-        self._repo.checkout(branch, strategy=pygit2.GIT_CHECKOUT_FORCE)
+        branch = self._repo.branches[branch]
+        branch.set_reference(remote.refs[branch.name])
+        self._repo.head.reset(index=True, working_tree=True)
 
 
 def _include_file(