Allow shallow cloning

DanielChabrowski · DanielChabrowski · commit e74253d2def6 · 2019-03-03T01:35:53.000+01:00
diff --git a/pre_commit/store.py b/pre_commit/store.py
@@ -10,6 +10,7 @@
 import pre_commit.constants as C
 from pre_commit import file_lock
 from pre_commit import git
+from pre_commit.util import CalledProcessError
 from pre_commit.util import clean_path_on_failure
 from pre_commit.util import cmd_output
 from pre_commit.util import resource_text
@@ -121,10 +122,7 @@ def _get_result():
                 return result
 
             logger.info('Initializing environment for {}.'.format(repo))
-
-            directory = tempfile.mkdtemp(prefix='repo', dir=self.directory)
-            with clean_path_on_failure(directory):
-                make_strategy(directory)
+            directory = make_strategy()
 
             # Update our db with the created repo
             with self.connect() as db:
@@ -134,19 +132,50 @@ def _get_result():
                 )
         return directory
 
-    def clone(self, repo, ref, deps=()):
-        """Clone the given url and checkout the specific ref."""
-        def clone_strategy(directory):
-            env = git.no_git_env()
+    def _perform_safe_clone(self, clone_strategy):
+        directory = tempfile.mkdtemp(prefix='repo', dir=self.directory)
+        with clean_path_on_failure(directory):
+            clone_strategy(directory)
+        return directory
 
-            cmd = ('git', 'clone', '--no-checkout', repo, directory)
-            cmd_output(*cmd, env=env)
+    def _complete_clone(self, repo, ref, directory):
+        """Perform a complete clone of a repository and its submodules """
+        env = git.no_git_env()
 
-            def _git_cmd(*args):
-                return cmd_output('git', *args, cwd=directory, env=env)
+        cmd = ('git', 'clone', '--no-checkout', repo, directory)
+        cmd_output(*cmd, env=env)
+
+        def _git_cmd(*args):
+            return cmd_output('git', *args, cwd=directory, env=env)
+
+        _git_cmd('reset', ref, '--hard')
+        _git_cmd('submodule', 'update', '--init', '--recursive')
 
-            _git_cmd('reset', ref, '--hard')
-            _git_cmd('submodule', 'update', '--init', '--recursive')
+    def _shallow_clone(self, repo, ref, directory):
+        """Perform a shallow clone of a repository and its submodules """
+        env = git.no_git_env()
+
+        def _git_cmd(*args):
+            return cmd_output('git', *args, cwd=directory, env=env)
+
+        _git_cmd('init', '.')
+        _git_cmd('remote', 'add', 'origin', repo)
+        _git_cmd('fetch', 'origin', ref, '--depth=1')
+        _git_cmd('checkout', ref)
+        _git_cmd('submodule', 'update', '--init', '--recursive', '--depth=1')
+
+    def clone(self, repo, ref, deps=()):
+        """Clone the given url and checkout the specific ref."""
+
+        def clone_strategy():
+            try:
+                def shallow_clone(directory):
+                    self._shallow_clone(repo, ref, directory)
+                return self._perform_safe_clone(shallow_clone)
+            except CalledProcessError:
+                def complete_clone(directory):
+                    self._complete_clone(repo, ref, directory)
+                return self._perform_safe_clone(complete_clone)
 
         return self._new_repo(repo, ref, deps, clone_strategy)
 
@@ -173,8 +202,11 @@ def _git_cmd(*args):
             _git_cmd('add', '.')
             git.commit(repo=directory)
 
+        def make_strategy():
+            return self._perform_safe_clone(make_local_strategy)
+
         return self._new_repo(
-            'local', C.LOCAL_REPO_VERSION, deps, make_local_strategy,
+            'local', C.LOCAL_REPO_VERSION, deps, make_strategy,
         )
 
     def _create_config_table_if_not_exists(self, db):
diff --git a/testing/util.py b/testing/util.py
@@ -142,3 +142,8 @@ def git_commit(*args, **kwargs):
     if msg is not None:  # allow skipping `-m` with `msg=None`
         cmd += ('-m', msg)
     return fn(*cmd, **kwargs)
+
+
+def git_ref_count(repo):
+    _, out, _ = cmd_output('git', 'rev-list', '--all', '--count', cwd=repo)
+    return int(out.split()[0])
diff --git a/tests/store_test.py b/tests/store_test.py
@@ -12,9 +12,11 @@
 from pre_commit import git
 from pre_commit.store import _get_default_directory
 from pre_commit.store import Store
+from pre_commit.util import CalledProcessError
 from testing.fixtures import git_dir
 from testing.util import cwd
 from testing.util import git_commit
+from testing.util import git_ref_count
 
 
 def test_our_session_fixture_works():
@@ -81,6 +83,7 @@ def test_clone(store, tempdir_factory, log_info_mock):
     assert dirname.startswith('repo')
     # Should be checked out to the rev we specified
     assert git.head_rev(ret) == rev
+    assert git_ref_count(ret) == 1
 
     # Assert there's an entry in the sqlite db for this
     assert store.select_all_repos() == [(path, rev, ret)]
@@ -111,6 +114,41 @@ def test_clone_when_repo_already_exists(store):
     assert store.clone('fake_repo', 'fake_ref') == 'fake_path'
 
 
+def test_clone_shallow_failure_fallback_to_complete(
+    store, tempdir_factory,
+    log_info_mock,
+):
+    path = git_dir(tempdir_factory)
+    with cwd(path):
+        git_commit()
+        rev = git.head_rev(path)
+        git_commit()
+
+    # Force shallow clone failure
+    def fake_shallow_clone(self, *args, **kwargs):
+        raise CalledProcessError(None, None, None)
+    store._shallow_clone = fake_shallow_clone
+
+    ret = store.clone(path, rev)
+
+    # Should have printed some stuff
+    assert log_info_mock.call_args_list[0][0][0].startswith(
+        'Initializing environment for ',
+    )
+
+    # Should return a directory inside of the store
+    assert os.path.exists(ret)
+    assert ret.startswith(store.directory)
+    # Directory should start with `repo`
+    _, dirname = os.path.split(ret)
+    assert dirname.startswith('repo')
+    # Should be checked out to the rev we specified
+    assert git.head_rev(ret) == rev
+
+    # Assert there's an entry in the sqlite db for this
+    assert store.select_all_repos() == [(path, rev, ret)]
+
+
 def test_create_when_directory_exists_but_not_db(store):
     # In versions <= 0.3.5, there was no sqlite db causing a need for
     # backward compatibility
