We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
gitpython sometimes fails when checking for submodules
## Details

I noticed the other week that our builds have actually been failing for a few months but I'm not sure how to debug it. Locally the docs generate as expected when I run `sphinx-build -b dirhtml . _build/dirhtml`

* Read the Docs project URL: https://readthedocs.org/projects/chassis/
* Build URL (if applicable): https://readthedocs.org/projects/chassis/builds/7462561/
* Read the Docs username (if applicable): BronsonQuick

## Expected Result

A successful build.

## Actual Result

>There was a problem with Read the Docs while building your documentation. Please report this to us with your build id (7462561).

Thanks very much for all your work on this project!


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `readthedocs/projects/exceptions.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 
3 """Project exceptions."""
4 
5 from django.conf import settings
6 from django.utils.translation import ugettext_noop as _
7 
8 from readthedocs.doc_builder.exceptions import BuildEnvironmentError
9 
10 
11 class ProjectConfigurationError(BuildEnvironmentError):
12 
13     """Error raised trying to configure a project for build."""
14 
15     NOT_FOUND = _(
16         'A configuration file was not found. '
17         'Make sure you have a conf.py file in your repository.',
18     )
19 
20     MULTIPLE_CONF_FILES = _(
21         'We found more than one conf.py and are not sure which one to use. '
22         'Please, specify the correct file under the Advanced settings tab '
23         "in the project's Admin.",
24     )
25 
26 
27 class RepositoryError(BuildEnvironmentError):
28 
29     """Failure during repository operation."""
30 
31     PRIVATE_ALLOWED = _(
32         'There was a problem connecting to your repository, '
33         'ensure that your repository URL is correct.',
34     )
35     PRIVATE_NOT_ALLOWED = _(
36         'There was a problem connecting to your repository, '
37         'ensure that your repository URL is correct and your repository is public. '
38         'Private repositories are not supported.',
39     )
40 
41     INVALID_SUBMODULES = _(
42         'One or more submodule URLs are not valid: {}, '
43         'git/ssh URL schemas for submodules are not supported.'
44     )
45     INVALID_SUBMODULES_PATH = _(
46         'One or more submodule paths are not valid. '
47         'Check that all your submodules in .gitmodules are used.'
48     )
49 
50     DUPLICATED_RESERVED_VERSIONS = _(
51         'You can not have two versions with the name latest or stable.',
52     )
53 
54     FAILED_TO_CHECKOUT = _('Failed to checkout revision: {}')
55 
56     def get_default_message(self):
57         if settings.ALLOW_PRIVATE_REPOS:
58             return self.PRIVATE_ALLOWED
59         return self.PRIVATE_NOT_ALLOWED
60 
61 
62 class ProjectSpamError(Exception):
63 
64     """
65     Error raised when a project field has detected spam.
66 
67     This error is not raised to users, we use this for banning users in the
68     background.
69     """
70
```

Path: `readthedocs/vcs_support/backends/git.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 
3 """Git-related utilities."""
4 
5 import logging
6 import os
7 import re
8 
9 import git
10 from django.core.exceptions import ValidationError
11 from git.exc import BadName, InvalidGitRepositoryError
12 
13 from readthedocs.builds.constants import EXTERNAL
14 from readthedocs.config import ALL
15 from readthedocs.projects.constants import GITHUB_PR_PULL_PATTERN
16 from readthedocs.projects.exceptions import RepositoryError
17 from readthedocs.projects.validators import validate_submodule_url
18 from readthedocs.vcs_support.base import BaseVCS, VCSVersion
19 
20 
21 log = logging.getLogger(__name__)
22 
23 
24 class Backend(BaseVCS):
25 
26     """Git VCS backend."""
27 
28     supports_tags = True
29     supports_branches = True
30     supports_submodules = True
31     fallback_branch = 'master'  # default branch
32     repo_depth = 50
33 
34     def __init__(self, *args, **kwargs):
35         super().__init__(*args, **kwargs)
36         self.token = kwargs.get('token', None)
37         self.repo_url = self._get_clone_url()
38 
39     def _get_clone_url(self):
40         if '://' in self.repo_url:
41             hacked_url = self.repo_url.split('://')[1]
42             hacked_url = re.sub('.git$', '', hacked_url)
43             clone_url = 'https://%s' % hacked_url
44             if self.token:
45                 clone_url = 'https://{}@{}'.format(self.token, hacked_url)
46                 return clone_url
47             # Don't edit URL because all hosts aren't the same
48             # else:
49             #     clone_url = 'git://%s' % (hacked_url)
50         return self.repo_url
51 
52     def set_remote_url(self, url):
53         return self.run('git', 'remote', 'set-url', 'origin', url)
54 
55     def update(self):
56         """Clone or update the repository."""
57         super().update()
58         if self.repo_exists():
59             self.set_remote_url(self.repo_url)
60             return self.fetch()
61         self.make_clean_working_dir()
62         # A fetch is always required to get external versions properly
63         if self.version_type == EXTERNAL:
64             self.clone()
65             return self.fetch()
66         return self.clone()
67 
68     def repo_exists(self):
69         try:
70             git.Repo(self.working_dir)
71         except InvalidGitRepositoryError:
72             return False
73         return True
74 
75     def are_submodules_available(self, config):
76         """Test whether git submodule checkout step should be performed."""
77         # TODO remove this after users migrate to a config file
78         from readthedocs.projects.models import Feature
79         submodules_in_config = (
80             config.submodules.exclude != ALL or config.submodules.include
81         )
82         if (
83             self.project.has_feature(Feature.SKIP_SUBMODULES) or
84             not submodules_in_config
85         ):
86             return False
87 
88         # Keep compatibility with previous projects
89         return bool(self.submodules)
90 
91     def validate_submodules(self, config):
92         """
93         Returns the submodules and check that its URLs are valid.
94 
95         .. note::
96 
97            Allways call after `self.are_submodules_available`.
98 
99         :returns: tuple(bool, list)
100 
101         Returns `True` if all required submodules URLs are valid.
102         Returns a list of all required submodules:
103         - Include is `ALL`, returns all submodules available.
104         - Include is a list, returns just those.
105         - Exclude is `ALL` - this should never happen.
106         - Exlude is a list, returns all available submodules
107           but those from the list.
108 
109         Returns `False` if at least one submodule is invalid.
110         Returns the list of invalid submodules.
111         """
112         submodules = {sub.path: sub for sub in self.submodules}
113 
114         for sub_path in config.submodules.exclude:
115             path = sub_path.rstrip('/')
116             if path in submodules:
117                 del submodules[path]
118 
119         if config.submodules.include != ALL and config.submodules.include:
120             submodules_include = {}
121             for sub_path in config.submodules.include:
122                 path = sub_path.rstrip('/')
123                 submodules_include[path] = submodules[path]
124             submodules = submodules_include
125 
126         invalid_submodules = []
127         for path, submodule in submodules.items():
128             try:
129                 validate_submodule_url(submodule.url)
130             except ValidationError:
131                 invalid_submodules.append(path)
132 
133         if invalid_submodules:
134             return False, invalid_submodules
135         return True, submodules.keys()
136 
137     def use_shallow_clone(self):
138         """
139         Test whether shallow clone should be performed.
140 
141         .. note::
142 
143             Temporarily, we support skipping this option as builds that rely on
144             git history can fail if using shallow clones. This should
145             eventually be configurable via the web UI.
146         """
147         from readthedocs.projects.models import Feature
148         return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE)
149 
150     def fetch(self):
151         cmd = ['git', 'fetch', 'origin',
152                '--tags', '--prune', '--prune-tags']
153 
154         if self.use_shallow_clone():
155             cmd.extend(['--depth', str(self.repo_depth)])
156 
157         if (
158             self.verbose_name and
159             self.version_type == EXTERNAL and
160             'github.com' in self.repo_url
161         ):
162             cmd.append(
163                 GITHUB_PR_PULL_PATTERN.format(id=self.verbose_name)
164             )
165 
166         code, stdout, stderr = self.run(*cmd)
167         if code != 0:
168             raise RepositoryError
169         return code, stdout, stderr
170 
171     def checkout_revision(self, revision=None):
172         if not revision:
173             branch = self.default_branch or self.fallback_branch
174             revision = 'origin/%s' % branch
175 
176         code, out, err = self.run('git', 'checkout', '--force', revision)
177         if code != 0:
178             raise RepositoryError(
179                 RepositoryError.FAILED_TO_CHECKOUT.format(revision),
180             )
181         return [code, out, err]
182 
183     def clone(self):
184         """Clones the repository."""
185         cmd = ['git', 'clone', '--no-single-branch']
186 
187         if self.use_shallow_clone():
188             cmd.extend(['--depth', str(self.repo_depth)])
189 
190         cmd.extend([self.repo_url, '.'])
191 
192         code, stdout, stderr = self.run(*cmd)
193         if code != 0:
194             raise RepositoryError
195         return code, stdout, stderr
196 
197     @property
198     def tags(self):
199         versions = []
200         repo = git.Repo(self.working_dir)
201         for tag in repo.tags:
202             try:
203                 versions.append(VCSVersion(self, str(tag.commit), str(tag)))
204             except ValueError:
205                 # ValueError: Cannot resolve commit as tag TAGNAME points to a
206                 # blob object - use the `.object` property instead to access it
207                 # This is not a real tag for us, so we skip it
208                 # https://github.com/rtfd/readthedocs.org/issues/4440
209                 log.warning('Git tag skipped: %s', tag, exc_info=True)
210                 continue
211         return versions
212 
213     @property
214     def branches(self):
215         repo = git.Repo(self.working_dir)
216         versions = []
217         branches = []
218 
219         # ``repo.remotes.origin.refs`` returns remote branches
220         if repo.remotes:
221             branches += repo.remotes.origin.refs
222 
223         for branch in branches:
224             verbose_name = branch.name
225             if verbose_name.startswith('origin/'):
226                 verbose_name = verbose_name.replace('origin/', '')
227             if verbose_name == 'HEAD':
228                 continue
229             versions.append(VCSVersion(self, str(branch), verbose_name))
230         return versions
231 
232     @property
233     def commit(self):
234         if self.repo_exists():
235             _, stdout, _ = self.run('git', 'rev-parse', 'HEAD')
236             return stdout.strip()
237         return None
238 
239     @property
240     def submodules(self):
241         try:
242             repo = git.Repo(self.working_dir)
243             return list(repo.submodules)
244         except InvalidGitRepositoryError:
245             raise RepositoryError(
246                 RepositoryError.INVALID_SUBMODULES_PATH,
247             )
248 
249     def checkout(self, identifier=None):
250         """Checkout to identifier or latest."""
251         super().checkout()
252         # Find proper identifier
253         if not identifier:
254             identifier = self.default_branch or self.fallback_branch
255 
256         identifier = self.find_ref(identifier)
257 
258         # Checkout the correct identifier for this branch.
259         code, out, err = self.checkout_revision(identifier)
260         if code != 0:
261             return code, out, err
262 
263         # Clean any remains of previous checkouts
264         self.run('git', 'clean', '-d', '-f', '-f')
265         return code, out, err
266 
267     def update_submodules(self, config):
268         if self.are_submodules_available(config):
269             valid, submodules = self.validate_submodules(config)
270             if valid:
271                 self.checkout_submodules(submodules, config)
272             else:
273                 raise RepositoryError(
274                     RepositoryError.INVALID_SUBMODULES.format(submodules),
275                 )
276 
277     def checkout_submodules(self, submodules, config):
278         """Checkout all repository submodules."""
279         self.run('git', 'submodule', 'sync')
280         cmd = [
281             'git',
282             'submodule',
283             'update',
284             '--init',
285             '--force',
286         ]
287         if config.submodules.recursive:
288             cmd.append('--recursive')
289         cmd += submodules
290         self.run(*cmd)
291 
292     def find_ref(self, ref):
293         # Check if ref starts with 'origin/'
294         if ref.startswith('origin/'):
295             return ref
296 
297         # Check if ref is a branch of the origin remote
298         if self.ref_exists('remotes/origin/' + ref):
299             return 'origin/' + ref
300 
301         return ref
302 
303     def ref_exists(self, ref):
304         try:
305             r = git.Repo(self.working_dir)
306             if r.commit(ref):
307                 return True
308         except (BadName, ValueError):
309             return False
310         return False
311 
312     @property
313     def env(self):
314         env = super().env
315         env['GIT_DIR'] = os.path.join(self.working_dir, '.git')
316         # Don't prompt for username, this requires Git 2.3+
317         env['GIT_TERMINAL_PROMPT'] = '0'
318         return env
319
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/readthedocs/projects/exceptions.py b/readthedocs/projects/exceptions.py
--- a/readthedocs/projects/exceptions.py
+++ b/readthedocs/projects/exceptions.py
@@ -37,16 +37,10 @@
         'ensure that your repository URL is correct and your repository is public. '
         'Private repositories are not supported.',
     )
-
     INVALID_SUBMODULES = _(
         'One or more submodule URLs are not valid: {}, '
         'git/ssh URL schemas for submodules are not supported.'
     )
-    INVALID_SUBMODULES_PATH = _(
-        'One or more submodule paths are not valid. '
-        'Check that all your submodules in .gitmodules are used.'
-    )
-
     DUPLICATED_RESERVED_VERSIONS = _(
         'You can not have two versions with the name latest or stable.',
     )
diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py
--- a/readthedocs/vcs_support/backends/git.py
+++ b/readthedocs/vcs_support/backends/git.py
@@ -238,13 +238,8 @@
 
     @property
     def submodules(self):
-        try:
-            repo = git.Repo(self.working_dir)
-            return list(repo.submodules)
-        except InvalidGitRepositoryError:
-            raise RepositoryError(
-                RepositoryError.INVALID_SUBMODULES_PATH,
-            )
+        repo = git.Repo(self.working_dir)
+        return list(repo.submodules)
 
     def checkout(self, identifier=None):
         """Checkout to identifier or latest."""

{"golden_diff": "diff --git a/readthedocs/projects/exceptions.py b/readthedocs/projects/exceptions.py\n--- a/readthedocs/projects/exceptions.py\n+++ b/readthedocs/projects/exceptions.py\n@@ -37,16 +37,10 @@\n         'ensure that your repository URL is correct and your repository is public. '\n         'Private repositories are not supported.',\n     )\n-\n     INVALID_SUBMODULES = _(\n         'One or more submodule URLs are not valid: {}, '\n         'git/ssh URL schemas for submodules are not supported.'\n     )\n-    INVALID_SUBMODULES_PATH = _(\n-        'One or more submodule paths are not valid. '\n-        'Check that all your submodules in .gitmodules are used.'\n-    )\n-\n     DUPLICATED_RESERVED_VERSIONS = _(\n         'You can not have two versions with the name latest or stable.',\n     )\ndiff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py\n--- a/readthedocs/vcs_support/backends/git.py\n+++ b/readthedocs/vcs_support/backends/git.py\n@@ -238,13 +238,8 @@\n \n     @property\n     def submodules(self):\n-        try:\n-            repo = git.Repo(self.working_dir)\n-            return list(repo.submodules)\n-        except InvalidGitRepositoryError:\n-            raise RepositoryError(\n-                RepositoryError.INVALID_SUBMODULES_PATH,\n-            )\n+        repo = git.Repo(self.working_dir)\n+        return list(repo.submodules)\n \n     def checkout(self, identifier=None):\n         \"\"\"Checkout to identifier or latest.\"\"\"\n", "issue": "gitpython sometimes fails when checking for submodules\n## Details\r\n\r\nI noticed the other week that our builds have actually been failing for a few months but I'm not sure how to debug it. Locally the docs generate as expected when I run `sphinx-build -b dirhtml . _build/dirhtml`\r\n\r\n* Read the Docs project URL: https://readthedocs.org/projects/chassis/\r\n* Build URL (if applicable): https://readthedocs.org/projects/chassis/builds/7462561/\r\n* Read the Docs username (if applicable): BronsonQuick\r\n\r\n## Expected Result\r\n\r\nA successful build.\r\n\r\n## Actual Result\r\n\r\n>There was a problem with Read the Docs while building your documentation. Please report this to us with your build id (7462561).\r\n\r\nThanks very much for all your work on this project!\r\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n\n\"\"\"Project exceptions.\"\"\"\n\nfrom django.conf import settings\nfrom django.utils.translation import ugettext_noop as _\n\nfrom readthedocs.doc_builder.exceptions import BuildEnvironmentError\n\n\nclass ProjectConfigurationError(BuildEnvironmentError):\n\n    \"\"\"Error raised trying to configure a project for build.\"\"\"\n\n    NOT_FOUND = _(\n        'A configuration file was not found. '\n        'Make sure you have a conf.py file in your repository.',\n    )\n\n    MULTIPLE_CONF_FILES = _(\n        'We found more than one conf.py and are not sure which one to use. '\n        'Please, specify the correct file under the Advanced settings tab '\n        \"in the project's Admin.\",\n    )\n\n\nclass RepositoryError(BuildEnvironmentError):\n\n    \"\"\"Failure during repository operation.\"\"\"\n\n    PRIVATE_ALLOWED = _(\n        'There was a problem connecting to your repository, '\n        'ensure that your repository URL is correct.',\n    )\n    PRIVATE_NOT_ALLOWED = _(\n        'There was a problem connecting to your repository, '\n        'ensure that your repository URL is correct and your repository is public. '\n        'Private repositories are not supported.',\n    )\n\n    INVALID_SUBMODULES = _(\n        'One or more submodule URLs are not valid: {}, '\n        'git/ssh URL schemas for submodules are not supported.'\n    )\n    INVALID_SUBMODULES_PATH = _(\n        'One or more submodule paths are not valid. '\n        'Check that all your submodules in .gitmodules are used.'\n    )\n\n    DUPLICATED_RESERVED_VERSIONS = _(\n        'You can not have two versions with the name latest or stable.',\n    )\n\n    FAILED_TO_CHECKOUT = _('Failed to checkout revision: {}')\n\n    def get_default_message(self):\n        if settings.ALLOW_PRIVATE_REPOS:\n            return self.PRIVATE_ALLOWED\n        return self.PRIVATE_NOT_ALLOWED\n\n\nclass ProjectSpamError(Exception):\n\n    \"\"\"\n    Error raised when a project field has detected spam.\n\n    This error is not raised to users, we use this for banning users in the\n    background.\n    \"\"\"\n", "path": "readthedocs/projects/exceptions.py"}, {"content": "# -*- coding: utf-8 -*-\n\n\"\"\"Git-related utilities.\"\"\"\n\nimport logging\nimport os\nimport re\n\nimport git\nfrom django.core.exceptions import ValidationError\nfrom git.exc import BadName, InvalidGitRepositoryError\n\nfrom readthedocs.builds.constants import EXTERNAL\nfrom readthedocs.config import ALL\nfrom readthedocs.projects.constants import GITHUB_PR_PULL_PATTERN\nfrom readthedocs.projects.exceptions import RepositoryError\nfrom readthedocs.projects.validators import validate_submodule_url\nfrom readthedocs.vcs_support.base import BaseVCS, VCSVersion\n\n\nlog = logging.getLogger(__name__)\n\n\nclass Backend(BaseVCS):\n\n    \"\"\"Git VCS backend.\"\"\"\n\n    supports_tags = True\n    supports_branches = True\n    supports_submodules = True\n    fallback_branch = 'master'  # default branch\n    repo_depth = 50\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.token = kwargs.get('token', None)\n        self.repo_url = self._get_clone_url()\n\n    def _get_clone_url(self):\n        if '://' in self.repo_url:\n            hacked_url = self.repo_url.split('://')[1]\n            hacked_url = re.sub('.git$', '', hacked_url)\n            clone_url = 'https://%s' % hacked_url\n            if self.token:\n                clone_url = 'https://{}@{}'.format(self.token, hacked_url)\n                return clone_url\n            # Don't edit URL because all hosts aren't the same\n            # else:\n            #     clone_url = 'git://%s' % (hacked_url)\n        return self.repo_url\n\n    def set_remote_url(self, url):\n        return self.run('git', 'remote', 'set-url', 'origin', url)\n\n    def update(self):\n        \"\"\"Clone or update the repository.\"\"\"\n        super().update()\n        if self.repo_exists():\n            self.set_remote_url(self.repo_url)\n            return self.fetch()\n        self.make_clean_working_dir()\n        # A fetch is always required to get external versions properly\n        if self.version_type == EXTERNAL:\n            self.clone()\n            return self.fetch()\n        return self.clone()\n\n    def repo_exists(self):\n        try:\n            git.Repo(self.working_dir)\n        except InvalidGitRepositoryError:\n            return False\n        return True\n\n    def are_submodules_available(self, config):\n        \"\"\"Test whether git submodule checkout step should be performed.\"\"\"\n        # TODO remove this after users migrate to a config file\n        from readthedocs.projects.models import Feature\n        submodules_in_config = (\n            config.submodules.exclude != ALL or config.submodules.include\n        )\n        if (\n            self.project.has_feature(Feature.SKIP_SUBMODULES) or\n            not submodules_in_config\n        ):\n            return False\n\n        # Keep compatibility with previous projects\n        return bool(self.submodules)\n\n    def validate_submodules(self, config):\n        \"\"\"\n        Returns the submodules and check that its URLs are valid.\n\n        .. note::\n\n           Allways call after `self.are_submodules_available`.\n\n        :returns: tuple(bool, list)\n\n        Returns `True` if all required submodules URLs are valid.\n        Returns a list of all required submodules:\n        - Include is `ALL`, returns all submodules available.\n        - Include is a list, returns just those.\n        - Exclude is `ALL` - this should never happen.\n        - Exlude is a list, returns all available submodules\n          but those from the list.\n\n        Returns `False` if at least one submodule is invalid.\n        Returns the list of invalid submodules.\n        \"\"\"\n        submodules = {sub.path: sub for sub in self.submodules}\n\n        for sub_path in config.submodules.exclude:\n            path = sub_path.rstrip('/')\n            if path in submodules:\n                del submodules[path]\n\n        if config.submodules.include != ALL and config.submodules.include:\n            submodules_include = {}\n            for sub_path in config.submodules.include:\n                path = sub_path.rstrip('/')\n                submodules_include[path] = submodules[path]\n            submodules = submodules_include\n\n        invalid_submodules = []\n        for path, submodule in submodules.items():\n            try:\n                validate_submodule_url(submodule.url)\n            except ValidationError:\n                invalid_submodules.append(path)\n\n        if invalid_submodules:\n            return False, invalid_submodules\n        return True, submodules.keys()\n\n    def use_shallow_clone(self):\n        \"\"\"\n        Test whether shallow clone should be performed.\n\n        .. note::\n\n            Temporarily, we support skipping this option as builds that rely on\n            git history can fail if using shallow clones. This should\n            eventually be configurable via the web UI.\n        \"\"\"\n        from readthedocs.projects.models import Feature\n        return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE)\n\n    def fetch(self):\n        cmd = ['git', 'fetch', 'origin',\n               '--tags', '--prune', '--prune-tags']\n\n        if self.use_shallow_clone():\n            cmd.extend(['--depth', str(self.repo_depth)])\n\n        if (\n            self.verbose_name and\n            self.version_type == EXTERNAL and\n            'github.com' in self.repo_url\n        ):\n            cmd.append(\n                GITHUB_PR_PULL_PATTERN.format(id=self.verbose_name)\n            )\n\n        code, stdout, stderr = self.run(*cmd)\n        if code != 0:\n            raise RepositoryError\n        return code, stdout, stderr\n\n    def checkout_revision(self, revision=None):\n        if not revision:\n            branch = self.default_branch or self.fallback_branch\n            revision = 'origin/%s' % branch\n\n        code, out, err = self.run('git', 'checkout', '--force', revision)\n        if code != 0:\n            raise RepositoryError(\n                RepositoryError.FAILED_TO_CHECKOUT.format(revision),\n            )\n        return [code, out, err]\n\n    def clone(self):\n        \"\"\"Clones the repository.\"\"\"\n        cmd = ['git', 'clone', '--no-single-branch']\n\n        if self.use_shallow_clone():\n            cmd.extend(['--depth', str(self.repo_depth)])\n\n        cmd.extend([self.repo_url, '.'])\n\n        code, stdout, stderr = self.run(*cmd)\n        if code != 0:\n            raise RepositoryError\n        return code, stdout, stderr\n\n    @property\n    def tags(self):\n        versions = []\n        repo = git.Repo(self.working_dir)\n        for tag in repo.tags:\n            try:\n                versions.append(VCSVersion(self, str(tag.commit), str(tag)))\n            except ValueError:\n                # ValueError: Cannot resolve commit as tag TAGNAME points to a\n                # blob object - use the `.object` property instead to access it\n                # This is not a real tag for us, so we skip it\n                # https://github.com/rtfd/readthedocs.org/issues/4440\n                log.warning('Git tag skipped: %s', tag, exc_info=True)\n                continue\n        return versions\n\n    @property\n    def branches(self):\n        repo = git.Repo(self.working_dir)\n        versions = []\n        branches = []\n\n        # ``repo.remotes.origin.refs`` returns remote branches\n        if repo.remotes:\n            branches += repo.remotes.origin.refs\n\n        for branch in branches:\n            verbose_name = branch.name\n            if verbose_name.startswith('origin/'):\n                verbose_name = verbose_name.replace('origin/', '')\n            if verbose_name == 'HEAD':\n                continue\n            versions.append(VCSVersion(self, str(branch), verbose_name))\n        return versions\n\n    @property\n    def commit(self):\n        if self.repo_exists():\n            _, stdout, _ = self.run('git', 'rev-parse', 'HEAD')\n            return stdout.strip()\n        return None\n\n    @property\n    def submodules(self):\n        try:\n            repo = git.Repo(self.working_dir)\n            return list(repo.submodules)\n        except InvalidGitRepositoryError:\n            raise RepositoryError(\n                RepositoryError.INVALID_SUBMODULES_PATH,\n            )\n\n    def checkout(self, identifier=None):\n        \"\"\"Checkout to identifier or latest.\"\"\"\n        super().checkout()\n        # Find proper identifier\n        if not identifier:\n            identifier = self.default_branch or self.fallback_branch\n\n        identifier = self.find_ref(identifier)\n\n        # Checkout the correct identifier for this branch.\n        code, out, err = self.checkout_revision(identifier)\n        if code != 0:\n            return code, out, err\n\n        # Clean any remains of previous checkouts\n        self.run('git', 'clean', '-d', '-f', '-f')\n        return code, out, err\n\n    def update_submodules(self, config):\n        if self.are_submodules_available(config):\n            valid, submodules = self.validate_submodules(config)\n            if valid:\n                self.checkout_submodules(submodules, config)\n            else:\n                raise RepositoryError(\n                    RepositoryError.INVALID_SUBMODULES.format(submodules),\n                )\n\n    def checkout_submodules(self, submodules, config):\n        \"\"\"Checkout all repository submodules.\"\"\"\n        self.run('git', 'submodule', 'sync')\n        cmd = [\n            'git',\n            'submodule',\n            'update',\n            '--init',\n            '--force',\n        ]\n        if config.submodules.recursive:\n            cmd.append('--recursive')\n        cmd += submodules\n        self.run(*cmd)\n\n    def find_ref(self, ref):\n        # Check if ref starts with 'origin/'\n        if ref.startswith('origin/'):\n            return ref\n\n        # Check if ref is a branch of the origin remote\n        if self.ref_exists('remotes/origin/' + ref):\n            return 'origin/' + ref\n\n        return ref\n\n    def ref_exists(self, ref):\n        try:\n            r = git.Repo(self.working_dir)\n            if r.commit(ref):\n                return True\n        except (BadName, ValueError):\n            return False\n        return False\n\n    @property\n    def env(self):\n        env = super().env\n        env['GIT_DIR'] = os.path.join(self.working_dir, '.git')\n        # Don't prompt for username, this requires Git 2.3+\n        env['GIT_TERMINAL_PROMPT'] = '0'\n        return env\n", "path": "readthedocs/vcs_support/backends/git.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n\n\"\"\"Project exceptions.\"\"\"\n\nfrom django.conf import settings\nfrom django.utils.translation import ugettext_noop as _\n\nfrom readthedocs.doc_builder.exceptions import BuildEnvironmentError\n\n\nclass ProjectConfigurationError(BuildEnvironmentError):\n\n    \"\"\"Error raised trying to configure a project for build.\"\"\"\n\n    NOT_FOUND = _(\n        'A configuration file was not found. '\n        'Make sure you have a conf.py file in your repository.',\n    )\n\n    MULTIPLE_CONF_FILES = _(\n        'We found more than one conf.py and are not sure which one to use. '\n        'Please, specify the correct file under the Advanced settings tab '\n        \"in the project's Admin.\",\n    )\n\n\nclass RepositoryError(BuildEnvironmentError):\n\n    \"\"\"Failure during repository operation.\"\"\"\n\n    PRIVATE_ALLOWED = _(\n        'There was a problem connecting to your repository, '\n        'ensure that your repository URL is correct.',\n    )\n    PRIVATE_NOT_ALLOWED = _(\n        'There was a problem connecting to your repository, '\n        'ensure that your repository URL is correct and your repository is public. '\n        'Private repositories are not supported.',\n    )\n    INVALID_SUBMODULES = _(\n        'One or more submodule URLs are not valid: {}, '\n        'git/ssh URL schemas for submodules are not supported.'\n    )\n    DUPLICATED_RESERVED_VERSIONS = _(\n        'You can not have two versions with the name latest or stable.',\n    )\n\n    FAILED_TO_CHECKOUT = _('Failed to checkout revision: {}')\n\n    def get_default_message(self):\n        if settings.ALLOW_PRIVATE_REPOS:\n            return self.PRIVATE_ALLOWED\n        return self.PRIVATE_NOT_ALLOWED\n\n\nclass ProjectSpamError(Exception):\n\n    \"\"\"\n    Error raised when a project field has detected spam.\n\n    This error is not raised to users, we use this for banning users in the\n    background.\n    \"\"\"\n", "path": "readthedocs/projects/exceptions.py"}, {"content": "# -*- coding: utf-8 -*-\n\n\"\"\"Git-related utilities.\"\"\"\n\nimport logging\nimport os\nimport re\n\nimport git\nfrom django.core.exceptions import ValidationError\nfrom git.exc import BadName, InvalidGitRepositoryError\n\nfrom readthedocs.builds.constants import EXTERNAL\nfrom readthedocs.config import ALL\nfrom readthedocs.projects.constants import GITHUB_PR_PULL_PATTERN\nfrom readthedocs.projects.exceptions import RepositoryError\nfrom readthedocs.projects.validators import validate_submodule_url\nfrom readthedocs.vcs_support.base import BaseVCS, VCSVersion\n\n\nlog = logging.getLogger(__name__)\n\n\nclass Backend(BaseVCS):\n\n    \"\"\"Git VCS backend.\"\"\"\n\n    supports_tags = True\n    supports_branches = True\n    supports_submodules = True\n    fallback_branch = 'master'  # default branch\n    repo_depth = 50\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.token = kwargs.get('token', None)\n        self.repo_url = self._get_clone_url()\n\n    def _get_clone_url(self):\n        if '://' in self.repo_url:\n            hacked_url = self.repo_url.split('://')[1]\n            hacked_url = re.sub('.git$', '', hacked_url)\n            clone_url = 'https://%s' % hacked_url\n            if self.token:\n                clone_url = 'https://{}@{}'.format(self.token, hacked_url)\n                return clone_url\n            # Don't edit URL because all hosts aren't the same\n            # else:\n            #     clone_url = 'git://%s' % (hacked_url)\n        return self.repo_url\n\n    def set_remote_url(self, url):\n        return self.run('git', 'remote', 'set-url', 'origin', url)\n\n    def update(self):\n        \"\"\"Clone or update the repository.\"\"\"\n        super().update()\n        if self.repo_exists():\n            self.set_remote_url(self.repo_url)\n            return self.fetch()\n        self.make_clean_working_dir()\n        # A fetch is always required to get external versions properly\n        if self.version_type == EXTERNAL:\n            self.clone()\n            return self.fetch()\n        return self.clone()\n\n    def repo_exists(self):\n        try:\n            git.Repo(self.working_dir)\n        except InvalidGitRepositoryError:\n            return False\n        return True\n\n    def are_submodules_available(self, config):\n        \"\"\"Test whether git submodule checkout step should be performed.\"\"\"\n        # TODO remove this after users migrate to a config file\n        from readthedocs.projects.models import Feature\n        submodules_in_config = (\n            config.submodules.exclude != ALL or config.submodules.include\n        )\n        if (\n            self.project.has_feature(Feature.SKIP_SUBMODULES) or\n            not submodules_in_config\n        ):\n            return False\n\n        # Keep compatibility with previous projects\n        return bool(self.submodules)\n\n    def validate_submodules(self, config):\n        \"\"\"\n        Returns the submodules and check that its URLs are valid.\n\n        .. note::\n\n           Allways call after `self.are_submodules_available`.\n\n        :returns: tuple(bool, list)\n\n        Returns `True` if all required submodules URLs are valid.\n        Returns a list of all required submodules:\n        - Include is `ALL`, returns all submodules available.\n        - Include is a list, returns just those.\n        - Exclude is `ALL` - this should never happen.\n        - Exlude is a list, returns all available submodules\n          but those from the list.\n\n        Returns `False` if at least one submodule is invalid.\n        Returns the list of invalid submodules.\n        \"\"\"\n        submodules = {sub.path: sub for sub in self.submodules}\n\n        for sub_path in config.submodules.exclude:\n            path = sub_path.rstrip('/')\n            if path in submodules:\n                del submodules[path]\n\n        if config.submodules.include != ALL and config.submodules.include:\n            submodules_include = {}\n            for sub_path in config.submodules.include:\n                path = sub_path.rstrip('/')\n                submodules_include[path] = submodules[path]\n            submodules = submodules_include\n\n        invalid_submodules = []\n        for path, submodule in submodules.items():\n            try:\n                validate_submodule_url(submodule.url)\n            except ValidationError:\n                invalid_submodules.append(path)\n\n        if invalid_submodules:\n            return False, invalid_submodules\n        return True, submodules.keys()\n\n    def use_shallow_clone(self):\n        \"\"\"\n        Test whether shallow clone should be performed.\n\n        .. note::\n\n            Temporarily, we support skipping this option as builds that rely on\n            git history can fail if using shallow clones. This should\n            eventually be configurable via the web UI.\n        \"\"\"\n        from readthedocs.projects.models import Feature\n        return not self.project.has_feature(Feature.DONT_SHALLOW_CLONE)\n\n    def fetch(self):\n        cmd = ['git', 'fetch', 'origin',\n               '--tags', '--prune', '--prune-tags']\n\n        if self.use_shallow_clone():\n            cmd.extend(['--depth', str(self.repo_depth)])\n\n        if (\n            self.verbose_name and\n            self.version_type == EXTERNAL and\n            'github.com' in self.repo_url\n        ):\n            cmd.append(\n                GITHUB_PR_PULL_PATTERN.format(id=self.verbose_name)\n            )\n\n        code, stdout, stderr = self.run(*cmd)\n        if code != 0:\n            raise RepositoryError\n        return code, stdout, stderr\n\n    def checkout_revision(self, revision=None):\n        if not revision:\n            branch = self.default_branch or self.fallback_branch\n            revision = 'origin/%s' % branch\n\n        code, out, err = self.run('git', 'checkout', '--force', revision)\n        if code != 0:\n            raise RepositoryError(\n                RepositoryError.FAILED_TO_CHECKOUT.format(revision),\n            )\n        return [code, out, err]\n\n    def clone(self):\n        \"\"\"Clones the repository.\"\"\"\n        cmd = ['git', 'clone', '--no-single-branch']\n\n        if self.use_shallow_clone():\n            cmd.extend(['--depth', str(self.repo_depth)])\n\n        cmd.extend([self.repo_url, '.'])\n\n        code, stdout, stderr = self.run(*cmd)\n        if code != 0:\n            raise RepositoryError\n        return code, stdout, stderr\n\n    @property\n    def tags(self):\n        versions = []\n        repo = git.Repo(self.working_dir)\n        for tag in repo.tags:\n            try:\n                versions.append(VCSVersion(self, str(tag.commit), str(tag)))\n            except ValueError:\n                # ValueError: Cannot resolve commit as tag TAGNAME points to a\n                # blob object - use the `.object` property instead to access it\n                # This is not a real tag for us, so we skip it\n                # https://github.com/rtfd/readthedocs.org/issues/4440\n                log.warning('Git tag skipped: %s', tag, exc_info=True)\n                continue\n        return versions\n\n    @property\n    def branches(self):\n        repo = git.Repo(self.working_dir)\n        versions = []\n        branches = []\n\n        # ``repo.remotes.origin.refs`` returns remote branches\n        if repo.remotes:\n            branches += repo.remotes.origin.refs\n\n        for branch in branches:\n            verbose_name = branch.name\n            if verbose_name.startswith('origin/'):\n                verbose_name = verbose_name.replace('origin/', '')\n            if verbose_name == 'HEAD':\n                continue\n            versions.append(VCSVersion(self, str(branch), verbose_name))\n        return versions\n\n    @property\n    def commit(self):\n        if self.repo_exists():\n            _, stdout, _ = self.run('git', 'rev-parse', 'HEAD')\n            return stdout.strip()\n        return None\n\n    @property\n    def submodules(self):\n        repo = git.Repo(self.working_dir)\n        return list(repo.submodules)\n\n    def checkout(self, identifier=None):\n        \"\"\"Checkout to identifier or latest.\"\"\"\n        super().checkout()\n        # Find proper identifier\n        if not identifier:\n            identifier = self.default_branch or self.fallback_branch\n\n        identifier = self.find_ref(identifier)\n\n        # Checkout the correct identifier for this branch.\n        code, out, err = self.checkout_revision(identifier)\n        if code != 0:\n            return code, out, err\n\n        # Clean any remains of previous checkouts\n        self.run('git', 'clean', '-d', '-f', '-f')\n        return code, out, err\n\n    def update_submodules(self, config):\n        if self.are_submodules_available(config):\n            valid, submodules = self.validate_submodules(config)\n            if valid:\n                self.checkout_submodules(submodules, config)\n            else:\n                raise RepositoryError(\n                    RepositoryError.INVALID_SUBMODULES.format(submodules),\n                )\n\n    def checkout_submodules(self, submodules, config):\n        \"\"\"Checkout all repository submodules.\"\"\"\n        self.run('git', 'submodule', 'sync')\n        cmd = [\n            'git',\n            'submodule',\n            'update',\n            '--init',\n            '--force',\n        ]\n        if config.submodules.recursive:\n            cmd.append('--recursive')\n        cmd += submodules\n        self.run(*cmd)\n\n    def find_ref(self, ref):\n        # Check if ref starts with 'origin/'\n        if ref.startswith('origin/'):\n            return ref\n\n        # Check if ref is a branch of the origin remote\n        if self.ref_exists('remotes/origin/' + ref):\n            return 'origin/' + ref\n\n        return ref\n\n    def ref_exists(self, ref):\n        try:\n            r = git.Repo(self.working_dir)\n            if r.commit(ref):\n                return True\n        except (BadName, ValueError):\n            return False\n        return False\n\n    @property\n    def env(self):\n        env = super().env\n        env['GIT_DIR'] = os.path.join(self.working_dir, '.git')\n        # Don't prompt for username, this requires Git 2.3+\n        env['GIT_TERMINAL_PROMPT'] = '0'\n        return env\n", "path": "readthedocs/vcs_support/backends/git.py"}]}