We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Retry requests.exceptions.ConnectionError
Hi Team,

Could you add a retry to this exception? We are running this code in Cloud Function and GKE infrastructure from time to time we get these errors

Bigquery SDK == google-cloud-bigquery==3.18.0

```
Error Type: <class 'requests.exceptions.ConnectionError'> error: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')) 
Traceback (most recent call last): File "/workspace/visionCommon/common.py", line 117, 
in wrapper return func(*args, **kwargs) File "/workspace/main.py", line 576, in controller current_step = check_eligibility() File "/workspace/main.py", line 314, 
in check_eligibility total_rows = service.execute_query(query).result().total_rows File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py", line 1595, 
in result do_get_result() File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py", line 293, 
in retry_wrapped_func return retry_target( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py", line 153, 
in retry_target _retry_error_helper( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_base.py", line 212, 
in _retry_error_helper raise final_exc from source_exc File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py", line 144, 
in retry_target result = target() File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py", line 1584, 
in do_get_result super(QueryJob, self).result(retry=retry, timeout=timeout) File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/base.py", line 971, 
in result return super(_AsyncJob, self).result(timeout=timeout, **kwargs) File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/future/polling.py", line 256, 
in result self._blocking_poll(timeout=timeout, retry=retry, polling=polling) File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py", line 1326, 
in _blocking_poll super(QueryJob, self)._blocking_poll(timeout=timeout, **kwargs) File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/future/polling.py", line 137, 
in _blocking_poll polling(self._done_or_raise)(retry=retry) File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py", line 293, 
in retry_wrapped_func return retry_target( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py", line 153, 
in retry_target _retry_error_helper( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_base.py", line 212, 
in _retry_error_helper raise final_exc from source_exc File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py", line 144, 
in retry_target result = target() File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py", line 1448, 
in _done_or_raise self._reload_query_results(retry=retry, timeout=transport_timeout) File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py", line 1429, 
in _reload_query_results self._query_results = self._client._get_query_results( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/client.py", line 1936, 
in _get_query_results resource = self._call_api( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/client.py", line 827, 
in _call_api return call() File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/_http/__init__.py", line 482, 
in api_request response = self._make_request( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/_http/__init__.py", line 341, 
in _make_request return self._do_request( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/_http/__init__.py", line 379, 
in _do_request return self.http.request( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/auth/transport/requests.py", line 541, 
in request response = super(AuthorizedSession, self).request( File "/layers/google.python.pip/pip/lib/python3.9/site-packages/requests/sessions.py", line 589, 
in request resp = self.send(prep, **send_kwargs) File "/layers/google.python.pip/pip/lib/python3.9/site-packages/requests/sessions.py", line 703, 
in send r = adapter.send(request, **kwargs) File "/layers/google.python.pip/pip/lib/python3.9/site-packages/requests/adapters.py", line 501, 
in send raise ConnectionError(err, request=request) requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
```

 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `google/cloud/bigquery/retry.py`
Content:
```
1 # Copyright 2018 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 from google.api_core import exceptions
16 from google.api_core import retry
17 from google.auth import exceptions as auth_exceptions  # type: ignore
18 import requests.exceptions
19 
20 
21 _RETRYABLE_REASONS = frozenset(
22     ["rateLimitExceeded", "backendError", "internalError", "badGateway"]
23 )
24 
25 _UNSTRUCTURED_RETRYABLE_TYPES = (
26     ConnectionError,
27     exceptions.TooManyRequests,
28     exceptions.InternalServerError,
29     exceptions.BadGateway,
30     exceptions.ServiceUnavailable,
31     requests.exceptions.ChunkedEncodingError,
32     requests.exceptions.ConnectionError,
33     requests.exceptions.Timeout,
34     auth_exceptions.TransportError,
35 )
36 
37 _DEFAULT_RETRY_DEADLINE = 10.0 * 60.0  # 10 minutes
38 
39 # Ambiguous errors (e.g. internalError, backendError, rateLimitExceeded) retry
40 # until the full `_DEFAULT_RETRY_DEADLINE`. This is because the
41 # `jobs.getQueryResults` REST API translates a job failure into an HTTP error.
42 #
43 # TODO(https://github.com/googleapis/python-bigquery/issues/1903): Investigate
44 # if we can fail early for ambiguous errors in `QueryJob.result()`'s call to
45 # the `jobs.getQueryResult` API.
46 #
47 # We need `_DEFAULT_JOB_DEADLINE` to be some multiple of
48 # `_DEFAULT_RETRY_DEADLINE` to allow for a few retries after the retry
49 # timeout is reached.
50 #
51 # Note: This multiple should actually be a multiple of
52 # (2 * _DEFAULT_RETRY_DEADLINE). After an ambiguous exception, the first
53 # call from `job_retry()` refreshes the job state without actually restarting
54 # the query. The second `job_retry()` actually restarts the query. For a more
55 # detailed explanation, see the comments where we set `restart_query_job = True`
56 # in `QueryJob.result()`'s  inner `is_job_done()` function.
57 _DEFAULT_JOB_DEADLINE = 2.0 * (2.0 * _DEFAULT_RETRY_DEADLINE)
58 
59 
60 def _should_retry(exc):
61     """Predicate for determining when to retry.
62 
63     We retry if and only if the 'reason' is 'backendError'
64     or 'rateLimitExceeded'.
65     """
66     if not hasattr(exc, "errors") or len(exc.errors) == 0:
67         # Check for unstructured error returns, e.g. from GFE
68         return isinstance(exc, _UNSTRUCTURED_RETRYABLE_TYPES)
69 
70     reason = exc.errors[0]["reason"]
71     return reason in _RETRYABLE_REASONS
72 
73 
74 DEFAULT_RETRY = retry.Retry(predicate=_should_retry, deadline=_DEFAULT_RETRY_DEADLINE)
75 """The default retry object.
76 
77 Any method with a ``retry`` parameter will be retried automatically,
78 with reasonable defaults. To disable retry, pass ``retry=None``.
79 To modify the default retry behavior, call a ``with_XXX`` method
80 on ``DEFAULT_RETRY``. For example, to change the deadline to 30 seconds,
81 pass ``retry=bigquery.DEFAULT_RETRY.with_deadline(30)``.
82 """
83 
84 # Note: Take care when updating DEFAULT_TIMEOUT to anything but None. We
85 # briefly had a default timeout, but even setting it at more than twice the
86 # theoretical server-side default timeout of 2 minutes was not enough for
87 # complex queries. See:
88 # https://github.com/googleapis/python-bigquery/issues/970#issuecomment-921934647
89 DEFAULT_TIMEOUT = None
90 """The default API timeout.
91 
92 This is the time to wait per request. To adjust the total wait time, set a
93 deadline on the retry object.
94 """
95 
96 job_retry_reasons = (
97     "rateLimitExceeded",
98     "backendError",
99     "internalError",
100     "jobRateLimitExceeded",
101 )
102 
103 
104 def _job_should_retry(exc):
105     # Sometimes we have ambiguous errors, such as 'backendError' which could
106     # be due to an API problem or a job problem. For these, make sure we retry
107     # our is_job_done() function.
108     #
109     # Note: This won't restart the job unless we know for sure it's because of
110     # the job status and set restart_query_job = True in that loop. This means
111     # that we might end up calling this predicate twice for the same job
112     # but from different paths: (1) from jobs.getQueryResults RetryError and
113     # (2) from translating the job error from the body of a jobs.get response.
114     #
115     # Note: If we start retrying job types other than queries where we don't
116     # call the problematic getQueryResults API to check the status, we need
117     # to provide a different predicate, as there shouldn't be ambiguous
118     # errors in those cases.
119     if isinstance(exc, exceptions.RetryError):
120         exc = exc.cause
121 
122     if not hasattr(exc, "errors") or len(exc.errors) == 0:
123         return False
124 
125     reason = exc.errors[0]["reason"]
126     return reason in job_retry_reasons
127 
128 
129 DEFAULT_JOB_RETRY = retry.Retry(
130     predicate=_job_should_retry, deadline=_DEFAULT_JOB_DEADLINE
131 )
132 """
133 The default job retry object.
134 """
135
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/google/cloud/bigquery/retry.py b/google/cloud/bigquery/retry.py
--- a/google/cloud/bigquery/retry.py
+++ b/google/cloud/bigquery/retry.py
@@ -119,6 +119,14 @@
     if isinstance(exc, exceptions.RetryError):
         exc = exc.cause
 
+    # Per https://github.com/googleapis/python-bigquery/issues/1929, sometimes
+    # retriable errors make their way here. Because of the separate
+    # `restart_query_job` logic to make sure we aren't restarting non-failed
+    # jobs, it should be safe to continue and not totally fail our attempt at
+    # waiting for the query to complete.
+    if _should_retry(exc):
+        return True
+
     if not hasattr(exc, "errors") or len(exc.errors) == 0:
         return False

{"golden_diff": "diff --git a/google/cloud/bigquery/retry.py b/google/cloud/bigquery/retry.py\n--- a/google/cloud/bigquery/retry.py\n+++ b/google/cloud/bigquery/retry.py\n@@ -119,6 +119,14 @@\n     if isinstance(exc, exceptions.RetryError):\n         exc = exc.cause\n \n+    # Per https://github.com/googleapis/python-bigquery/issues/1929, sometimes\n+    # retriable errors make their way here. Because of the separate\n+    # `restart_query_job` logic to make sure we aren't restarting non-failed\n+    # jobs, it should be safe to continue and not totally fail our attempt at\n+    # waiting for the query to complete.\n+    if _should_retry(exc):\n+        return True\n+\n     if not hasattr(exc, \"errors\") or len(exc.errors) == 0:\n         return False\n", "issue": "Retry requests.exceptions.ConnectionError\nHi Team,\r\n\r\nCould you add a retry to this exception? We are running this code in Cloud Function and GKE infrastructure from time to time we get these errors\r\n\r\nBigquery SDK == google-cloud-bigquery==3.18.0\r\n\r\n```\r\nError Type: <class 'requests.exceptions.ConnectionError'> error: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')) \r\nTraceback (most recent call last): File \"/workspace/visionCommon/common.py\", line 117, \r\nin wrapper return func(*args, **kwargs) File \"/workspace/main.py\", line 576, in controller current_step = check_eligibility() File \"/workspace/main.py\", line 314, \r\nin check_eligibility total_rows = service.execute_query(query).result().total_rows File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py\", line 1595, \r\nin result do_get_result() File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py\", line 293, \r\nin retry_wrapped_func return retry_target( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py\", line 153, \r\nin retry_target _retry_error_helper( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_base.py\", line 212, \r\nin _retry_error_helper raise final_exc from source_exc File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py\", line 144, \r\nin retry_target result = target() File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py\", line 1584, \r\nin do_get_result super(QueryJob, self).result(retry=retry, timeout=timeout) File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/base.py\", line 971, \r\nin result return super(_AsyncJob, self).result(timeout=timeout, **kwargs) File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/future/polling.py\", line 256, \r\nin result self._blocking_poll(timeout=timeout, retry=retry, polling=polling) File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py\", line 1326, \r\nin _blocking_poll super(QueryJob, self)._blocking_poll(timeout=timeout, **kwargs) File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/future/polling.py\", line 137, \r\nin _blocking_poll polling(self._done_or_raise)(retry=retry) File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py\", line 293, \r\nin retry_wrapped_func return retry_target( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py\", line 153, \r\nin retry_target _retry_error_helper( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_base.py\", line 212, \r\nin _retry_error_helper raise final_exc from source_exc File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry/retry_unary.py\", line 144, \r\nin retry_target result = target() File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py\", line 1448, \r\nin _done_or_raise self._reload_query_results(retry=retry, timeout=transport_timeout) File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/job/query.py\", line 1429, \r\nin _reload_query_results self._query_results = self._client._get_query_results( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/client.py\", line 1936, \r\nin _get_query_results resource = self._call_api( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/bigquery/client.py\", line 827, \r\nin _call_api return call() File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/_http/__init__.py\", line 482, \r\nin api_request response = self._make_request( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/_http/__init__.py\", line 341, \r\nin _make_request return self._do_request( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/_http/__init__.py\", line 379, \r\nin _do_request return self.http.request( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/google/auth/transport/requests.py\", line 541, \r\nin request response = super(AuthorizedSession, self).request( File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/requests/sessions.py\", line 589, \r\nin request resp = self.send(prep, **send_kwargs) File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/requests/sessions.py\", line 703, \r\nin send r = adapter.send(request, **kwargs) File \"/layers/google.python.pip/pip/lib/python3.9/site-packages/requests/adapters.py\", line 501, \r\nin send raise ConnectionError(err, request=request) requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))\r\n```\r\n\r\n \n", "before_files": [{"content": "# Copyright 2018 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom google.api_core import exceptions\nfrom google.api_core import retry\nfrom google.auth import exceptions as auth_exceptions  # type: ignore\nimport requests.exceptions\n\n\n_RETRYABLE_REASONS = frozenset(\n    [\"rateLimitExceeded\", \"backendError\", \"internalError\", \"badGateway\"]\n)\n\n_UNSTRUCTURED_RETRYABLE_TYPES = (\n    ConnectionError,\n    exceptions.TooManyRequests,\n    exceptions.InternalServerError,\n    exceptions.BadGateway,\n    exceptions.ServiceUnavailable,\n    requests.exceptions.ChunkedEncodingError,\n    requests.exceptions.ConnectionError,\n    requests.exceptions.Timeout,\n    auth_exceptions.TransportError,\n)\n\n_DEFAULT_RETRY_DEADLINE = 10.0 * 60.0  # 10 minutes\n\n# Ambiguous errors (e.g. internalError, backendError, rateLimitExceeded) retry\n# until the full `_DEFAULT_RETRY_DEADLINE`. This is because the\n# `jobs.getQueryResults` REST API translates a job failure into an HTTP error.\n#\n# TODO(https://github.com/googleapis/python-bigquery/issues/1903): Investigate\n# if we can fail early for ambiguous errors in `QueryJob.result()`'s call to\n# the `jobs.getQueryResult` API.\n#\n# We need `_DEFAULT_JOB_DEADLINE` to be some multiple of\n# `_DEFAULT_RETRY_DEADLINE` to allow for a few retries after the retry\n# timeout is reached.\n#\n# Note: This multiple should actually be a multiple of\n# (2 * _DEFAULT_RETRY_DEADLINE). After an ambiguous exception, the first\n# call from `job_retry()` refreshes the job state without actually restarting\n# the query. The second `job_retry()` actually restarts the query. For a more\n# detailed explanation, see the comments where we set `restart_query_job = True`\n# in `QueryJob.result()`'s  inner `is_job_done()` function.\n_DEFAULT_JOB_DEADLINE = 2.0 * (2.0 * _DEFAULT_RETRY_DEADLINE)\n\n\ndef _should_retry(exc):\n    \"\"\"Predicate for determining when to retry.\n\n    We retry if and only if the 'reason' is 'backendError'\n    or 'rateLimitExceeded'.\n    \"\"\"\n    if not hasattr(exc, \"errors\") or len(exc.errors) == 0:\n        # Check for unstructured error returns, e.g. from GFE\n        return isinstance(exc, _UNSTRUCTURED_RETRYABLE_TYPES)\n\n    reason = exc.errors[0][\"reason\"]\n    return reason in _RETRYABLE_REASONS\n\n\nDEFAULT_RETRY = retry.Retry(predicate=_should_retry, deadline=_DEFAULT_RETRY_DEADLINE)\n\"\"\"The default retry object.\n\nAny method with a ``retry`` parameter will be retried automatically,\nwith reasonable defaults. To disable retry, pass ``retry=None``.\nTo modify the default retry behavior, call a ``with_XXX`` method\non ``DEFAULT_RETRY``. For example, to change the deadline to 30 seconds,\npass ``retry=bigquery.DEFAULT_RETRY.with_deadline(30)``.\n\"\"\"\n\n# Note: Take care when updating DEFAULT_TIMEOUT to anything but None. We\n# briefly had a default timeout, but even setting it at more than twice the\n# theoretical server-side default timeout of 2 minutes was not enough for\n# complex queries. See:\n# https://github.com/googleapis/python-bigquery/issues/970#issuecomment-921934647\nDEFAULT_TIMEOUT = None\n\"\"\"The default API timeout.\n\nThis is the time to wait per request. To adjust the total wait time, set a\ndeadline on the retry object.\n\"\"\"\n\njob_retry_reasons = (\n    \"rateLimitExceeded\",\n    \"backendError\",\n    \"internalError\",\n    \"jobRateLimitExceeded\",\n)\n\n\ndef _job_should_retry(exc):\n    # Sometimes we have ambiguous errors, such as 'backendError' which could\n    # be due to an API problem or a job problem. For these, make sure we retry\n    # our is_job_done() function.\n    #\n    # Note: This won't restart the job unless we know for sure it's because of\n    # the job status and set restart_query_job = True in that loop. This means\n    # that we might end up calling this predicate twice for the same job\n    # but from different paths: (1) from jobs.getQueryResults RetryError and\n    # (2) from translating the job error from the body of a jobs.get response.\n    #\n    # Note: If we start retrying job types other than queries where we don't\n    # call the problematic getQueryResults API to check the status, we need\n    # to provide a different predicate, as there shouldn't be ambiguous\n    # errors in those cases.\n    if isinstance(exc, exceptions.RetryError):\n        exc = exc.cause\n\n    if not hasattr(exc, \"errors\") or len(exc.errors) == 0:\n        return False\n\n    reason = exc.errors[0][\"reason\"]\n    return reason in job_retry_reasons\n\n\nDEFAULT_JOB_RETRY = retry.Retry(\n    predicate=_job_should_retry, deadline=_DEFAULT_JOB_DEADLINE\n)\n\"\"\"\nThe default job retry object.\n\"\"\"\n", "path": "google/cloud/bigquery/retry.py"}], "after_files": [{"content": "# Copyright 2018 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom google.api_core import exceptions\nfrom google.api_core import retry\nfrom google.auth import exceptions as auth_exceptions  # type: ignore\nimport requests.exceptions\n\n\n_RETRYABLE_REASONS = frozenset(\n    [\"rateLimitExceeded\", \"backendError\", \"internalError\", \"badGateway\"]\n)\n\n_UNSTRUCTURED_RETRYABLE_TYPES = (\n    ConnectionError,\n    exceptions.TooManyRequests,\n    exceptions.InternalServerError,\n    exceptions.BadGateway,\n    exceptions.ServiceUnavailable,\n    requests.exceptions.ChunkedEncodingError,\n    requests.exceptions.ConnectionError,\n    requests.exceptions.Timeout,\n    auth_exceptions.TransportError,\n)\n\n_DEFAULT_RETRY_DEADLINE = 10.0 * 60.0  # 10 minutes\n\n# Ambiguous errors (e.g. internalError, backendError, rateLimitExceeded) retry\n# until the full `_DEFAULT_RETRY_DEADLINE`. This is because the\n# `jobs.getQueryResults` REST API translates a job failure into an HTTP error.\n#\n# TODO(https://github.com/googleapis/python-bigquery/issues/1903): Investigate\n# if we can fail early for ambiguous errors in `QueryJob.result()`'s call to\n# the `jobs.getQueryResult` API.\n#\n# We need `_DEFAULT_JOB_DEADLINE` to be some multiple of\n# `_DEFAULT_RETRY_DEADLINE` to allow for a few retries after the retry\n# timeout is reached.\n#\n# Note: This multiple should actually be a multiple of\n# (2 * _DEFAULT_RETRY_DEADLINE). After an ambiguous exception, the first\n# call from `job_retry()` refreshes the job state without actually restarting\n# the query. The second `job_retry()` actually restarts the query. For a more\n# detailed explanation, see the comments where we set `restart_query_job = True`\n# in `QueryJob.result()`'s  inner `is_job_done()` function.\n_DEFAULT_JOB_DEADLINE = 2.0 * (2.0 * _DEFAULT_RETRY_DEADLINE)\n\n\ndef _should_retry(exc):\n    \"\"\"Predicate for determining when to retry.\n\n    We retry if and only if the 'reason' is 'backendError'\n    or 'rateLimitExceeded'.\n    \"\"\"\n    if not hasattr(exc, \"errors\") or len(exc.errors) == 0:\n        # Check for unstructured error returns, e.g. from GFE\n        return isinstance(exc, _UNSTRUCTURED_RETRYABLE_TYPES)\n\n    reason = exc.errors[0][\"reason\"]\n    return reason in _RETRYABLE_REASONS\n\n\nDEFAULT_RETRY = retry.Retry(predicate=_should_retry, deadline=_DEFAULT_RETRY_DEADLINE)\n\"\"\"The default retry object.\n\nAny method with a ``retry`` parameter will be retried automatically,\nwith reasonable defaults. To disable retry, pass ``retry=None``.\nTo modify the default retry behavior, call a ``with_XXX`` method\non ``DEFAULT_RETRY``. For example, to change the deadline to 30 seconds,\npass ``retry=bigquery.DEFAULT_RETRY.with_deadline(30)``.\n\"\"\"\n\n# Note: Take care when updating DEFAULT_TIMEOUT to anything but None. We\n# briefly had a default timeout, but even setting it at more than twice the\n# theoretical server-side default timeout of 2 minutes was not enough for\n# complex queries. See:\n# https://github.com/googleapis/python-bigquery/issues/970#issuecomment-921934647\nDEFAULT_TIMEOUT = None\n\"\"\"The default API timeout.\n\nThis is the time to wait per request. To adjust the total wait time, set a\ndeadline on the retry object.\n\"\"\"\n\njob_retry_reasons = (\n    \"rateLimitExceeded\",\n    \"backendError\",\n    \"internalError\",\n    \"jobRateLimitExceeded\",\n)\n\n\ndef _job_should_retry(exc):\n    # Sometimes we have ambiguous errors, such as 'backendError' which could\n    # be due to an API problem or a job problem. For these, make sure we retry\n    # our is_job_done() function.\n    #\n    # Note: This won't restart the job unless we know for sure it's because of\n    # the job status and set restart_query_job = True in that loop. This means\n    # that we might end up calling this predicate twice for the same job\n    # but from different paths: (1) from jobs.getQueryResults RetryError and\n    # (2) from translating the job error from the body of a jobs.get response.\n    #\n    # Note: If we start retrying job types other than queries where we don't\n    # call the problematic getQueryResults API to check the status, we need\n    # to provide a different predicate, as there shouldn't be ambiguous\n    # errors in those cases.\n    if isinstance(exc, exceptions.RetryError):\n        exc = exc.cause\n\n    # Per https://github.com/googleapis/python-bigquery/issues/1929, sometimes\n    # retriable errors make their way here. Because of the separate\n    # `restart_query_job` logic to make sure we aren't restarting non-failed\n    # jobs, it should be safe to continue and not totally fail our attempt at\n    # waiting for the query to complete.\n    if _should_retry(exc):\n        return True\n\n    if not hasattr(exc, \"errors\") or len(exc.errors) == 0:\n        return False\n\n    reason = exc.errors[0][\"reason\"]\n    return reason in job_retry_reasons\n\n\nDEFAULT_JOB_RETRY = retry.Retry(\n    predicate=_job_should_retry, deadline=_DEFAULT_JOB_DEADLINE\n)\n\"\"\"\nThe default job retry object.\n\"\"\"\n", "path": "google/cloud/bigquery/retry.py"}]}