Skip to content
Navigation Menu
{{ message }}
forked from googleapis/python-api-core
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtimeout.py
More file actions
284 lines (219 loc) · 9.46 KB
/
Copy pathtimeout.py
File metadata and controls
284 lines (219 loc) · 9.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
# Copyright 2017 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Decorators for applying timeout arguments to functions.
These decorators are used to wrap API methods to apply either a
Deadline-dependent (recommended), constant (DEPRECATED) or exponential
(DEPRECATED) timeout argument.
For example, imagine an API method that can take a while to return results,
such as one that might block until a resource is ready:
.. code-block:: python
def is_thing_ready(timeout=None):
response = requests.get('https://example.com/is_thing_ready')
response.raise_for_status()
return response.json()
This module allows a function like this to be wrapped so that timeouts are
automatically determined, for example:
.. code-block:: python
timeout_ = timeout.ExponentialTimeout()
is_thing_ready_with_timeout = timeout_(is_thing_ready)
for n in range(10):
try:
is_thing_ready_with_timeout({'example': 'data'})
except:
pass
In this example the first call to ``is_thing_ready`` will have a relatively
small timeout (like 1 second). If the resource is available and the request
completes quickly, the loop exits. But, if the resource isn't yet available
and the request times out, it'll be retried - this time with a larger timeout.
In the broader context these decorators are typically combined with
:mod:`google.api_core.retry` to implement API methods with a signature that
matches ``api_method(request, timeout=None, retry=None)``.
"""
from __future__ import unicode_literals
import datetime
import functools
from google.api_core import datetime_helpers
_DEFAULT_INITIAL_TIMEOUT = 5.0 # seconds
_DEFAULT_MAXIMUM_TIMEOUT = 30.0 # seconds
_DEFAULT_TIMEOUT_MULTIPLIER = 2.0
# If specified, must be in seconds. If none, deadline is not used in the
# timeout calculation.
_DEFAULT_DEADLINE = None
class TimeToDeadlineTimeout(object):
"""A decorator that decreases timeout set for an RPC based on how much time
has left till its deadline. The deadline is calculated as
``now + initial_timeout`` when this decorator is first called for an rpc.
In other words this decorator implements deadline semantics in terms of a
sequence of decreasing timeouts t0 > t1 > t2 ... tn >= 0.
Args:
timeout (Optional[float]): the timeout (in seconds) to applied to the
wrapped function. If `None`, the target function is expected to
never timeout.
"""
def __init__(self, timeout=None, clock=datetime_helpers.utcnow):
self._timeout = timeout
self._clock = clock
def __call__(self, func):
"""Apply the timeout decorator.
Args:
func (Callable): The function to apply the timeout argument to.
This function must accept a timeout keyword argument.
Returns:
Callable: The wrapped function.
"""
first_attempt_timestamp = self._clock().timestamp()
@functools.wraps(func)
def func_with_timeout(*args, **kwargs):
"""Wrapped function that adds timeout."""
remaining_timeout = self._timeout
if remaining_timeout is not None:
# All calculations are in seconds
now_timestamp = self._clock().timestamp()
# To avoid usage of nonlocal but still have round timeout
# numbers for first attempt (in most cases the only attempt made
# for an RPC.
if now_timestamp - first_attempt_timestamp < 0.001:
now_timestamp = first_attempt_timestamp
time_since_first_attempt = now_timestamp - first_attempt_timestamp
# Avoid setting negative timeout
kwargs["timeout"] = max(0, self._timeout - time_since_first_attempt)
return func(*args, **kwargs)
return func_with_timeout
def __str__(self):
return "<TimeToDeadlineTimeout timeout={:.1f}>".format(self._timeout)
class ConstantTimeout(object):
"""A decorator that adds a constant timeout argument.
DEPRECATED: use ``TimeToDeadlineTimeout`` instead.
This is effectively equivalent to
``functools.partial(func, timeout=timeout)``.
Args:
timeout (Optional[float]): the timeout (in seconds) to applied to the
wrapped function. If `None`, the target function is expected to
never timeout.
"""
def __init__(self, timeout=None):
self._timeout = timeout
def __call__(self, func):
"""Apply the timeout decorator.
Args:
func (Callable): The function to apply the timeout argument to.
This function must accept a timeout keyword argument.
Returns:
Callable: The wrapped function.
"""
@functools.wraps(func)
def func_with_timeout(*args, **kwargs):
"""Wrapped function that adds timeout."""
kwargs["timeout"] = self._timeout
return func(*args, **kwargs)
return func_with_timeout
def __str__(self):
return "<ConstantTimeout timeout={:.1f}>".format(self._timeout)
def _exponential_timeout_generator(initial, maximum, multiplier, deadline):
"""A generator that yields exponential timeout values.
Args:
initial (float): The initial timeout.
maximum (float): The maximum timeout.
multiplier (float): The multiplier applied to the timeout.
deadline (float): The overall deadline across all invocations.
Yields:
float: A timeout value.
"""
if deadline is not None:
deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
seconds=deadline
)
else:
deadline_datetime = datetime.datetime.max
timeout = initial
while True:
now = datetime_helpers.utcnow()
yield min(
# The calculated timeout based on invocations.
timeout,
# The set maximum timeout.
maximum,
# The remaining time before the deadline is reached.
float((deadline_datetime - now).seconds),
)
timeout = timeout * multiplier
class ExponentialTimeout(object):
"""A decorator that adds an exponentially increasing timeout argument.
DEPRECATED: the concept of incrementing timeout exponentially has been
deprecated. Use ``TimeToDeadlineTimeout`` instead.
This is useful if a function is called multiple times. Each time the
function is called this decorator will calculate a new timeout parameter
based on the the number of times the function has been called.
For example
.. code-block:: python
Args:
initial (float): The initial timeout to pass.
maximum (float): The maximum timeout for any one call.
multiplier (float): The multiplier applied to the timeout for each
invocation.
deadline (Optional[float]): The overall deadline across all
invocations. This is used to prevent a very large calculated
timeout from pushing the overall execution time over the deadline.
This is especially useful in conjunction with
:mod:`google.api_core.retry`. If ``None``, the timeouts will not
be adjusted to accommodate an overall deadline.
"""
def __init__(
self,
initial=_DEFAULT_INITIAL_TIMEOUT,
maximum=_DEFAULT_MAXIMUM_TIMEOUT,
multiplier=_DEFAULT_TIMEOUT_MULTIPLIER,
deadline=_DEFAULT_DEADLINE,
):
self._initial = initial
self._maximum = maximum
self._multiplier = multiplier
self._deadline = deadline
def with_deadline(self, deadline):
"""Return a copy of this timeout with the given deadline.
Args:
deadline (float): The overall deadline across all invocations.
Returns:
ExponentialTimeout: A new instance with the given deadline.
"""
return ExponentialTimeout(
initial=self._initial,
maximum=self._maximum,
multiplier=self._multiplier,
deadline=deadline,
)
def __call__(self, func):
"""Apply the timeout decorator.
Args:
func (Callable): The function to apply the timeout argument to.
This function must accept a timeout keyword argument.
Returns:
Callable: The wrapped function.
"""
timeouts = _exponential_timeout_generator(
self._initial, self._maximum, self._multiplier, self._deadline
)
@functools.wraps(func)
def func_with_timeout(*args, **kwargs):
"""Wrapped function that adds timeout."""
kwargs["timeout"] = next(timeouts)
return func(*args, **kwargs)
return func_with_timeout
def __str__(self):
return (
"<ExponentialTimeout initial={:.1f}, maximum={:.1f}, "
"multiplier={:.1f}, deadline={:.1f}>".format(
self._initial, self._maximum, self._multiplier, self._deadline
)
)
You can’t perform that action at this time.