Skip to content
Snippets Groups Projects
Commit 434838f9 authored by ale's avatar ale
Browse files

Improve the implementation of timeouts

Now we no longer rely on 'backoff' re-raising the urllib3-specific
TimeoutError, but we use the 'outer loop' to implement the request
timeout, possibly repeating the backoff loop multiple times, with an
accuracy determined by the max backoff delay.

This way the timeout is at least vaguely accurate.
parent 900de095
No related branches found
No related tags found
1 merge request!1Refactor rpc library
...@@ -18,9 +18,7 @@ DNS_CACHE_TTL = 60 ...@@ -18,9 +18,7 @@ DNS_CACHE_TTL = 60
DEFAULT_REQUEST_TIMEOUT = 10 DEFAULT_REQUEST_TIMEOUT = 10
DEFAULT_MAX_TIMEOUT = 30 DEFAULT_MAX_TIMEOUT = 30
DEFAULT_MAX_BACKOFF_INTERVAL = 3
def _get_global_max_timeout():
return DEFAULT_MAX_TIMEOUT
HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL'] HTTP_URL = attributes_helper.COMMON_ATTRIBUTES['HTTP_URL']
...@@ -211,10 +209,10 @@ class ClientStub(): ...@@ -211,10 +209,10 @@ class ClientStub():
RetriableStatusError), RetriableStatusError),
base=1.4142, base=1.4142,
factor=0.1, factor=0.1,
max_value=5, max_value=DEFAULT_MAX_BACKOFF_INTERVAL,
max_time=_get_global_max_timeout) max_time=DEFAULT_MAX_TIMEOUT)
def _do_single_request(self, parsed_url, targets, request_data, def _do_single_request(self, parsed_url, targets, request_data,
headers, response_decoder): headers, response_decoder, deadline):
"""Issue a request, retry on temporary errors. """Issue a request, retry on temporary errors.
Temporary errors include all HTTP status codes >500, and the Temporary errors include all HTTP status codes >500, and the
...@@ -231,6 +229,12 @@ class ClientStub(): ...@@ -231,6 +229,12 @@ class ClientStub():
""" """
# By checking the timestamp now, we can bail out early
# even if the 'backoff' retry loop is still running (with
# an approximate correctness of +/- max_value).
if time.time() > deadline:
raise TimeoutError()
target_addr, target_port = targets.next() target_addr, target_port = targets.next()
tracer = execution_context.get_opencensus_tracer() tracer = execution_context.get_opencensus_tracer()
...@@ -317,13 +321,17 @@ class ClientStub(): ...@@ -317,13 +321,17 @@ class ClientStub():
else: else:
targets = _Targets.resolve(parsed_url.host, parsed_url.port) targets = _Targets.resolve(parsed_url.host, parsed_url.port)
# Run an outer loop to satisfy the overall request timeout,
# calling the backoff inner loop multiple times during the
# process. We do this because the backoff timeout is set once
# globally in the decorator, and we can't influence it
# per-request.
ignore_cache = False ignore_cache = False
deadline = time.time() + self._timeout deadline = time.time() + self._timeout
while True: while True:
now = time.time()
if now > deadline:
raise TimeoutError()
# Acquire a SSO token if the service requires it by
# calling the sso_exchange function.
if self._sso_service and self._sso_exchange: if self._sso_service and self._sso_exchange:
kwargs['sso'] = self._sso_exchange( kwargs['sso'] = self._sso_exchange(
self._sso_service, ignore_cache=ignore_cache) self._sso_service, ignore_cache=ignore_cache)
...@@ -335,11 +343,11 @@ class ClientStub(): ...@@ -335,11 +343,11 @@ class ClientStub():
request_encoder(kwargs), request_encoder(kwargs),
headers, headers,
response_decoder, response_decoder,
deadline,
) )
except urllib3.exceptions.TimeoutError: except urllib3.exceptions.TimeoutError:
# Rewrite the urllib3 exception as a nicer # Rely on the deadline check in _do_single_request().
# TimeoutError. pass
raise TimeoutError()
except AuthenticationError: except AuthenticationError:
if ignore_cache: if ignore_cache:
raise raise
......
...@@ -18,7 +18,8 @@ from cryptography.x509.oid import NameOID ...@@ -18,7 +18,8 @@ from cryptography.x509.oid import NameOID
from ai_web_common.rpc import core from ai_web_common.rpc import core
core.DEFAULT_MAX_TIMEOUT = 3 # Set a shorter timeout for tests.
core.DEFAULT_REQUEST_TIMEOUT = 3
def generate_ca(): def generate_ca():
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment