Description
This test failed!
To configure my behavior, see the Flaky Bot documentation.
If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.
commit: 2487800
buildURL: Build Status, Sponge
status: failed
Test output
target = functools.partial(>) predicate = .if_exception_type_predicate at 0x7fe795a5a5e0> sleep_generator = deadline = 60, on_error = Nonedef retry_target(target, predicate, sleep_generator, deadline, on_error=None): """Call a function and retry if it fails. This is the lowest-level retry helper. Generally, you'll use the higher-level retry helper :class:`Retry`. Args: target(Callable): The function to call and retry. This must be a nullary function - apply arguments with `functools.partial`. predicate (Callable[Exception]): A callable used to determine if an exception raised by the target should be considered retryable. It should return True to retry or False otherwise. sleep_generator (Iterable[float]): An infinite iterator that determines how long to sleep between retries. deadline (float): How long to keep retrying the target. The last sleep period is shortened as necessary, so that the last retry runs at ``deadline`` (and not considerably beyond it). on_error (Callable[Exception]): A function to call while processing a retryable exception. Any error raised by this function will *not* be caught. Returns: Any: the return value of the target function. Raises: google.api_core.RetryError: If the deadline is exceeded while retrying. ValueError: If the sleep generator stops yielding values. Exception: If the target raises a method that isn't retryable. """ if deadline is not None: deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta( seconds=deadline ) else: deadline_datetime = None last_exc = None for sleep in sleep_generator: try:return target().nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189:
self = <google.api_core.operation.Operation object at 0x7fe795521b20>
retry = <google.api_core.retry.Retry object at 0x7fe795a4ca30>def _done_or_raise(self, retry=DEFAULT_RETRY): """Check if the future is done and raise if it's not.""" kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry} if not self.done(**kwargs):raise _OperationNotComplete()E google.api_core.future.polling._OperationNotComplete
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:87: _OperationNotComplete
The above exception was the direct cause of the following exception:
self = <google.api_core.operation.Operation object at 0x7fe795521b20>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7fe795a4ca30>def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY): """Poll and wait for the Future to be resolved. Args: timeout (int): How long (in seconds) to wait for the operation to complete. If None, wait indefinitely. """ if self._result_set: return retry_ = self._retry.with_deadline(timeout) try: kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}retry_(self._done_or_raise)(**kwargs).nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108:
args = (), kwargs = {}
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fe795521b20>>)
sleep_generator = <generator object exponential_sleep_generator at 0x7fe7954feb30>@general_helpers.wraps(func) def retry_wrapped_func(*args, **kwargs): """A wrapper that calls target function with retry.""" target = functools.partial(func, *args, **kwargs) sleep_generator = exponential_sleep_generator( self._initial, self._maximum, multiplier=self._multiplier )return retry_target(target, self._predicate, sleep_generator, self._deadline, on_error=on_error, ).nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286:
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fe795521b20>>)
predicate = <function if_exception_type..if_exception_type_predicate at 0x7fe795a5a5e0>
sleep_generator = <generator object exponential_sleep_generator at 0x7fe7954feb30>
deadline = 60, on_error = Nonedef retry_target(target, predicate, sleep_generator, deadline, on_error=None): """Call a function and retry if it fails. This is the lowest-level retry helper. Generally, you'll use the higher-level retry helper :class:`Retry`. Args: target(Callable): The function to call and retry. This must be a nullary function - apply arguments with `functools.partial`. predicate (Callable[Exception]): A callable used to determine if an exception raised by the target should be considered retryable. It should return True to retry or False otherwise. sleep_generator (Iterable[float]): An infinite iterator that determines how long to sleep between retries. deadline (float): How long to keep retrying the target. The last sleep period is shortened as necessary, so that the last retry runs at ``deadline`` (and not considerably beyond it). on_error (Callable[Exception]): A function to call while processing a retryable exception. Any error raised by this function will *not* be caught. Returns: Any: the return value of the target function. Raises: google.api_core.RetryError: If the deadline is exceeded while retrying. ValueError: If the sleep generator stops yielding values. Exception: If the target raises a method that isn't retryable. """ if deadline is not None: deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta( seconds=deadline ) else: deadline_datetime = None last_exc = None for sleep in sleep_generator: try: return target() # pylint: disable=broad-except # This function explicitly must deal with broad exceptions. except Exception as exc: if not predicate(exc): raise last_exc = exc if on_error is not None: on_error(exc) now = datetime_helpers.utcnow() if deadline_datetime is not None: if deadline_datetime <= now:six.raise_from(exceptions.RetryError( "Deadline of {:.1f}s exceeded while calling {}".format( deadline, target ), last_exc, ), last_exc, ).nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:204:
value = None, from_value = _OperationNotComplete()
???
E google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fe795521b20>>), last exception::3: RetryError
During handling of the above exception, another exception occurred:
def setUpModule(): if USE_EMULATOR: from google.auth.credentials import AnonymousCredentials emulator_project = os.getenv("GCLOUD_PROJECT", "emulator-test-project") Config.CLIENT = Client( project=emulator_project, credentials=AnonymousCredentials() ) else: Config.CLIENT = Client() retry = RetryErrors(exceptions.ServiceUnavailable) configs = list(retry(Config.CLIENT.list_instance_configs)()) instances = retry(_list_instances)() EXISTING_INSTANCES[:] = instances # Delete test instances that are older than an hour. cutoff = int(time.time()) - 1 * 60 * 60 instance_pbs = Config.CLIENT.list_instances("labels.python-spanner-systests:true") for instance_pb in instance_pbs: instance = Instance.from_pb(instance_pb, Config.CLIENT) if "created" not in instance.labels: continue create_time = int(instance.labels["created"]) if create_time > cutoff: continue # Instance cannot be deleted while backups exist. for backup_pb in instance.list_backups(): backup = Backup.from_pb(backup_pb, instance) backup.delete() instance.delete() if CREATE_INSTANCE: if not USE_EMULATOR: # Defend against back-end returning configs for regions we aren't # actually allowed to use. configs = [config for config in configs if "-us-" in config.name] if not configs: raise ValueError("List instance configs failed in module set up.") Config.INSTANCE_CONFIG = configs[0] config_name = configs[0].name create_time = str(int(time.time())) labels = {"python-spanner-systests": "true", "created": create_time} Config.INSTANCE = Config.CLIENT.instance( INSTANCE_ID, config_name, labels=labels ) created_op = Config.INSTANCE.create()created_op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS ) # block until completiontests/system/test_system.py:159:
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result
self._blocking_poll(timeout=timeout, **kwargs)
self = <google.api_core.operation.Operation object at 0x7fe795521b20>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7fe795a4ca30>def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY): """Poll and wait for the Future to be resolved. Args: timeout (int): How long (in seconds) to wait for the operation to complete. If None, wait indefinitely. """ if self._result_set: return retry_ = self._retry.with_deadline(timeout) try: kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry} retry_(self._done_or_raise)(**kwargs) except exceptions.RetryError:raise concurrent.futures.TimeoutError("Operation did not complete within the designated " "timeout." )E concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: TimeoutError