Cant invoke the endpoint! (104 Connection rest by peer) #448
Description
Please fill out the form below.
System Information
- Framework (e.g. TensorFlow) / Algorithm (e.g. KMeans): Tensorflow
- Framework Version: 1.10
- Python Version: 2
- CPU or GPU: CPU
- Python SDK Version:
- Are you using a custom image: No
Describe the problem
Describe the problem or feature request clearly here.
When I try to invoke my newly created endoint, I get a 104 error.
Is this a timeout issue? It is a fairly large network
Minimal repro / logs
`---------------------------------------------------------------------------
ConnectionResetError Traceback (most recent call last)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
543 timeout=timeout_obj,
--> 544 body=body, headers=headers)
545
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
348 # urllib3.request. It also calls makefile (recv) on the socket.
--> 349 conn.request(method, url, **httplib_request_kw)
350
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
1238 """Send a complete request to the server."""
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
1240
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in _send_request(self, method, url, body, headers, *args, **kwargs)
129 rval = HTTPConnection._send_request(
--> 130 self, method, url, body, headers, *args, **kwargs)
131 self._expect_header_set = False
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
1284 body = _encode(body, 'body')
-> 1285 self.endheaders(body, encode_chunked=encode_chunked)
1286
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in endheaders(self, message_body, encode_chunked)
1233 raise CannotSendHeader()
-> 1234 self._send_output(message_body, encode_chunked=encode_chunked)
1235
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in _send_output(self, message_body, *args, **kwargs)
156 message_body = None
--> 157 self.send(msg)
158 if self._expect_header_set:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in send(self, str)
241 return
--> 242 return HTTPConnection.send(self, str)
243
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in send(self, data)
985 try:
--> 986 self.sock.sendall(data)
987 except TypeError:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in sendall(self, data, flags)
971 while count < amount:
--> 972 v = self.send(byte_view[count:])
973 count += v
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in send(self, data, flags)
940 self.class)
--> 941 return self._sslobj.write(data)
942 else:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in write(self, data)
641 """
--> 642 return self._sslobj.write(data)
643
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
ProtocolError Traceback (most recent call last)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
369 retries=self.max_retries,
--> 370 timeout=timeout
371 )
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
596 retries = retries.increment(method, url, error=e, _pool=self,
--> 597 _stacktrace=sys.exc_info()[2])
598 retries.sleep()
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
244 if read is False:
--> 245 raise six.reraise(type(error), error, _stacktrace)
246 elif read is not None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/packages/six.py in reraise(tp, value, tb)
308 if value.traceback is not tb:
--> 309 raise value.with_traceback(tb)
310 raise value
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
543 timeout=timeout_obj,
--> 544 body=body, headers=headers)
545
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
348 # urllib3.request. It also calls makefile (recv) on the socket.
--> 349 conn.request(method, url, **httplib_request_kw)
350
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in request(self, method, url, body, headers, encode_chunked)
1238 """Send a complete request to the server."""
-> 1239 self._send_request(method, url, body, headers, encode_chunked)
1240
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in _send_request(self, method, url, body, headers, *args, **kwargs)
129 rval = HTTPConnection._send_request(
--> 130 self, method, url, body, headers, *args, **kwargs)
131 self._expect_header_set = False
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in _send_request(self, method, url, body, headers, encode_chunked)
1284 body = _encode(body, 'body')
-> 1285 self.endheaders(body, encode_chunked=encode_chunked)
1286
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in endheaders(self, message_body, encode_chunked)
1233 raise CannotSendHeader()
-> 1234 self._send_output(message_body, encode_chunked=encode_chunked)
1235
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in _send_output(self, message_body, *args, **kwargs)
156 message_body = None
--> 157 self.send(msg)
158 if self._expect_header_set:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/awsrequest.py in send(self, str)
241 return
--> 242 return HTTPConnection.send(self, str)
243
~/anaconda3/envs/tensorflow_p36/lib/python3.6/http/client.py in send(self, data)
985 try:
--> 986 self.sock.sendall(data)
987 except TypeError:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in sendall(self, data, flags)
971 while count < amount:
--> 972 v = self.send(byte_view[count:])
973 count += v
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in send(self, data, flags)
940 self.class)
--> 941 return self._sslobj.write(data)
942 else:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/ssl.py in write(self, data)
641 """
--> 642 return self._sslobj.write(data)
643
ProtocolError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
in ()
----> 1 endpoint.predict(json.dumps(test_dict))
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/sagemaker/predictor.py in predict(self, data)
85 request_args['Accept'] = self.accept
86
---> 87 response = self.sagemaker_session.sagemaker_runtime_client.invoke_endpoint(**request_args)
88
89 response_body = response['Body']
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
312 "%s() only accepts keyword arguments." % py_operation_name)
313 # The "self" in this scope is referring to the BaseClient.
--> 314 return self._make_api_call(operation_name, kwargs)
315
316 _api_call.name = str(py_operation_name)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
597 else:
598 http, parsed_response = self._endpoint.make_request(
--> 599 operation_model, request_dict)
600
601 self.meta.events.emit(
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
146 logger.debug("Making request for %s (verify_ssl=%s) with params: %s",
147 operation_model, self.verify, request_dict)
--> 148 return self._send_request(request_dict, operation_model)
149
150 def create_request(self, params, operation_model=None):
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
175 request, operation_model, attempts)
176 while self._needs_retry(attempts, operation_model, request_dict,
--> 177 success_response, exception):
178 attempts += 1
179 # If there is a stream associated with the request, we need
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
271 event_name, response=response, endpoint=self,
272 operation=operation_model, attempts=attempts,
--> 273 caught_exception=caught_exception, request_dict=request_dict)
274 handler_response = first_non_none_response(responses)
275 if handler_response is None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
225 handlers.
226 """
--> 227 return self._emit(event_name, kwargs)
228
229 def emit_until_response(self, event_name, **kwargs):
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
358 aliased_event_name = self._alias_event_name(event_name)
359 return super(AliasedEventEmitter, self)._emit(
--> 360 aliased_event_name, kwargs, stop_on_response
361 )
362
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
208 for handler in handlers_to_call:
209 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 210 response = handler(**kwargs)
211 responses.append((handler, response))
212 if stop_on_response and response is not None:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempts, response, caught_exception, **kwargs)
181
182 """
--> 183 if self._checker(attempts, response, caught_exception):
184 result = self._action(attempts=attempts)
185 logger.debug("Retry needed, action of: %s", result)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
249 def call(self, attempt_number, response, caught_exception):
250 should_retry = self._should_retry(attempt_number, response,
--> 251 caught_exception)
252 if should_retry:
253 if attempt_number >= self._max_attempts:
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
275 # If we've exceeded the max attempts we just let the exception
276 # propogate if one has occurred.
--> 277 return self._checker(attempt_number, response, caught_exception)
278
279
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
315 for checker in self._checkers:
316 checker_response = checker(attempt_number, response,
--> 317 caught_exception)
318 if checker_response:
319 return checker_response
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in call(self, attempt_number, response, caught_exception)
221 elif caught_exception is not None:
222 return self._check_caught_exception(
--> 223 attempt_number, caught_exception)
224 else:
225 raise ValueError("Both response and caught_exception are None.")
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
357 # the MaxAttemptsDecorator is not interested in retrying the exception
358 # then this exception just propogates out past the retry code.
--> 359 raise caught_exception
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/endpoint.py in _get_response(self, request, operation_model, attempts)
220 request, verify=self.verify,
221 stream=streaming,
--> 222 proxies=self.proxies, timeout=self.timeout)
223 except ConnectionError as e:
224 # For a connection error, if it looks like it's a DNS
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/sessions.py in send(self, request, **kwargs)
571
572 # Send the request
--> 573 r = adapter.send(request, **kwargs)
574
575 # Total elapsed time of the request (approximately)
~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/botocore/vendored/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
413
414 except (ProtocolError, socket.error) as err:
--> 415 raise ConnectionError(err, request=request)
416
417 except MaxRetryError as e:
ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))`
- Exact command to reproduce:
endpoint = my_estimator.deploy(initial_instance_count=1, instance_type="ml.m4.xlarge")
endpoint.predict(test_dict)