Flaky test: boto error connecting to endpoint

[alexowens90]

==================================== ERRORS ====================================
_______________ ERROR at setup of test_add_to_snapshot_atomicity _______________
[gw1] linux -- Python 3.10.14 /home/runner/micromamba/envs/arcticdb/bin/python

self = <botocore.awsrequest.AWSHTTPConnection object at 0x7fd8fc969e40>

    def _new_conn(self) -> socket.socket:
        """Establish a socket connection and set nodelay settings on it.
    
        :return: New socket connection.
        """
        try:
>           sock = connection.create_connection(
                (self._dns_host, self.port),
                self.timeout,
                source_address=self.source_address,
                socket_options=self.socket_options,
            )

../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/connection.py:196: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/util/connection.py:85: in create_connection
    raise err
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

address = ('localhost', 13167), timeout = 60, source_address = None
socket_options = [(6, 1, 1)]

    def create_connection(
        address: tuple[str, int],
        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
        source_address: tuple[str, int] | None = None,
        socket_options: _TYPE_SOCKET_OPTIONS | None = None,
    ) -> socket.socket:
        """Connect to *address* and return the socket object.
    
        Convenience function.  Connect to *address* (a 2-tuple ``(host,
        port)``) and return the socket object.  Passing the optional
        *timeout* parameter will set the timeout on the socket instance
        before attempting to connect.  If no *timeout* is supplied, the
        global default timeout setting returned by :func:`socket.getdefaulttimeout`
        is used.  If *source_address* is set it must be a tuple of (host, port)
        for the socket to bind as a source address before making the connection.
        An host of '' or port 0 tells the OS to use the default.
        """
    
        host, port = address
        if host.startswith("["):
            host = host.strip("[]")
        err = None
    
        # Using the value from allowed_gai_family() in the context of getaddrinfo lets
        # us select whether to work with IPv4 DNS records, IPv6 records, or both.
        # The original create_connection function always returns all records.
        family = allowed_gai_family()
    
        try:
            host.encode("idna")
        except UnicodeError:
            raise LocationParseError(f"'{host}', label empty or too long") from None
    
        for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
            af, socktype, proto, canonname, sa = res
            sock = None
            try:
                sock = socket.socket(af, socktype, proto)
    
                # If provided, set socket level options before connecting.
                _set_socket_options(sock, socket_options)
    
                if timeout is not _DEFAULT_TIMEOUT:
                    sock.settimeout(timeout)
                if source_address:
                    sock.bind(source_address)
>               sock.connect(sa)
E               ConnectionRefusedError: [Errno 111] Connection refused

../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/util/connection.py:73: ConnectionRefusedError

The above exception was the direct cause of the following exception:

self = <botocore.httpsession.URLLib3Session object at 0x7fd8fc92be50>
request = <AWSPreparedRequest stream_output=False, method=PUT, url=http://localhost:13167/test_bucket_0, headers={'User-Agent': ...nvocation-id': b'c946d3da-b688-49d0-84c3-daafab30cb51', 'amz-sdk-request': b'attempt=5; max=5', 'Content-Length': '0'}>

    def send(self, request):
        try:
            proxy_url = self._proxy_config.proxy_url_for(request.url)
            manager = self._get_connection_manager(request.url, proxy_url)
            conn = manager.connection_from_url(request.url)
            self._setup_ssl_cert(conn, request.url, self._verify)
            if ensure_boolean(
                os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '')
            ):
                # This is currently an "experimental" feature which provides
                # no guarantees of backwards compatibility. It may be subject
                # to change or removal in any patch version. Anyone opting in
                # to this feature should strictly pin botocore.
                host = urlparse(request.url).hostname
                conn.proxy_headers['host'] = host
    
            request_target = self._get_request_target(request.url, proxy_url)
>           urllib_response = conn.urlopen(
                method=request.method,
                url=request_target,
                body=request.body,
                headers=request.headers,
                retries=Retry(False),
                assert_same_host=False,
                preload_content=False,
                decode_content=False,
                chunked=self._chunked(request.headers),
            )

../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/httpsession.py:464: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/connectionpool.py:843: in urlopen
    retries = retries.increment(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/util/retry.py:449: in increment
    raise reraise(type(error), error, _stacktrace)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/util/util.py:39: in reraise
    raise value
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/connectionpool.py:789: in urlopen
    response = self._make_request(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/connectionpool.py:495: in _make_request
    conn.request(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/awsrequest.py:96: in request
    rval = super().request(method, url, body, headers, *args, **kwargs)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/connection.py:398: in request
    self.endheaders()
../../../../micromamba/envs/arcticdb/lib/python3.10/http/client.py:1278: in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/awsrequest.py:123: in _send_output
    self.send(msg)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/awsrequest.py:223: in send
    return super().send(str)
../../../../micromamba/envs/arcticdb/lib/python3.10/http/client.py:976: in send
    self.connect()
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/connection.py:236: in connect
    self.sock = self._new_conn()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <botocore.awsrequest.AWSHTTPConnection object at 0x7fd8fc969e40>

    def _new_conn(self) -> socket.socket:
        """Establish a socket connection and set nodelay settings on it.
    
        :return: New socket connection.
        """
        try:
            sock = connection.create_connection(
                (self._dns_host, self.port),
                self.timeout,
                source_address=self.source_address,
                socket_options=self.socket_options,
            )
        except socket.gaierror as e:
            raise NameResolutionError(self.host, self, e) from e
        except SocketTimeout as e:
            raise ConnectTimeoutError(
                self,
                f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
            ) from e
    
        except OSError as e:
>           raise NewConnectionError(
                self, f"Failed to establish a new connection: {e}"
            ) from e
E           urllib3.exceptions.NewConnectionError: <botocore.awsrequest.AWSHTTPConnection object at 0x7fd8fc969e40>: Failed to establish a new connection: [Errno 111] Connection refused

../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/urllib3/connection.py:211: NewConnectionError

During handling of the above exception, another exception occurred:

s3_bucket_versioning_storage_factory = <arcticdb.storage_fixtures.s3.MotoS3StorageFixtureFactory object at 0x7fd9060e11e0>

    @pytest.fixture
    def s3_bucket_versioning_storage(s3_bucket_versioning_storage_factory):
>       with s3_bucket_versioning_storage_factory.create_fixture() as f:

tests/conftest.py:159: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
arcticdb/storage_fixtures/s3.py:400: in create_fixture
    self._s3_admin.create_bucket(Bucket=bucket)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/client.py:565: in _api_call
    return self._make_api_call(operation_name, kwargs)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/client.py:999: in _make_api_call
    http, parsed_response = self._make_request(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/client.py:1023: in _make_request
    return self._endpoint.make_request(operation_model, request_dict)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/endpoint.py:119: in make_request
    return self._send_request(request_dict, operation_model)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/endpoint.py:200: in _send_request
    while self._needs_retry(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/endpoint.py:352: in _needs_retry
    responses = self._event_emitter.emit(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/hooks.py:412: in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/hooks.py:256: in emit
    return self._emit(event_name, kwargs)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/hooks.py:239: in _emit
    response = handler(**kwargs)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/retryhandler.py:207: in __call__
    if self._checker(**checker_kwargs):
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/retryhandler.py:284: in __call__
    should_retry = self._should_retry(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/retryhandler.py:320: in _should_retry
    return self._checker(attempt_number, response, caught_exception)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/retryhandler.py:363: in __call__
    checker_response = checker(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/retryhandler.py:247: in __call__
    return self._check_caught_exception(
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/retryhandler.py:416: in _check_caught_exception
    raise caught_exception
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/endpoint.py:279: in _do_get_response
    http_response = self._send(request)
../../../../micromamba/envs/arcticdb/lib/python3.10/site-packages/botocore/endpoint.py:375: in _send
    return self.http_session.send(request)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <botocore.httpsession.URLLib3Session object at 0x7fd8fc92be50>
request = <AWSPreparedRequest stream_output=False, method=PUT, url=http://localhost:13167/test_bucket_0, headers={'User-Agent': ...nvocation-id': b'c946d3da-b688-49d0-84c3-daafab30cb51', 'amz-sdk-request': b'attempt=5; max=5', 'Content-Length': '0'}>

    def send(self, request):
        try:
            proxy_url = self._proxy_config.proxy_url_for(request.url)
            manager = self._get_connection_manager(request.url, proxy_url)
            conn = manager.connection_from_url(request.url)
            self._setup_ssl_cert(conn, request.url, self._verify)
            if ensure_boolean(
                os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '')
            ):
                # This is currently an "experimental" feature which provides
                # no guarantees of backwards compatibility. It may be subject
                # to change or removal in any patch version. Anyone opting in
                # to this feature should strictly pin botocore.
                host = urlparse(request.url).hostname
                conn.proxy_headers['host'] = host
    
            request_target = self._get_request_target(request.url, proxy_url)
            urllib_response = conn.urlopen(
                method=request.method,
                url=request_target,
                body=request.body,
                headers=request.headers,
                retries=Retry(False),
                assert_same_host=False,
                preload_content=False,
                decode_content=False,
                chunked=self._chunked(request.headers),
            )
    
            http_response = botocore.awsrequest.AWSResponse(
                request.url,
                urllib_response.status,
                urllib_response.headers,
                urllib_response,
            )
    
            if not request.stream_output:
                # Cause the raw stream to be exhausted immediately. We do it
                # this way instead of using preload_content because
                # preload_content will never buffer chunked responses
                http_response.content
    
            return http_response
        except URLLib3SSLError as e:
            raise SSLError(endpoint_url=request.url, error=e)
        except (NewConnectionError, socket.gaierror) as e:
>           raise EndpointConnectionError(endpoint_url=request.url, error=e)
E           botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL: "http://localhost:13167/test_bucket_0"