8000 Try anon=True if no credentials are supplied or found (#823) · fsspec/s3fs@8a87309 · GitHub
[go: up one dir, main page]
Skip to content

Commit 8a87309

Browse files
BENR0martindurant
BENR0
and
martindurant
authored
Try anon=True if no credentials are supplied or found (#823)
 Co-authored-by: Martin Durant <martin.durant@alumni.utoronto.ca>
1 parent 94ef88e commit 8a87309

File tree

4 files changed

+71
-43
lines changed

4 files changed

+71
-43
lines changed

‎s3fs/core.py

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# -*- coding: utf-8 -*-
22
import asyncio
33
import errno
4+
import inspect
45
import logging
56
import mimetypes
67
import os
@@ -30,6 +31,7 @@
3031
from aiobotocore.config import AioConfig
3132
from botocore.exceptions import ClientError, HTTPClientError, ParamValidationError
3233
from botocore.parsers import ResponseParserError
34+
from botocore.credentials import create_credential_resolver
3335

3436
from s3fs.errors import translate_boto_error
3537
from s3fs.utils import S3BucketRegionCache, ParamKwargsHelper, _get_brange, FileExpired
@@ -137,6 +139,18 @@ async def _error_wrapper(func, *, args=(), kwargs=None, retries):
137139
except Exception as e:
138140
err = e
139141
err = translate_boto_error(err)
142+
143+
if inspect.ismethod(func):
144+
s3 = func.__self__
145+
try:
146+
is_anon = s3._client_config.signature_version == botocore.UNSIGNED
147+
except AttributeError:
148+
is_anon = False
149+
if isinstance(err, PermissionError) and is_anon:
150+
raise PermissionError(
151+
"Access failed in anonymous mode. You may need to provide credentials."
152+
) from err
153+
140154
raise err
141155

142156

@@ -464,6 +478,36 @@ async def set_session(self, refresh=False, kwargs={}):
464478
return self._s3
465479
logger.debug("Setting up s3fs instance")
466480

481+
if self.session is None:
482+
self.session = aiobotocore.session.AioSession(**self.kwargs)
483+
484+
drop_keys = {
485+
"aws_access_key_id",
486+
"aws_secret_access_key",
487+
"aws_session_token",
488+
}
489+
if (
490+
not self.anon
491+
and (self.key or self.secret or self.token) is None
492+
and not drop_keys.intersection(set(self.client_kwargs))
493+
):
494+
# creating credentials resolver which enables loading credentials from configs/environment variables see
495+
# https://github.com/boto/botocore/blob/develop/botocore/credentials.py#L2043
496+
# tests whether any creds are available at all; if not, default to anonymous
497+
cred_resolver = create_credential_resolver(
498+
self.session, region_name=self.session._last_client_region_used
499+
)
500+
credentials = cred_resolver.load_credentials()
501+
502+
if credentials is None:
503+
logger.debug("No credentials given/found, setting `anon` to True.")
504+
self.anon = True
505+
else:
506+
# by stashing these, we avoid doing the lookup again
507+
self.key = credentials.access_key
508+
self.secret = credentials.secret_key
509+
self.token = credentials.token
510+
467511
client_kwargs = self.client_kwargs.copy()
468512
init_kwargs = dict(
469513
aws_access_key_id=self.key,
@@ -479,14 +523,10 @@ async def set_session(self, refresh=False, kwargs={}):
479523
if "use_ssl" not in client_kwargs.keys():
480524
init_kwargs["use_ssl"] = self.use_ssl
481525
config_kwargs = self._prepare_config_kwargs()
526+
482527
if self.anon:
483528
from botocore import UNSIGNED
484529

485-
drop_keys = {
486-
"aws_access_key_id",
487-
"aws_secret_access_key",
488-
"aws_session_token",
489-
}
490530
init_kwargs = {
491531
key: value for key, value in init_kwargs.items() if key not in drop_keys
492532
}
@@ -498,8 +538,6 @@ async def set_session(self, refresh=False, kwargs={}):
498538
config_kwargs["signature_version"] = UNSIGNED
499539

500540
conf = AioConfig(**config_kwargs)
501-
if self.session is None:
502-
self.session = aiobotocore.session.AioSession(**self.kwargs)
503541

504542
for parameters in (config_kwargs, self.kwargs, init_kwargs, client_kwargs):
505543
for option in ("region_name", "endpoint_url"):

‎s3fs/tests/test_mapping.py

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -52,22 +52,6 @@ def test_with_data(s3):
5252
assert list(d) == []
5353

5454

55-
def test_complex_keys(s3):
56-
d = s3.get_mapper(root)
57-
d[1] = b"hello"
58-
assert d[1] == b"hello"
59-
del d[1]
60-
61-
d[1, 2] = b"world"
62-
assert d[1, 2] == b"world"
63-
del d[1, 2]
64-
65-
d["x", 1, 2] = b"hello world"
66-
assert d["x", 1, 2] == b"hello world"
67-
68-
assert ("x", 1, 2) in d
69-
70-
7155
def test_clear_empty(s3):
7256
d = s3.get_mapper(root)
7357
d.clear()

‎s3fs/tests/test_s3fs.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,16 @@ def test_simple(s3):
166166
assert out == data
167167

168168

169+
def test_auto_anon(s3, monkeypatch):
170+
monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False)
171+
monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False)
172+
monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False)
173+
174+
fs = S3FileSystem(skip_instance_cache=True, endpoint_url=endpoint_uri)
175+
fs.s3
176+
assert fs.anon
177+
178+
169179
def test_with_size(s3):
170180
data = b"a" * (10 * 2**20)
171181

‎s3fs/utils.py

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import errno
22
import logging
33
from contextlib import contextmanager, AsyncExitStack
4-
from botocore.exceptions import ClientError
54

65

76
logger = logging.getLogger("s3fs")
@@ -31,26 +30,13 @@ async def get_bucket_client(self, bucket_name=None):
3130
if bucket_name in self._buckets:
3231
return self._buckets[bucket_name]
3332

34-
general_client = await self.get_client()
3533
if bucket_name is None:
36-
return general_client
37-
38-
try:
39-
response = await general_client.head_bucket(Bucket=bucket_name)
40-
except ClientError as e:
41-
region = (
42-
e.response["ResponseMetadata"]
43-
.get("HTTPHeaders", {})
44-
.get("x-amz-bucket-region")
45-
)
46-
if not region:
47-
logger.debug(
48-
"RC: HEAD_BUCKET call for %r has failed, returning the general client",
49-
bucket_name,
50-
)
51-
return general_client
52-
else:
53-
region = response["ResponseMetadata"]["HTTPHeaders"]["x-amz-bucket-region"]
34+
# general client
35+
return await self.get_client()
36+
37+
region = get_bucket_region(
38+
bucket_name
39+
) # this is sync - matters? can reuse some aiohttp session?
5440

5541
if region not in self._regions:
5642
logger.debug(
@@ -68,6 +54,7 @@ async def get_bucket_client(self, bucket_name=None):
6854
return client
6955

7056
async def get_client(self):
57+
# general, non-regional client
< 9E9B code>7158
if not self._client:
7259
self._client = await self._stack.enter_async_context(
7360
self._session.create_client("s3", **self._client_kwargs)
@@ -88,6 +75,15 @@ async def __aexit__(self, *exc_args):
8875
await self.clear()
8976

9077

78+
def get_bucket_region(bucket):
79+
"""Simple way to locate bucket"""
80+
import requests
81+
82+
return requests.head(f"https://s3.amazonaws.com/{bucket}").headers[
83+
"x-amz-bucket-region"
84+
]
85+
86+
9187
class FileExpired(IOError):
9288
"""
9389
Is raised, when the file content has been changed from a different process after

0 commit comments

Comments
 (0)
0