8000 Update azure blob SDK version (#1515) · andreikop/botbuilder-python@51a6bde · GitHub
[go: up one dir, main page]
Skip to content

Commit 51a6bde

Browse files
mdrichardsonMichael Richardsonaxelsrz
authored
Update azure blob SDK version (microsoft#1515)
* update azure blob SDK version * black fixes * pylint fixes * more pylint fixes * don't try to encode blob name Co-authored-by: Michael Richardson <v-micric@microsoft.com> Co-authored-by: Axel Suárez <axsuarez@microsoft.com>
1 parent b5f8b42 commit 51a6bde

File tree

3 files changed

+137
-58
lines changed

3 files changed

+137
-58
lines changed

‎libraries/botbuilder-azure/botbuilder/azure/blob_storage.py

Lines changed: 128 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,34 @@
33

44
from jsonpickle import encode
55
from jsonpickle.unpickler import Unpickler
6-
from azure.storage.blob import BlockBlobService, Blob, PublicAccess
6+
from azure.core import MatchConditions
7+
from azure.core.exceptions import (
8+
HttpResponseError,
9+
ResourceExistsError,
10+
ResourceNotFoundError,
11+
)
12+
from azure.storage.blob.aio import (
13+
BlobServiceClient,
14+
BlobClient,
15+
StorageStreamDownloader,
16+
)
717
from botbuilder.core import Storage
818

9-
# TODO: sanitize_blob_name
10-
1119

1220
class BlobStorageSettings:
21+
"""The class for Azure Blob configuration for the Azure Bot Framework.
22+
23+
:param container_name: Name of the Blob container.
24+
:type container_name: str
25+
:param account_name: Name of the Blob Storage account. Required if not using connection_string.
26+
:type account_name: str
27+
:param account_key: Key of the Blob Storage account. Required if not using connection_string.
28+
:type account_key: str
29+
:param connection_string: Connection string of the Blob Storage account.
30+
Required if not using account_name and account_key.
31+
:type connection_string: str
32+
"""
33+
1334
def __init__(
1435
self,
1536
container_name: str,
@@ -23,56 +44,105 @@ def __init__(
2344
self.connection_string = connection_string
2445

2546

47+
# New Azure Blob SDK only allows connection strings, but our SDK allows key+name.
48+
# This is here for backwards compatibility.
49+
def convert_account_name_and_key_to_connection_string(settings: BlobStorageSettings):
50+
if not settings.account_name or not settings.account_key:
51+
raise Exception(
52+
"account_name and account_key are both required for BlobStorageSettings if not using a connections string."
53+
)
54+
return (
55+
f"DefaultEndpointsProtocol=https;AccountName={settings.account_name};"
56+
f"AccountKey={settings.account_key};EndpointSuffix=core.windows.net"
57+
)
58+
59+
2660
class BlobStorage(Storage):
61+
"""An Azure Blob based storage provider for a bot.
62+
63+
This class uses a single Azure Storage Blob Container.
64+
Each entity or StoreItem is serialized into a JSON string and stored in an individual text blob.
65+
Each blob is named after the store item key, which is encoded so that it conforms a valid blob name.
66+
If an entity is an StoreItem, the storage object will set the entity's e_tag
67+
property value to the blob's e_tag upon read. Afterward, an match_condition with the ETag value
68+
will be generated during Write. New entities start with a null e_tag.
69+
70+
:param settings: Settings used to instantiate the Blob service.
71+
:type settings: :class:`botbuilder.azure.BlobStorageSettings`
72+
"""
73+
2774
def __init__(self, settings: BlobStorageSettings):
75+
if not settings.container_name:
76+
raise Exception("Container name is required.")
77+
2878
if settings.connection_string:
29-
client = BlockBlobService(connection_string=settings.connection_string)
30-
elif settings.account_name and settings.account_key:
31-
client = BlockBlobService(
32-
account_name=settings.account_name, account_key=settings.account_key
79+
blob_service_client = BlobServiceClient.from_connection_string(
80+
settings.connection_string
3381
)
3482
else:
35-
raise Exception(
36-
"Connection string should be provided if there are no account name and key"
83+
blob_service_client = BlobServiceClient.from_connection_string(
84+
convert_account_name_and_key_to_connection_string(settings)
3785
)
3886

39-
self.client = client
40-
self.settings = settings
87+
self.__container_client = blob_service_client.get_container_client(
88+
settings.container_name
89+
)
90+
91+
self.__initialized = False
92+
93+
async def _initialize(self):
94+
if self.__initialized is False:
95+
# This should only happen once - assuming this is a singleton.
96+
# ContainerClient.exists() method is available in an unreleased version of the SDK. Until then, we use:
97+
try:
98+
await self.__container_client.create_container()
99+
except ResourceExistsError:
100+
pass
101+
self.__initialized = True
102+
return self.__initialized
41103

42104
async def read(self, keys: List[str]) -> Dict[str, object]:
105+
"""Retrieve entities from the configured blob container.
106+
107+
:param keys: An array of entity keys.
108+
:type keys: Dict[str, object]
109+
:return dict:
110+
"""
43111
if not keys:
44112
raise Exception("Keys are required when reading")
45113

46-
self.client.create_container(self.settings.container_name)
47-
self.client.set_container_acl(
48-
self.settings.container_name, public_access=PublicAccess.Container
49-
)
114+
await self._initialize()
115+
50116
items = {}
51117

52118
for key in keys:
53-
if self.client.exists(
54-
container_name=self.settings.container_name, blob_name=key
55-
):
56-
items[key] = self._blob_to_store_item(
57-
self.client.get_blob_to_text(
58-
container_name=self.settings.container_name, blob_name=key
59-
)
60-
)
119+
blob_client = self.__container_client.get_blob_client(key)
120+
121+
try:
122+
items[key] = await self._inner_read_blob(blob_client)
123+
except HttpResponseError as err:
124+
if err.status_code == 404:
125+
continue
61126

62127
return items
63128

64129
async def write(self, changes: Dict[str, object]):
130+
"""Stores a new entity in the configured blob container.
131+
132+
:param changes: The changes to write to storage.
133+
:type changes: Dict[str, object]
134+
:return:
135+
"""
65136
if changes is None:
66137
raise Exception("Changes are required when writing")
67138
if not changes:
68139
return
69140

70-
self.client.create_container(self.settings.container_name)
71-
self.client.set_container_acl(
72-
self.settings.container_name, public_access=PublicAccess.Container
73-
)
141+
await self._initialize()
74142

75143
for (name, item) in changes.items():
144+
blob_reference = self.__container_client.get_blob_client(name)
145+
76146
e_tag = None
77147
if isinstance(item, dict):
78148
e_tag = item.get("e_tag", None)
@@ -81,39 +151,46 @@ async def write(self, changes: Dict[str, object]):
81151
e_tag = None if e_tag == "*" else e_tag
82152
if e_tag == "":
83153
raise Exception("blob_storage.write(): etag missing")
154+
84155
item_str = self._store_item_to_str(item)
85-
try:
86-
self.client.create_blob_from_text(
87-
container_name=self.settings.container_name,
88-
blob_name=name,
89-
text=item_str,
90-
if_match=e_tag,
156+
157+
if e_tag:
158+
await blob_reference.upload_blob(
159+
item_str, match_condition=MatchConditions.IfNotModified, etag=e_tag
91160
)
92-
except Exception as error:
93-
raise error
161+
else:
162+
await blob_reference.upload_blob(item_str, overwrite=True)
94163

95164
async def delete(self, keys: List[str]):
165+
"""Deletes entity blobs from the configured container.
166+
167+
:param keys: An array of entity keys.
168+
:type keys: Dict[str, object]
169+
"""
96170
if keys is None:
97171
raise Exception("BlobStorage.delete: keys parameter can't be null")
98172

99-
self.client.create_container(self.settings.container_name)
100-
self.client.set_container_acl(
101-
self.settings.container_name, public_access=PublicAccess.Container
102-
)
173+
await self._initialize()
103174

104175
for key in keys:
105-
if self.client.exists(
106-
container_name=self.settings.container_name, blob_name=key
107-
):
108-
self.client.delete_blob(
109-
container_name=self.settings.container_name, blob_name=key
110-
)
111-
112-
def _blob_to_store_item(self, blob: Blob) -> object:
113-
item = json.loads(blob.content)
114-
item["e_tag"] = blob.properties.etag
115-
result = Unpickler().restore(item)
116-
return result
176+
blob_client = self.__container_client.get_blob_client(key)
177+
try:
178+
await blob_client.delete_blob()
179+
# We can't delete what's already gone.
180+
except ResourceNotFoundError:
181+
pass
117182

118183
def _store_item_to_str(self, item: object) -> str:
119184
return encode(item)
185+
186+
async def _inner_read_blob(self, blob_client: BlobClient):
187+
blob = await blob_client.download_blob()
188+
189+
return await self._blob_to_store_item(blob)
190+
191+
@staticmethod
192+
async def _blob_to_store_item(blob: StorageStreamDownloader) -> object:
193+
item = json.loads(await blob.content_as_text())
194+
item["e_tag"] = blob.properties.etag.replace('"', "")
195+
result = Unpickler().restore(item)
196+
return result

‎libraries/botbuilder-azure/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
REQUIRES = [
88
"azure-cosmos==3.2.0",
9-
"azure-storage-blob==2.1.0",
9+
"azure-storage-blob==12.7.0",
1010
"botbuilder-schema==4.12.0",
1111
"botframework-connector==4.12.0",
1212
"jsonpickle==1.2",

‎libraries/botbuilder-azure/tests/test_blob_storage.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
# Licensed under the MIT License.
33

44
import pytest
5+
from azure.core.exceptions import ResourceNotFoundError
6+
from azure.storage.blob.aio import BlobServiceClient
57
from botbuilder.core import StoreItem
68
from botbuilder.azure import BlobStorage, BlobStorageSettings
79
from botbuilder.testing import StorageBaseTests
@@ -26,12 +28,12 @@ def get_storage():
2628

2729

2830
async def reset():
29-
storage = get_storage()
31+
storage = BlobServiceClient.from_connection_string(
32+
BLOB_STORAGE_SETTINGS.connection_string
33+
)
3034
try:
31-
await storage.client.delete_container(
32-
container_name=BLOB_STORAGE_SETTINGS.container_name
33-
)
34-
except Exception:
35+
await storage.delete_container(BLOB_STORAGE_SETTINGS.container_name)
36+
except ResourceNotFoundError:
3537
pass
3638

3739

@@ -44,7 +46,7 @@ def __init__(self, counter=1, e_tag="*"):
4446

4547
class TestBlobStorageConstructor:
4648
@pytest.mark.asyncio
47-
async def test_blob_storage_init_should_error_without_cosmos_db_config(self):
49+
async def test_blob_storage_init_should_error_without_blob_config(self):
4850
try:
4951
BlobStorage(BlobStorageSettings()) # pylint: disable=no-value-for-parameter
5052
except Exception as error:

0 commit comments

Comments
 (0)
0