8000 Axsuarez/blob storage (#294) · sherlock666/botbuilder-python@9632e44 · GitHub
[go: up one dir, main page]
Skip to content

Commit 9632e44

Browse files
axelsrzaxelsrz
authored
Axsuarez/blob storage (microsoft#294)
* Blob Storage read done, missing write and delete * Blob storage implemented * BlobStorage tested * renamed blob tests
1 parent 9381c1b commit 9632e44

File tree

5 files changed

+302
-2
lines changed

5 files changed

+302
-2
lines changed

‎libraries/botbuilder-azure/botbuilder/azure/__init__.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,13 @@
77

88
from .about import __version__
99
from .cosmosdb_storage import CosmosDbStorage, CosmosDbConfig, CosmosDbKeyEscape
10+
from .blob_storage import BlobStorage, BlobStorageSettings
1011

11-
__all__ = ["CosmosDbStorage", "CosmosDbConfig", "CosmosDbKeyEscape", "__version__"]
12+
__all__ = [
13+
"BlobStorage",
14+
"BlobStorageSettings",
15+
"CosmosDbStorage",
16+
"CosmosDbConfig",
17+
"CosmosDbKeyEscape",
18+
"__version__",
19+
]

‎libraries/botbuilder-azure/botbuilder/azure/blob_storage.py

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
import json
2+
from typing import Dict, List
3+
4+
from azure.storage.blob import BlockBlobService, Blob, PublicAccess
5+
from botbuilder.core import Storage, StoreItem
6+
7+
# TODO: sanitize_blob_name
8+
9+
10+
class BlobStorageSettings:
11+
def __init__(
12+
self,
13+
container_name: str,
14+
account_name: str = "",
15+
account_key: str = "",
16+
connection_string: str = "",
17+
):
18+
self.container_name = container_name
19+
self.account_name = account_name
20+
self.account_key = account_key
21+
self.connection_string = connection_string
22+
23+
24+
class BlobStorage(Storage):
25+
def __init__(self, settings: BlobStorageSettings):
26+
if settings.connection_string:
27+
client = BlockBlobService(connection_string=settings.connection_string)
28+
elif settings.account_name and settings.account_key:
29+
client = BlockBlobService(
30+
account_name=settings.account_name, account_key=settings.account_key
31+
)
32+
else:
33+
raise Exception(
34+
"Connection string should be provided if there are no account name and key"
35+
)
36+
37+
self.client = client
38+
self.settings = settings
39+
40+
async def read(self, keys: List[str]) -> Dict[str, object]:
41+
if not keys:
42+
raise Exception("Please provide at least one key to read from storage.")
43+
44+
self.client.create_container(self.settings.container_name)
45+
self.client.set_container_acl(
46+
self.settings.container_name, public_access=PublicAccess.Container
47+
)
48+
items = {}
49+
50+
for key in keys:
51+
if self.client.exists(
52+
container_name=self.settings.container_name, blob_name=key
53+
):
54+
items[key] = self._blob_to_store_item(
55+
self.client.get_blob_to_text(
56+
container_name=self.settings.container_name, blob_name=key
57+
)
58+
)
59+
60+
return items
61+
62+
async def write(self, changes: Dict[str, StoreItem]):
63+
self.client.create_container(self.settings.container_name)
64+
self.client.set_container_acl(
65+
self.settings.container_name, public_access=PublicAccess.Container
66+
)
67+
68+
for name, item in changes.items():
69+
e_tag = (
70+
None if not hasattr(item, "e_tag") or item.e_tag == "*" else item.e_tag
71+
)
72+
if e_tag:
73+
item.e_tag = e_tag.replace('"', '\\"')
74+
self.client.create_blob_from_text(
75+
container_name=self.settings.container_name,
76+
blob_name=name,
77+
text=str(item),
78+
if_match=e_tag,
79+
)
80+
81+
async def delete(self, keys: List[str]):
82+
if keys is None:
83+
raise Exception("BlobStorage.delete: keys parameter can't be null")
84+
85+
self.client.create_container(self.settings.container_name)
86+
self.client.set_container_acl(
87+
self.settings.container_name, public_access=PublicAccess.Container
88+
)
89+
90+
for key in keys:
91+
if self.client.exists(
92+
container_name=self.settings.container_name, blob_name=key
93+
):
94+
self.client.delete_blob(
95+
container_name=self.settings.container_name, blob_name=key
96+
)
97+
98+
def _blob_to_store_item(self, blob: Blob) -> StoreItem:
99+
item = json.loads(blob.content)
100+
item["e_tag"] = blob.properties.etag
101+
item["id"] = blob.name
102+
return StoreItem(**item)

‎libraries/botbuilder-azure/setup.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
REQUIRES = [
88
"azure-cosmos>=3.0.0",
9+
"azure-storage-blob>=2.1.0",
910
"botbuilder-schema>=4.4.0b1",
1011
"botframework-connector>=4.4.0b1",
1112
]

‎libraries/botbuilder-azure/tests/test_blob_storage.py

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License.
3+
4+
import pytest
5+
from botbuilder.core import StoreItem
6+
from botbuilder.azure import BlobStorage, BlobStorageSettings
7+
8+
# local blob emulator instance blob
9+
BLOB_STORAGE_SETTINGS = BlobStorageSettings(
10+
account_name="", account_key="", container_name="test"
11+
)
12+
EMULATOR_RUNNING = False
13+
14+
15+
async def reset():
16+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
17+
try:
18+
await storage.client.delete_container(
19+
container_name=BLOB_STORAGE_SETTINGS.container_name
20+
)
21+
except Exception:
22+
pass
23+
24+
25+
class SimpleStoreItem(StoreItem):
26+
def __init__(self, counter=1, e_tag="*"):
27+
super(SimpleStoreItem, self).__init__()
28+
self.counter = counter
29+
self.e_tag = e_tag
30+
31+
32+
class TestBlobStorage:
33+
@pytest.mark.asyncio
34+
async def test_blob_storage_init_should_error_without_cosmos_db_config(self):
35+
try:
36+
BlobStorage(BlobStorageSettings()) # pylint: disable=no-value-for-parameter
37+
except Exception as error:
38+
assert error
39+
40+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
41+
@pytest.mark.asyncio
42+
async def test_blob_storage_read_should_return_data_with_valid_key(self):
43+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
44+
await storage.write({"user": SimpleStoreItem()})
45+
46+
data = await storage.read(["user"])
47+
assert "user" in data
48+
assert data["user"].counter == "1"
49+
assert len(data.keys()) == 1
50+
51+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
52+
@pytest.mark.asyncio
53+
async def test_blob_storage_read_update_should_return_new_etag(self):
54+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
55+
await storage.write({"test": SimpleStoreItem(counter=1)})
56+
data_result = await storage.read(["test"])
57+
data_result["test"].counter = 2
58+
await storage.write(data_result)
59+
data_updated = await storage.read(["test"])
60+
assert data_updated["test"].counter == "2"
61+
assert data_updated["test"].e_tag != data_result["test"].e_tag
62+
63+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
64+
@pytest.mark.asyncio
65+
async def test_blob_storage_read_no_key_should_throw(self):
66+
try:
67+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
68+
await storage.read([])
69+
except Exception as error:
70+
assert error
71+
72+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
73+
@pytest.mark.asyncio
74+
async def test_blob_storage_write_should_add_new_value(self):
75+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
76+
await storage.write({"user": SimpleStoreItem(counter=1)})
77+
78+
data = await storage.read(["user"])
79+
assert "user" in data
80+
assert data["user"].counter == "1"
81+
82+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
83+
@pytest.mark.asyncio
84+
async def test_blob_storage_write_should_overwrite_when_new_e_tag_is_an_asterisk(
85+
self
86+
):
87+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
88+
await storage.write({"user": SimpleStoreItem()})
89+
90+
await storage.write({"user": SimpleStoreItem(counter=10, e_tag="*")})
91+
data = await storage.read(["user"])
92+
assert data["user"].counter == "10"
93+
94+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
95+
@pytest.mark.asyncio
96+
async def test_blob_storage_write_batch_operation(self):
97+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
98+
await storage.write(
99+
{
100+
"batch1": SimpleStoreItem(counter=1),
101+
"batch2": SimpleStoreItem(counter=1),
102+
"batch3": SimpleStoreItem(counter=1),
103+
}
104+
)
105+
data = await storage.read(["batch1", "batch2", "batch3"])
106+
assert len(data.keys()) == 3
107+
assert data["batch1"]
108+
assert data["batch2"]
109+
assert data["batch3"]
110+
assert data["batch1"].counter == "1"
111+
assert data["batch2"].counter == "1"
112+
assert data["batch3"].counter == "1"
113+
assert data["batch1"].e_tag
114+
assert data["batch2"].e_tag
115+
assert data["batch3"].e_tag
116+
await storage.delete(["batch1", "batch2", "batch3"])
117+
data = await storage.read(["batch1", "batch2", "batch3"])
118+
assert not data.keys()
119+
120+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
121+
@pytest.mark.asyncio
122+
async def test_blob_storage_delete_should_delete_according_cached_data(self):
123+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
124+
await storage.write({"test": SimpleStoreItem()})
125+
try:
126+
await storage.delete(["test"])
127+
except Exception as error:
128+
raise error
129+
else:
130+
data = await storage.read(["test"])
131+
132+
assert isinstance(data, dict)
133+
assert not data.keys()
134+
135+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
136+
@pytest.mark.asyncio
137+
async def test_blob_storage_delete_should_delete_multiple_values_when_given_multiple_valid_keys(
138+
self
139+
):
140+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
141+
await storage.write({"test": SimpleStoreItem(), "test2": SimpleStoreItem(2)})
142+
143+
await storage.delete(["test", "test2"])
144+
data = await storage.read(["test", "test2"])
145+
assert not data.keys()
146+
147+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
148+
@pytest.mark.asyncio
149+
async def test_blob_storage_delete_should_delete_values_when_given_multiple_valid_keys_and_ignore_other_data(
150+
self
151+
):
152+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
153+
await storage.write(
154+
{
155+
"test": SimpleStoreItem(),
156+
"test2": SimpleStoreItem(counter=2),
157+
"test3": SimpleStoreItem(counter=3),
158+
}
159+
)
160+
161+
await storage.delete(["test", "test2"])
162+
data = await storage.read(["test", "test2", "test3"])
163+
assert len(data.keys()) == 1
164+
165+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
166+
@pytest.mark.asyncio
167+
async def test_blob_storage_delete_invalid_key_should_do_nothing_and_not_affect_cached_data(
168+
self
169+
):
170+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
171+
await storage.write({"test": SimpleStoreItem()})
172+
173+
await storage.delete(["foo"])
174+
data = await storage.read(["test"])
175+
assert len(data.keys()) == 1
176+
data = await storage.read(["foo"])
177+
assert not data.keys()
178+
179+
@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
180+
@pytest.mark.asyncio
181+
async def test_blob_storage_delete_invalid_keys_should_do_nothing_and_not_affect_cached_data(
182+
self
183+
):
184+
storage = BlobStorage(BLOB_STORAGE_SETTINGS)
185+
await storage.write({"test": SimpleStoreItem()})
186+
187+
await storage.delete(["foo", "bar"])
188+
data = await storage.read(["test"])
189+
assert len(data.keys()) == 1

‎libraries/botbuilder-core/botbuilder/core/storage.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ def __str__(self):
4949
output = (
5050
"{"
5151
+ ",".join(
52-
[f" '{attr}': '{getattr(self, attr)}'" for attr in non_magic_attributes]
52+
[f' "{attr}": "{getattr(self, attr)}"' for attr in non_magic_attributes]
5353
341A )
5454
+ " }"
5555
)

0 commit comments

Comments
 (0)
0