8000 Add customer-supplied encryption to storage (#1844) · googleapis/google-cloud-python@437cc29 · GitHub
[go: up one dir, main page]
Skip to content

Commit 437cc29

Browse files
daspecsterdaspecster
authored
Add customer-supplied encryption to storage (#1844)
* Add customer-supplied encryption to storage.
1 parent 1ab2169 commit 437cc29

File tree

4 files changed

+300
-11
lines changed

4 files changed

+300
-11
lines changed

‎gcloud/_helpers.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,27 @@ def _to_bytes(value, encoding='ascii'):
394394
raise TypeError('%r could not be converted to bytes' % (value,))
395395

396396

397+
def _bytes_to_unicode(value):
398+
"""Converts bytes to a unicode value, if necessary.
399+
400+
:type value: bytes
401+
:param value: bytes value to attempt string conversion on.
402+
403+
:rtype: str
404+
:returns: The original value converted to unicode (if bytes) or as passed
405+
in if it started out as unicode.
406+
407+
:raises: :class:`ValueError` if the value could not be converted to
408+
unicode.
409+
"""
410+
result = (value.decode('utf-8')
411+
if isinstance(value, six.binary_type) else value)
412+
if isinstance(result, six.text_type):
413+
return result
414+
else:
415+
raise ValueError('%r could not be converted to unicode' % (value,))
416+
417+
397418
def _pb_timestamp_to_datetime(timestamp):
398419
"""Convert a Timestamp protobuf to a datetime object.
399420

‎gcloud/storage/blob.py

Lines changed: 108 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414

1515
"""Create / interact with Google Cloud Storage blobs."""
1616

17+
import base64
1718
import copy
19+
import hashlib
1820
from io import BytesIO
1921
import json
2022
import mimetypes
@@ -26,6 +28,8 @@
2628
from six.moves.urllib.parse import quote
2729

2830
from gcloud._helpers import _rfc3339_to_datetime
31+
from gcloud._helpers import _to_bytes
32+
from gcloud._helpers import _bytes_to_unicode
2933
from gcloud.credentials import generate_signed_url
3034
from gcloud.exceptions import NotFound
3135
from gcloud.exceptions import make_exception
@@ -276,17 +280,41 @@ def delete(self, client=None):
276280
"""
277281
return self.bucket.delete_blob(self.name, client=client)
278282

279-
def download_to_file(self, file_obj, client=None):
283+
def download_to_file(self, file_obj, encryption_key=None, client=None):
280284
"""Download the contents of this blob into a file-like object.
281285
282286
.. note::
283287
284288
If the server-set property, :attr:`media_link`, is not yet
285289
initialized, makes an additional API request to load it.
286290
291+
Downloading a file that has been encrypted with a `customer-supplied`_
292+
encryption key::
293+
294+
>>> from gcloud import storage
295+
>>> from gcloud.storage import Blob
296+
297+
>>> client = storage.Client(project='my-project')
298+
>>> bucket = client.get_bucket('my-bucket')
299+
>>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
300+
>>> blob = Blob('secure-data', bucket)
301+
>>> with open('/tmp/my-secure-file', 'wb') as file_obj:
302+
>>> blob.download_to_file(file_obj,
303+
... encryption_key=encryption_key)
304+
305+
The ``encryption_key`` should be a str or bytes with a length of at
306+
least 32.
307+
308+
.. _customer-supplied: https://cloud.google.com/storage/docs/\
309+
encryption#customer-supplied
310+
287311
:type file_obj: file
288312
:param file_obj: A file handle to which to write the blob's data.
289313
314+
:type encryption_key: str or bytes
315+
:param encryption_key: Optional 32 byte encryption key for
316+
customer-supplied encryption.
317+
290318
:type client: :class:`gcloud.storage.client.Client` or ``NoneType``
291319
:param client: Optional. The client to use. If not passed, falls back
292320
to the ``client`` stored on the blob's bucket.
@@ -305,7 +333,11 @@ def download_to_file(self, file_obj, client=None):
305333
if self.chunk_size is not None:
306334
download.chunksize = self.chunk_size
307335

308-
request = Request(download_url, 'GET')
336+
headers = {}
337+
if encryption_key:
338+
_set_encryption_headers(encryption_key, headers)
339+
340+
request = Request(download_url, 'GET', headers)
309341

310342
# Use the private ``_connection`` rather than the public
311343
# ``.connection``, since the public connection may be a batch. A
@@ -315,27 +347,36 @@ def download_to_file(self, file_obj, client=None):
315347
# it has all three (http, API_BASE_URL and build_api_url).
316348
download.initialize_download(request, client._connection.http)
317349

318-
def download_to_filename(self, filename, client=None):
350+
def download_to_filename(self, filename, encryption_key=None, client=None):
319351
"""Download the contents of this blob into a named file.
320352
321353
:type filename: string
322354
:param filename: A filename to be passed to ``open``.
323355
356+
:type encryption_key: str or bytes
357+
:param encryption_key: Optional 32 byte encryption key for
358+
customer-supplied encryption.
359+
324360
:type client: :class:`gcloud.storage.client.Client` or ``NoneType``
325361
:param client: Optional. The client to use. If not passed, falls back
326362
to the ``client`` stored on the blob's bucket.
327363
328364
:raises: :class:`gcloud.exceptions.NotFound`
329365
"""
330366
with open(filename, 'wb') as file_obj:
331-
self.download_to_file(file_obj, client=client)
367+
self.download_to_file(file_obj, encryption_key=encryption_key,
368+
client=client)
332369

333370
mtime = time.mktime(self.updated.timetuple())
334371
os.utime(file_obj.name, (mtime, mtime))
335372

336-
def download_as_string(self, client=None):
373+
def download_as_string(self, encryption_key=None, client=None):
337374
"""Download the contents of this blob as a string.
338375
376+
:type encryption_key: str or bytes
377+
:param encryption_key: Optional 32 byte encryption key for
378+
customer-supplied encryption.
379+
339380
:type client: :class:`gcloud.storage.client.Client` or ``NoneType``
340381
:param client: Optional. The client to use. If not passed, falls back
341382
to the ``client`` stored on the blob's bucket.
@@ -345,7 +386,8 @@ def download_as_string(self, client=None):
345386
:raises: :class:`gcloud.exceptions.NotFound`
346387
"""
347388
string_buffer = BytesIO()
348-
self.download_to_file(string_buffer, client=client)
389+
self.download_to_file(string_buffer, encryption_key=encryption_key,
390+
client=client)
349391
return string_buffer.getvalue()
350392

351393
@staticmethod
@@ -358,8 +400,10 @@ def _check_response_error(request, http_response):
358400
raise make_exception(faux_response, http_response.content,
359401
error_info=request.url)
360402

403+
# pylint: disable=too-many-locals
361404
def upload_from_file(self, file_obj, rewind=False, size=None,
362-
content_type=None, num_retries=6, client=None):
405+
encryption_key=None, content_type=None, num_retries=6,
406+
client=None):
363407
"""Upload the contents of this blob from a file-like object.
364408
365409
The content type of the upload will either be
@@ -378,6 +422,25 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
378422
`lifecycle <https://cloud.google.com/storage/docs/lifecycle>`_
379423
API documents for details.
380424
425+
Uploading a file with a `customer-supplied`_ encryption key::
426+
427+
>>> from gcloud import storage
428+
>>> from gcloud.storage import Blob
429+
430+
>>> client = storage.Client(project='my-project')
431+
>>> bucket = client.get_bucket('my-bucket')
432+
>>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
433+
>>> blob = Blob('secure-data', bucket)
434+
>>> with open('my-file', 'rb') as my_file:
435+
>>> blob.upload_from_file(my_file,
436+
... encryption_key=encryption_key)
437+
438+
The ``encryption_key`` should be a str or bytes with a length of at
439+
least 32.
440+
441+
.. _customer-supplied: https://cloud.google.com/storage/docs/\
442+
encryption#customer-supplied
443+
381444
:type file_obj: file
382445
:param file_obj: A file handle open for reading.
383446
@@ -391,6 +454,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
391454
:func:`os.fstat`. (If the file handle is not from the
392455
filesystem this won't be possible.)
393456
457+
:type encryption_key: str or bytes
458+
:param encryption_key: Optional 32 byte encryption key for
459+
customer-supplied encryption.
460+
394461
:type content_type: string or ``NoneType``
395462
:param content_type: Optional type of content being uploaded.
396463
@@ -434,6 +501,9 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
434501
'User-Agent': connection.USER_AGENT,
435502
}
436503

504+
if encryption_key:
505+
_set_encryption_headers(encryption_key, headers)
506+
437507
upload = Upload(file_obj, content_type, total_bytes,
438508
auto_transfer=False)
439509

@@ -473,9 +543,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
473543
six.string_types): # pragma: NO COVER Python3
474544
response_content = response_content.decode('utf-8')
475545
self._set_properties(json.loads(response_content))
546+
# pylint: enable=too-many-locals
476547

477548
def upload_from_filename(self, filename, content_type=None,
478-
client=None):
549+
encryption_key=None, client=None):
479550
"""Upload this blob's contents from the content of a named file.
480551
481552
The content type of the upload will either be
@@ -500,6 +571,10 @@ def upload_from_filename(self, filename, content_type=None,
500571
:type content_type: string or ``NoneType``
501572
:param content_type: Optional type of content being uploaded.
502573
574+
:type encryption_key: str or bytes
575+
:param encryption_key: Optional 32 byte encryption key for
576+
customer-supplied encryption.
577+
503578
:type client: :class:`gcloud.storage.client.Client` or ``NoneType``
504579
:param client: Optional. The client to use. If not passed, falls back
505580
to the ``client`` stored on the blob's bucket.
@@ -510,10 +585,10 @@ def upload_from_filename(self, filename, content_type=None,
510585

511586
with open(filename, 'rb') as file_obj:
512587
self.upload_from_file(file_obj, content_type=content_type,
513-
client=client)
588+
encryption_key=encryption_key, client=client)
514589

515590
def upload_from_string(self, data, content_type='text/plain',
516-
client=None):
591+
encryption_key=None, client=None):
517592
"""Upload contents of this blob from the provided string.
518593
519 F438 594
.. note::
@@ -535,6 +610,10 @@ def upload_from_string(self, data, content_type='text/plain',
535610
:param content_type: Optional type of content being uploaded. Defaults
536611
to ``'text/plain'``.
537612
613+
:type encryption_key: str or bytes
614+
:param encryption_key: Optional 32 byte encryption key for
615+
customer-supplied encryption.
616+
538617
:type client: :class:`gcloud.storage.client.Client` or ``NoneType``
539618
:param client: Optional. The client to use. If not passed, falls back
540619
to the ``client`` stored on the blob's bucket.
@@ -545,7 +624,7 @@ def upload_from_string(self, data, content_type='text/plain',
545624
string_buffer.write(data)
546625
self.upload_from_file(file_obj=string_buffer, rewind=True,
547626
size=len(data), content_type=content_type,
548-
client=client)
627+
encryption_key=encryption_key, client=client)
549628

550629
def make_public(self, client=None):
551630
"""Make this blob public giving all users read access.
@@ -838,3 +917,21 @@ def __init__(self, bucket_name, object_name):
838917
self.query_params = {'name': object_name}
839918
self._bucket_name = bucket_name
840919
self._relative_path = ''
920+
921+
922+
def _set_encryption_headers(key, headers):
923+
"""Builds customer encyrption key headers
924+
925+
:type key: str or bytes
926+
:param key: 32 byte key to build request key and hash.
927+
928+
:type headers: dict
929+
:param headers: dict of HTTP headers being sent in request.
930+
"""
931+
key = _to_bytes(key)
932+
sha256_key = hashlib.sha256(key).digest()
933+
key_hash = base64.b64encode(sha256_key).rstrip()
934+
encoded_key = base64.b64encode(key).rstrip()
935+
headers['X-Goog-Encryption-Algorithm'] = 'AES256'
936+
headers['X-Goog-Encryption-Key'] = _bytes_to_unicode(encoded_key)
937+
headers['X-Goog-Encryption-Key-Sha256'] = _bytes_to_unicode(key_hash)

0 commit comments

Comments
 (0)
0