8000 Merge pull request #2806 from tseaver/2229-bigquery-marshal-bytes-tim… · richkadel/google-cloud-python@024ea0a · GitHub
[go: up one dir, main page]
Skip to content

Commit 024ea0a

Browse files
tseavertseaver
authored
Merge pull request googleapis#2806 from tseaver/2229-bigquery-marshal-bytes-time-types
Marshal 'BYTES' and 'TIME' column / paramter types
2 parents 5f04239 + 10f6813 commit 024ea0a

File tree

5 files changed

+195
-24
lines changed

5 files changed

+195
-24
lines changed

‎bigquery/google/cloud/bigquery/_helpers.py‎

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
"""Shared helper functions for BigQuery API classes."""
1616

17+
import base64
1718
from collections import OrderedDict
1819
import datetime
1920

@@ -22,6 +23,7 @@
2223
from google.cloud._helpers import _datetime_to_rfc3339
2324
from google.cloud._helpers import _microseconds_from_datetime
2425
from google.cloud._helpers import _RFC3339_NO_FRACTION
26+
from google.cloud._helpers import _time_from_iso8601_time_naive
2527

2628

2729
def _not_null(value, field):
@@ -47,6 +49,17 @@ def _bool_from_json(value, field):
4749
return value.lower() in ['t', 'true', '1']
4850

4951

52+
def _string_from_json(value, _):
53+
"""NOOP string -> string coercion"""
54+
return value
55+
56+
57+
def _bytes_from_json(value, field):
58+
"""Base64-decode value"""
59+
if _not_null(value, field):
60+
return base64.decodestring(value)
61+
62+
5063
def _timestamp_from_json(value, field):
5164
"""Coerce 'value' to a datetime, if set or not nullable."""
5265
if _not_null(value, field):
@@ -64,9 +77,17 @@ def _datetime_from_json(value, field):
6477
def _date_from_json(value, field):
6578
"""Coerce 'value' to a datetime date, if set or not nullable"""
6679
if _not_null(value, field):
80+
# value will be a string, in YYYY-MM-DD form.
6781
return _date_from_iso8601_date(value)
6882

6983

84+
def _time_from_json(value, field):
85+
"""Coerce 'value' to a datetime date, if set or not nullable"""
86+
if _not_null(value, field):
87+
# value will be a string, in HH:MM:SS form.
88+
return _time_from_iso8601_time_naive(value)
89+
90+
7091
def _record_from_json(value, field):
7192
"""Coerce 'value' to a mapping, if set or not nullable."""
7293
if _not_null(value, field):
@@ -82,23 +103,20 @@ def _record_from_json(value, field):
82103
return record
83104

84105

85-
def _string_from_json(value, _):
86-
"""NOOP string -> string coercion"""
87-
return value
88-
89-
90106
_CELLDATA_FROM_JSON = {
91107
'INTEGER': _int_from_json,
92108
'INT64': _int_from_json,
93109
'FLOAT': _float_from_json,
94110
'FLOAT64': _float_from_json,
95111
'BOOLEAN': _bool_from_json,
96112
'BOOL': _bool_from_json,
113+
'STRING': _string_from_json,
114+
'BYTES': _bytes_from_json,
97115
'TIMESTAMP': _timestamp_from_json,
98116
'DATETIME': _datetime_from_json,
99117
'DATE': _date_from_json,
118+
'TIME': _time_from_json,
100119
'RECORD': _record_from_json,
101-
'STRING': _string_from_json,
102120
}
103121

104122

@@ -121,6 +139,13 @@ def _bool_to_json(value):
121139
return value
122140

123141

142+
def _bytes_to_json(value):
143+
"""Coerce 'value' to an JSON-compatible representation."""
144+
if isinstance(value, bytes):
145+
value = base64.encodestring(value)
146+
return value
147+
148+
124149
def _timestamp_to_json(value):
125150
"""Coerce 'value' to an JSON-compatible representation."""
126151
if isinstance(value, datetime.datetime):
@@ -142,16 +167,25 @@ def _date_to_json(value):
142167
return value
143168

144169

170+
def _time_to_json(value):
171+
"""Coerce 'value' to an JSON-compatible representation."""
172+
if isinstance(value, datetime.time):
173+
value = value.isoformat()
174+
return value
175+
176+
145177
_SCALAR_VALUE_TO_JSON = {
146178
'INTEGER': _int_to_json,
147179
'INT64': _int_to_json,
148180
'FLOAT': _float_to_json,
149181
'FLOAT64': _float_to_json,
150182
'BOOLEAN': _bool_to_json,
151183
'BOOL': _bool_to_json,
184+
'BYTES': _bytes_to_json,
152185
'TIMESTAMP': _timestamp_to_json,
153186
'DATETIME': _datetime_to_json,
154187
'DATE': _date_to_json,
188+
'TIME': _time_to_json,
155189
}
156190

157191

‎bigquery/unit_tests/test__helpers.py‎

Lines changed: 92 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,44 @@ def test_w_value_other(self):
105105
self.assertFalse(coerced)
106106

107107

108+
class Test_string_from_json(unittest.TestCase):
109+
110+
def _call_fut(self, value, field):
111+
from google.cloud.bigquery._helpers import _string_from_json
112+
return _string_from_json(value, field)
113+
114+
def test_w_none_nullable(self):
115+
self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
116+
117+
def test_w_none_required(self):
118+
self.assertIsNone(self._call_fut(None, _Field('REQUIRED')))
119+
120+
def test_w_string_value(self):
121+
coerced = self._call_fut('Wonderful!', object())
122+
self.assertEqual(coerced, 'Wonderful!')
123+
124+
125+
class Test_bytes_from_json(unittest.TestCase):
126+
127+
def _call_fut(self, value, field):
128+
from google.cloud.bigquery._helpers import _bytes_from_json
129+
return _bytes_from_json(value, field)
130+
131+
def test_w_none_nullable(self):
132+
self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
133+
134+
def test_w_none_required(self):
135+
with self.assertRaises(TypeError):
136+
self._call_fut(None, _Field('REQUIRED'))
137+
138+
def test_w_base64_encoded_value(self):
139+
import base64
140+
expected = b'Wonderful!'
141+
encoded = base64.encodestring(expected)
142+
coerced = self._call_fut(encoded, object())
143+
self.assertEqual(coerced, expected)
144+
145+
108146
class Test_timestamp_from_json(unittest.TestCase):
109147

110148
def _call_fut(self, value, field):
@@ -177,6 +215,27 @@ def test_w_string_value(self):
177215
datetime.date(1987, 9, 22))
178216

179217

218+
class Test_time_from_json(unittest.TestCase):
219+
220+
def _call_fut(self, value, field):
221+
from google.cloud.bigquery._helpers import _time_from_json
222+
return _time_from_json(value, field)
223+
224+
def test_w_none_nullable(self):
225+
self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
226+
227+
def test_w_none_required(self):
228+
with self.assertRaises(TypeError):
229+
self._call_fut(None, _Field('REQUIRED'))
230+
231+
def test_w_string_value(self):
232+
import datetime
233+
coerced = self._call_fut('12:12:27', object())
234+
self.assertEqual(
235+
coerced,
236+
datetime.time(12, 12, 27))
237+
238+
180239
class Test_record_from_json(unittest.TestCase):
181240

182241
def _call_fut(self, value, field):
@@ -238,23 +297,6 @@ def test_w_record_subfield(self):
238297
self.assertEqual(coerced, expected)
239298

240299

241-
class Test_string_from_json(unittest.TestCase):
242-
243-
def _call_fut(self, value, field):
244-
from google.cloud.bigquery._helpers import _string_from_json
245-
return _string_from_json(value, field)
246-
247-
def test_w_none_nullable(self):
248-
self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
249-
250-
def test_w_none_required(self):
251-
self.assertIsNone(self._call_fut(None, _Field('RECORD')))
252-
253-
def test_w_string_value(self):
254-
coerced = self._call_fut('Wonderful!', object())
255-
self.assertEqual(coerced, 'Wonderful!')
256-
257-
258300
class Test_row_from_json(unittest.TestCase):
259301

260302
def _call_fut(self, row, schema):
@@ -471,6 +513,23 @@ def test_w_string(self):
471513
self.assertEqual(self._call_fut('false'), 'false')
472514

473515

516+
class Test_bytes_to_json(unittest.TestCase):
517+
518+
def _call_fut(self, value):
519+
from google.cloud.bigquery._helpers import _bytes_to_json
520+
return _bytes_to_json(value)
521+
522+
def test_w_non_bytes(self):
523+
non_bytes = object()
524+
self.assertIs(self._call_fut(non_bytes), non_bytes)
525+
526+
def test_w_bytes(self):
527+
import base64
528+
source = b'source'
529+
expected = base64.encodestring(source)
530+
self.assertEqual(self._call_fut(source), expected)
531+
532+
474533
class Test_timestamp_to_json(unittest.TestCase):
475534

476535
def _call_fut(self, value):
@@ -522,6 +581,22 @@ def test_w_datetime(self):
522581
self.assertEqual(self._call_fut(when), '2016-12-03')
523582

524583

584+
class Test_time_to_json(unittest.TestCase):
585+
586+
def _call_fut(self, value):
587+
from google.cloud.bigquery._helpers import _time_to_json
588+
return _time_to_json(value)
589+
590+
def test_w_string(self):
591+
RFC3339 = '12:13:41'
592+
self.assertEqual(self._call_fut(RFC3339), RFC3339)
593+
594+
def test_w_datetime(self):
595+
import datetime
596+
when = datetime.time(12, 13, 41)
597+
self.assertEqual(self._call_fut(when), '12:13:41')
598+
599+
525600
class Test_ConfigurationProperty(unittest.TestCase):
526601

527602
@staticmethod

‎core/google/cloud/_helpers.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,19 @@ def _date_from_iso8601_date(value):
251251
return datetime.datetime.strptime(value, '%Y-%m-%d').date()
252252

253253

254+
def _time_from_iso8601_time_naive(value):
255+
"""Convert a zoneless ISO8601 time string to naive datetime time
256+
257+
:type value: str
258+
:param value: The time string to convert
259+
260+
:rtype: :class:`datetime.time`
261+
:returns: A datetime time object created from the string
262+
263+
"""
264+
return datetime.datetime.strptime(value, '%H:%M:%S').time()
265+
266+
254267
def _rfc3339_to_datetime(dt_str):
255268
"""Convert a microsecond-precision timetamp to a native datetime.
256269

‎core/unit_tests/test__helpers.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,18 @@ def test_todays_date(self):
265265
self.assertEqual(self._call_fut(TODAY.strftime("%Y-%m-%d")), TODAY)
266266

267267

268+
class Test___time_from_iso8601_time_naive(unittest.TestCase):
269+
270+
def _call_fut(self, value):
271+
from google.cloud._helpers import _time_from_iso8601_time_naive
272+
return _time_from_iso8601_time_naive(value)
273+
274+
def test_todays_date(self):
275+
import datetime
276+
WHEN = datetime.time(12, 9, 42)
277+
self.assertEqual(self._call_fut(("12:09:42")), WHEN)
278+
279+
268280
class Test__rfc3339_to_datetime(unittest.TestCase):
269281

270282
def _call_fut(self, dt_str):

‎system_tests/bigquery.py‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -479,12 +479,49 @@ def _job_done(instance):
479479
# raise an error, and that the job completed (in the `retry()`
480480
# above).
481481

482-
def test_sync_query_w_nested_arrays_and_structs(self):
482+
def test_sync_query_w_standard_sql_types(self):
483+
import datetime
484+
from google.cloud._helpers import UTC
485+
naive = datetime.datetime(2016, 12, 5, 12, 41, 9)
486+
stamp = "%s %s" % (naive.date().isoformat(), naive.time().isoformat())
487+
zoned = naive.replace(tzinfo=UTC)
483488
EXAMPLES = [
484489
{
485490
'sql': 'SELECT 1',
486491
'expected': 1,
487492
},
493+
{
494+
'sql': 'SELECT 1.3',
495+
'expected': 1.3,
496+
},
497+
{
498+
'sql': 'SELECT TRUE',
499+
'expected': True,
500+
},
501+
{
502+
'sql': 'SELECT "ABC"',
503+
'expected': 'ABC',
504+
},
505+
{
506+
'sql': 'SELECT CAST("foo" AS BYTES)',
507+
'expected': b'foo',
508+
},
509+
{
510+
'sql': 'SELECT TIMESTAMP "%s"' % (stamp,),
511+
'expected': zoned,
512+
},
513+
{
514+
'sql': 'SELECT DATETIME(TIMESTAMP "%s")' % (stamp,),
515+
'expected': naive,
516+
},
517+
{
518+
'sql': 'SELECT DATE(TIMESTAMP "%s")' % (stamp,),
519+
'expected': naive.date(),
520+
},
521+
{
522+
'sql': 'SELECT TIME(TIMESTAMP "%s")' % (stamp,),
523+
'expected': naive.time(),
524+
},
488525
{
489526
'sql': 'SELECT (1, 2)',
490527
'expected': {'_field_1': 1, '_field_2': 2},

0 commit comments

Comments
 (0)
0