to_dataframe fails with Decimal type with precision 77 does not fit when REST API is used to download extreme BIGNUMERIC values #812
Description
Discovered in #786, but I think it's unrelated to the dtype changes I'm making there.
Steps to reproduce
- Create a table with max / min NUMERIC values.
{"bool_col": true, "bytes_col": "abcd", "date_col": "9999-12-31", "datetime_col": "9999-12-31 23:59:59.999999", "geography_col": "POINT(-122.0838511 37.3860517)", "int64_col": "9223372036854775807", "numeric_col": "9.9999999999999999999999999999999999999E+28", "bignumeric_col": "5.7896044618658097711785492504343953926634992332820282019728792003956564819967E+38", "float64_col": "+inf", "string_col": "Hello, World", "time_col": "23:59:59.99999", "timestamp_col": "9999-12-31T23:59:59.999999Z"}
{"bool_col": false, "bytes_col": "abcd", "date_col": "0001-01-01", "datetime_col": "0001-01-01 00:00:00", "geography_col": "POINT(-122.0838511 37.3860517)", "int64_col": "-9223372036854775808", "numeric_col": "-9.9999999999999999999999999999999999999E+28", "bignumeric_col": "-5.7896044618658097711785492504343953926634992332820282019728792003956564819968E+38", "float64_col": "-inf", "string_col": "Hello, World", "time_col": "00:00:00", "timestamp_col": "0001-01-01T00:00:00.000000Z"}
{"bool_col": false, "bytes_col": "", "date_col": "1970-01-01", "datetime_col": "1970-01-01 00:00:00", "geography_col": "POINT(0 0)", "int64_col": "0", "numeric_col": "0.0", "bignumeric_col": "0.0", "float64_col": 0.0, "string_col": "", "time_col": "12:00:00", "timestamp_col": "1970-01-01T00:00:00.000000Z"}
{"bool_col": null, "bytes_col": null, "date_col": null, "datetime_col": null, "geography_col": null, "int64_col": null, "numeric_col": null, "bignumeric_col": null, "float64_col": null, "string_col": null, "time_col": null, "timestamp_col": null}
- Download table with
list_rows, but setmax_results=10so that REST API is used. - Call
to_dataframe
Stack trace
bigquery_client = <google.cloud.bigquery.client.Client object at 0x7fcdafac9bb0>
scalars_extreme_table = 'precise-truck-742.python_bigquery_tests_system_20210723213441_aa9f85.scalars_extreme'
max_results = 10
@pytest.mark.parametrize(
("max_results",), ((None,), (10,),) # Use BQ Storage API. # Use REST API.
)
def test_list_rows_nullable_scalars_extreme_dtypes(
bigquery_client, scalars_extreme_table, max_results
):
> df = bigquery_client.list_rows(
scalars_extreme_table, max_results=max_results
).to_dataframe()
tests/system/test_pandas.py:828:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
google/cloud/bigquery/table.py:1903: in to_dataframe
record_batch = self.to_arrow(
google/cloud/bigquery/table.py:1718: in to_arrow
for record_batch in self._to_arrow_iterable(
google/cloud/bigquery/table.py:1618: in _to_page_iterable
yield from result_pages
google/cloud/bigquery/_pandas_helpers.py:570: in download_arrow_row_iterator
yield _row_iterator_page_to_arrow(page, column_names, arrow_types)
google/cloud/bigquery/_pandas_helpers.py:543: in _row_iterator_page_to_arrow
arrays.append(pyarrow.array(page._columns[column_index], type=arrow_type))
pyarrow/array.pxi:306: in pyarrow.lib.array
???
pyarrow/array.pxi:39: in pyarrow.lib._sequence_to_array
???
pyarrow/error.pxi:141: in pyarrow.lib.pyarrow_internal_check_status
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E pyarrow.lib.ArrowInvalid: Decimal type with precision 77 does not fit into precision inferred from first array element: 76