8000 ENH: support non default indexes in writing to Parquet by dhirschfeld · Pull Request #18629 · pandas-dev/pandas · GitHub
[go: up one dir, main page]
Skip to content

ENH: support non default indexes in writing to Parquet #18629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Dec 11, 2017
Merged

ENH: support non default indexes in writing to Parquet #18629

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
4bf7f56
Allow non-default indexes in to_parquet.
dhirschfeld Dec 9, 2017
7a6d9a9
Fixed test_write_with_index on Windows
dhirschfeld Dec 9, 2017
1496ef6
Fix test_write_with_index to work with fastparquet
dhirschfeld Dec 9, 2017
9c8cfb7
Fix linting issues
dhirschfeld Dec 10, 2017
60124ac
Added wahtsnew entry for 0.21.1
dhirschfeld Dec 10, 2017
03941b1
Merge remote-tracking branch 'upstream/master' into pr/18629
jorisvandenbossche Dec 10, 2017
d8a27e1
move whatsnew note to parquet section
jorisvandenbossche Dec 10, 2017
cd26110
undo version changes in ci and install docs
jorisvandenbossche Dec 10, 2017
1acb00c
undo testing changes for old pyarrow versions
jorisvandenbossche Dec 10, 2017
0f7b526
fix test for older versions of pyarrow
jorisvandenbossche Dec 10, 2017
67dffde
use read_pandas for pyarrow < 0.7
jorisvandenbossche Dec 10, 2017
79de86b
further clean-up
jorisvandenbossche Dec 10, 2017
0fbdc14
style update
jorisvandenbossche Dec 11, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
further clean-up
  • Loading branch information
@jorisvandenbossche
jorisvandenbossche committed Dec 10, 2017
commit 79de86b989dfcacf9f1f0ffd4b30833843bc2bc7
40 changes: 18 additions & 22 deletions pandas/io/parquet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,7 @@ def write(self, df, path, compression='snappy',
coerce_timestamps='ms', **kwargs):
self.validate_dataframe(df)
if self._pyarrow_lt_070:
self._validate_write_lt_070(
df, path, compression, coerce_timestamps, **kwargs
)
self._validate_write_lt_070(df)
path, _, _ = get_filepath_or_buffer(path)

if self._pyarrow_lt_060:
Expand All @@ -116,48 +114,46 @@ def write(self, df, path, compression='snappy',

def read(self, path, columns=None, **kwargs):
path, _, _ = get_filepath_or_buffer(path)
parquet_file = self.api.parquet.ParquetFile(path)
if self._pyarrow_lt_070:
return self._read_lt_070(path, parquet_file, columns, **kwargs)
return self.api.parquet.read_pandas(path, columns=columns,
**kwargs).to_pandas()
kwargs['use_pandas_metadata'] = True
return parquet_file.read(columns=columns, **kwargs).to_pandas()
return self.api.parquet.read_table(path, columns=columns,
**kwargs).to_pandas()

def _validate_write_lt_070(self, df, path, compression='snappy',
coerce_timestamps='ms', **kwargs):
def _validate_write_lt_070(self, df):
# Compatibility shim for pyarrow < 0.7.0
# TODO: Remove in pandas 0.22.0
from pandas.core.indexes.multi import MultiIndex
if isinstance(df.index, MultiIndex):
msg = (
"Mulit-index DataFrames are only supported "
"Multi-index DataFrames are only supported "
"with pyarrow >= 0.7.0"
)
raise ValueError(msg)
# Validate index
if not isinstance(df.index, Int64Index):
msg = (
"parquet does not support serializing {} for the index;"
"you can .reset_index() to make the index into column(s)"
"pyarrow < 0.7.0 does not support serializing {} for the "
"index; you can .reset_index() to make the index into "
"column(s), or install the latest version of pyarrow or "
"fastparquet."
)
raise ValueError(msg.format(type(df.index)))
if not df.index.equals(RangeIndex(len(df))):
raise ValueError(
"parquet does not support serializing a non-default index "
"for the index; you can .reset_index() to make the index "
"into column(s)"
"pyarrow < 0.7.0 does not support serializing a non-default "
"index; you can .reset_index() to make the index into "
"column(s), or install the latest version of pyarrow or "
"fastparquet."
)
if df.index.name is not None:
raise ValueError(
"parquet does not serialize index meta-data "
"on a default index"
"pyarrow < 0.7.0 does not serialize indexes with a name; you "
"can set the index.name to None or install the latest version "
"of pyarrow or fastparquet."
)

def _read_lt_070(self, path, parquet_file, columns, **kwargs):
# Compatibility shim for pyarrow < 0.7.0
# TODO: Remove in pandas 0.22.0
kwargs['columns'] = columns
return self.api.parquet.read_pandas(path, **kwargs).to_pandas()


class FastParquetImpl(BaseImpl):

Expand Down
9 changes: 4 additions & 5 deletions pandas/tests/io/test_parquet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from warnings import catch_warnings

import numpy as np
from numpy.random import randn
import pandas as pd
from pandas.compat import PY3, is_platform_windows
from pandas.io.parquet import (to_parquet, read_parquet, get_engine,
Expand Down Expand Up @@ -324,7 +323,7 @@ def test_write_index(self, engine):
# index with meta-data
df.index = [0, 1, 2]
df.index.name = 'foo'
self.check_round_trip( df, engine, write_kwargs={'compression': None})
self.check_round_trip(df, engine, write_kwargs={'compression': None})

def test_write_multiindex(self, pa_ge_070):
# Not suppoprted in fastparquet as of 0.1.3 or older pyarrow version
Expand All @@ -345,11 +344,11 @@ def test_multiindex_with_columns(self, pa_ge_070):

engine = pa_ge_070
dates = pd.date_range('01-Jan-2018', '01-Dec-2018', freq='MS')
df = pd.DataFrame(randn(2 * len(dates), 3), columns=list('ABC'))
df = pd.DataFrame(np.random.randn(2 * len(dates), 3),
columns=list('ABC'))
index1 = pd.MultiIndex.from_product(
[['Level1', 'Level2'], dates],
names=['level', 'date']
)
names=['level', 'date'])
index2 = index1.copy(names=None)
for index in [index1, index2]:
df.index = index
Expand Down
0