10000 BUG: df.agg, df.transform and df.apply use different methods when axis=1 than when axis=0 by topper-123 · Pull Request #21224 · pandas-dev/pandas · GitHub
[go: up one dir, main page]
Skip to content

BUG: df.agg, df.transform and df.apply use different methods when axis=1 than when axis=0 #21224

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 28, 2018
Merged

BUG: df.agg, df.transform and df.apply use different methods when axis=1 than when axis=0 #21224

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3d95bfc
Fix bug where df.agg(..., axis=1) gives wrong result
May 18, 2018
262bd3e
Fix tests for bug where df.agg(..., axis=1) gives wrong result
May 30, 2018
ed43757
changed according to comments
Jun 23, 2018
2be3747
correct apply(axis=1) and related bugs
Jul 16, 2018
b6382d4
'index' and 'columns' added to fixture and related changes.
Jul 20, 2018
5ad024c
add conftest cython_table_items + a few corrections
Jul 22, 2018
39ced29
clarified according to comments
topper-123 Jul 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
'index' and 'columns' added to fixture and related changes.
  • Loading branch information
@topper-123
tp authored and topper-123 committed Jul 26, 2018
commit b6382d4487534b033c5cf4fddd9748cfc69c451e
8 changes: 3 additions & 5 deletions doc/source/whatsnew/v0.24.0.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -475,11 +475,9 @@ Numeric
- Bug in :class:`Series` ``__rmatmul__`` doesn't support matrix vector multiplication (:issue:`21530`)
- Bug in :func:`factorize` fails with read-only array (:issue:`12813`)
- Fixed bug in :func:`unique` handled signed zeros inconsistently: for some inputs 0.0 and -0.0 were treated as equal and for some inputs as different. Now they are treated as equal for all inputs (:issue:`21866`)
- Bug in :meth:`DataFrame.agg`, :meth:`DataFrame.transform` and :meth:`DataFrame.apply` when ``axis=1``.
Using ``apply`` with a list of functions and axis=1 (e.g. ``df.apply(['abs'], axis=1)``)
previously gave a TypeError. This fixes that issue.
As ``agg`` and ``transform`` in some cases delegate to ``apply``, this also
fixed this issue for them (:issue:`16679`).
- Bug in :meth:`DataFrame.agg`, :meth:`DataFrame.transform` and :meth:`DataFrame.apply` where,
when supplied with a list of functions and ``axis=1`` (e.g. ``df.apply(['sum', 'mean'], axis=1)``),
a ``TypeError`` was wrongly raised. For all three methods such calculation are now done correctly. (:issue:`16679`).
-

Strings
Expand Down
6 changes: 3 additions & 3 deletions pandas/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,16 @@ def spmatrix(request):
return getattr(sparse, request.param + '_matrix')


@pytest.fixture(params=[0, 1],
ids=lambda x: "axis {}".format(x))
@pytest.fixture(params=[0, 1, 'index', 'columns'],
ids=lambda x: "axis {!r}".format(x))
def axis(request):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe also add axis_frame just for consistency? (its exactly equal to axis)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure, ok

"""
Fixture for returning the axis numbers of a dataframe.
"""
return request.param


@pytest.fixture(params=[0], ids=lambda x: "axis {}".format(x))
@pytest.fixture(params=[0, 'index'], ids=lambda x: "axis {!r}".format(x))
def axis_series(request):
"""
Fixture for returning the axis numbers of a series.
Expand Down
4 changes: 3 additions & 1 deletion pandas/core/apply.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from pandas.core.dtypes.generic import ABCSeries
from pandas.core.dtypes.common import (
is_extension_type,
is_dict_like,
is_list_like,
is_sequence)
from pandas.util._decorators import cache_readonly

Expand Down Expand Up @@ -106,7 +108,7 @@ def get_result(self):
""" compute the results """

# dispatch to agg
if isinstance(self.f, (list, dict)):
if is_list_like(self.f) or is_dict_like(self.f):
return self.obj.aggregate(self.f, axis=self.axis,
*self.args, **self.kwds)

Expand Down
4 changes: 2 additions & 2 deletions pandas/core/frame.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6080,7 +6080,7 @@ def aggregate(self, func, axis=0, *args, **kwargs):
return result

def _aggregate(self, arg, axis=0, *args, **kwargs):
if axis == 1:
if axis in {1, 'columns'}:
result, how = (super(DataFrame, self.T)
._aggregate(arg, *args, **kwargs))
result = result.T if result is not None else result
Expand All @@ -6090,7 +6090,7 @@ def _aggregate(self, arg, axis=0, *args, **kwargs):
agg = aggregate

def transform(self, func, axis=0, *args, **kwargs):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this doc-string get updated somewhere?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add the Appender

if axis == 1:
if axis in {1, 'columns'}:
return super(DataFrame, self.T).transform(func, *args, **kwargs).T
return super(DataFrame, self).transform(func, *args, **kwargs)

Expand Down
14 changes: 7 additions & 7 deletions pandas/tests/frame/test_apply.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -869,7 +869,7 @@ def zip_frames(frames, axis=1):
class TestDataFrameAggregate(TestData):

def test_agg_transform(self, axis):
other_axis = abs(axis - 1)
other_axis = 1 if axis in {0, 'index'} else 0

with np.errstate(all='ignore'):

Expand All @@ -890,7 +890,7 @@ def test_agg_transform(self, axis):
# list-like
result = self.frame.apply([np.sqrt], axis=axis)
expected = f_sqrt.copy()
if axis == 0:
if axis in {0, 'index'}:
expected.columns = pd.MultiIndex.from_product(
[self.frame.columns, ['sqrt']])
else:
Expand All @@ -906,7 +906,7 @@ def test_agg_transform(self, axis):
# functions per series and then concatting
result = self.frame.apply([np.abs, np.sqrt], axis=axis)
expected = zip_frames([f_abs, f_sqrt], axis=other_axis)
if axis == 0:
if axis in {0, 'index'}:
expected.columns = pd.MultiIndex.from_product(
[self.frame.columns, ['absolute', 'sqrt']])
else:
Expand Down Expand Up @@ -1001,7 +1001,7 @@ def test_agg_dict_nested_renaming_depr(self):
'B': {'bar': 'max'}})

def test_agg_reduce(self, axis):
other_axis = abs(axis - 1)
other_axis = 1 if axis in {0, 'index'} else 0
name1, name2 = self.frame.axes[other_axis].unique()[:2].sort_values()

# all reducers
Expand All @@ -1010,7 +1010,7 @@ def test_agg_reduce(self, axis):
self.frame.sum(axis=axis),
], axis=1)
expected.columns = ['mean', 'max', 'sum']
expected = expected.T if axis == 0 else expected
expected = expected.T if axis in {0, 'index'} else expected

result = self.frame.agg(['mean', 'max', 'sum'], axis=axis)
assert_frame_equal(result, expected)
Expand All @@ -1031,7 +1031,7 @@ def test_agg_reduce(self, axis):
index=['mean']),
name2: Series([self.frame.loc(other_axis)[name2].sum()],
index=['sum'])})
expected = expected.T if axis == 1 else expected
expected = expected.T if axis in {1, 'columns'} else expected
assert_frame_equal(result, expected)

# dict input with lists with multiple
Expand All @@ -1045,7 +1045,7 @@ def test_agg_reduce(self, axis):
self.frame.loc(other_axis)[name2].max()],
index=['sum', 'max'])),
]))
expected = expected.T if axis == 1 else expected
expected = expected.T if axis in {1, 'columns'} else expected
assert_frame_equal(result, expected)

def test_nuiscance_columns(self):
Expand Down
26 changes: 13 additions & 13 deletions pandas/tests/generic/test_label_or_level_utils.py
View file
Open in desktop
BE9D
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def test_is_level_or_label_reference_df_simple(df_levels, axis):
expected_labels, expected_levels = get_labels_levels(df_levels)

# Transpose frame if axis == 1
if axis == 1:
if axis in {1, 'columns'}:
df_levels = df_levels.T

# Perform checks
Expand All @@ -93,7 +93,7 @@ def test_is_level_or_label_reference_df_simple(df_levels, axis):
def test_is_level_reference_df_ambig(df_ambig, axis):

# Transpose frame if axis == 1
if axis == 1:
if axis in {1, 'columns'}:
df_ambig = df_ambig.T

# df has both an on-axis level and off-axis label named L1
Expand Down Expand Up @@ -166,7 +166,7 @@ def test_is_label_or_level_reference_panel_error(panel):
def test_check_label_or_level_ambiguity_df(df_ambig, axis):

# Transpose frame if axis == 1
if axis == 1:
if axis in {1, 'columns'}:
df_ambig = df_ambig.T

# df_ambig has both an on-axis level and off-axis label named L1
Expand All @@ -176,7 +176,7 @@ def test_check_label_or_level_ambiguity_df(df_ambig, axis):

assert df_ambig._check_label_or_level_ambiguity('L1', axis=axis)
warning_msg = w[0].message.args[0]
if axis == 0:
if axis in {0, 'index'}:
assert warning_msg.startswith("'L1' is both an index level "
"and a column label")
else:
Expand Down Expand Up @@ -236,7 +236,7 @@ def test_check_label_or_level_ambiguity_panel_error(panel):
# ===============================
def assert_label_values(frame, labels, axis):
for label in labels:
if axis == 0:
if axis in {0, 'index'}:
expected = frame[label]._values
else:
expected = frame.loc[label]._values
Expand All @@ -248,7 +248,7 @@ def assert_label_values(frame, labels, axis):

def assert_level_values(frame, levels, axis):
for level in levels:
if axis == 0:
if axis in {0, 'index'}:
expected = frame.index.get_level_values(level=level)._values
else:
expected = (frame.columns
Expand All @@ -267,7 +267,7 @@ def test_get_label_or_level_values_df_simple(df_levels, axis):
expected_labels, expected_levels = get_labels_levels(df_levels)

# Transpose frame if axis == 1
if axis == 1:
if axis in {1, 'columns'}:
df_levels = df_levels.T

# Perform checks
Expand All @@ -278,7 +278,7 @@ def test_get_label_or_level_values_df_simple(df_levels, axis):
def test_get_label_or_level_values_df_ambig(df_ambig, axis):

# Transpose frame if axis == 1
if axis == 1:
if axis in {1, 'columns'}:
df_ambig = df_ambig.T

# df has both an on-axis level and off-axis label named L1
Expand All @@ -298,7 +298,7 @@ def test_get_label_or_level_values_df_ambig(df_ambig, axis):
def test_get_label_or_level_values_df_duplabels(df_duplabels, axis):

# Transpose frame if axis == 1
if axis == 1:
if axis in {1, 'columns'}:
df_duplabels = df_duplabels.T

# df has unambiguous level 'L1'
Expand All @@ -308,7 +308,7 @@ def test_get_label_or_level_values_df_duplabels(df_duplabels, axis):
assert_label_values(df_duplabels, ['L3'], axis=axis)

# df has duplicate labels 'L2'
if axis == 0:
if axis in {0, 'index'}:
expected_msg = "The column label 'L2' is not unique"
else:
expected_msg = "The index label 'L2' is not unique"
Expand Down Expand Up @@ -355,7 +355,7 @@ def assert_labels_dropped(frame, labels, axis):
for label in labels:
df_dropped = frame._drop_labels_or_levels(label, axis=axis)

if axis == 0:
if axis in {0, 'index'}:
assert label in frame.columns
assert label not in df_dropped.columns
else:
Expand All @@ -367,7 +367,7 @@ def assert_levels_dropped(frame, levels, axis):
for level in levels:
df_dropped = frame._drop_labels_or_levels(level, axis=axis)

if axis == 0:
if axis in {0, 'index'}:
assert level in frame.index.names
assert level not in df_dropped.index.names
else:
Expand All @@ -383,7 +383,7 @@ def test_drop_labels_or_levels_df(df_levels, axis):
expected_labels, expected_levels = get_labels_levels(df_levels)

# Transpose frame if axis == 1
if axis == 1:
if axis in {1, 'columns'}:
df_levels = df_levels.T

# Perform checks
Expand Down
0