8000 [MRG+1] Fixes n_iter_without_progress and min_grad_norm in TSNE by ssaeger · Pull Request #6497 · scikit-learn/scikit-learn · GitHub
[go: up one dir, main page]

Skip to content

[MRG+1] Fixes n_iter_without_progress and min_grad_norm in TSNE #6497

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions sklearn/manifold/t_sne.py
Original file line number Diff line number Diff line change
Expand Up @@ -545,15 +545,19 @@ class TSNE(BaseEstimator):
least 200.

n_iter_without_progress : int, optional (default: 30)
Only used if method='exact'
Maximum number of iterations without progress before we abort the
optimization.
optimization. If method='barnes_hut' this parameter is fixed to
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This behavior is somewhat peculiar and we could work around it, but let's leave it for now.

a value of 30 and cannot be changed.

.. versionadded:: 0.17
parameter *n_iter_without_progress* to control stopping criteria.

min_grad_norm : float, optional (default: 1E-7)
min_grad_norm : float, optional (default: 1e-7)
Only used if method='exact'
If the gradient norm is below this threshold, the optimization will
be aborted.
be aborted. If method='barnes_hut' this parameter is fixed to a value
of 1e-3 and cannot be changed.

metric : string or callable, optional
The metric to use when calculating distance between instances in a
Expand Down Expand Up @@ -695,7 +699,8 @@ def _fit(self, X, skip_num_points=0):
'memory. Otherwise consider dimensionality '
'reduction techniques (e.g. TruncatedSVD)')
else:
X = check_array(X, accept_sparse=['csr', 'csc', 'coo'], dtype=np.float64)
X = check_array(X, accept_sparse=['csr', 'csc', 'coo'],
dtype=np.float64)
random_state = check_random_state(self.random_state)

if self.early_exaggeration < 1.0:
Expand Down Expand Up @@ -797,9 +802,9 @@ def _tsne(self, P, degrees_of_freedom, n_samples, random_state,
self.n_components)
params = X_embedded.ravel()

opt_args = {}
opt_args = {"n_iter": 50, "momentum": 0.5, "it": 0,
"learning_rate": self.learning_rate,
"n_iter_without_progress": self.n_iter_without_progress,
"verbose": self.verbose, "n_iter_check": 25,
"kwargs": dict(skip_num_points=skip_num_points)}
if self.method == 'barnes_hut':
Expand All @@ -824,7 +829,7 @@ def _tsne(self, P, degrees_of_freedom, n_samples, random_state,
opt_args['args'] = [P, degrees_of_freedom, n_samples,
self.n_components]
opt_args['min_error_diff'] = 0.0
opt_args['min_grad_norm'] = 0.0
opt_args['min_grad_norm'] = self.min_grad_norm

# Early exaggeration
P *= self.early_exaggeration
Expand Down
64 changes: 64 additions & 0 deletions sklearn/manifold/tests/test_t_sne.py
Original file line number Diff line number Diff line change
Expand Up @@ -542,3 +542,67 @@ def test_index_offset():
# Make sure translating between 1D and N-D indices are preserved
assert_equal(_barnes_hut_tsne.test_index2offset(), 1)
assert_equal(_barnes_hut_tsne.test_index_offset(), 1)


def test_n_iter_without_progress():
# Make sure that the parameter n_iter_without_progress is used correctly
random_state = check_random_state(0)
X = random_state.randn(100, 2)
tsne = TSNE(n_iter_without_progress=2, verbose=2,
random_state=0, method='exact')

old_stdout = sys.stdout
sys.stdout = StringIO()
try:
tsne.fit_transform(X)
finally:
out = sys.stdout.getvalue()
sys.stdout.close()
sys.stdout = old_stdout

# The output needs to contain the value of n_iter_without_progress
assert("did not make any progress during the "
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can use nose.tools.assert_in

"last 2 episodes. Finished." in out)


def test_min_grad_norm():
# Make sure that the parameter min_grad_norm is used correctly
random_state = check_random_state(0)
X = random_state.randn(100, 2)
min_grad_norm = 0.002
tsne = TSNE(min_grad_norm=min_grad_norm, verbose=2,
random_state=0, method='exact')

old_stdout = sys.stdout
sys.stdout = StringIO()
try:
tsne.fit_transform(X)
finally:
out = sys.stdout.getvalue()
sys.stdout.close()
sys.stdout = old_stdout

lines_out = out.split('\n')

# extract the gradient norm from the verbose output
gradient_norm_values = []
for line in lines_out:
# When the computation is Finished just an old gradient norm value
# is repeated that we do not need to store
if 'Finished' in line:
break

start_grad_norm = line.find('gradient norm')
if start_grad_norm >= 0:
line = line[start_grad_norm:]
line = line.replace('gradient norm = ', '')
gradient_norm_values.append(float(line))

# Compute how often the gradient norm is smaller than min_grad_norm
gradient_norm_values = np.array(gradient_norm_values)
n_smaller_gradient_norms = \
len(gradient_norm_values[gradient_norm_values <= min_grad_norm])

# The gradient norm can be smaller than min_grad_norm at most once,
# because in the moment it becomes smaller the optimization stops
assert_less_equal(n_smaller_gradient_norms, 1)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Needs newline at end of file.

0