8000 made switching to a big array (much) faster by not computing its min/max · gdementen/larray-editor@d4ba0af · GitHub
[go: up one dir, main page]
Skip to content

Commit d4ba0af

Browse files
gdementengdementen
committed
made switching to a big array (much) faster by not computing its min/max
on the whole array but on a sample (fixes larray-project#93) This means we can miss the actual min/max of the displayed part, so it complexifies the code. Most of this awful code will go away after : * we invert how changes work (store old values instead of new values) * we get decent buffering (in that case the min/max should only be updated when "moving" the buffer.
1 parent 4bdee96 commit d4ba0af

File tree

3 files changed

+85
-12
lines changed

3 files changed

+85
-12
lines changed

‎larray_editor/arraymodel.py

Lines changed: 44 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import numpy as np
44
from larray_editor.utils import (get_font, from_qvariant, to_qvariant, to_text_string,
55
is_float, is_number, LinearGradient, SUPPORTED_FORMATS, scale_to_01range,
6-
Product)
6+
Product, is_number_value, get_sample, get_sample_indices)
77
from qtpy.QtCore import Qt, QModelIndex, QAbstractTableModel
88
from qtpy.QtGui import QColor
99
from qtpy.QtWidgets import QMessageBox
@@ -256,8 +256,8 @@ def _set_data(self, data):
256256
self._compute_rows_cols_loaded()
257257

258258
def reset_minmax(self):
259-
data = self.get_values()
260259
try:
260+
data = self.get_values(sample=True)
261261
color_value = self.color_func(data) if self.color_func is not None else data
262262
# ignore nan, -inf, inf (setting them to 0 or to very large numbers is not an option)
263263
color_value = color_value[np.isfinite(color_value)]
@@ -336,8 +336,21 @@ def data(self, index, role=Qt.DisplayRole):
336336
elif role == Qt.BackgroundColorRole:
337337
if self.bgcolor_possible and self.bg_gradient is not None and value is not np.ma.masked:
338338
if self.bg_value is None:
339-
v = float(self.color_func(value) if self.color_func is not None else value)
340-
v = scale_to_01range(v, self.vmin, self.vmax)
339+
try:
340+
v = self.color_func(value) if self.color_func is not None else value
341+
if -np.inf < v < self.vmin:
342+
# TODO: this is suboptimal, as it can reset many times (though in practice, it is usually
343+
# ok). When we get buffering, we will need to compute vmin/vmax on the whole buffer
344+
# at once, eliminating this problem (and we could even compute final colors directly
345+
# all at once)
346+
self.vmin = v
347+
self.reset()
348+
elif self.vmax < v < np.inf:
349+
self.vmax = v
350+
self.reset()
351+
v = scale_to_01range(v, self.vmin, self.vmax)
352+
except TypeError:
353+
v = np.nan
341354
else:
342355
i, j = index.row(), index.column()
343356
v = self.bg_value[i, j]
@@ -346,26 +359,45 @@ def data(self, index, role=Qt.DisplayRole):
346359
# return to_qvariant("{}\n{}".format(repr(value),self.get_labels(index)))
347360
return to_qvariant()
348361

349-
def get_values(self, left=0, top=0, right=None, bottom=None):
362+
def get_values(self, left=0, top=0, right=None, bottom=None, sample=False):
350363
width, height = self.total_rows, self.total_cols
351364
if right is None:
352365
right = width
353366
if bottom is None:
354367
bottom = height
355-
values = self._data[left:right, top:bottom].copy()
356-
# both versions get the same result, but depending on inputs, the
357-
# speed difference can be large.
368+
# this whole bullshit will disappear when we implement undo/redo
369+
values = self._data[left:right, top:bottom]
370+
# both versions get the same result, but depending on inputs, the speed difference can be large.
358371
if values.size < len(self.changes):
372+
# changes are supposedly relatively small so this case should not be too slow even if we just want a sample
373+
values = values.copy()
359374
for i in range(left, right):
360375
for j in range(top, bottom):
361376
pos = i, j
362377
if pos in self.changes:
363378
values[i - left, j - top] = self.changes[pos]
379+
if sample:
380+
return get_sample(values, 500)
381+
else:
382+
return values
364383
else:
365-
for (i, j), value in self.changes.items():
366-
if left <= i < right and top <= j < bottom:
367-
values[i - left, j - top] = value
368-
return values
384+
if sample:
385+
sample_indices = get_sample_indices(values, 500)
386+
changes = self.changes
387+
388+
def get_val(idx):
389+
i, j = idx
390+
changes_idx = (i + left, j + top)
391+
return changes[changes_idx] if changes_idx in changes else values[idx]
392+
393+
# we need to keep the dtype, otherwise numpy might convert mixed object arrays to strings
394+
return np.array([get_val(idx) for idx in zip(*sample_indices)], dtype=values.dtype)
395+
else:
396+
values = values.copy()
397+
for (i, j), value in self.changes.items():
398+
if left <= i < right and top <= j < bottom:
399+
values[i - left, j - top] = value
400+
return values
369401

370402
def convert_value(self, value):
371403
"""

‎larray_editor/arraywidget.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -872,6 +872,7 @@ def _update_digits_scientific(self, data, scientific=None):
872872
def _get_sample(self, data):
873873
assert isinstance(data, la.LArray)
874874
data = data.data
875+
# TODO: use utils.get_sample instead
875876
size = data.size
876877
# this will yield a data sample of max 199
877878
step = (size // 100) if size > 100 else 1

‎larray_editor/utils.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import os
44
import sys
5+
import math
6+
57
import numpy as np
68

79
from qtpy import PYQT5
@@ -547,3 +549,41 @@ def scale_to_01range(value, vmin, vmax):
547549
else:
548550
assert vmin < vmax
549551
return (value - vmin) / (vmax - vmin)
552+
553+
554+
is_number_value = np.vectorize(lambda x: isinstance(x, (int, float, np.number)))
555+
556+
557+
def get_sample_step(data, maxsize):
558+
size = data.size
559+
if not size:
560+
return None
561+
return math.ceil(size / maxsize)
562+
563+
564+
def get_sample(data, maxsize):
565+
"""return sample. View in all cases.
566+
567+
if data.size < maxsize:
568+
sample_size == data.size
569+
else:
570+
(maxsize // 2) < sample_size <= maxsize
571+
572+
Parameters
573+
----------
574+
data
575+
maxsize
576+
577+
Returns
578+
-------
579+
view
580+
"""
581+
size = data.size
582+
if not size:
583+
return data
584+
return data.flat[::get_sample_step(data, maxsize)]
585+
586+
587+
def get_sample_indices(data, maxsize):
588+
flat_indices = np.arange(0, data.size, get_sample_step(data, maxsize))
589+
return np.unravel_index(flat_indices, data.shape)

0 commit comments

Comments
 (0)
0