8000 Jorwoods/query endpoint paging (#881) · tableau/server-client-python@6a56f08 · GitHub
[go: up one dir, main page]
Skip to content

Commit 6a56f08

Browse files
jorwoodsjacalata
jorwoods
authored and
jacalata
committed
Jorwoods/query endpoint paging (#881)
* Automatically paginate in QuerySet endpoints * Enable queryset operations for flows, groups, jobs, projects * Keep only page_size in memory * NULL result cache to ensure restarting from the first page * Use modulus to simplify idx calculation * Add support for slicing * Minimum page number is 1 * Properly support negative steps
1 parent 2194c0d commit 6a56f08

10 files changed

+138
-14
lines changed

‎tableauserverclient/models/workbook_item.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from .view_item import ViewItem
1717
from ..datetime_helpers import parse_datetime
1818

19+
1920
if TYPE_CHECKING:
2021
from .connection_item import ConnectionItem
2122
from .permissions_item import PermissionsRule

‎tableauserverclient/server/endpoint/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from .data_alert_endpoint import DataAlerts
44
from .databases_endpoint import Databases
55
from .datasources_endpoint import Datasources
6-
from .endpoint import Endpoint
6+
from .endpoint import Endpoint, QuerysetEndpoint
77
from .exceptions import (
88
ServerResponseError,
99
MissingRequiredFieldError,

‎tableauserverclient/server/endpoint/endpoint.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,9 @@ def all(self, *args, **kwargs):
228228
return queryset
229229

230230
@api(version="2.0")
231-
def filter(self, *args, **kwargs):
231+
def filter(self, *_, **kwargs):
232+
if _:
233+
raise RuntimeError("Only keyword arguments accepted.")
232234
queryset = QuerySet(self).filter(**kwargs)
233235
return queryset
234236

‎tableauserverclient/server/endpoint/flows_endpoint.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from typing import Iterable, List, Optional, TYPE_CHECKING, Tuple, Union
77

88
from .dqw_endpoint import _DataQualityWarningEndpoint
9-
from .endpoint import Endpoint, api
9+
from .endpoint import Endpoint, QuerysetEndpoint, api
1010
from .exceptions import InternalServerError, MissingRequiredFieldError
1111
from .permissions_endpoint import _PermissionsEndpoint
1212
from .resource_tagger import _ResourceTagger
@@ -30,7 +30,7 @@
3030
FilePath = Union[str, os.PathLike]
3131

3232

33-
class Flows(Endpoint):
33+
class Flows(QuerysetEndpoint):
3434
def __init__(self, parent_srv):
3535
super(Flows, self).__init__(parent_srv)
3636
self._resource_tagger = _ResourceTagger(parent_srv)

‎tableauserverclient/server/endpoint/groups_endpoint.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import logging
22

3-
from .endpoint import Endpoint, api
3+
from .endpoint import QuerysetEndpoint, api
44
from .exceptions import MissingRequiredFieldError
55
from .. import RequestFactory, GroupItem, UserItem, PaginationItem, JobItem
66
from ..pager import Pager
@@ -13,7 +13,7 @@
1313
from ..request_options import RequestOptions
1414

1515

16-
class Groups(Endpoint):
16+
class Groups(QuerysetEndpoint):
1717
@property
1818
def baseurl(self) -> str:
1919
return "{0}/sites/{1}/groups".format(self.parent_srv.baseurl, self.parent_srv.site_id)

‎tableauserverclient/server/endpoint/jobs_endpoint.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import logging
22

3-
from .endpoint import Endpoint, api
3+
from .endpoint import QuerysetEndpoint, api
44
from .exceptions import JobCancelledException, JobFailedException
55
from .. import JobItem, BackgroundJobItem, PaginationItem
66
from ..request_options import RequestOptionsBase
@@ -11,7 +11,7 @@
1111
from typing import List, Optional, Tuple, Union
1212

1313

14-
class Jobs(Endpoint):
14+
class Jobs(QuerysetEndpoint):
1515
@property
1616
def baseurl(self):
1717
return "{0}/sites/{1}/jobs".format(self.parent_srv.baseurl, self.parent_srv.site_id)

‎tableauserverclient/server/endpoint/projects_endpoint.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import logging
22

33
from .default_permissions_endpoint import _DefaultPermissionsEndpoint
4-
from .endpoint import api, Endpoint, XML_CONTENT_TYPE
4+
from .endpoint import QuerysetEndpoint, api, XML_CONTENT_TYPE
55
from .exceptions import MissingRequiredFieldError
66
from .permissions_endpoint import _PermissionsEndpoint
77
from .. import RequestFactory, RequestOptions, ProjectItem, PaginationItem, Permission
@@ -15,7 +15,7 @@
1515
from ..request_options import RequestOptions
1616

1717

18-
class Projects(Endpoint):
18+
class Projects(QuerysetEndpoint):
1919
def __init__(self, parent_srv: "Server") -> None:
2020
super(Projects, self).__init__(parent_srv)
2121

‎tableauserverclient/server/query.py

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from .filter import Filter
22
from .request_options import RequestOptions
33
from .sort import Sort
4+
import math
45

56

67
def to_camel_case(word):
@@ -15,11 +16,54 @@ def __init__(self, model):
1516
self._pagination_item = None
1617

1718
def __iter__(self):
18-
self._fetch_all()
19-
return iter(self._result_cache)
19+
self.request_options.pagenumber = 1
20+
self._result_cache = None
21+
total = self.total_available
22+
size = self.page_size
23+
yield from self._result_cache
24+
for page in range(1, math.ceil(total / size)):
25+
self.request_options.pagenumber = page + 1
26+
self._result_cache = None
27+
self._fetch_all()
28+
yield from self._result_cache
2029

2130
def __getitem__(self, k):
22-
return list(self)[k]
31+
page = self.page_number
32+
size = self.page_size
33+
34+
page_range = range((page - 1) * size, page * size)
35+
36+
if isinstance(k, slice):
37+
step = k.step if k.step is not None else 1
38+
start = k.start if k.start is not None else 0
39+
stop = k.stop if k.stop is not None else self.total_available
40+
if start < 0:
41+
start += self.total_available
42+
if stop < 0:
43+
stop += self.total_available
44+
if start < stop and step < 0:
45+
# Since slicing is left inclusive and right exclusive, shift
46+
# the start and stop values by 1 to keep that behavior
47+
start, stop = stop - 1, start - 1
48+
slice_stop = stop if stop > 0 else None
49+
k = slice(start, slice_stop, step)
50+
51+
k_range = range(start, stop, step)
52+
if all(i in page_range for i in k_range):
53+
return self._result_cache[k]
54+
return [self[i] for i in k_range]
55+
56+
if k < 0:
57+
k += self.total_available
58+
59+
if k in page_range:
60+
return self._result_cache[k % size]
61+
elif k in range(self.total_available):
62+
self._result_cache = None
63+
self.request_options.pagenumber = max(1, math.ceil(k / size))
64+
return self[k]
65+
else:
66+
raise IndexError
2367

2468
def _fetch_all(self):
2569
"""
@@ -43,7 +87,9 @@ def page_size(self):
4387
self._fetch_all()
4488
return self._pagination_item.page_size
4589

46-
def filter(self, **kwargs):
90+
def filter(self, *invalid, **kwargs):
91+
if invalid:
92+
raise RuntimeError(f"Only accepts keyword arguments.")
4793
for kwarg_key, value in kwargs.items():
4894
field_name, operator = self._parse_shorthand_filter(kwarg_key)
4995
self.request_options.filter.add(Filter(field_name, operator, value))

‎test/assets/request_option_slicing_queryset.xml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
<?xml version='1.0' encoding='UTF-8'?>
2+
<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
3+
<pagination pageNumber="1" pageSize="10" totalAvailable="10" />
4+
<views>
5+
<view id="d79634e1-6063-4ec9-95ff-50acbf609ff5" name="ENDANGERED SAFARI" contentUrl="SafariSample/sheets/ENDANGEREDSAFARI">
6+
<workbook id="3cc6cd06-89ce-4fdc-b935-5294135d6d42" />
7+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
8+
</view>
9+
<view id="fd252f73-593c-4c4e-8584-c032b8022adc" name="Overview" contentUrl="Superstore/sheets/Overview">
10+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
11+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
12+
</view>
13+
<view id="7ddf1576-4b15-4df5-98cb-03066f279059" name="Product" contentUrl="Superstore/sheets/Product">
14+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
15+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
16+
</view>
17+
<view id="c75f90a9-f10e-46be-8f01-07403780386a" name="Customers" contentUrl="Superstore/sheets/Customers">
18+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
19+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
20+
</view>
21+
<view id="880cd185-3f15-4063-816c-4a600883f4ed" name="Shipping" contentUrl="Superstore/sheets/Shipping">
22+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
23+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
24+
</view>
25+
<view id="29c73e36-94ed-474a-9533-e310d1e6a787" name="Performance" contentUrl="Superstore/sheets/Performance">
26+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
27+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
28+
</view>
29+
<view id="48ff745e-aeda-4d76-a162-2bc740b4f478" name="Commission Model" contentUrl="Superstore/sheets/CommissionModel">
30+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
31+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
32+
</view>
33+
<view id="2df55de2-3a2d-4e34-b515-6d4e70b830e9" name="Order Details" contentUrl="Superstore/sheets/OrderDetails">
34+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
35+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
36+
</view>
37+
<view id="c1513b2f-cb6f-4ae8-841f-4128fdd2a7dd" name="Forecast" contentUrl="Superstore/sheets/Forecast">
38+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
39+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
40+
</view>
41+
<view id="2e6d6c81-da71-4b41-892c-ba80d4e7a6d0" name="What If Forecast" contentUrl="Superstore/sheets/WhatIfForecast">
42+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" />
43+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
44+
</view>
45+
</views>
46+
</tsResponse>

‎test/test_request_option.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
FILTER_EQUALS = os.path.join(TEST_ASSET_DIR, "request_option_filter_equals.xml")
1515
FILTER_TAGS_IN = os.path.join(TEST_ASSET_DIR, "request_option_filter_tags_in.xml")
1616
FILTER_MULTIPLE = os.path.join(TEST_ASSET_DIR, "request_option_filter_tags_in.xml")
17+
SLICING_QUERYSET = os.path.join(TEST_ASSET_DIR, "request_option_slicing_queryset.xml")
1718

1819

1920
class RequestOptionTests(unittest.TestCase):
@@ -244,3 +245,31 @@ def test_multiple_filter_options_shorthand(self):
244245
for _ in range(100):
245246
matching_workbooks = self.server.workbooks.filter(tags__in=["sample", "safari", "weather"], name="foo")
246247
self.assertEqual(3, matching_workbooks.total_available)
248+
249+
def test_slicing_queryset(self):
250+
with open(SLICING_QUERYSET, "rb") as f:
251+
response_xml = f.read().decode("utf-8")
252+
with requests_mock.mock() as m:
253+
m.get(self.baseurl + "/views?pageNumber=1", text=response_xml)
254+
all_views = self.server.views.all()
255+
256+
self.assertEqual(10, len(all_views[::]))
257+
self.assertEqual(5, len(all_views[::2]))
258+
self.assertEqual(8, len(all_views[2:]))
259+
self.assertEqual(2, len(all_views[:2]))
260+
self.assertEqual(3, len(all_views[2:5]))
261+
self.assertEqual(3, len(all_views[-3:]))
262+
self.assertEqual(3, len(all_views[-6:-3]))
263+
self.assertEqual(3, len(all_views[3:6:-1]))
264+
self.assertEqual(3, len(all_views[6:3:-1]))
265+
self.assertEqual(10, len(all_views[::-1]))
266+
self.assertEqual(all_views[3:6], list(reversed(all_views[3:6:-1])))
267+
268+
self.assertEqual(all_views[-3].id, "2df55de2-3a2d-4e34-b515-6d4e70b830e9")
269+
270+
with self.assertRaises(IndexError):
271+
all_views[100]
272+
273+
def test_queryset_filter_args_error(self):
274+
with self.assertRaises(RuntimeError):
275+
workbooks = self.server.workbooks.filter("argument")

0 commit comments

Comments
 (0)
0