8000 Implement Pager for auto-paging requests (#90) · LGraber/server-client-python@a6d0ede · GitHub
[go: up one dir, main page]
Skip to content

Commit a6d0ede

Browse files
t8y8t8y8
authored
Implement Pager for auto-paging requests (tableau#90)
Add a `Pager` object that can wrap any `Endpoint` with a `.get` method. This returns a generator that can be used anywhere a standard iterator can. It also takes request_options and can start from any page, and use any page size. It will make a single call, and yield items (in proper order) until you reach the end of that page, it will then call `Endpoint.get` again and fetch the next page. * If you start midway (page 5 of 10) the iterator will only work from page 5 forward. * Sort and Filter are supported * If the count changes on the Server-side (eg someone deleted an item while you were iterating) it will raise `StopIteration` and exit gracefully * Tested with unittests and against a live server, ran sample too Initial Implementation based on sample by @RussTheAerialist
1 parent 101eedb commit a6d0ede

File tree

9 files changed

+193
-34
lines changed

9 files changed

+193
-34
lines changed

‎samples/pagination_sample.py

Lines changed: 24 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -17,37 +17,6 @@
1717
import tableauserverclient as TSC
1818

1919

20-
class pagination_generator(object):
21-
""" This class returns a generator that will iterate over all of the results.
22-
23-
server is the server object that will be used when calling the callback. It will be passed
24-
to the callback on each iteration
25-
26-
Callback is expected to take a server object and a request options and return two values, an array of results,
27-
and the pagination item from the current call. This will be used to build subsequent requests.
28-
"""
29-
30-
def __init__(self, fetch_more):
31-
self._fetch_more = fetch_more
32-
33-
def __call__(self):
34-
current_item_list, last_pagination_item = self._fetch_more(None) # Prime the generator
35-
count = 0
36-
37-
while count < last_pagination_item.total_available:
38-
if len(current_item_list) == 0:
39-
current_item_list, last_pagination_item = self._load_next_page(current_item_list, last_pagination_item)
40-
41-
yield current_item_list.pop(0)
42-
count += 1
43-
44-
def _load_next_page(self, current_item_list, last_pagination_item):
45-
next_page = last_pagination_item.page_number + 1
46-
opts = TSC.RequestOptions(pagenumber=next_page, pagesize=last_pagination_item.page_size)
47-
current_item_list, last_pagination_item = self._fetch_more(opts)
48-
return current_item_list, last_pagination_item
49-
50-
5120
def main():
5221

5322
parser = argparse.ArgumentParser(description='Return a list of all of the workbooks on your server')
@@ -70,10 +39,32 @@ def main():
7039
server = TSC.Server(args.server)
7140

7241
with server.auth.sign_in(tableau_auth):
73-
generator = pagination_generator(server.workbooks.get)
42+
43+
# Pager returns a generator that yields one item at a time fetching
44+
# from Server only when necessary. Pager takes a server Endpoint as its
45+
# first parameter. It will call 'get' on that endpoint. To get workbooks
46+
# pass `server.workbooks`, to get users pass` server.users`, etc
47+
# You can then loop over the generator to get the objects one at a time
48+
# Here we print the workbook id for each workbook
49+
7450
print("Your server contains the following workbooks:\n")
75-
for wb in generator():
51+
for wb in TSC.Pager(server.workbooks):
7652
print(wb.name)
7753

54+
# Pager can also be used in list comprehensions or generator expressions
55+
# for compactness and easy filtering. Generator expressions will use less
56+
# memory than list comprehsnsions. Consult the Python laguage documentation for
57+
# best practices on which are best for your use case. Here we loop over the
58+
# Pager and only keep workbooks where the name starts with the letter 'a'
59+
# >>> [wb for wb in TSC.Pager(server.workbooks) if wb.name.startswith('a')] # List Comprehension
60+
# >>> (wb for wb in TSC.Pager(server.workbooks) if wb.name.startswith('a')) # Generator Expression
61+
62+
# Since Pager is a generator it follows the standard conventions and can
63+
# be fed to a list if you really need all the workbooks in memory at once.
64+
# If you need everything, it may be faster to use a larger page size
65+
66+
# >>> request_options = TSC.RequestOptions(pagesize=1000)
67+
# >>> all_workbooks = list(TSC.Pager(server.workbooks, request_options))
68+
7869
if __name__ == '__main__':
7970
main()

‎tableauserverclient/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
SiteItem, TableauAuth, UserItem, ViewItem, WorkbookItem, UnpopulatedPropertyError, \
55
HourlyInterval, DailyInterval, WeeklyInterval, MonthlyInterval, IntervalItem
66
from .server import RequestOptions, Filter, Sort, Server, ServerResponseError,\
7-
MissingRequiredFieldError, NotSignedInError
7+
MissingRequiredFieldError, NotSignedInError, Pager
88

99
__version__ = '0.0.1'
1010
__VERSION__ = __version__

‎tableauserverclient/server/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@
88
from .endpoint import Auth, Datasources, Endpoint, Groups, Projects, Schedules, \
99
Sites, Users, Views, Workbooks, ServerResponseError, MissingRequiredFieldError
1010
from .server import Server
11+
from .pager import Pager
1112
from .exceptions import NotSignedInError

‎tableauserverclient/server/pager.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
from . import RequestOptions
2+
3+
4+
class Pager(object):
5+
"""
6+
Generator that takes an endpoint with `.get` and lazily loads items from Server.
7+
Supports all `RequestOptions` including starting on any page.
8+
"""
9+
10+
def __init__(self, endpoint, request_opts=None):
11+
self._endpoint = endpoint.get
12+
self._options = request_opts
13+
14+
# If we have options we could be starting on any page, backfill the count
15+
if self._options:
16+
self._count = ((self._options.pagenumber - 1) * self._options.pagesize)
17+
else:
18+
self._count = 0
19+
20+
def __iter__(self):
21+
# Fetch the first page
22+
current_item_list, last_pagination_item = self._endpoint(self._options)
23+
24+
# Get the rest on demand as a generator
25+
while self._count < last_pagination_item.total_available:
26+
if len(current_item_list) == 0:
27+
current_item_list, last_pagination_item = self._load_next_page(last_pagination_item)
28+
29+
try:
30+
yield current_item_list.pop(0)
31+
self._count += 1
32+
33+
except IndexError:
34+
# The total count on Server changed while fetching exit gracefully
35+
raise StopIteration
36+
37+
def _load_next_page(self, last_pagination_item):
38+
next_page = last_pagination_item.page_number + 1
39+
opts = RequestOptions(pagenumber=next_page, pagesize=last_pagination_item.page_size)
40+
if self._options is not None:
41+
opts.sort, opts.filter = self._options.sort, self._options.filter
42+
current_item_list, last_pagination_item = self._endpoint(opts)
43+
return current_item_list, last_pagination_item

‎tableauserverclient/server/server.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from .exceptions import NotSignedInError
22
from .endpoint import Sites, Views, Users, Groups, Workbooks, Datasources, Projects, Auth, Schedules, ServerInfo
3+
34
import requests
45

56

‎test/assets/workbook_get_page_1.xml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<?xml version='1.0' encoding='UTF-8'?>
2+
<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
3+
<pagination pageNumber="1" pageSize="1" totalAvailable="3" />
4+
<workbooks>
5+
<workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" name="Page1Workbook" contentUrl="Page1Workbook" showTabs="false" size="1" createdAt="2016-08-03T20:34:04Z" updatedAt="2016-08-04T17:56:41Z">
6+
<project id="ee8c6e70-43b6-11e6-af4f-f7b0d8e20760" name="default" />
7+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
8+
<tags />
9+
</workbook>
10+
</workbooks>
11+
</tsResponse>

‎test/assets/workbook_get_page_2.xml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?xml version='1.0' encoding='UTF-8'?>
2+
<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
3+
<pagination pageNumber="2" pageSize="1" totalAvailable="3" />
4+
<workbooks>
5+
<workbook id="3cc6cd06-89ce-4fdc-b935-5294135d6d42" name="Page2Workbook" contentUrl="Page2Workbook" showTabs="false" size="26" createdAt="2016-07-26T20:34:56Z" updatedAt="2016-07-26T20:35:05Z">
6+
<project id="ee8c6e70-43b6-11e6-af4f-f7b0d8e20760" name="default" />
7+
<owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
8+
<tags>
9+
<tag label="Safari" />
10+
<tag label="Sample" />
11+
</tags>
12+
</workbook>
13+
</workbooks>
14+
</tsResponse>

‎test/assets/workbook_get_page_3.xml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?xml version='1.0' encoding='UTF-8'?>
2+
<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
3+
<pagination pageNumber="3" pageSize="1" totalAvailable="3" />
4+
<workbooks>
5+
<workbook id="0413f2d6-387d-4fb6-9823-370d67b1276f" name="Page3Workbook" contentUrl="Page3Workbook" showTabs="false" size="26" createdAt="2016-07-26T20:34:56Z" updatedAt="2016-07-26T20:35:05Z">
6+
<project id="63ba565c-f352-41f4-87f5-ba4573fc7c2c" name="default" />
7+
<owner id="299fe064-51a5-4e11-93a1-76116239f082" />
8+
</workbook>
9+
</workbooks>
10+
</tsResponse>

‎test/test_pager.py

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import unittest
2+
import os
3+
import requests_mock
4+
import tableauserverclient as TSC
5+
6+
TEST_ASSET_DIR = os.path.join(os.path.dirname(__file__), 'assets')
7+
8+
GET_XML_PAGE1 = os.path.join(TEST_ASSET_DIR, 'workbook_get_page_1.xml')
9+
GET_XML_PAGE2 = os.path.join(TEST_ASSET_DIR, 'workbook_get_page_2.xml')
10+
GET_XML_PAGE3 = os.path.join(TEST_ASSET_DIR, 'workbook_get_page_3.xml')
11+
12+
13+
class PagerTests(unittest.TestCase):
14+
def setUp(self):
15+
self.server = TSC.Server('http://test')
16+
17+
# Fake sign in
18+
self.server._site_id = 'dad65087-b08b-4603-af4e-2887b8aafc67'
19+
self.server._auth_token = 'j80k54ll2lfMZ0tv97mlPvvSCRyD0DOM'
20+
21+
self.baseurl = self.server.workbooks.baseurl
22+
23+
def test_pager_with_no_options(self):
24+
with open(GET_XML_PAGE1, 'rb') as f:
25+
page_1 = f.read().decode('utf-8')
26+
with open(GET_XML_PAGE2, 'rb') as f:
27+
page_2 = f.read().decode('utf-8')
28+
with open(GET_XML_PAGE3, 'rb') as f:
29+
page_3 = f.read().decode('utf-8')
30+
with requests_mock.mock() as m:
31+
# Register Pager with default request options
32+
m.get(self.baseurl, text=page_1)
33+
34+
# Register Pager with some pages
35+
m.get(self.baseurl + "?pageNumber=1&pageSize=1", text=page_1)
36+
m.get(self.baseurl + "?pageNumber=2&pageSize=1", text=page_2)
37+
m.get(self.baseurl + "?pageNumber=3&pageSize=1", text=page_3)
38+
39+
# No options should get all 3
40+
workbooks = list(TSC.Pager(self.server.workbooks))
41+
self.assertTrue(len(workbooks) == 3)
42+
43+
# Let's check that workbook items aren't duplicates
44+
wb1, wb2, wb3 = workbooks
45+
self.assertEqual(wb1.name, 'Page1Workbook')
46+
self.assertEqual(wb2.name, 'Page2Workbook')
47+
self.assertEqual(wb3.name, 'Page3Workbook')
48+
49+
def test_pager_with_options(self):
50+
with open(GET_XML_PAGE1, 'rb') as f:
51+
page_1 = f.read().decode('utf-8')
52+
with open(GET_XML_PAGE2, 'rb') as f:
53+
page_2 = f.read().decode('utf-8')
54+
with open(GET_XML_PAGE3, 'rb') as f:
55+
page_3 = f.read().decode('utf-8')
56+
with requests_mock.mock() as m:
57+
# Register Pager with some pages
58+
m.get(self.baseurl + "?pageNumber=1&pageSize=1", text=page_1)
59+
m.get(self.baseurl + "?pageNumber=2&pageSize=1", text=page_2)
60+
m.get(self.baseurl + "?pageNumber=3&pageSize=1", text=page_3)
61+
m.get(self.baseurl + "?pageNumber=1&pageSize=3", text=page_1)
62+
63+
# Starting on page 2 should get 2 out of 3
64+
opts = TSC.RequestOptions(2, 1)
65+
workbooks = list(TSC.Pager(self.server.workbooks, opts))
66+
self.assertTrue(len(workbooks) == 2)
67+
68+
# Check that the workbooks are the 2 we think they should be
69+
wb2, wb3 = workbooks
70+
self.assertEqual(wb2.name, 'Page2Workbook')
71+
self.assertEqual(wb3.name, 'Page3Workbook')
72+
73+
# Starting on 1 with pagesize of 3 should get all 3
74+
opts = TSC.RequestOptions(1, 3)
75+
workbooks = list(TSC.Pager(self.server.workbooks, opts))
76+
self.assertTrue(len(workbooks) == 3)
77+
wb1, wb2, wb3 = workbooks
78+
self.assertEqual(wb1.name, 'Page1Workbook')
79+
self.assertEqual(wb2.name, 'Page2Workbook')
80+
self.assertEqual(wb3.name, 'Page3Workbook')
81+
82+
# Starting on 3 with pagesize of 1 should get the last item
83+
opts = TSC.RequestOptions(3, 1)
84+
workbooks = list(TSC.Pager(self.server.workbooks, opts))
85+
self.assertTrue(len(workbooks) == 1)
86+
# Should have the last workbook
87+
wb3 = workbooks.pop()
88+
self.assertEqual(wb3.name, 'Page3Workbook')

0 commit comments

Comments
 (0)
0