8000 fix: queryset support for flowruns · tableau/server-client-python@8c2d9bd · GitHub
[go: up one dir, main page]
Skip to content

Commit 8c2d9bd

Browse files
jorwoodsjorwoods
committed
fix: queryset support for flowruns
FlowRun's get endpoint does not return a PaginationItem. This provides a tweak to QuerySet to provide a workaround so all items matching whatever filters are supplied. It also corrects the return types of flowruns.get and fixes the XML test asset to reflect what is really returned by the server.
1 parent fad98bd commit 8c2d9bd

File tree

4 files changed

+87
-16
lines changed

4 files changed

+87
-16
lines changed

‎tableauserverclient/server/endpoint/flow_runs_endpoint.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import logging
2-
from typing import List, Optional, Tuple, TYPE_CHECKING
2+
from typing import List, Optional, TYPE_CHECKING
33

44
from tableauserverclient.server.endpoint.endpoint import QuerysetEndpoint, api
55
from tableauserverclient.server.endpoint.exceptions import FlowRunFailedException, FlowRunCancelledException
6-
from tableauserverclient.models import FlowRunItem, PaginationItem
6+
from tableauserverclient.models import FlowRunItem
77
from tableauserverclient.exponential_backoff import ExponentialBackoffTimer
88

99
from tableauserverclient.helpers.logging import logger
@@ -25,13 +25,15 @@ def baseurl(self) -> str:
2525

2626
# Get all flows
2727
@api(version="3.10")
28-
def get(self, req_options: Optional["RequestOptions"] = None) -> Tuple[List[FlowRunItem], PaginationItem]:
28+
# QuerysetEndpoint expects a PaginationItem to be returned, but FlowRuns
29+
# does not return a PaginationItem. Suppressing the mypy error because the
30+
# changes to the QuerySet class should permit this to function regardless.
31+
def get(self, req_options: Optional["RequestOptions"] = None) -> List[FlowRunItem]: # type: ignore[override]
2932
logger.info("Querying all flow runs on site")
3033
url = self.baseurl
3134
server_response = self.get_request(url, req_options)
32-
pagination_item = PaginationItem.from_response(server_response.content, self.parent_srv.namespace)
3335
all_flow_run_items = FlowRunItem.from_response(server_response.content, self.parent_srv.namespace)
34-
return all_flow_run_items, pagination_item
36+
return all_flow_run_items
3537

3638
# Get 1 flow by id
3739
@api(version="3.10")

‎tableauserverclient/server/query.py

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
from collections.abc import Sized
22
from itertools import count
3+
import sys
34
from typing import Iterable, Iterator, List, Optional, Protocol, Tuple, TYPE_CHECKING, TypeVar, overload
45
from tableauserverclient.config import config
56
from tableauserverclient.models.pagination_item import PaginationItem
7+
from tableauserverclient.server.endpoint.exceptions import ServerResponseError
68
from tableauserverclient.server.filter import Filter
79
from tableauserverclient.server.request_options import RequestOptions
810
from tableauserverclient.server.sort import Sort
@@ -34,6 +36,31 @@ def to_camel_case(word: str) -> str:
3436

3537

3638
class QuerySet(Iterable[T], Sized):
39+
"""
40+
QuerySet is a class that allows easy filtering, sorting, and iterating over
41+
many endpoints in TableauServerClient. It is designed to be used in a similar
42+
way to Django QuerySets, but with a more limited feature set.
43+
44+
QuerySet is an iterable, and can be used in for loops, list comprehensions,
45+
and other places where iterables are expected.
46+
47+
QuerySet is also a Sized, and can be used in places where the length of the
48+
QuerySet is needed. The length of the QuerySet is the total number of items
49+
available in the QuerySet, not just the number of items that have been
50+
fetched. If the endpoint does not return a total count of items, the length
51+
of the QuerySet will be None. If there is no total count, the QuerySet will
52+
continue to fetch items until there are no more items to fetch.
53+
54+
QuerySet is not re-entrant. It is not designed to be used in multiple places
55+
at the same time. If you need to use a QuerySet in multiple places, you
56+
should create a new QuerySet for each place you need to use it, convert it
57+
to a list, or create a deep copy of the QuerySet.
58+
59+
QuerySet's are also indexable, and can be sliced. If you try to access an
60+
item that has not been fetched, the QuerySet will fetch the page that
61+
contains the item you are looking for.
62+
"""
63+
3764
def __init__(self, model: "QuerysetEndpoint[T]", page_size: Optional[int] = None) -> None:
38 6D47 65
self.model = model
3966
self.request_options = RequestOptions(pagesize=page_size or config.PAGE_SIZE)
@@ -49,10 +76,20 @@ def __iter__(self: Self) -> Iterator[T]:
4976
for page in count(1):
5077
self.request_options.pagenumber = page
5178
self._result_cache = []
52-
self._fetch_all()
79+
try:
80+
self._fetch_all()
81+
except ServerResponseError as e:
82+
if e.code == "400006":
83+
# If the endpoint does not support pagination, it will end
84+
# up overrunning the total number of pages. Catch the
85+
# error and break out of the loop.
86+
raise StopIteration
5387
yield from self._result_cache
54-
# Set result_cache to empty so the fetch will populate
55-
if (page * self.page_size) >= len(self):
88+
# If the length of the QuerySet is unknown, continue fetching until
89+
# the result cache is empty.
90+
if (size := len(self)) == 0:
91+
continue
92+
if (page * self.page_size) >= size:
5693
return
5794

5895
@overload
@@ -115,10 +152,15 @@ def _fetch_all(self: Self) -> None:
115152
Retrieve the data and store result and pagination item in cache
116153
"""
117154
if not self._result_cache:
118-
self._result_cache, self._pagination_item = self.model.get(self.request_options)
155+
response = self.model.get(self.request_options)
156+
if isinstance(response, tuple):
157+
self._result_cache, self._pagination_item = response
158+
else:
159+
self._result_cache = response
160+
self._pagination_item = PaginationItem()
119161

120162
def __len__(self: Self) -> int:
121-
return self.total_available
163+
return self.total_available or 0
122164

123165
@property
124166
def total_available(self: Self) -> int:
@@ -128,12 +170,16 @@ def total_available(self: Self) -> int:
128170
@property
129171
def page_number(self: Self) -> int:
130172
self._fetch_all()
131-
return self._pagination_item.page_number
173+
# If the PaginationItem is not returned from the endpoint, use the
174+
# pagenumber from the RequestOptions.
175+
return self._pagination_item.page_number or self.request_options.pagenumber
132176

133177
@property
134178
def page_size(self: Self) -> int:
135179
self._fetch_all()
136-
return self._pagination_item.page_size
180+
# If the PaginationItem is not returned from the endpoint, use the
181+
# pagesize from the RequestOptions.
182+
return self._pagination_item.page_size or self.request_options.pagesize
137183

138184
def filter(self: Self, *invalid, page_size: Optional[int] = None, **kwargs) -> Self:
139185
if invalid:

‎test/assets/flow_runs_get.xml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-3.10.xsd">
2-
<pagination pageNumber="1" pageSize="100" totalAvailable="2"/>
32
<flowRuns>
43
<flowRuns id="cc2e652d-4a9b-4476-8c93-b238c45db968"
54
flowId="587daa37-b84d-4400-a9a2-aa90e0be7837"
@@ -16,4 +15,4 @@
1615
progress="100"
1716
backgroundJobId="1ad21a9d-2530-4fbf-9064-efd3c736e023"/>
1817
</flowRuns>
19-
</tsResponse>
18+
</tsResponse>

‎test/test_flowruns.py

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import unittest
2+
import xml.etree.ElementTree as ET
23

34
import requests_mock
45

@@ -13,6 +14,19 @@
1314
GET_BY_ID_INPROGRESS_XML = "flow_runs_get_by_id_inprogress.xml"
1415

1516

17+
def server_response_error_factory(code: str, summary: str, detail: str) -> str:
18+
root = ET.Element("tsResponse")
19+
error = ET.SubElement(root, "error")
20+
error.attrib["code"] = code
21+
22+
summary_element = ET.SubElement(error, "summary")
23+
summary_element.text = summary
24+
25+
detail_element = ET.SubElement(error, "detail")
26+
detail_element.text = detail
27+
return ET.tostring(root, encoding="utf-8").decode("utf-8")
28+
29+
1630
class FlowRunTests(unittest.TestCase):
1731
def setUp(self) -> None:
1832
self.server = TSC.Server("http://test", False)
@@ -28,9 +42,8 @@ def test_get(self) -> None:
2842
response_xml = read_xml_asset(GET_XML)
2943
with requests_mock.mock() as m:
3044
m.get(self.baseurl, text=response_xml)
31-
all_flow_runs, pagination_item = self.server.flow_runs.get()
45+
all_flow_runs = self.server.flow_runs.get()
3246

33-
self.assertEqual(2, pagination_item.total_available)
3447
self.assertEqual("cc2e652d-4a9b-4476-8c93-b238c45db968", all_flow_runs[0].id)
3548
self.assertEqual("2021-02-11T01:42:55Z", format_datetime(all_flow_runs[0].started_at))
3649
self.assertEqual("2021-02-11T01:57:38Z", format_datetime(all_flow_runs[0].completed_at))
@@ -98,3 +111,14 @@ def test_wait_for_job_timeout(self) -> None:
98111
m.get("{0}/{1}".format(self.baseurl, flow_run_id), text=response_xml)
99112
with self.assertRaises(TimeoutError):
100113
self.server.flow_runs.wait_for_job(flow_run_id, timeout=30)
114+
115+
def test_queryset(self) -> None:
116+
response_xml = read_xml_asset(GET_XML)
117+
error_response = server_response_error_factory(
118+
"400006", "Bad Request", "0xB4EAB088 : The start index '9900' is greater than or equal to the total count.)"
119+
)
120+
with requests_mock.mock() as m:
121+
m.get(f"{self.baseurl}?pageNumber=1", text=response_xml)
122+
m.get(f"{self.baseurl}?pageNumber=2", text=error_response)
123+
queryset = self.server.flow_runs.all()
124+
assert len(queryset) == 0

0 commit comments

Comments
 (0)
0