8000 fix: queryset support for flowruns (#1460) · tableau/server-client-python@6ec632e · GitHub
[go: up one dir, main page]
Skip to content

Commit 6ec632e

Browse files
jorwoodsjorwoods
authored
fix: queryset support for flowruns (#1460)
* fix: queryset support for flowruns FlowRun's get endpoint does not return a PaginationItem. This provides a tweak to QuerySet to provide a workaround so all items matching whatever filters are supplied. It also corrects the return types of flowruns.get and fixes the XML test asset to reflect what is really returned by the server. * fix: set unknown size to sys.maxsize Users may length check a QuerySet as part of a normal workflow. A len of 0 would be misleading, indicating to the user that there are no matches for the endpoint and/or filters they supplied. __len__ must return a non-negative int. Sentinel values such as -1 or None do not work. This only leaves maxsize as the possible flag. * fix: docstring on QuerySet * refactor(test): extract error factory to _utils * chore(typing): flowruns.cancel can also accept a FlowRunItem * style: black --------- Co-authored-by: Jordan Woods <13803242+jorwoods@users.noreply.github.com>
1 parent 2a7fb2b commit 6ec632e

File tree

5 files changed

+92
-20
lines changed

5 files changed

+92
-20
lines changed

‎tableauserverclient/server/endpoint/flow_runs_endpoint.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import logging
2-
from typing import Optional, TYPE_CHECKING
2+
from typing import Optional, TYPE_CHECKING, Union
33

44
from tableauserverclient.server.endpoint.endpoint import QuerysetEndpoint, api
55
from tableauserverclient.server.endpoint.exceptions import FlowRunFailedException, FlowRunCancelledException
6-
from tableauserverclient.models import FlowRunItem, PaginationItem
6+
from tableauserverclient.models import FlowRunItem
77
from tableauserverclient.exponential_backoff import ExponentialBackoffTimer
88

99
from tableauserverclient.helpers.logging import logger
@@ -25,13 +25,15 @@ def baseurl(self) -> str:
2525

2626
# Get all flows
2727
@api(version="3.10")
28-
def get(self, req_options: Optional["RequestOptions"] = None) -> tuple[list[FlowRunItem], PaginationItem]:
28+
# QuerysetEndpoint expects a PaginationItem to be returned, but FlowRuns
29+
# does not return a PaginationItem. Suppressing the mypy error because the
30+
# changes to the QuerySet class should permit this to function regardless.
31+
def get(self, req_options: Optional["RequestOptions"] = None) -> list[FlowRunItem]: # type: ignore[override]
2932
logger.info("Querying all flow runs on site")
3033
url = self.baseurl
3134
server_response = self.get_request(url, req_options)
32-
pagination_item = PaginationItem.from_response(server_response.content, self.parent_srv.namespace)
3335
all_flow_run_items = FlowRunItem.from_response(server_response.content, self.parent_srv.namespace)
34-
return all_flow_run_items, pagination_item
36+
return all_flow_run_items
3537

3638
# Get 1 flow by id
3739
@api(version="3.10")
@@ -46,7 +48,7 @@ def get_by_id(self, flow_run_id: str) -> FlowRunItem:
4648

4749
# Cancel 1 flow run by id
4850
@api(version="3.10")
49-
def cancel(self, flow_run_id: str) -> None:
51+
def cancel(self, flow_run_id: Union[str, FlowRunItem]) -> None:
5052
if not flow_run_id:
5153
error = "Flow ID undefined."
5254
raise ValueError(error)

‎tableauserverclient/server/query.py

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
from collections.abc import Sized
1+
from collections.abc import Iterable, Iterator, Sized
22
from itertools import count
33
from typing import Optional, Protocol, TYPE_CHECKING, TypeVar, overload
4-
from collections.abc import Iterable, Iterator
4+
import sys
55
from tableauserverclient.config import config
66
from tableauserverclient.models.pagination_item import PaginationItem
7+
from tableauserverclient.server.endpoint.exceptions import ServerResponseError
78
from tableauserverclient.server.filter import Filter
89
from tableauserverclient.server.request_options import RequestOptions
910
from tableauserverclient.server.sort import Sort
@@ -35,6 +36,32 @@ def to_camel_case(word: str) -> str:
3536

3637

3738
class QuerySet(Iterable[T], Sized):
39+
"""
40+
QuerySet is a class that allows easy filtering, sorting, and iterating over
41+
many endpoints in TableauServerClient. It is designed to be used in a similar
42+
way to Django QuerySets, but with a more limited feature set.
43+
44+
QuerySet is an iterable, and can be used in for loops, list comprehensions,
45+
and other places where iterables are expected.
46+
47+
QuerySet is also Sized, and can be used in places where the length of the
48+
QuerySet is needed. The length of the QuerySet is the total number of items
49+
available in the QuerySet, not just the number of items that have been
50+
fetched. If the endpoint does not return a total count of items, the length
51+
of the QuerySet will be sys.maxsize. If there is no total count, the
52+
QuerySet will continue to fetch items until there are no more items to
53+
fetch.
54+
55+
QuerySet is not re-entrant. It is not designed to be used in multiple places
56+
at the same time. If you need to use a QuerySet in multiple places, you
57+
should create a new QuerySet for each place you need to use it, convert it
58+
to a list, or create a deep copy of the QuerySet.
59+
60+
QuerySets are also indexable, and can be sliced. If you try to access an
61+
index that has not been fetched, the QuerySet will fetch the page that
62+
contains the item you are looking for.
63+
"""
64+
3865
def __init__(self, model: "QuerysetEndpoint[T]", page_size: Optional[int] = None) -> None:
3966
self.model = model
4067
self.request_options = RequestOptions(pagesize=page_size or config.PAGE_SIZE)
@@ -50,10 +77,20 @@ def __iter__(self: Self) -> Iterator[T]:
5077
for page in count(1):
5178
self.request_options.pagenumber = page
5279
self._result_cache = []
53-
self._fetch_all()
80+
try:
81+
self._fetch_all()
82+
except ServerResponseError as e:
83+
if e.code == "400006":
84+
# If the endpoint does not support pagination, it will end
85+
# up overrunning the total number of pages. Catch the
86+
# error and break out of the loop.
87+
raise StopIteration
5488
yield from self._result_cache
55-
# Set result_cache to empty so the fetch will populate
56-
if (page * self.page_size) >= len(self):
89+
# If the length of the QuerySet is unknown, continue fetching until
90+
# the result cache is empty.
91+
if (size := len(self)) == 0:
92+
continue
93+
if (page * self.page_size) >= size:
5794
return
5895

5996
@overload
@@ -114,10 +151,15 @@ def _fetch_all(self: Self) -> None:
114151
Retrieve the data and store result and pagination item in cache
115152
"""
116153
if not self._result_cache:
117-
self._result_cache, self._pagination_item = self.model.get(self.request_options)
154+
response = self.model.get(self.request_options)
155+
if isinstance(response, tuple):
156+
self._result_cache, self._pagination_item = response
157+
else:
158+
self._result_cache = response
159+
self._pagination_item = PaginationItem()
118160

119161
def __len__(self: Self) -> int:
120-
return self.total_available
162+
return self.total_available or sys.maxsize
121163

122164
@property
123165
def total_available(self: Self) -> int:
@@ -127,12 +169,16 @@ def total_available(self: Self) -> int:
127169
@property
128170
def page_number(self: Self) -> int:
129171
self._fetch_all()
130-
return self._pagination_item.page_number
172+
# If the PaginationItem is not returned from the endpoint, use the
173+
# pagenumber from the RequestOptions.
174+
return self._pagination_item.page_number or self.request_options.pagenumber
131175

132176
@property
133177
def page_size(self: Self) -> int:
134178
self._fetch_all()
135-
return self._pagination_item.page_size
179+
# If the PaginationItem is not returned from the endpoint, use the
180+
# pagesize from the RequestOptions.
181+
return self._pagination_item.page_size or self.request_options.pagesize
136182

137183
def filter(self: Self, *invalid, page_size: Optional[int] = None, **kwargs) -> Self:
138184
if invalid:

‎test/_utils.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import os.path
22
import unittest
3+
from xml.etree import ElementTree as ET
34
from contextlib import contextmanager
45

56
TEST_ASSET_DIR = os.path.join(os.path.dirname(__file__), "assets")
@@ -18,6 +19,19 @@ def read_xml_assets(*args):
1819
return map(read_xml_asset, args)
1920

2021

22+
def server_response_error_factory(code: str, summary: str, detail: str) -> str:
23+
root = ET.Element("tsResponse")
24+
error = ET.SubElement(root, "error")
25+
error.attrib["code"] = code
26+
27+
summary_element = ET.SubElement(error, "summary")
28+
summary_element.text = summary
29+
30+
detail_element = ET.SubElement(error, "detail")
31+
detail_element.text = detail
32+
return ET.tostring(root, encoding="utf-8").decode("utf-8")
33+
34+
2135
@contextmanager
2236
def mocked_time():
2337
mock_time = 0

‎test/assets/flow_runs_get.xml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-3.10.xsd">
2-
<pagination pageNumber="1" pageSize="100" totalAvailable="2"/>
32
<flowRuns>
43
<flowRuns id="cc2e652d-4a9b-4476-8c93-b238c45db968"
54
flowId="587daa37-b84d-4400-a9a2-aa90e0be7837"
@@ -16,4 +15,4 @@
1615
progress="100"
1716
backgroundJobId="1ad21a9d-2530-4fbf-9064-efd3c736e023"/>
1817
</flowRuns>
19-
</tsResponse>
18+
</tsResponse>

‎test/test_flowruns.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1+
import sys
12
import unittest
23

34
import requests_mock
45

56
import tableauserverclient as TSC
67
from tableauserverclient.datetime_helpers import format_datetime
78
from tableauserverclient.server.endpoint.exceptions import FlowRunFailedException
8-
from ._utils import read_xml_asset, mocked_time
9+
from ._utils import read_xml_asset, mocked_time, server_response_error_factory
910

1011
GET_XML = "flow_runs_get.xml"
1112
GET_BY_ID_XML = "flow_runs_get_by_id.xml"
@@ -28,9 +29,8 @@ def test_get(self) -> None:
2829
response_xml = read_xml_asset(GET_XML)
2930
with requests_mock.mock() as m:
3031
m.get(self.baseurl, text=response_xml)
31-
all_flow_runs, pagination_item = self.server.flow_runs.get()
32+
all_flow_runs = self.server.flow_runs.get()
3233

33-
self.assertEqual(2, pagination_item.total_available)
3434
self.assertEqual("cc2e652d-4a9b-4476-8c93-b238c45db968", all_flow_runs[0].id)
3535
self.assertEqual("2021-02-11T01:42:55Z", format_datetime(all_flow_runs[0].started_at))
3636
self.assertEqual("2021-02-11T01:57:38Z", format_datetime(all_flow_runs[0].completed_at))
@@ -98,3 +98,14 @@ def test_wait_for_job_timeout(self) -> None:
9898
m.get(f"{self.baseurl}/{flow_run_id}", text=response_xml)
9999
with self.assertRaises(TimeoutError):
100100
self.server.flow_runs.wait_for_job(flow_run_id, timeout=30)
101+
102+
def test_queryset(self) -> None:
103+
response_xml = read_xml_asset(GET_XML)
104+
error_response = server_response_error_factory(
105+
"400006", "Bad Request", "0xB4EAB088 : The start index '9900' is greater than or equal to the total count.)"
106+
)
107+
with requests_mock.mock() as m:
108+
m.get(f"{self.baseurl}?pageNumber=1", text=response_xml)
109+
m.get(f"{self.baseurl}?pageNumber=2", text=error_response)
110+
queryset = self.server.flow_runs.all()
111+
assert len(queryset) == sys.maxsize

0 commit comments

Comments
 (0)
0