8000 feat: get page and chunk size from env vars · Der-Henning/server-client-python@78291d6 · GitHub
[go: up one dir, main page]
Skip to content

Commit 78291d6

Browse files
jorwoodsjorwoods
committed
feat: get page and chunk size from env vars
1 parent 3a53d00 commit 78291d6

File tree

6 files changed

+46
-11
lines changed

6 files changed

+46
-11
lines changed

‎tableauserverclient/config.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,23 @@
1-
# TODO: check for env variables, else set default values
1+
import os
22

33
ALLOWED_FILE_EXTENSIONS = ["tds", "tdsx", "tde", "hyper", "parquet"]
44

55
BYTES_PER_MB = 1024 * 1024
66

7-
# For when a datasource is over 64MB, break it into 5MB(standard chunk size) chunks
8-
CHUNK_SIZE_MB = 5 * 10 # 5MB felt too slow, upped it to 50
9-
107
DELAY_SLEEP_SECONDS = 0.1
118

129
# The maximum size of a file that can be published in a single request is 64MB
1310
FILESIZE_LIMIT_MB = 64
11+
12+
class Config:
13+
# For when a datasource is over 64MB, break it into 5MB(standard chunk size) chunks
14+
@property
15+
def CHUNK_SIZE_MB(self):
16+
return int(os.getenv("TSC_CHUNK_SIZE_MB", 5 * 10)) # 5MB felt too slow, upped it to 50
17+
18+
# Default page size
19+
@property
20+
def PAGE_SIZE(self):
21+
return int(os.getenv("TSC_PAGE_SIZE", 100))
22+
23+
config = Config()

‎tableauserverclient/server/endpoint/datasources_endpoint.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from tableauserverclient.server.endpoint.permissions_endpoint import _PermissionsEndpoint
2222
from tableauserverclient.server.endpoint.resource_tagger import _ResourceTagger
2323

24-
from tableauserverclient.config import ALLOWED_FILE_EXTENSIONS, FILESIZE_LIMIT_MB, BYTES_PER_MB, CHUNK_SIZE_MB
24+
from tableauserverclient.config import ALLOWED_FILE_EXTENSIONS, FILESIZE_LIMIT_MB, BYTES_PER_MB, config
2525
from tableauserverclient.filesys_helpers import (
2626
make_download_path,
2727
get_file_type,
@@ -272,7 +272,7 @@ def publish(
272272
if file_size >= FILESIZE_LIMIT_MB * BYTES_PER_MB:
273273
logger.info(
274274
"Publishing {} to server with chunking method (datasource over {}MB, chunk size {}MB)".format(
275-
filename, FILESIZE_LIMIT_MB, CHUNK_SIZE_MB
275+
filename, FILESIZE_LIMIT_MB, config.CHUNK_SIZE_MB
276276
)
277277
)
278278
upload_session_id = self.parent_srv.fileuploads.upload(file)

‎tableauserverclient/server/endpoint/fileuploads_endpoint.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
from tableauserverclient import datetime_helpers as datetime
33
from tableauserverclient.helpers.logging import logger
44

5-
from tableauserverclient.config import BYTES_PER_MB, CHUNK_SIZE_MB
5+
from tableauserverclient.config import BYTES_PER_MB, config
66
from tableauserverclient.models import FileuploadItem
77
from tableauserverclient.server import RequestFactory
88

@@ -41,7 +41,7 @@ def _read_chunks(self, file):
4141

4242
try:
4343
while True:
44-
chunked_content = file_content.read(CHUNK_SIZE_MB * BYTES_PER_MB)
44+
chunked_content = file_content.read(config.CHUNK_SIZE_MB * BYTES_PER_MB)
4545
if not chunked_content:
4646
break
4747
yield chunked_content

‎tableauserverclient/server/query.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from collections.abc import Sized
22
from itertools import count
33
from typing import Iterable, Iterator, List, Optional, Protocol, Tuple, TYPE_CHECKING, TypeVar, overload
4+
from tableauserverclient.config import config
45
from tableauserverclient.models.pagination_item import PaginationItem
56
from tableauserverclient.server.filter import Filter
67
from tableauserverclient.server.request_options import RequestOptions
@@ -35,7 +36,7 @@ def to_camel_case(word: str) -> str:
3536
class QuerySet(Iterable[T], Sized):
3637
def __init__(self, model: "QuerysetEndpoint[T]", page_size: Optional[int] = None) -> None:
3738
self.model = model
38-
self.request_options = RequestOptions(pagesize=page_size or 100)
39+
self.request_options = RequestOptions(pagesize=page_size or config.PAGE_SIZE)
3940
self._result_cache: List[T] = []
4041
self._pagination_item = PaginationItem()
4142

‎tableauserverclient/server/request_options.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from typing_extensions import Self
44

5+
from tableauserverclient.config import config
56
from tableauserverclient.models.property_decorators import property_is_int
67
import logging
78

@@ -115,9 +116,9 @@ class Direction:
115116
Desc = "desc"
116117
Asc = "asc"
117118

118-
def __init__(self, pagenumber=1, pagesize=100):
119+
def __init__(self, pagenumber=1, pagesize=None):
119120
self.pagenumber = pagenumber
120-
self.pagesize = pagesize
121+
self.pagesize = pagesize or config.PAGE_SIZE
121122
self.sort = set()
122123
self.filter = set()
123124

‎test/test_pager.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,27 @@
1+
import contextlib
12
import os
23
import unittest
34

45
import requests_mock
56

67
import tableauserverclient as TSC
8+
from tableauserverclient.config import config
79

810
TEST_ASSET_DIR = os.path.join(os.path.dirname(__file__), "assets")
911

1012
GET_XML_PAGE1 = os.path.join(TEST_ASSET_DIR, "workbook_get_page_1.xml")
1113
GET_XML_PAGE2 = os.path.join(TEST_ASSET_DIR, "workbook_get_page_2.xml")
1214
GET_XML_PAGE3 = os.path.join(TEST_ASSET_DIR, "workbook_get_page_3.xml")
1315

16+
@contextlib.contextmanager
17+
def set_env(**environ):
18+
old_environ = dict(os.environ)
19+
os.environ.update(environ)
20+
try:
21+
yield
22+
finally:
23+
os.environ.clear()
24+
os.environ.update(old_environ)
1425

1526
class PagerTests(unittest.TestCase):
1627
def setUp(self):
@@ -88,3 +99,15 @@ def test_pager_with_options(self):
8899
# Should have the last workbook
89100
wb3 = workbooks.pop()
90101
self.assertEqual(wb3.name, "Page3Workbook")
102+
103+
def test_pager_with_env_var(self) -> None:
104+
with set_env(TSC_PAGE_SIZE="1000"):
105+
assert config.PAGE_SIZE == 1000
106+
loop = TSC.Pager(self.server.workbooks)
107+
assert loop._options.pagesize == 1000
108+
109+
def test_queryset_with_env_var(self) -> None:
110+
with set_env(TSC_PAGE_SIZE="1000"):
111+
assert config.PAGE_SIZE == 1000
112+
loop = self.server.workbooks.all()
113+
assert loop.request_options.pagesize == 1000

0 commit comments

Comments
 (0)
0