8000 Merge pull request #1439 from jorwoods/jorwoods/env_var_config · Der-Henning/server-client-python@ced5fe3 · GitHub
[go: up one dir, main page]
Skip to content

Commit ced5fe3

Browse files
jacalatajacalata
authored
Merge pull request tableau#1439 from jorwoods/jorwoods/env_var_config
feat: get page and chunk size from env vars
2 parents 9e23d31 + afc293e commit ced5fe3

File tree

7 files changed

+75
-11
lines changed

7 files changed

+75
-11
lines changed

‎tableauserverclient/config.py

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,25 @@
1-
# TODO: check for env variables, else set default values
1+
import os
22

33
ALLOWED_FILE_EXTENSIONS = ["tds", "tdsx", "tde", "hyper", "parquet"]
44

55
BYTES_PER_MB = 1024 * 1024
66

7-
# For when a datasource is over 64MB, break it into 5MB(standard chunk size) chunks
8-
CHUNK_SIZE_MB = 5 * 10 # 5MB felt too slow, upped it to 50
9-
107
DELAY_SLEEP_SECONDS = 0.1
118

129
# The maximum size of a file that can be published in a single request is 64MB
1310
FILESIZE_LIMIT_MB = 64
11+
12+
13+
class Config:
14+
# For when a datasource is over 64MB, break it into 5MB(standard chunk size) chunks
15+
@property
16+
def CHUNK_SIZE_MB(self):
17+
return int(os.getenv("TSC_CHUNK_SIZE_MB", 5 * 10)) # 5MB felt too slow, upped it to 50
18+
19+
# Default page size
20+
@property
21+
def PAGE_SIZE(self):
22+
return int(os.getenv("TSC_PAGE_SIZE", 100))
23+
24+
25+
config = Config()

‎tableauserverclient/server/endpoint/datasources_endpoint.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
from tableauserverclient.server.endpoint.permissions_endpoint import _PermissionsEndpoint
2323
from tableauserverclient.server.endpoint.resource_tagger import TaggingMixin
2424

25-
from tableauserverclient.config import ALLOWED_FILE_EXTENSIONS, FILESIZE_LIMIT_MB, BYTES_PER_MB, CHUNK_SIZE_MB
25+
from tableauserverclient.config import ALLOWED_FILE_EXTENSIONS, FILESIZE_LIMIT_MB, BYTES_PER_MB, config
2626
from tableauserverclient.filesys_helpers import (
2727
make_download_path,
2828
get_file_type,
@@ -272,7 +272,7 @@ def publish(
272272
if file_size >= FILESIZE_LIMIT_MB * BYTES_PER_MB:
273273
logger.info(
274274
"Publishing {} to server with chunking method (datasource over {}MB, chunk size {}MB)".format(
275-
filename, FILESIZE_LIMIT_MB, CHUNK_SIZE_MB
275+
filename, FILESIZE_LIMIT_MB, config.CHUNK_SIZE_MB
276276
)
277277
)
278278
upload_session_id = self.parent_srv.fileuploads.upload(file)

‎tableauserverclient/server/endpoint/fileuploads_endpoint.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
from tableauserverclient import datetime_helpers as datetime
33
from tableauserverclient.helpers.logging import logger
44

5-
from tableauserverclient.config import BYTES_PER_MB, CHUNK_SIZE_MB
5+
from tableauserverclient.config import BYTES_PER_MB, config
66
from tableauserverclient.models import FileuploadItem
77
from tableauserverclient.server import RequestFactory
88

@@ -41,7 +41,7 @@ def _read_chunks(self, file):
4141

4242
try:
4343
while True:
44-
chunked_content = file_content.read(CHUNK_SIZE_MB * BYTES_PER_MB)
44+
chunked_content = file_content.read(config.CHUNK_SIZE_MB * BYTES_PER_MB)
4545
if not chunked_content:
4646
break
4747
yield chunked_content

‎tableauserverclient/server/query.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from collections.abc import Sized
22
from itertools import count
33
from typing import Iterable, Iterator, List, Optional, Protocol, Tuple, TYPE_CHECKING, TypeVar, overload
4+
from tableauserverclient.config import config
45
from tableauserverclient.models.pagination_item import PaginationItem
56
from tableauserverclient.server.filter import Filter
67
from tableauserverclient.server.request_options import RequestOptions
@@ -35,7 +36,7 @@ def to_camel_case(word: str) -> str:
3536
class QuerySet(Iterable[T], Sized):
3637
def __init__(self, model: "QuerysetEndpoint[T]", page_size: Optional[int] = None) -> None:
3738
self.model = model
38-
self.request_options = RequestOptions(pagesize=page_size or 100)
39+
self.request_options = RequestOptions(pagesize=page_size or config.PAGE_SIZE)
3940
self._result_cache: List[T] = []
4041
self._pagination_item = PaginationItem()
4142

‎tableauserverclient/server/request_options.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from typing_extensions import Self
44

5+
from tableauserverclient.config import config
56
from tableauserverclient.models.property_decorators import property_is_int
67
import logging
78

@@ -116,9 +117,9 @@ class Direction:
116117
Desc = "desc"
117118
Asc = "asc"
118119

119-
def __init__(self, pagenumber=1, pagesize=100):
120+
def __init__(self, pagenumber=1, pagesize=None):
120121
self.pagenumber = pagenumber
121-
self.pagesize = pagesize
122+
self.pagesize = pagesize or config.PAGE_SIZE
122123
self.sort = set()
123124
self.filter = set()
124125

‎test/test_fileuploads.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1+
import contextlib
2+
import io
13
import os
24
import unittest
35

46
import requests_mock
57

8+
from tableauserverclient.config import BYTES_PER_MB, config
69
from tableauserverclient.server import Server
710
from ._utils import asset
811

@@ -11,6 +14,17 @@
1114
FILEUPLOAD_APPEND = os.path.join(TEST_ASSET_DIR, "fileupload_append.xml")
1215

1316

17+
@contextlib.contextmanager
18+
def set_env(**environ):
19+
old_environ = dict(os.environ)
20+
os.environ.update(environ)
21+
try:
22+
yield
23+
finally:
24+
os.environ.clear()
25+
os.environ.update(old_environ)
26+
27+
1428
class FileuploadsTests(unittest.TestCase):
1529
def setUp(self):
1630
self.server = Server("http://test", False)
@@ -62,3 +76,14 @@ def test_upload_chunks_file_object(self):
6276
actual = self.server.fileuploads.upload(file_content)
6377

6478
self.assertEqual(upload_id, actual)
79+
80+
def test_upload_chunks_config(self):
81+
data = io.BytesIO()
82+
data.write(b"1" * (config.CHUNK_SIZE_MB * BYTES_PER_MB + 1))
83+
data.seek(0)
84+
with set_env(TSC_CHUNK_SIZE_MB="1"):
85+
chunker = self.server.fileuploads._read_chunks(data)
86+
chunk = next(chunker)
87+
assert len(chunk) == config.CHUNK_SIZE_MB * BYTES_PER_MB
88+
data.seek(0)
89+
assert len(chunk) < len(data.read())

‎test/test_pager.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1+
import contextlib
12
import os
23
import unittest
34

45
import requests_mock
56

67
import tableauserverclient as TSC
8+
from tableauserverclient.config import config
79

810
TEST_ASSET_DIR = os.path.join(os.path.dirname(__file__), "assets")
911

@@ -12,6 +14,17 @@
1214
GET_XML_PAGE3 = os.path.join(TEST_ASSET_DIR, "workbook_get_page_3.xml")
1315

1416

17+
@contextlib.contextmanager
18+
def set_env(**environ):
19+
old_environ = dict(os.environ)
20+
os.environ.update(environ)
21+
try:
22+
yield
23+
finally:
24+
os.environ.clear()
25+
os.environ.update(old_environ)
26+
27+
1528
class PagerTests(unittest.TestCase):
1629
def setUp(self):
1730
self.server = TSC.Server("http://test", False)
@@ -88,3 +101,15 @@ def test_pager_with_options(self):
88101
# Should have the last workbook
89102
wb3 = workbooks.pop()
90103
self.assertEqual(wb3.name, "Page3Workbook")
104+
105+
def test_pager 6F6D _with_env_var(self):
106+
with set_env(TSC_PAGE_SIZE="1000"):
107+
assert config.PAGE_SIZE == 1000
108+
loop = TSC.Pager(self.server.workbooks)
109+
assert loop._options.pagesize == 1000
110+
111+
def test_queryset_with_env_var(self):
112+
with set_env(TSC_PAGE_SIZE="1000"):
113+
assert config.PAGE_SIZE == 1000
114+
loop = self.server.workbooks.all()
115+
assert loop.request_options.pagesize == 1000

0 commit comments

Comments
 (0)
0