10000 implement csv import with validation · tableau/server-client-python@5eb8f1e · GitHub
[go: up one dir, main page]
Skip to content

Commit 5eb8f1e

Browse files
jacalatajacalata
committed
implement csv import with validation
created enums to hold the expected values for each field included some simple rules like no repeating @, included tests
1 parent 2cc27c6 commit 5eb8f1e

File tree

8 files changed

+82
-87
lines changed

8 files changed

+82
-87
lines changed

‎samples/create_group.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ def main():
7070
print("Add users to site from file {}:".format(filepath))
7171
added: List[TSC.UserItem]
7272
failed: List[TSC.UserItem, TSC.ServerResponseError]
73-
added, failed = server.users.create_from_file(filepath)
73+
added, failed = TSC.UserItem.create_from_file(filepath)
7474
for user, error in failed:
7575
print(user, error.code)
7676
if error.code == "409017":

‎tableauserverclient/models/user_item.py

Lines changed: 41 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import io
21
import xml.etree.ElementTree as ET
2+
from csv import reader
33
from datetime import datetime
44
from enum import IntEnum
55
from typing import Dict, List, Optional, TYPE_CHECKING, Tuple
@@ -298,13 +298,13 @@ class ColumnType(IntEnum):
298298

299299
MAX = 7
300300

301-
# Read a csv line and create a user item populated by the given attributes
301+
# Take in a list of strings in expected order
302+
# and create a user item populated by the given attributes
302303
@staticmethod
303-
def create_user_from_line(line: str):
304-
if line is None or line is False or line == "\n" or line == "":
305-
return None
306-
line = line.strip().lower()
307-
values: List[str] = list(map(str.strip, line.split(",")))
304+
def create_user_model_from_line(line_values: List[str], logger) -> "UserItem":
305+
UserItem.CSVImport._validate_import_line_or_throw(line_values, logger)
306+
values: List[str] = list(map(lambda x: x.strip(), line_values))
307+
values = list(map(lambda x: x.lower(), values))
308308
user = UserItem(values[UserItem.C B41A SVImport.ColumnType.USERNAME])
309309
if len(values) > 1:
310310
if len(values) > UserItem.CSVImport.ColumnType.MAX:
@@ -330,30 +330,36 @@ def create_user_from_line(line: str):
330330
)
331331
return user
332332

333-
# Read through an entire CSV file meant for user import
334-
# Return the number of valid lines and a list of all the invalid lines
333+
# helper method: validates an import file and creates user models for valid lines
334+
# result: (users[], valid_lines[], (line, error)[])
335335
@staticmethod
336-
def validate_file_for_import(csv_file: io.TextIOWrapper, logger) -> Tuple[int, List[str]]:
337-
num_valid_lines = 0
338-
invalid_lines = []
339-
csv_file.seek(0) # set to start of file in case it has been read earlier
340-
line: str = csv_file.readline()
341-
while line and line != "":
342-
try:
343-
# do not print passwords
344-
logger.info("Reading user {}".format(line[:4]))
345-
UserItem.CSVImport._validate_import_line_or_throw(line, logger)
346-
num_valid_lines += 1
347-
except Exception as exc:
348-
logger.info("Error parsing {}: {}".format(line[:4], exc))
349-
invalid_lines.append(line)
350-
line = csv_file.readline()
351-
return num_valid_lines, invalid_lines
336+
def process_file_for_import(
337+
filepath: str, logger, validate_only=False
338+
) -> Tuple[List["UserItem"], List[str], List[Tuple[str, Exception]]]:
339+
users: List[UserItem] = []
340+
failed: List[Tuple[str, Exception]] = []
341+
if not filepath.find("csv"):
342+
raise ValueError("Only csv files are accepted")
343+
344+
with open(filepath, encoding="utf-8-sig") as csv_file:
345+
csv_file.seek(0) # set to start of file in case it has been read earlier
346+
csv_data = reader(csv_file, delimiter=",")
347+
valid: [str] = []
348+
for line in csv_data:
349+
try:
350+
UserItem.CSVImport._validate_import_line_or_throw(line, logger)
351+
if not validate_only:
352+
user: UserItem = UserItem.CSVImport.create_user_model_from_line(line, logger)
353+
users.append(user)
354+
valid.append(line)
355+
except Exception as e:
356+
failed.append((" ".join(line), e))
357+
return users, valid, failed
352358

353359
# Some fields in the import file are restricted to specific values
354360
# Iterate through each field and validate the given value against hardcoded constraints
355361
@staticmethod
356-
def _validate_import_line_or_throw(incoming, logger) -> None:
362+
def _validate_import_line_or_throw(line, logger) -> None:
357363
_valid_attributes: List[List[str]] = [
358364
[],
359365
[],
@@ -365,7 +371,9 @@ def _validate_import_line_or_throw(incoming, logger) -> None:
365371
[UserItem.Auth.SAML, UserItem.Auth.OpenID, UserItem.Auth.ServerDefault], # auth
366372
]
367373

368-
line = list(map(str.strip, incoming.split(",")))
374+
if line is None or line is False or len(line) == 0 or line == "":
375+
raise AttributeError("Empty line")
376+
369377
if len(line) > UserItem.CSVImport.ColumnType.MAX:
370378
raise AttributeError("Too many attributes in line")
371379
username = line[UserItem.CSVImport.ColumnType.USERNAME.value]
@@ -383,9 +391,14 @@ def _validate_attribute_value(item: str, possible_values: List[str], column_type
383391
if item is None or item == "":
384392
# value can be empty for any column except user, which is checked elsewhere
385393
return
394+
item = item.strip()
386395
if item in possible_values or possible_values == []:
387396
return
388-
raise AttributeError("Invalid value {} for {}".format(item, column_type))
397+
raise AttributeError(
398+
"Invalid value {} for {}. Valid values: {}".format(
399+
item, UserItem.CSVImport.ColumnType(column_type).name, possible_values
400+
)
401+
)
389402

390403
# https://help.tableau.com/current/server/en-us/csvguidelines.htm#settings_and_site_roles
391404
# This logic is hardcoded to match the existing rules for import csv files

‎tableauserverclient/server/endpoint/users_endpoint.py

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -93,29 +93,13 @@ def add_all(self, users: List[UserItem]):
9393
failed.append(user)
9494
return created, failed
9595

96-
# helping the user by parsing a file they could have used to add users through the UI
96+
# takes in a csv file of the same format used to add users through the UI
9797
# line format: Username [required], password, display name, license, admin, publish
9898
@api(version="2.0")
99-
def create_from_file(self, filepath: str) -> Tuple[List[UserItem], List[Tuple[UserItem, ServerResponseError]]]:
100-
created = []
101-
failed = []
102-
if not filepath.find("csv"):
103-
raise ValueError("Only csv files are accepted")
104-
105-
with open(filepath) as csv_file:
106-
csv_file.seek(0) # set to start of file in case it has been read earlier
107-
line: str = csv_file.readline()
108-
while line and line != "":
109-
user: UserItem = UserItem.CSVImport.create_user_from_line(line)
110-
try:
111-
print(user)
112-
result = self.add(user)
113-
created.append(result)
114-
except ServerResponseError as serverError:
115-
print("failed")
116-
failed.append((user, serverError))
117-
line = csv_file.readline()
118-
return created, failed
99+
def create_from_file(self, filepath: str) -> Tuple[List[UserItem], List[Tuple[UserItem, Exception]]]:
100+
user_models, valid_lines, errors = UserItem.CSVImport.process_file_for_import(filepath, logger)
101+
users, server_errors = self.add_all(user_models)
102+
return users, server_errors + errors
119103

120104
# Get workbooks for user
121105
@api(version="2.0")

‎tableauserverclient/server/request_factory.py

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -782,12 +782,6 @@ def set_versioned_flow_attributes(self, flows_all, flows_edit, flows_schedule, p
782782
if site_item.flows_enabled is not None:
783783
flows_edit = flows_edit or flows_all
784784
flows_schedule = flows_schedule or flows_all
785-
import warnings
786-
787-
warnings.warn(
788-
"FlowsEnabled has been removed and become two options:"
789-
" SchedulingFlowsEnabled and EditingFlowsEnabled"
790-
)
791785
if site_item.editing_flows_enabled is not None:
792786
site_element.attrib["editingFlowsEnabled"] = flows_edit
793787
if site_item.scheduling_flows_enabled is not None:

‎test/assets/Data/user_details.csv

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
username, pword, , yes, email
1+
username, pword, , yes, none, email
2+
username, pword, ,viewer, none, yes, email

‎test/assets/Data/users_import_2.csv

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
line1, pword, fname, creator, site, yes, email
2+
line2, pword, fname, explorer, none, no, email
3+
line3, pword, fname, yes, , ,
4+
line4@me@me, pword

‎test/test_user.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,20 +219,27 @@ def test_populate_groups(self) -> None:
219219
self.assertEqual("TableauExample", group_list[2].name)
220220
self.assertEqual("local", group_list[2].domain_name)
221221

222+
# these tests are weird. The input file USERNAMES will be parsed and invalid lines put in 'failures'
223+
# Then we will send the valid lines to the server, and the response from that, ADD_XML, is our 'users'.
224+
# not covered: the server rejects one of our 'valid' lines
222225
def test_get_usernames_from_file(self):
223226
with open(ADD_XML, "rb") as f:
224227
response_xml = f.read().decode("utf-8")
225228
with requests_mock.mock() as m:
226229
m.post(self.server.users.baseurl, text=response_xml)
227230
user_list, failures = self.server.users.create_from_file(USERNAMES)
231+
assert failures != [], failures
232+
assert len(failures) == 2, failures
233+
assert user_list is not None, user_list
228234
assert user_list[0].name == "Cassie", user_list
229-
assert failures == [], failures
230235

231236
def test_get_users_from_file(self):
232237
with open(ADD_XML, "rb") as f:
233238
response_xml = f.read().decode("utf-8")
234239
with requests_mock.mock() as m:
235240
m.post(self.server.users.baseurl, text=response_xml)
236241
users, failures = self.server.users.create_from_file(USERS)
242+
assert failures != [], failures
243+
assert len(failures) == 1, failures
244+
assert users != [], users
237245
assert users[0].name == "Cassie", users
238-
assert failures == []

‎test/test_user_model.py

Lines changed: 20 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -81,48 +81,40 @@ def test_evaluate_role(self):
8181

8282
def test_get_user_detail_empty_line(self):
8383
test_line = ""
84-
test_user = TSC.UserItem.CSVImport.create_user_from_line(test_line)
85-
assert test_user is None
84+
with self.assertRaises(AttributeError):
85+
test_user = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger)
8686

8787
def test_get_user_detail_standard(self):
88-
test_line = "username, pword, fname, license, admin, pub, email"
89-
test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line)
88+
test_line = ["username", "pword", "fname", "unlicensed", "no", "no", "email"]
89+
test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger)
9090
assert test_user.name == "username", test_user.name
9191
assert test_user.fullname == "fname", test_user.fullname
9292
assert test_user.site_role == "Unlicensed", test_user.site_role
9393
assert test_user.email == "email", test_user.email
9494

9595
def test_get_user_details_only_username(self):
96-
test_line = "username"
97-
test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line)
96+
test_line = ["username"]
97+
test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger)
9898

9999
def test_populate_user_details_only_some(self):
100-
values = "username, , , creator, admin"
101-
user = TSC.UserItem.CSVImport.create_user_from_line(values)
100+
values = ["username", "", "", "creator", "site"]
101+
user = TSC.UserItem.CSVImport.create_user_model_from_line(values, UserDataTest.logger)
102102
assert user.name == "username"
103103

104104
def test_validate_user_detail_standard(self):
105-
test_line = "username, pword, fname, creator, site, 1, email"
106-
TSC.UserItem.CSVImport._validate_import_line_or_throw(test_line, UserDataTest.logger)
107-
TSC.UserItem.CSVImport.create_user_from_line(test_line)
108-
109-
# for file handling
110-
def _mock_file_content(self, content: List[str]) -> io.TextIOWrapper:
111-
# the empty string represents EOF
112-
# the tests run through the file twice, first to validate then to fetch
113-
mock = MagicMock(io.TextIOWrapper)
114-
content. 341A append("") # EOF
115-
mock.readline.side_effect = content
116-
mock.name = "file-mock"
117-
return mock
105+
test_line = ["username", "pword", "fname", "creator", "site", "1", "email"]
106+
TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger)
118107

119108
def test_validate_import_file(self):
120-
test_data = self._mock_file_content(UserDataTest.valid_import_content)
121-
valid, invalid = TSC.UserItem.CSVImport.validate_file_for_import(test_data, UserDataTest.logger)
122-
assert valid == 2, "Expected two lines to be parsed, got {}".format(valid)
123-
assert invalid == [], "Expected no failures, got {}".format(invalid)
109+
users, valid, invalid = TSC.UserItem.CSVImport.process_file_for_import(
110+
"test/assets/data/users_import_2.csv", UserDataTest.logger
111+
)
112+
assert len(valid) == 2, "Expected two lines to be valid, got {}".format(len(valid))
113+
assert invalid is not None, invalid
114+
assert len(invalid) == 2, "Expected 2 failures, got {}".format(len(invalid))
124115

125116
def test_validate_usernames_file(self):
126-
test_data = self._mock_file_content(UserDataTest.usernames)
127-
valid, invalid = TSC.UserItem.CSVImport.validate_file_for_import(test_data, UserDataTest.logger)
128-
assert valid == 5, "Exactly 5 of the lines were valid, counted {}".format(valid + invalid)
117+
users, valid_lines, errors = TSC.UserItem.CSVImport.process_file_for_import(
118+
"test/assets/data/usernames.csv", UserDataTest.logger
119+
)
120+
assert len(users) == 5, "Expected 5 of the lines to be valid, counted {}".format(len(users))

0 commit comments

Comments
 (0)
0