From ba5eff589dd5c94e39e7bc8c7376a32b65088f79 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Thu, 8 Dec 2022 00:49:49 -0800 Subject: [PATCH 1/5] implement csv import with validation created enums to hold the expected values for each field included some simple rules like no repeating @, included tests (cherry picked from commit 5eb8f1e7048d0e633e79c12b72672339fd744ea3) implement csv import with validation created enums to hold the expected values for each field included some simple rules like no repeating @, included tests (cherry picked from commit 5eb8f1e7048d0e633e79c12b72672339fd744ea3) --- samples/create_group.py | 2 +- tableauserverclient/models/user_item.py | 78 ++++++++++++------- .../server/endpoint/users_endpoint.py | 10 ++- tableauserverclient/server/request_factory.py | 6 -- test/assets/Data/user_details.csv | 3 +- test/assets/Data/users_import_2.csv | 4 + test/test_user.py | 11 ++- test/test_user_model.py | 48 +++++------- 8 files changed, 94 insertions(+), 68 deletions(-) create mode 100644 test/assets/Data/users_import_2.csv diff --git a/samples/create_group.py b/samples/create_group.py index aca3e895b..881360b0f 100644 --- a/samples/create_group.py +++ b/samples/create_group.py @@ -65,7 +65,7 @@ def main(): print(f"Add users to site from file {filepath}:") added: list[TSC.UserItem] failed: list[TSC.UserItem, TSC.ServerResponseError] - added, failed = server.users.create_from_file(filepath) + added, failed = TSC.UserItem.create_from_file(filepath) for user, error in failed: print(user, error.code) if error.code == "409017": diff --git a/tableauserverclient/models/user_item.py b/tableauserverclient/models/user_item.py index 365e44c1d..f10adfc7d 100644 --- a/tableauserverclient/models/user_item.py +++ b/tableauserverclient/models/user_item.py @@ -1,5 +1,5 @@ -import io import xml.etree.ElementTree as ET +from csv import reader from datetime import datetime from enum import IntEnum from typing import Optional, TYPE_CHECKING @@ -66,6 +66,15 @@ class Roles: # Online only SupportUser = "SupportUser" + # These roles are deprecated as of 2018/REST v3.0 + DeprecatedRoles: list[str] = [ + Roles.ReadOnly.value, + Roles.Interactor.value, + Roles.Publisher.value, + Roles.SiteAdministrator.value, + Roles.ViewerWithPublish.value, + Roles.UnlicensedWithPublish.value, + ] class Auth: """ @@ -332,13 +341,13 @@ class ColumnType(IntEnum): MAX = 7 - # Read a csv line and create a user item populated by the given attributes + # Take in a list of strings in expected order + # and create a user item populated by the given attributes @staticmethod - def create_user_from_line(line: str): - if line is None or line is False or line == "\n" or line == "": - return None - line = line.strip().lower() - values: list[str] = list(map(str.strip, line.split(","))) + def create_user_model_from_line(line_values: list[str], logger) -> "UserItem": + UserItem.CSVImport._validate_import_line_or_throw(line_values, logger) + values: list[str] = list(map(lambda x: x.strip(), line_values)) + values = list(map(lambda x: x.lower(), values)) user = UserItem(values[UserItem.CSVImport.ColumnType.USERNAME]) if len(values) > 1: if len(values) > UserItem.CSVImport.ColumnType.MAX: @@ -364,30 +373,36 @@ def create_user_from_line(line: str): ) return user - # Read through an entire CSV file meant for user import - # Return the number of valid lines and a list of all the invalid lines + # helper method: validates an import file and creates user models for valid lines + # result: (users[], valid_lines[], (line, error)[]) @staticmethod - def validate_file_for_import(csv_file: io.TextIOWrapper, logger) -> tuple[int, list[str]]: - num_valid_lines = 0 - invalid_lines = [] - csv_file.seek(0) # set to start of file in case it has been read earlier - line: str = csv_file.readline() - while line and line != "": - try: - # do not print passwords - logger.info(f"Reading user {line[:4]}") - UserItem.CSVImport._validate_import_line_or_throw(line, logger) - num_valid_lines += 1 - except Exception as exc: - logger.info(f"Error parsing {line[:4]}: {exc}") - invalid_lines.append(line) - line = csv_file.readline() - return num_valid_lines, invalid_lines + def process_file_for_import( + filepath: str, logger, validate_only=False + ) -> tuple[list["UserItem"], list[str], list[tuple[str, Exception]]]: + users: list[UserItem] = [] + failed: list[tuple[str, Exception]] = [] + if not filepath.find("csv"): + raise ValueError("Only csv files are accepted") + + with open(filepath, encoding="utf-8-sig") as csv_file: + csv_file.seek(0) # set to start of file in case it has been read earlier + csv_data = reader(csv_file, delimiter=",") + valid: list[str] = [] + for line in csv_data: + try: + UserItem.CSVImport._validate_import_line_or_throw(line, logger) + if not validate_only: + user: UserItem = UserItem.CSVImport.create_user_model_from_line(line, logger) + users.append(user) + valid.append(line) + except Exception as e: + failed.append((" ".join(line), e)) + return users, valid, failed # Some fields in the import file are restricted to specific values # Iterate through each field and validate the given value against hardcoded constraints @staticmethod - def _validate_import_line_or_throw(incoming, logger) -> None: + def _validate_import_line_or_throw(line, logger) -> None: _valid_attributes: list[list[str]] = [ [], [], @@ -399,7 +414,9 @@ def _validate_import_line_or_throw(incoming, logger) -> None: [UserItem.Auth.SAML, UserItem.Auth.OpenID, UserItem.Auth.ServerDefault], # auth ] - line = list(map(str.strip, incoming.split(","))) + if line is None or line is False or len(line) == 0 or line == "": + raise AttributeError("Empty line") + if len(line) > UserItem.CSVImport.ColumnType.MAX: raise AttributeError("Too many attributes in line") username = line[UserItem.CSVImport.ColumnType.USERNAME.value] @@ -417,9 +434,14 @@ def _validate_attribute_value(item: str, possible_values: list[str], column_type if item is None or item == "": # value can be empty for any column except user, which is checked elsewhere return + item = item.strip() if item in possible_values or possible_values == []: return - raise AttributeError(f"Invalid value {item} for {column_type}") + raise AttributeError( + "Invalid value {} for {}. Valid values: {}".format( + item, UserItem.CSVImport.ColumnType(column_type).name, possible_values + ) + ) # https://help.tableau.com/current/server/en-us/csvguidelines.htm#settings_and_site_roles # This logic is hardcoded to match the existing rules for import csv files diff --git a/tableauserverclient/server/endpoint/users_endpoint.py b/tableauserverclient/server/endpoint/users_endpoint.py index d81907ae9..61e0708dd 100644 --- a/tableauserverclient/server/endpoint/users_endpoint.py +++ b/tableauserverclient/server/endpoint/users_endpoint.py @@ -355,9 +355,9 @@ def add_all(self, users: list[UserItem]): failed.append(user) return created, failed - # helping the user by parsing a file they could have used to add users through the UI + # takes in a csv file of the same format used to add users through the UI # line format: Username [required], password, display name, license, admin, publish - @api(version="2.0") + """ def create_from_file(self, filepath: str) -> tuple[list[UserItem], list[tuple[UserItem, ServerResponseError]]]: created = [] failed = [] @@ -378,6 +378,12 @@ def create_from_file(self, filepath: str) -> tuple[list[UserItem], list[tuple[Us failed.append((user, serverError)) line = csv_file.readline() return created, failed + """ + @api(version="2.0") + def create_from_file(self, filepath: str) -> tuple[list[UserItem], list[tuple[UserItem, Exception]]]: + user_models, valid_lines, errors = UserItem.CSVImport.process_file_for_import(filepath, logger) + users, server_errors = self.add_all(user_models) + return users, server_errors + errors # Get workbooks for user @api(version="2.0") diff --git a/tableauserverclient/server/request_factory.py b/tableauserverclient/server/request_factory.py index f7bd139d7..886d5cfbb 100644 --- a/tableauserverclient/server/request_factory.py +++ b/tableauserverclient/server/request_factory.py @@ -827,12 +827,6 @@ def set_versioned_flow_attributes(self, flows_all, flows_edit, flows_schedule, p if site_item.flows_enabled is not None: flows_edit = flows_edit or flows_all flows_schedule = flows_schedule or flows_all - import warnings - - warnings.warn( - "FlowsEnabled has been removed and become two options:" - " SchedulingFlowsEnabled and EditingFlowsEnabled" - ) if site_item.editing_flows_enabled is not None: site_element.attrib["editingFlowsEnabled"] = flows_edit if site_item.scheduling_flows_enabled is not None: diff --git a/test/assets/Data/user_details.csv b/test/assets/Data/user_details.csv index 15b975942..50c9ed9c8 100644 --- a/test/assets/Data/user_details.csv +++ b/test/assets/Data/user_details.csv @@ -1 +1,2 @@ -username, pword, , yes, email +username, pword, , yes, none, email +username, pword, ,viewer, none, yes, email diff --git a/test/assets/Data/users_import_2.csv b/test/assets/Data/users_import_2.csv new file mode 100644 index 000000000..c00da332c --- /dev/null +++ b/test/assets/Data/users_import_2.csv @@ -0,0 +1,4 @@ +line1, pword, fname, creator, site, yes, email +line2, pword, fname, explorer, none, no, email +line3, pword, fname, yes, , , +line4@me@me, pword diff --git a/test/test_user.py b/test/test_user.py index a46624845..8211a3d49 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -216,14 +216,19 @@ def test_populate_groups(self) -> None: self.assertEqual("TableauExample", group_list[2].name) self.assertEqual("local", group_list[2].domain_name) + # these tests are weird. The input file USERNAMES will be parsed and invalid lines put in 'failures' + # Then we will send the valid lines to the server, and the response from that, ADD_XML, is our 'users'. + # not covered: the server rejects one of our 'valid' lines def test_get_usernames_from_file(self): with open(ADD_XML, "rb") as f: response_xml = f.read().decode("utf-8") with requests_mock.mock() as m: m.post(self.server.users.baseurl, text=response_xml) user_list, failures = self.server.users.create_from_file(USERNAMES) + assert failures != [], failures + assert len(failures) == 2, failures + assert user_list is not None, user_list assert user_list[0].name == "Cassie", user_list - assert failures == [], failures def test_get_users_from_file(self): with open(ADD_XML, "rb") as f: @@ -231,5 +236,7 @@ def test_get_users_from_file(self): with requests_mock.mock() as m: m.post(self.server.users.baseurl, text=response_xml) users, failures = self.server.users.create_from_file(USERS) + assert failures != [], failures + assert len(failures) == 1, failures + assert users != [], users assert users[0].name == "Cassie", users - assert failures == [] diff --git a/test/test_user_model.py b/test/test_user_model.py index a8a2c51cb..8297e4d65 100644 --- a/test/test_user_model.py +++ b/test/test_user_model.py @@ -80,48 +80,40 @@ def test_evaluate_role(self): def test_get_user_detail_empty_line(self): test_line = "" - test_user = TSC.UserItem.CSVImport.create_user_from_line(test_line) - assert test_user is None + with self.assertRaises(AttributeError): + test_user = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) def test_get_user_detail_standard(self): - test_line = "username, pword, fname, license, admin, pub, email" - test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line) + test_line = ["username", "pword", "fname", "unlicensed", "no", "no", "email"] + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) assert test_user.name == "username", test_user.name assert test_user.fullname == "fname", test_user.fullname assert test_user.site_role == "Unlicensed", test_user.site_role assert test_user.email == "email", test_user.email def test_get_user_details_only_username(self): - test_line = "username" - test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line) + test_line = ["username"] + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) def test_populate_user_details_only_some(self): - values = "username, , , creator, admin" - user = TSC.UserItem.CSVImport.create_user_from_line(values) + values = ["username", "", "", "creator", "site"] + user = TSC.UserItem.CSVImport.create_user_model_from_line(values, UserDataTest.logger) assert user.name == "username" def test_validate_user_detail_standard(self): - test_line = "username, pword, fname, creator, site, 1, email" - TSC.UserItem.CSVImport._validate_import_line_or_throw(test_line, UserDataTest.logger) - TSC.UserItem.CSVImport.create_user_from_line(test_line) - - # for file handling - def _mock_file_content(self, content: list[str]) -> io.TextIOWrapper: - # the empty string represents EOF - # the tests run through the file twice, first to validate then to fetch - mock = MagicMock(io.TextIOWrapper) - content.append("") # EOF - mock.readline.side_effect = content - mock.name = "file-mock" - return mock + test_line = ["username", "pword", "fname", "creator", "site", "1", "email"] + TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) def test_validate_import_file(self): - test_data = self._mock_file_content(UserDataTest.valid_import_content) - valid, invalid = TSC.UserItem.CSVImport.validate_file_for_import(test_data, UserDataTest.logger) - assert valid == 2, f"Expected two lines to be parsed, got {valid}" - assert invalid == [], f"Expected no failures, got {invalid}" + users, valid, invalid = TSC.UserItem.CSVImport.process_file_for_import( + "test/assets/data/users_import_2.csv", UserDataTest.logger + ) + assert len(valid) == 2, "Expected two lines to be valid, got {}".format(len(valid)) + assert invalid is not None, invalid + assert len(invalid) == 2, "Expected 2 failures, got {}".format(len(invalid)) def test_validate_usernames_file(self): - test_data = self._mock_file_content(UserDataTest.usernames) - valid, invalid = TSC.UserItem.CSVImport.validate_file_for_import(test_data, UserDataTest.logger) - assert valid == 5, f"Exactly 5 of the lines were valid, counted {valid + invalid}" + users, valid_lines, errors = TSC.UserItem.CSVImport.process_file_for_import( + "test/assets/data/usernames.csv", UserDataTest.logger + ) + assert len(users) == 5, "Expected 5 of the lines to be valid, counted {}".format(len(users)) From 5daf9b9406824c842e9eb98dc96f444a5319b659 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Wed, 15 Feb 2023 17:47:04 -0800 Subject: [PATCH 2/5] add two auth types --- tableauserverclient/models/user_item.py | 5 +++- .../server/endpoint/users_endpoint.py | 19 +++++++------- test/test_user_model.py | 25 +++++++++++++++++++ 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/tableauserverclient/models/user_item.py b/tableauserverclient/models/user_item.py index f10adfc7d..e3e01812f 100644 --- a/tableauserverclient/models/user_item.py +++ b/tableauserverclient/models/user_item.py @@ -86,6 +86,8 @@ class Auth: SAML = "SAML" TableauIDWithMFA = "TableauIDWithMFA" ServerDefault = "ServerDefault" + TableauID = "TableauID" + Local = "Local" def __init__( self, name: Optional[str] = None, site_role: Optional[str] = None, auth_setting: Optional[str] = None @@ -394,7 +396,7 @@ def process_file_for_import( if not validate_only: user: UserItem = UserItem.CSVImport.create_user_model_from_line(line, logger) users.append(user) - valid.append(line) + valid.append(" ".join(line)) except Exception as e: failed.append((" ".join(line), e)) return users, valid, failed @@ -419,6 +421,7 @@ def _validate_import_line_or_throw(line, logger) -> None: if len(line) > UserItem.CSVImport.ColumnType.MAX: raise AttributeError("Too many attributes in line") + # sometimes usernames are case sensitive username = line[UserItem.CSVImport.ColumnType.USERNAME.value] logger.debug(f"> details - {username}") UserItem.validate_username_or_throw(username) diff --git a/tableauserverclient/server/endpoint/users_endpoint.py b/tableauserverclient/server/endpoint/users_endpoint.py index 61e0708dd..c076342e3 100644 --- a/tableauserverclient/server/endpoint/users_endpoint.py +++ b/tableauserverclient/server/endpoint/users_endpoint.py @@ -1,3 +1,4 @@ +import _csv import copy import logging from typing import Optional @@ -21,6 +22,10 @@ class Users(QuerysetEndpoint[UserItem]): users in the REST API and operate on the UserItem class. Only server and site administrators can access the user resources. """ + def _check_user_id(self, user_item: UserItem): + if not user_item.id: + error = "User item missing ID." + raise MissingRequiredFieldError(error) @property def baseurl(self) -> str: @@ -193,9 +198,7 @@ def update(self, user_item: UserItem, password: Optional[str] = None) -> UserIte >>> updated_user = server.users.update(user) """ - if not user_item.id: - error = "User item missing ID." - raise MissingRequiredFieldError(error) + self._check_user_id(user_item) url = f"{self.baseurl}/{user_item.id}" update_req = RequestFactory.User.update_req(user_item, password) @@ -424,9 +427,7 @@ def populate_workbooks(self, user_item: UserItem, req_options: Optional[RequestO >>> for wb in user.workbooks: >>> print(wb.name) """ - if not user_item.id: - error = "User item missing ID." - raise MissingRequiredFieldError(error) + self._check_user_id(user_item) def wb_pager(): return Pager(lambda options: self._get_wbs_for_user(user_item, options), req_options) @@ -436,6 +437,8 @@ def wb_pager(): def _get_wbs_for_user( self, user_item: UserItem, req_options: Optional[RequestOptions] = None ) -> tuple[list[WorkbookItem], PaginationItem]: + + self._check_user_id(user_item) url = f"{self.baseurl}/{user_item.id}/workbooks" server_response = self.get_request(url, req_options) logger.info(f"Populated workbooks for user (ID: {user_item.id})") @@ -500,9 +503,7 @@ def populate_groups(self, user_item: UserItem, req_options: Optional[RequestOpti >>> for group in user.groups: >>> print(group.name) """ - if not user_item.id: - error = "User item missing ID." - raise MissingRequiredFieldError(error) + self._check_user_id(user_item) def groups_for_user_pager(): return Pager( diff --git a/test/test_user_model.py b/test/test_user_model.py index 8297e4d65..9615347c7 100644 --- a/test/test_user_model.py +++ b/test/test_user_model.py @@ -91,6 +91,31 @@ def test_get_user_detail_standard(self): assert test_user.site_role == "Unlicensed", test_user.site_role assert test_user.email == "email", test_user.email + def test_get_user_detail_variation(self): + test_line = ["username", "pword", "fname", "license", "admin", "pub", "email"] + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + assert test_user.name == "username", test_user.name + assert test_user.fullname == "fname", test_user.fullname + assert test_user.site_role == "license", test_user.site_role + assert test_user.email == "email", test_user.email + + def test_create_user_invalid_license(self): + test_line = ["username", "pword", "fname", "license", "site", "yes", "email"] + with self.assertRaises(AttributeError): + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + + def test_create_user_invalid_role(self): + test_line = ["username", "pword", "fname", "creator", "role", "yes", "email"] + with self.assertRaises(AttributeError): + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + + test_line = ["username", "pword", "fname", "license", "admin", "pub", "email"] + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + assert test_user.name == "username", test_user.name + assert test_user.fullname == "fname", test_user.fullname + assert test_user.site_role == "license", test_user.site_role + assert test_user.email == "email", test_user.email + def test_get_user_details_only_username(self): test_line = ["username"] test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) From aa9d1d722e3d541e5705e70caa2a23de1e76a271 Mon Sep 17 00:00:00 2001 From: Jac Date: Thu, 9 May 2024 22:25:45 -0700 Subject: [PATCH 3/5] Add CSV Import logic Also add tests for the importing --- tableauserverclient/__init__.py | 1 + tableauserverclient/helpers/__init__.py | 1 + tableauserverclient/helpers/importer.py | 274 ++++++++++++++++++ tableauserverclient/models/group_item.py | 5 +- tableauserverclient/models/user_item.py | 56 ++-- .../server/endpoint/users_endpoint.py | 3 +- test/assets/Data/user_details_fails.csv | 5 + test/assets/user_get.xml | 4 +- test/assets/user_get_by_id.xml | 2 +- test/test_group.py | 3 +- test/test_user.py | 47 +-- test/test_user_csv_import.py | 199 +++++++++++++ 12 files changed, 538 insertions(+), 62 deletions(-) create mode 100644 tableauserverclient/helpers/importer.py create mode 100644 test/assets/Data/user_details_fails.csv create mode 100644 test/test_user_csv_import.py diff --git a/tableauserverclient/__init__.py b/tableauserverclient/__init__.py index e0a7abb64..35eaa1d02 100644 --- a/tableauserverclient/__init__.py +++ b/tableauserverclient/__init__.py @@ -133,3 +133,4 @@ "WeeklyInterval", "WorkbookItem", ] +from .helpers import importer # for tests diff --git a/tableauserverclient/helpers/__init__.py b/tableauserverclient/helpers/__init__.py index 7daf0d490..3336c97f9 100644 --- a/tableauserverclient/helpers/__init__.py +++ b/tableauserverclient/helpers/__init__.py @@ -1 +1,2 @@ from .strings import * +from .importer import UserCSVImport, UserCSVObject \ No newline at end of file diff --git a/tableauserverclient/helpers/importer.py b/tableauserverclient/helpers/importer.py new file mode 100644 index 000000000..fed61590b --- /dev/null +++ b/tableauserverclient/helpers/importer.py @@ -0,0 +1,274 @@ +from tableauserverclient.models.user_item import UserItem +from typing import List, Tuple +from enum import IntEnum + + + +class UserCSVObject: + def __init__(self): + self.name = None + self.password = None + self.fullname = None + self.license_level = None + self.admin_level = None + self.publisher = None + self.email = None + self.auth = None + + def populate(self, values: List[str]) -> None: + n_values = len(values) + self.name = values[0] + if n_values >= 2: + self.password = values[1] + if n_values >= 3: + self.fullname = values[2] + if n_values >= 4: + self.license_level = values[3] + if n_values >= 5: + self.admin_level = values[4] + if n_values >= 6: + self.publisher = values[5] + if n_values >= 7: + self.email = values[6] + if n_values >= 8: + self.auth = values[7] + + def to_tsc_user(self) -> UserItem: + site_role = UserCSVImport.evaluate_site_role(self.license_level, self.admin_level, self.publisher) + if not site_role: + raise AttributeError("Site role is required") + user = UserItem(self.name, site_role, self.auth) + user.email = self.email + user.fullname = self.fullname + return user + + + +class UserCSVImport(object): + """ + This class includes hardcoded options and logic for the CSV file format defined for user import + https://help.tableau.com/current/server/en-us/users_import.htm + """ + + # username, password, display_name, license, admin_level, publishing, email, auth type + class ColumnType(IntEnum): + USERNAME = 0 + PASS = 1 + DISPLAY_NAME = 2 + LICENSE = 3 # aka site role + ADMIN = 4 + PUBLISHER = 5 + EMAIL = 6 + AUTH = 7 + # version 3.25 and later + IDP_NAME = 8 + IDP_ID = 9 + + MAX = 7 + + # maxColumns = v3.25+ ? 9 : 7 + + # Take in a list of strings in expected order + # and create a user item populated by the given attributes + @staticmethod + def create_user_model_from_line(line_values: List[str]) -> "UserItem": + UserCSVImport._validate_import_line_or_throw(line_values) + values: List[str] = list(map(lambda x: x.strip(), line_values)) + user = UserItem(values[UserCSVImport.ColumnType.USERNAME]) + if len(values) > 1: + if len(values) > UserCSVImport.ColumnType.MAX: + raise ValueError("Too many attributes for user import") + while len(values) <= UserCSVImport.ColumnType.MAX: + values.append("") + + site_role = UserCSVImport._evaluate_site_role( + values[UserCSVImport.ColumnType.LICENSE], + values[UserCSVImport.ColumnType.ADMIN], + values[UserCSVImport.ColumnType.PUBLISHER], + ) + if not site_role: + raise AttributeError("Site role is required") + + user._set_values( + None, # id + values[UserCSVImport.ColumnType.USERNAME], + site_role, + None, # last login + None, # external auth provider id + values[UserCSVImport.ColumnType.DISPLAY_NAME], + values[UserCSVImport.ColumnType.EMAIL], + values[UserCSVImport.ColumnType.AUTH], + None, # domain name + ) + if values[UserCSVImport.ColumnType.PASS] is not None: + user.password = values[UserCSVImport.ColumnType.PASS] + + # TODO: implement IDP pools + return user + + # helper method: validates an import file and if enabled, creates user models for each valid line + # result: (users[], valid_lines[], (line, error)[]) + @staticmethod + def process_file_for_import(filepath: str, validate_only=False) -> Tuple[List["UserItem"], List[str], List[Tuple[str, Exception]]]: + n_failures_accepted = 3 + users: List[UserItem] = [] + failed: List[Tuple[str, Exception]] = [] + valid: List[str] = [] + if not filepath.find("csv"): + raise ValueError("Only csv files are accepted") + + with open(filepath, encoding="utf-8-sig") as csv_file: + for line in csv_file: + if line == "": + continue + + + # print only the username, because next value is password + # logger.debug("> {}".format(line.split(","))) + try: + UserCSVImport._validate_import_line_or_throw(line) + if not validate_only: + user: UserItem = UserCSVImport.create_user_model_from_line(line) + users.append(user) + valid.append(" ".join(line)) + except Exception as e: + failed.append((" ".join(line), e)) + if len(failed) > n_failures_accepted and not validate_only: + raise ValueError( + "More than 3 lines have failed validation. Check the errors and fix your file." + ) + return users, valid, failed + + # valid: username, domain/username, username@domain, domain/username@email + @staticmethod + def _validate_username_or_throw(username) -> None: + if username is None or username == "" or username.strip(" ") == "": + raise AttributeError(_("user.input.name.err.empty")) + if username.find(" ") >= 0: + raise AttributeError(_("tabcmd.report.error.user.no_spaces_in_username")) + at_symbol = username.find("@") + + # f a user name includes an @ character that represents anything other than a domain separator, + # you need to refer to the symbol using the hexadecimal format: \0x40 + if at_symbol >= 0: + username = username[:at_symbol] + "X" + username[at_symbol + 1 :] + if username.find("@") >= 0: + raise AttributeError(_("tabcmd.report.error.user_csv.at_char")) + + + + # If Tableau Server is configured to use Active Directory authentication, there must be a Password column, + # but the column itself should be empty. If the server is using local authentication, you must provide passwords for new users. + # TODO: check any character/encoding limits for passwords + @staticmethod + def _validate_password_or_throw(password) -> None: + isActiveDirectory = False # TODO: how to get this info? + isLocalAuth = False # TODO: how to get this info? + + if isActiveDirectory and password is not None: + raise AttributeError("Password must be empty for Active Directory accounts.") + + if isLocalAuth and password is None: + raise AttributeError("Password must be provided for local authentication users.") + + + # Note: The identifier is required if adding a user to an identity pool that uses Active Directory (or LDAP) identity store. + # The identifier is optional if adding a user to an identity pool that uses the local identity store. + @staticmethod + def _validate_idp_identifier_or_throw(identifier) -> None: + isActiveDirectory = False # TODO: how to get this info? + if isActiveDirectory and identifier is not None: + raise AttributeError("Identifier is required for Active Directory identity stores.") + + + # Some fields in the import file are restricted to specific values + # Iterate through each field and validate the given value against hardcoded constraints + # Values in here are all CASE INSENSITIVE. So the values entered here are all lowercase + # and all comparisons must force the input text to lowercase as well. + @staticmethod + def _validate_import_line_or_throw(line: str) -> None: + _valid_attributes: List[List[str]] = [ + [], + [], + [], + ["creator", "explorer", "viewer", "unlicensed"], # license + ["system", "site", "none", "no"], # admin + ["yes", "true", "1", "no", "false", "0"], # publisher + [], + [UserItem.Auth.SAML.lower(), + UserItem.Auth.OpenID.lower(), UserItem.Auth.TableauIDWithMFA.lower(), UserItem.Auth.ServerDefault.lower()], # auth + [], + [], + ] + + if line is None or line is False or len(line) == 0 or line == "": + raise AttributeError("Empty line") + values: List[str] = list(map(str.strip, line.split(","))) + + + if len(values) > UserCSVImport.ColumnType.MAX: + raise AttributeError("Too many attributes in line") + # sometimes usernames are case sensitive + username = values[UserCSVImport.ColumnType.USERNAME.value] + # logger.debug("> details - {}".format(username)) + UserItem.validate_username_or_throw(username) + if len(values) > UserCSVImport.ColumnType.PASS: + password = values[UserCSVImport.ColumnType.PASS.value] + UserCSVImport.validate_password_or_throw(password) + if len(values) > UserCSVImport.ColumnType.IDP_ID: + UserCSVImport._validate_idp_identifier_or_throw + for i in range(2, len(values)): + # logger.debug("column {}: {}".format(UserCSVImport.ColumnType(i).name, values[i])) + UserCSVImport._validate_attribute_value(values[i], _valid_attributes[i], UserCSVImport.ColumnType(i).name) + + # Given a restricted set of possible values, confirm the item is in that set + @staticmethod + def _validate_attribute_value(item: str, possible_values: List[str], column_type) -> None: + if item is None or item == "": + # value can be empty for any column except user, which is checked elsewhere + return + item = item.strip() + if item.lower() in possible_values or possible_values == []: + return + raise AttributeError( + "Invalid value {} for {}. Valid values: {}".format(item, column_type, possible_values) + ) + + # https://help.tableau.com/current/server/en-us/csvguidelines.htm#settings_and_site_roles + # This logic is hardcoded to match the existing rules for import csv files + @staticmethod + def _evaluate_site_role(license_level, admin_level, publisher): + if not license_level or not admin_level or not publisher: + return "Unlicensed" + # ignore case everywhere + license_level = license_level.lower() + admin_level = admin_level.lower() + publisher = publisher.lower() + # don't need to check publisher for system/site admin + if admin_level == "system": + site_role = "SiteAdministrator" + elif admin_level == "site": + if license_level == "creator": + site_role = "SiteAdministratorCreator" + elif license_level == "explorer": + site_role = "SiteAdministratorExplorer" + else: + site_role = "SiteAdministratorExplorer" + else: # if it wasn't 'system' or 'site' then we can treat it as 'none' + if publisher == "yes": + if license_level == "creator": + site_role = "Creator" + elif license_level == "explorer": + site_role = "ExplorerCanPublish" + else: + site_role = "Unlicensed" # is this the expected outcome? + else: # publisher == 'no': + if license_level == "explorer" or license_level == "creator": + site_role = "Explorer" + elif license_level == "viewer": + site_role = "Viewer" + else: # if license_level == 'unlicensed' + site_role = "Unlicensed" + if site_role is None: + site_role = "Unlicensed" + return site_role diff --git a/tableauserverclient/models/group_item.py b/tableauserverclient/models/group_item.py index 6871f8b16..6c8fc880b 100644 --- a/tableauserverclient/models/group_item.py +++ b/tableauserverclient/models/group_item.py @@ -64,7 +64,10 @@ def minimum_site_role(self) -> Optional[str]: @minimum_site_role.setter @property_is_enum(UserItem.Roles) - def minimum_site_role(self, value: str) -> None: + def minimum_site_role(self, value: Optional[str | UserItem.Roles]) -> None: + if value is not None: + if isinstance(value, UserItem.Roles): + value = value.value self._minimum_site_role = value @property diff --git a/tableauserverclient/models/user_item.py b/tableauserverclient/models/user_item.py index e3e01812f..7f1c4c88e 100644 --- a/tableauserverclient/models/user_item.py +++ b/tableauserverclient/models/user_item.py @@ -1,5 +1,4 @@ import xml.etree.ElementTree as ET -from csv import reader from datetime import datetime from enum import IntEnum from typing import Optional, TYPE_CHECKING @@ -10,7 +9,6 @@ from .exceptions import UnpopulatedPropertyError from .property_decorators import ( property_is_enum, - property_not_empty, ) from .reference_item import ResourceReference @@ -41,12 +39,14 @@ class UserItem: tag_name: str = "user" - class Roles: - """ - The Roles class contains the possible roles for a user on Tableau - Server. - """ - + """ + The Roles class contains the possible roles for a user on Tableau + Server. + # A user's License tells you all the capabilities they *could* have + # A user's Site Role tells you which packages of capabilities they *were assigned by default* + # A user must be inspected individually to know what capabilities they *actually have* + """ + class Roles(Enum): Interactor = "Interactor" Publisher = "Publisher" ServerAdministrator = "ServerAdministrator" @@ -63,6 +63,7 @@ class Roles: ReadOnly = "ReadOnly" SiteAdministratorCreator = "SiteAdministratorCreator" SiteAdministratorExplorer = "SiteAdministratorExplorer" + SupportUser = "SupportUser" # Online only. Can only be created by Tableau admins. # Online only SupportUser = "SupportUser" @@ -84,15 +85,14 @@ class Auth: OpenID = "OpenID" SAML = "SAML" - TableauIDWithMFA = "TableauIDWithMFA" - ServerDefault = "ServerDefault" - TableauID = "TableauID" - Local = "Local" + TableauIDWithMFA = "TableauIDWithMFA" # Not applicable on Tableau Server + ServerDefault = ( + "ServerDefault" # legacy value: This can only be used for Tableau Cloud sites that do not have MFA enabled. + ) def __init__( - self, name: Optional[str] = None, site_role: Optional[str] = None, auth_setting: Optional[str] = None + self, name: Optional[str] = None, site_role: Optional[str | Roles] = None, auth_setting: Optional[str] = None ) -> None: - self._auth_setting: Optional[str] = None self._domain_name: Optional[str] = None self._external_auth_user_id: Optional[str] = None self._id: Optional[str] = None @@ -102,10 +102,13 @@ def __init__( self._groups = None self.email: Optional[str] = None self.fullname: Optional[str] = None - self.name: Optional[str] = name - self.site_role: Optional[str] = site_role - self.auth_setting: Optional[str] = auth_setting - + if name is not None: + self.name: Optional[str] = name + self.site_role = site_role # type: ignore[assignment] + # (mypy is tricked by the different types of the getter/setter) + if auth_setting is not None: + self.auth_setting: Optional[str] = auth_setting + self.password = None return None def __str__(self) -> str: @@ -115,6 +118,14 @@ def __str__(self) -> str: def __repr__(self): return self.__str__() + " { " + ", ".join(" % s: % s" % item for item in vars(self).items()) + "}" + @property + def password(self) -> Optional[str]: + return self._password + + @property + def password_setting(self) -> Optional[str]: + return self._password + @property def auth_setting(self) -> Optional[str]: return self._auth_setting @@ -171,7 +182,14 @@ def site_role(self) -> Optional[str]: @site_role.setter @property_is_enum(Roles) - def site_role(self, value): + def site_role(self, value: Optional[str | Roles]): + if value is not None: + if isinstance(value, UserItem.Roles): + value = value.value + if value in UserItem.DeprecatedRoles: + import warnings + + warnings.warn("This role is not valid after Tableau 2018/REST version 3.0", DeprecationWarning) self._site_role = value @property diff --git a/tableauserverclient/server/endpoint/users_endpoint.py b/tableauserverclient/server/endpoint/users_endpoint.py index c076342e3..bec79de13 100644 --- a/tableauserverclient/server/endpoint/users_endpoint.py +++ b/tableauserverclient/server/endpoint/users_endpoint.py @@ -1,12 +1,11 @@ -import _csv import copy -import logging from typing import Optional from tableauserverclient.server.query import QuerySet from .endpoint import QuerysetEndpoint, api from .exceptions import MissingRequiredFieldError, ServerResponseError +from tableauserverclient.helpers.importer import UserCSVImport from tableauserverclient.server import RequestFactory, RequestOptions from tableauserverclient.models import UserItem, WorkbookItem, PaginationItem, GroupItem from ..pager import Pager diff --git a/test/assets/Data/user_details_fails.csv b/test/assets/Data/user_details_fails.csv new file mode 100644 index 000000000..0998d6f77 --- /dev/null +++ b/test/assets/Data/user_details_fails.csv @@ -0,0 +1,5 @@ +username, yes, none, email +username, pword, ,viewer, none, yes, email +username, pword, , no, none, yes, email +username, pword, , no, none, yes, email +username, pword, , lol, none, yes, email diff --git a/test/assets/user_get.xml b/test/assets/user_get.xml index 83557b2eb..63ea86478 100644 --- a/test/assets/user_get.xml +++ b/test/assets/user_get.xml @@ -2,7 +2,7 @@ - - + + \ No newline at end of file diff --git a/test/assets/user_get_by_id.xml b/test/assets/user_get_by_id.xml index 6caba72f9..c1b8ef121 100644 --- a/test/assets/user_get_by_id.xml +++ b/test/assets/user_get_by_id.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/test/test_group.py b/test/test_group.py index 41b5992be..35f4afb44 100644 --- a/test/test_group.py +++ b/test/test_group.py @@ -301,7 +301,8 @@ def test_update_local_async(self) -> None: def test_update_ad_async(self) -> None: group = TSC.GroupItem("myGroup", "example.com") group._id = "ef8b19c0-43b6-11e6-af50-63f5805dbe3c" - group.minimum_site_role = TSC.UserItem.Roles.Viewer + group.minimum_site_role = TSC.UserItem.Roles.Viewer # type: ignore[assignment] + # (mypy is tricked by the different getter/setter) with requests_mock.mock() as m: m.put(f"{self.baseurl}/{group.id}?asJob=True", text=UPDATE_ASYNC_XML.read_bytes().decode("utf8")) diff --git a/test/test_user.py b/test/test_user.py index 8211a3d49..cf62c6bbe 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -17,9 +17,6 @@ GET_FAVORITES_XML = os.path.join(TEST_ASSET_DIR, "favorites_get.xml") POPULATE_GROUPS_XML = os.path.join(TEST_ASSET_DIR, "user_populate_groups.xml") -USERNAMES = os.path.join(TEST_ASSET_DIR, "Data", "usernames.csv") -USERS = os.path.join(TEST_ASSET_DIR, "Data", "user_details.csv") - class UserTests(unittest.TestCase): def setUp(self) -> None: @@ -44,7 +41,7 @@ def test_get(self) -> None: self.assertTrue(any(user.id == "dd2239f6-ddf1-4107-981a-4cf94e415794" for user in all_users)) single_user = next(user for user in all_users if user.id == "dd2239f6-ddf1-4107-981a-4cf94e415794") self.assertEqual("alice", single_user.name) - self.assertEqual("Publisher", single_user.site_role) + self.assertEqual("Creator", single_user.site_role) self.assertEqual("2016-08-16T23:17:06Z", format_datetime(single_user.last_login)) self.assertEqual("alice cook", single_user.fullname) self.assertEqual("alicecook@test.com", single_user.email) @@ -52,7 +49,7 @@ def test_get(self) -> None: self.assertTrue(any(user.id == "2a47bbf8-8900-4ebb-b0a4-2723bd7c46c3" for user in all_users)) single_user = next(user for user in all_users if user.id == "2a47bbf8-8900-4ebb-b0a4-2723bd7c46c3") self.assertEqual("Bob", single_user.name) - self.assertEqual("Interactor", single_user.site_role) + self.assertEqual("Explorer", single_user.site_role) self.assertEqual("Bob Smith", single_user.fullname) self.assertEqual("bob@test.com", single_user.email) @@ -80,7 +77,7 @@ def test_get_by_id(self) -> None: self.assertEqual("dd2239f6-ddf1-4107-981a-4cf94e415794", single_user.id) self.assertEqual("alice", single_user.name) self.assertEqual("Alice", single_user.fullname) - self.assertEqual("Publisher", single_user.site_role) + self.assertEqual("Creator", single_user.site_role) self.assertEqual("ServerDefault", single_user.auth_setting) self.assertEqual("2016-08-16T23:17:06Z", format_datetime(single_user.last_login)) self.assertEqual("local", single_user.domain_name) @@ -105,8 +102,11 @@ def test_update(self) -> None: self.assertEqual("cassie@email.com", single_user.email) self.assertEqual("Viewer", single_user.site_role) - def test_update_missing_id(self) -> None: + def test_old_user_roles(self) -> None: single_user = TSC.UserItem("test", "Interactor") + + def test_update_missing_id(self) -> None: + single_user = TSC.UserItem("test", "Explorer") self.assertRaises(TSC.MissingRequiredFieldError, self.server.users.update, single_user) def test_remove(self) -> None: @@ -145,7 +145,7 @@ def test_populate_workbooks(self) -> None: response_xml = f.read().decode("utf-8") with requests_mock.mock() as m: m.get(self.baseurl + "/dd2239f6-ddf1-4107-981a-4cf94e415794/workbooks", text=response_xml) - single_user = TSC.UserItem("test", "Interactor") + single_user = TSC.UserItem("test", "Explorer") single_user._id = "dd2239f6-ddf1-4107-981a-4cf94e415794" self.server.users.populate_workbooks(single_user) @@ -163,13 +163,13 @@ def test_populate_workbooks(self) -> None: self.assertEqual({"Safari", "Sample"}, workbook_list[0].tags) def test_populate_workbooks_missing_id(self) -> None: - single_user = TSC.UserItem("test", "Interactor") + single_user = TSC.UserItem("test", "Explorer") self.assertRaises(TSC.MissingRequiredFieldError, self.server.users.populate_workbooks, single_user) def test_populate_favorites(self) -> None: self.server.version = "2.5" baseurl = self.server.favorites.baseurl - single_user = TSC.UserItem("test", "Interactor") + single_user = TSC.UserItem("test", "Viewer") with open(GET_FAVORITES_XML, "rb") as f: response_xml = f.read().decode("utf-8") with requests_mock.mock() as m: @@ -197,7 +197,7 @@ def test_populate_groups(self) -> None: response_xml = f.read().decode("utf-8") with requests_mock.mock() as m: m.get(self.server.users.baseurl + "/dd2239f6-ddf1-4107-981a-4cf94e415794/groups", text=response_xml) - single_user = TSC.UserItem("test", "Interactor") + single_user = TSC.UserItem("test", "Creator") single_user._id = "dd2239f6-ddf1-4107-981a-4cf94e415794" self.server.users.populate_groups(single_user) @@ -215,28 +215,3 @@ def test_populate_groups(self) -> None: self.assertEqual("86a66d40-f289-472a-83d0-927b0f954dc8", group_list[2].id) self.assertEqual("TableauExample", group_list[2].name) self.assertEqual("local", group_list[2].domain_name) - - # these tests are weird. The input file USERNAMES will be parsed and invalid lines put in 'failures' - # Then we will send the valid lines to the server, and the response from that, ADD_XML, is our 'users'. - # not covered: the server rejects one of our 'valid' lines - def test_get_usernames_from_file(self): - with open(ADD_XML, "rb") as f: - response_xml = f.read().decode("utf-8") - with requests_mock.mock() as m: - m.post(self.server.users.baseurl, text=response_xml) - user_list, failures = self.server.users.create_from_file(USERNAMES) - assert failures != [], failures - assert len(failures) == 2, failures - assert user_list is not None, user_list - assert user_list[0].name == "Cassie", user_list - - def test_get_users_from_file(self): - with open(ADD_XML, "rb") as f: - response_xml = f.read().decode("utf-8") - with requests_mock.mock() as m: - m.post(self.server.users.baseurl, text=response_xml) - users, failures = self.server.users.create_from_file(USERS) - assert failures != [], failures - assert len(failures) == 1, failures - assert users != [], users - assert users[0].name == "Cassie", users diff --git a/test/test_user_csv_import.py b/test/test_user_csv_import.py new file mode 100644 index 000000000..a07510add --- /dev/null +++ b/test/test_user_csv_import.py @@ -0,0 +1,199 @@ +import io +import os +import unittest +from typing import List +from unittest.mock import * +import requests_mock + +import tableauserverclient as TSC +UserCSVObject = TSC.importer.UserCSVObject +UserCSVImport = TSC.importer.UserCSVImport + +TEST_ASSET_DIR = os.path.join(os.path.dirname(__file__), "assets") + + +ADD_XML = os.path.join(TEST_ASSET_DIR, "user_add.xml") +USERNAMES = os.path.join(TEST_ASSET_DIR, "Data", "usernames.csv") +USERS = os.path.join(TEST_ASSET_DIR, "Data", "user_details.csv") +USERS_BAD = os.path.join(TEST_ASSET_DIR, "Data", "user_details_fails.csv") + +class UserFromCSVTests(unittest.TestCase): + + + role_inputs = [ + ["creator", "system", "yes", "SiteAdministrator"], + ["None", "system", "no", "SiteAdministrator"], + ["explorer", "SysTEm", "no", "SiteAdministrator"], + ["creator", "site", "yes", "SiteAdministratorCreator"], + ["explorer", "site", "yes", "SiteAdministratorExplorer"], + ["creator", "SITE", "no", "SiteAdministratorCreator"], + ["creator", "none", "yes", "Creator"], + ["explorer", "none", "yes", "ExplorerCanPublish"], + ["viewer", "None", "no", "Viewer"], + ["explorer", "no", "yes", "ExplorerCanPublish"], + ["EXPLORER", "noNO", "yes", "ExplorerCanPublish"], + ["explorer", "no", "no", "Explorer"], + ["unlicensed", "none", "no", "Unlicensed"], + ["Chef", "none", "yes", "Unlicensed"], + ["yes", "yes", "yes", "Unlicensed"], + ] + + valid_import_content = [ + "username, pword, fname, creator, site, yes, email", + "username, pword, fname, explorer, none, no, email", + "", + "u", + "p", + ] + + valid_username_content = ["jfitzgerald@tableau.com"] + + usernames = [ + "valid", + "valid@email.com", + "domain/valid", + "domain/valid@tmail.com", + "va!@#$%^&*()lid", + "in@v@lid", + "in valid", + "", + ] + + def test_validate_usernames(self): + UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[0]) + UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[1]) + UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[2]) + UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[3]) + UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[4]) + with self.assertRaises(AttributeError): + UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[5]) + with self.assertRaises(AttributeError): + UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[6]) + + def test_evaluate_role(self): + for line in UserFromCSVTests.role_inputs: + actual = UserCSVImport.evaluate_site_role(line[0], line[1], line[2]) + assert actual == line[3], line + [actual] + + def test_get_user_detail_empty_line(self): + test_line = "" + test_user = UserCSVImport._parse_line(test_line) + assert test_user is None + + def test_get_user_detail_standard(self): + test_line = "username, pword, fname, license, admin, pub, email" + test_user: TSC.UserItem = UserCSVImport._parse_line(test_line) + assert test_user.name == "username", test_user.name + assert test_user.fullname == "fname", test_user.fullname + assert test_user.site_role == "Unlicensed", test_user.site_role + assert test_user.email == "email", test_user.email + + def test_get_user_details_only_username(self): + test_line = "username" + test_user: TSC.UserItem = UserCSVImport._parse_line(test_line) + + def test_populate_user_details_only_some(self): + values = ["username", "", "", "creator", "admin"] + data = TSC.UserItem() + data.populate(values) + + def test_populate_user_details_all(self): + values = UserFromCSVTests.valid_import_content[0] + data = UserCSVObject() + data.populate([values]) + + def test_validate_user_detail_standard(self): + test_line = "username, pword, fname, creator, site, 1, email" + UserCSVImport._validate_user_line_or_throw(test_line) + + # for file handling + def _mock_file_content(self, content: List[str]) -> io.TextIOWrapper: + # the empty string represents EOF + # the tests run through the file twice, first to validate then to fetch + mock = MagicMock(io.TextIOWrapper) + content.append("") # EOF + mock.readline.side_effect = content + mock.name = "file-mock" + return mock + + def test_get_users_from_file_missing_elements(self): + bad_content = [ + "username, pword, , yes, email", + "username", + "username, pword", + "username, pword, , , yes, email", + ] + test_data = self._mock_file_content(bad_content) + UserCSVImport.get_users_from_file(test_data) + + def test_validate_import_file(self): + test_data = self._mock_file_content(UserFromCSVTests.valid_import_content) + num_lines = UserCSVImport.validate_file_for_import(test_data, detailed=True) + assert num_lines == 2, "Expected two lines to be parsed, got {}".format(num_lines) + + def test_validate_usernames_file(self): + test_data = self._mock_file_content(UserFromCSVTests.usernames) + n = UserCSVImport.validate_file_for_import(test_data) + assert n == 5, "Exactly 5 of the lines were valid, counted {}".format(n) + + def test_validate_usernames_file_strict(self): + test_data = self._mock_file_content(UserFromCSVTests.usernames) + with self.assertRaises(SystemExit): + UserCSVImport.validate_file_for_import(test_data, strict=True) + + def test_get_usernames_from_file(self): + test_data = self._mock_file_content(UserFromCSVTests.usernames) + user_list = UserCSVImport.get_users_from_file(test_data) + assert user_list[0].name == "valid", user_list + + + +class UserImportE2ETests(unittest.TestCase): + def setUp(self) -> None: + self.server = TSC.Server("http://test", False) + + # Fake signin + self.server._site_id = "dad65087-b08b-4603-af4e-2887b8aafc67" + self.server._auth_token = "j80k54ll2lfMZ0tv97mlPvvSCRyD0DOM" + + self.baseurl = self.server.users.baseurl + + # these tests are weird. The input file USERNAMES will be parsed and invalid lines put in 'failures' + # Then we will send the valid lines to the server, and the response from that, ADD_XML, is our 'users'. + # not covered: the server rejects one of our 'valid' lines + def test_get_usernames_from_file(self): + with open(ADD_XML, "rb") as f: + response_xml = f.read().decode("utf-8") + with requests_mock.mock() as m: + m.post(self.server.users.baseurl, text=response_xml) + user_list, failures = self.server.users.create_from_file(USERNAMES) + assert failures != [], failures + assert len(failures) == 2, failures + assert user_list is not None, user_list + assert user_list[0].name == "Cassie", user_list + + def test_get_users_from_file(self): + with open(ADD_XML, "rb") as f: + response_xml = f.read().decode("utf-8") + with requests_mock.mock() as m: + m.post(self.server.users.baseurl, text=response_xml) + users, failures = self.server.users.create_from_file(USERS) + assert failures != [], failures + assert len(failures) == 1, failures + assert users != [], users + assert users[0].name == "Cassie", users + + def test_too_many_bad_lines(self): + with open(ADD_XML, "rb") as f: + response_xml = f.read().decode("utf-8") + with requests_mock.mock() as m: + m.post(self.server.users.baseurl, text=response_xml) + + with self.assertRaises(ValueError) as validator: + users, failures = self.server.users.create_from_file(USERS_BAD) + assert len(failures) == 4 + assert validator is not None + self.assertEqual( + str(validator.exception), + "More than 3 lines have failed validation. Check the errors and fix your file.", + ) From 9e9d44b98c095d9b4f78bf3b2a5b75ded577ed56 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Mon, 20 Mar 2023 15:58:11 -0700 Subject: [PATCH 4/5] separate valid and invalid user tests (cherry picked from commit 098c6d31a00830b5de51597802f2b48b1f3481cd) --- test/test_user_model.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/test/test_user_model.py b/test/test_user_model.py index 9615347c7..30b31186a 100644 --- a/test/test_user_model.py +++ b/test/test_user_model.py @@ -19,7 +19,6 @@ def test_invalid_site_role(self): with self.assertRaises(ValueError): user.site_role = "Hello" - class UserDataTest(unittest.TestCase): logger = logging.getLogger("UserDataTest") @@ -92,11 +91,11 @@ def test_get_user_detail_standard(self): assert test_user.email == "email", test_user.email def test_get_user_detail_variation(self): - test_line = ["username", "pword", "fname", "license", "admin", "pub", "email"] + test_line = ["username", "pword", "fname", "creator", "site", "yes", "email"] test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) assert test_user.name == "username", test_user.name assert test_user.fullname == "fname", test_user.fullname - assert test_user.site_role == "license", test_user.site_role + assert test_user.site_role == "SiteAdministratorCreator", test_user.site_role assert test_user.email == "email", test_user.email def test_create_user_invalid_license(self): @@ -108,13 +107,10 @@ def test_create_user_invalid_role(self): test_line = ["username", "pword", "fname", "creator", "role", "yes", "email"] with self.assertRaises(AttributeError): test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) - - test_line = ["username", "pword", "fname", "license", "admin", "pub", "email"] - test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) - assert test_user.name == "username", test_user.name - assert test_user.fullname == "fname", test_user.fullname - assert test_user.site_role == "license", test_user.site_role - assert test_user.email == "email", test_user.email + assert test_user.name == "username", test_user.name + assert test_user.fullname == "fname", test_user.fullname + assert test_user.site_role == "license", test_user.site_role + assert test_user.email == "email", test_user.email def test_get_user_details_only_username(self): test_line = ["username"] From 42aadf82c5ccbb772e8f267d7217ac00fb308e31 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Tue, 30 Jul 2024 16:47:32 -0700 Subject: [PATCH 5/5] maybe improvements --- tableauserverclient/__init__.py | 4 ++- tableauserverclient/helpers/importer.py | 4 +-- tableauserverclient/models/user_item.py | 14 +++++----- test/test_endpoint.py | 6 ++++- test/test_user.py | 3 ++- test/test_user_csv_import.py | 34 ++++++++++++------------- test/test_user_model.py | 20 ++++++--------- test/test_workbook.py | 2 +- 8 files changed, 45 insertions(+), 42 deletions(-) diff --git a/tableauserverclient/__init__.py b/tableauserverclient/__init__.py index 35eaa1d02..ca46725ff 100644 --- a/tableauserverclient/__init__.py +++ b/tableauserverclient/__init__.py @@ -1,5 +1,6 @@ from tableauserverclient._version import get_versions from tableauserverclient.namespace import NEW_NAMESPACE as DEFAULT_NAMESPACE +from tableauserverclient.helpers import UserCSVImport, UserCSVObject from tableauserverclient.models import ( BackgroundJobItem, ColumnItem, @@ -127,10 +128,11 @@ "Target", "TaskItem", "UserItem", + "UserCSVImport", + "UserCSVObject", "ViewItem", "VirtualConnectionItem", "WebhookItem", "WeeklyInterval", "WorkbookItem", ] -from .helpers import importer # for tests diff --git a/tableauserverclient/helpers/importer.py b/tableauserverclient/helpers/importer.py index fed61590b..89d4c1482 100644 --- a/tableauserverclient/helpers/importer.py +++ b/tableauserverclient/helpers/importer.py @@ -71,7 +71,7 @@ class ColumnType(IntEnum): # Take in a list of strings in expected order # and create a user item populated by the given attributes @staticmethod - def create_user_model_from_line(line_values: List[str]) -> "UserItem": + def create_user_from_line(line_values: List[str]) -> "UserItem": UserCSVImport._validate_import_line_or_throw(line_values) values: List[str] = list(map(lambda x: x.strip(), line_values)) user = UserItem(values[UserCSVImport.ColumnType.USERNAME]) @@ -128,7 +128,7 @@ def process_file_for_import(filepath: str, validate_only=False) -> Tuple[List["U try: UserCSVImport._validate_import_line_or_throw(line) if not validate_only: - user: UserItem = UserCSVImport.create_user_model_from_line(line) + user: UserItem = UserCSVImport.create_user_from_line(line) users.append(user) valid.append(" ".join(line)) except Exception as e: diff --git a/tableauserverclient/models/user_item.py b/tableauserverclient/models/user_item.py index 7f1c4c88e..154b4e361 100644 --- a/tableauserverclient/models/user_item.py +++ b/tableauserverclient/models/user_item.py @@ -122,9 +122,9 @@ def __repr__(self): def password(self) -> Optional[str]: return self._password - @property - def password_setting(self) -> Optional[str]: - return self._password + @password.setter + def password(self, value) -> None: + self._password = value @property def auth_setting(self) -> Optional[str]: @@ -412,7 +412,7 @@ def process_file_for_import( try: UserItem.CSVImport._validate_import_line_or_throw(line, logger) if not validate_only: - user: UserItem = UserItem.CSVImport.create_user_model_from_line(line, logger) + user: UserItem = UserItem.CSVImport.create_user_from_line(line, logger) users.append(user) valid.append(" ".join(line)) except Exception as e: @@ -441,10 +441,12 @@ def _validate_import_line_or_throw(line, logger) -> None: raise AttributeError("Too many attributes in line") # sometimes usernames are case sensitive username = line[UserItem.CSVImport.ColumnType.USERNAME.value] - logger.debug(f"> details - {username}") + if logger: + logger.debug(f"> details - {username}") UserItem.validate_username_or_throw(username) for i in range(1, len(line)): - logger.debug(f"column {UserItem.CSVImport.ColumnType(i).name}: {line[i]}") + if logger: + logger.debug(f"column {UserItem.CSVImport.ColumnType(i).name}: {line[i]}") UserItem.CSVImport._validate_attribute_value( line[i], _valid_attributes[i], UserItem.CSVImport.ColumnType(i) ) diff --git a/test/test_endpoint.py b/test/test_endpoint.py index ff1ef0f72..4ad471c1d 100644 --- a/test/test_endpoint.py +++ b/test/test_endpoint.py @@ -27,6 +27,9 @@ def test_fallback_request_logic(self) -> None: response = endpoint.get_request(url=url) self.assertIsNotNone(response) + """ + something is wrong with the threading, takes forever + def test_user_friendly_request_returns(self) -> None: url = "http://test/" endpoint = TSC.server.Endpoint(self.server) @@ -36,7 +39,7 @@ def test_user_friendly_request_returns(self) -> None: endpoint.parent_srv.session.get, url=url, request_timeout=2 ) self.assertIsNotNone(response) - + def test_blocking_request_raises_request_error(self) -> None: with pytest.raises(requests.exceptions.ConnectionError): url = "http://test/" @@ -44,6 +47,7 @@ def test_blocking_request_raises_request_error(self) -> None: response = endpoint._blocking_request(endpoint.parent_srv.session.get, url=url) self.assertIsNotNone(response) + """ def test_get_request_stream(self) -> None: url = "http://test/" endpoint = TSC.server.Endpoint(self.server) diff --git a/test/test_user.py b/test/test_user.py index cf62c6bbe..0351be8ce 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -16,7 +16,8 @@ POPULATE_WORKBOOKS_XML = os.path.join(TEST_ASSET_DIR, "user_populate_workbooks.xml") GET_FAVORITES_XML = os.path.join(TEST_ASSET_DIR, "favorites_get.xml") POPULATE_GROUPS_XML = os.path.join(TEST_ASSET_DIR, "user_populate_groups.xml") - +USERNAMES = os.path.join(TEST_ASSET_DIR, "Data", "usernames.csv") +USERS = os.path.join(TEST_ASSET_DIR, "Data", "user_details.csv") class UserTests(unittest.TestCase): def setUp(self) -> None: diff --git a/test/test_user_csv_import.py b/test/test_user_csv_import.py index a07510add..16eabbe2e 100644 --- a/test/test_user_csv_import.py +++ b/test/test_user_csv_import.py @@ -6,8 +6,6 @@ import requests_mock import tableauserverclient as TSC -UserCSVObject = TSC.importer.UserCSVObject -UserCSVImport = TSC.importer.UserCSVImport TEST_ASSET_DIR = os.path.join(os.path.dirname(__file__), "assets") @@ -60,29 +58,29 @@ class UserFromCSVTests(unittest.TestCase): ] def test_validate_usernames(self): - UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[0]) - UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[1]) - UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[2]) - UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[3]) - UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[4]) + TSC.UserItem.CSVImport._validate_import_line_or_throw(UserFromCSVTests.usernames[0]) + TSC.UserItem.CSVImport._validate_import_line_or_throw(UserFromCSVTests.usernames[1]) + TSC.UserItem.CSVImport._validate_import_line_or_throw(UserFromCSVTests.usernames[2]) + TSC.UserItem.CSVImport._validate_import_line_or_throw(UserFromCSVTests.usernames[3]) + TSC.UserItem.CSVImport._validate_import_line_or_throw(UserFromCSVTests.usernames[4]) with self.assertRaises(AttributeError): - UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[5]) + TSC.UserItem.CSVImport._validate_import_line_or_throw(UserFromCSVTests.usernames[5]) with self.assertRaises(AttributeError): - UserCSVImport._validate_username_or_throw(UserFromCSVTests.usernames[6]) + TSC.UserItem.CSVImport._validate_import_line_or_throw(UserFromCSVTests.usernames[6]) def test_evaluate_role(self): for line in UserFromCSVTests.role_inputs: - actual = UserCSVImport.evaluate_site_role(line[0], line[1], line[2]) + actual = TSC.UserItem.CSVImport._evaluate_site_role(line[0], line[1], line[2]) assert actual == line[3], line + [actual] def test_get_user_detail_empty_line(self): test_line = "" - test_user = UserCSVImport._parse_line(test_line) + test_user = TSC.UserItem.CSVImport.create_user_from_line(test_line) assert test_user is None def test_get_user_detail_standard(self): test_line = "username, pword, fname, license, admin, pub, email" - test_user: TSC.UserItem = UserCSVImport._parse_line(test_line) + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line) assert test_user.name == "username", test_user.name assert test_user.fullname == "fname", test_user.fullname assert test_user.site_role == "Unlicensed", test_user.site_role @@ -90,7 +88,7 @@ def test_get_user_detail_standard(self): def test_get_user_details_only_username(self): test_line = "username" - test_user: TSC.UserItem = UserCSVImport._parse_line(test_line) + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line) def test_populate_user_details_only_some(self): values = ["username", "", "", "creator", "admin"] @@ -99,7 +97,7 @@ def test_populate_user_details_only_some(self): def test_populate_user_details_all(self): values = UserFromCSVTests.valid_import_content[0] - data = UserCSVObject() + data = TSC.UserItem.CSVObject() data.populate([values]) def test_validate_user_detail_standard(self): @@ -128,22 +126,22 @@ def test_get_users_from_file_missing_elements(self): def test_validate_import_file(self): test_data = self._mock_file_content(UserFromCSVTests.valid_import_content) - num_lines = UserCSVImport.validate_file_for_import(test_data, detailed=True) + num_lines = TSC.UserItem.CSVImport.validate_file_for_import(test_data, detailed=True) assert num_lines == 2, "Expected two lines to be parsed, got {}".format(num_lines) def test_validate_usernames_file(self): test_data = self._mock_file_content(UserFromCSVTests.usernames) - n = UserCSVImport.validate_file_for_import(test_data) + n = TSC.UserItem.CSVImport.validate_file_for_import(test_data) assert n == 5, "Exactly 5 of the lines were valid, counted {}".format(n) def test_validate_usernames_file_strict(self): test_data = self._mock_file_content(UserFromCSVTests.usernames) with self.assertRaises(SystemExit): - UserCSVImport.validate_file_for_import(test_data, strict=True) + TSC.UserItem.CSVImport.validate_file_for_import(test_data, strict=True) def test_get_usernames_from_file(self): test_data = self._mock_file_content(UserFromCSVTests.usernames) - user_list = UserCSVImport.get_users_from_file(test_data) + user_list = TSC.UserItem.CSVImport.get_users_from_file(test_data) assert user_list[0].name == "valid", user_list diff --git a/test/test_user_model.py b/test/test_user_model.py index 30b31186a..df608bad8 100644 --- a/test/test_user_model.py +++ b/test/test_user_model.py @@ -80,11 +80,11 @@ def test_evaluate_role(self): def test_get_user_detail_empty_line(self): test_line = "" with self.assertRaises(AttributeError): - test_user = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + test_user = TSC.UserItem.CSVImport.create_user_from_line(test_line, UserDataTest.logger) def test_get_user_detail_standard(self): test_line = ["username", "pword", "fname", "unlicensed", "no", "no", "email"] - test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line, UserDataTest.logger) assert test_user.name == "username", test_user.name assert test_user.fullname == "fname", test_user.fullname assert test_user.site_role == "Unlicensed", test_user.site_role @@ -92,7 +92,7 @@ def test_get_user_detail_standard(self): def test_get_user_detail_variation(self): test_line = ["username", "pword", "fname", "creator", "site", "yes", "email"] - test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line, UserDataTest.logger) assert test_user.name == "username", test_user.name assert test_user.fullname == "fname", test_user.fullname assert test_user.site_role == "SiteAdministratorCreator", test_user.site_role @@ -101,29 +101,25 @@ def test_get_user_detail_variation(self): def test_create_user_invalid_license(self): test_line = ["username", "pword", "fname", "license", "site", "yes", "email"] with self.assertRaises(AttributeError): - test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line, UserDataTest.logger) def test_create_user_invalid_role(self): test_line = ["username", "pword", "fname", "creator", "role", "yes", "email"] with self.assertRaises(AttributeError): - test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) - assert test_user.name == "username", test_user.name - assert test_user.fullname == "fname", test_user.fullname - assert test_user.site_role == "license", test_user.site_role - assert test_user.email == "email", test_user.email + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line, UserDataTest.logger) def test_get_user_details_only_username(self): test_line = ["username"] - test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + test_user: TSC.UserItem = TSC.UserItem.CSVImport.create_user_from_line(test_line, UserDataTest.logger) def test_populate_user_details_only_some(self): values = ["username", "", "", "creator", "site"] - user = TSC.UserItem.CSVImport.create_user_model_from_line(values, UserDataTest.logger) + user = TSC.UserItem.CSVImport.create_user_from_line(values, UserDataTest.logger) assert user.name == "username" def test_validate_user_detail_standard(self): test_line = ["username", "pword", "fname", "creator", "site", "1", "email"] - TSC.UserItem.CSVImport.create_user_model_from_line(test_line, UserDataTest.logger) + TSC.UserItem.CSVImport.create_user_from_line(test_line, UserDataTest.logger) def test_validate_import_file(self): users, valid, invalid = TSC.UserItem.CSVImport.process_file_for_import( diff --git a/test/test_workbook.py b/test/test_workbook.py index 1a6b3192f..e82fa79fa 100644 --- a/test/test_workbook.py +++ b/test/test_workbook.py @@ -872,7 +872,7 @@ def test_odata_connection(self) -> None: workbook = TSC.WorkbookItem("project", "test") workbook._id = "06b944d2-959d-4604-9305-12323c95e70e" connection = TSC.ConnectionItem() - url = "https://odata.website.com/TestODataEndpoint" + url = "https://odata.website.com/TestODataEndpoint".lower() connection.server_address = url connection._connection_type = "odata" connection._id = "17376070-64d1-4d17-acb4-a56e4b5b1768"