From adffc947865f5becaaaa745d440e55c5959d1be3 Mon Sep 17 00:00:00 2001 From: Sophia Rowland Date: Fri, 18 Apr 2025 10:16:45 -0400 Subject: [PATCH 01/12] Update pzmm_generate_complete_model_card.ipynb Our preprocess function was missing 'Education_Some_college' and 'Education_HS_grad' in the input_cols --- examples/pzmm_generate_complete_model_card.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pzmm_generate_complete_model_card.ipynb b/examples/pzmm_generate_complete_model_card.ipynb index 3a68271b..f49efd56 100644 --- a/examples/pzmm_generate_complete_model_card.ipynb +++ b/examples/pzmm_generate_complete_model_card.ipynb @@ -578,7 +578,7 @@ " \"MartialStatus_Married_AF_spouse\", 'MartialStatus_Married_civ_spouse', 'MartialStatus_Never_married', 'MartialStatus_Divorced', 'MartialStatus_Separated', \n", " 'MartialStatus_Widowed', 'Race_White', 'Race_Black', 'Race_Asian_Pac_Islander', 'Race_Amer_Indian_Eskimo', 'Race_Other', 'Relationship_Husband', \n", " 'Relationship_Not_in_family', 'Relationship_Own_child', 'Relationship_Unmarried', 'Relationship_Wife', 'Relationship_Other_relative', 'WorkClass_Private',\n", - " 'Education_Bachelors'\n", + " 'Education_Bachelors', 'Education_Some_college', 'Education_HS_grad'\n", " ]\n", " # OHE columns must be removed after data combination\n", " predictor_columns = ['Age', 'HoursPerWeek', 'WorkClass_Private', 'WorkClass_Self', 'WorkClass_Gov', \n", From 7298b2c2d60e1d0a3a9cfca346a26d943ab6e868 Mon Sep 17 00:00:00 2001 From: djm21 Date: Mon, 28 Apr 2025 11:46:42 -0700 Subject: [PATCH 02/12] feat: upload local model without any extra file generation (PMMODEL-682) --- src/sasctl/tasks.py | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/sasctl/tasks.py b/src/sasctl/tasks.py index d466c10f..b08fb570 100644 --- a/src/sasctl/tasks.py +++ b/src/sasctl/tasks.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import Union from warnings import warn +import zipfile import pandas as pd @@ -998,3 +999,55 @@ def score_model_with_cas( print(score_execution_poll) score_results = se.get_score_execution_results(score_execution, use_cas_gateway) return score_results + +def upload_local_model( + path: Union[str, Path], + model_name: str, + project_name: str, + repo_name: Union[str, dict] = None, + version: str = "latest", + ): + """A barebones function to upload a model and any associated files to the model repository. + Parameters + ---------- + path : Union[str, Path] + The path to the model and any associated files. + model_name : str + The name of the model. + project_name : str + The name of the project to which the model will be uploaded. + """ + # Use default repository if not specified + try: + if repo_name is None: + repository = mr.default_repository() + else: + repository = mr.get_repository(repo_name) + except HTTPError as e: + if e.code == 403: + raise AuthorizationError( + "Unable to register model. User account does not have read permissions " + "for the /modelRepository/repositories/ URL. Please contact your SAS " + "Viya administrator." + ) + raise e + + # Unable to find or create the repo. + if repository is None and repo_name is None: + raise ValueError("Unable to find a default repository") + + if repository is None: + raise ValueError("Unable to find repository '{}'".format(repository)) + p = mr.get_project(project_name) + if p is None: + mr.create_project(project_name, repository) + zip_name = str(Path(path) / (model_name + ".zip")) + file_names = sorted(Path(path).glob("*[!zip]")) + with zipfile.ZipFile( + str(zip_name), mode="w" + ) as zFile: + for file in file_names: + zFile.write(str(file), arcname=file.name) + with open(zip_name, "rb") as zip_file: + model = mr.import_model_from_zip(model_name, project_name, zip_file, version=version) + return model From b3c2912e51bf18376bda9e32b98e56e421de4ad9 Mon Sep 17 00:00:00 2001 From: djm21 Date: Mon, 28 Apr 2025 12:17:18 -0700 Subject: [PATCH 03/12] black reformatting --- src/sasctl/tasks.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/sasctl/tasks.py b/src/sasctl/tasks.py index b08fb570..44c7b430 100644 --- a/src/sasctl/tasks.py +++ b/src/sasctl/tasks.py @@ -1000,13 +1000,14 @@ def score_model_with_cas( score_results = se.get_score_execution_results(score_execution, use_cas_gateway) return score_results + def upload_local_model( - path: Union[str, Path], - model_name: str, - project_name: str, - repo_name: Union[str, dict] = None, - version: str = "latest", - ): + path: Union[str, Path], + model_name: str, + project_name: str, + repo_name: Union[str, dict] = None, + version: str = "latest", +): """A barebones function to upload a model and any associated files to the model repository. Parameters ---------- @@ -1043,11 +1044,11 @@ def upload_local_model( mr.create_project(project_name, repository) zip_name = str(Path(path) / (model_name + ".zip")) file_names = sorted(Path(path).glob("*[!zip]")) - with zipfile.ZipFile( - str(zip_name), mode="w" - ) as zFile: - for file in file_names: - zFile.write(str(file), arcname=file.name) + with zipfile.ZipFile(str(zip_name), mode="w") as zFile: + for file in file_names: + zFile.write(str(file), arcname=file.name) with open(zip_name, "rb") as zip_file: - model = mr.import_model_from_zip(model_name, project_name, zip_file, version=version) + model = mr.import_model_from_zip( + model_name, project_name, zip_file, version=version + ) return model From 6ac153b3c9638087072e586bc74702d9da3e3b1f Mon Sep 17 00:00:00 2001 From: djm21 Date: Tue, 29 Apr 2025 08:33:03 -0700 Subject: [PATCH 04/12] updates to error handling for repository finding --- src/sasctl/tasks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sasctl/tasks.py b/src/sasctl/tasks.py index 44c7b430..dfa07e64 100644 --- a/src/sasctl/tasks.py +++ b/src/sasctl/tasks.py @@ -1034,11 +1034,10 @@ def upload_local_model( raise e # Unable to find or create the repo. - if repository is None and repo_name is None: + if not repository and not repo_name: raise ValueError("Unable to find a default repository") - - if repository is None: - raise ValueError("Unable to find repository '{}'".format(repository)) + elif not repository: + raise ValueError(f"Unable to find repository '{repo_name}'") p = mr.get_project(project_name) if p is None: mr.create_project(project_name, repository) From c5deb291f804dae061dab4dd91b520cf9033ef46 Mon Sep 17 00:00:00 2001 From: djm21 Date: Tue, 29 Apr 2025 22:29:03 -0700 Subject: [PATCH 05/12] prep for release --- CHANGELOG.md | 5 +++++ src/sasctl/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f955f4e9..8e0f6724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v1.11.3 (2025-04-29) +-------------------- +**Improvements** +- Added `upload_local_model` to `tasks.py`, which can be used to upload local directories to SAS Model Manager without any extra file generation. + v1.11.2 (2025-04-08) -------------------- **Bugfixes** diff --git a/src/sasctl/__init__.py b/src/sasctl/__init__.py index e4c7a0e1..aa942471 100644 --- a/src/sasctl/__init__.py +++ b/src/sasctl/__init__.py @@ -4,7 +4,7 @@ # Copyright © 2019, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -__version__ = "1.11.2" +__version__ = "1.11.3" __author__ = "SAS" __credits__ = [ "Yi Jian Ching", From 77666aa5416ab8a2882d1d1e856835de17bf0723 Mon Sep 17 00:00:00 2001 From: djm21 Date: Tue, 29 Apr 2025 22:59:24 -0700 Subject: [PATCH 06/12] prep for release --- src/sasctl/tasks.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/sasctl/tasks.py b/src/sasctl/tasks.py index dfa07e64..364042e3 100644 --- a/src/sasctl/tasks.py +++ b/src/sasctl/tasks.py @@ -265,10 +265,9 @@ def _register_sas_model( out_var = [] in_var = [] import copy - import zipfile as zp zip_file_copy = copy.deepcopy(zip_file) - tmp_zip = zp.ZipFile(zip_file_copy) + tmp_zip = zipfile.ZipFile(zip_file_copy) if "outputVar.json" in tmp_zip.namelist(): out_var = json.loads( tmp_zip.read("outputVar.json").decode("utf=8") @@ -328,8 +327,8 @@ def _register_sas_model( if current_session().version_info() < 4: # Upload the model as a ZIP file if using Viya 3. - zipfile = utils.create_package(model, input=input) - model = mr.import_model_from_zip(name, project, zipfile, version=version) + zip_file = utils.create_package(model, input=input) + model = mr.import_model_from_zip(name, project, zip_file, version=version) else: # If using Viya 4, just upload the raw AStore and Model Manager will handle inspection. astore = cas.astore.download(rstore=model) @@ -1008,7 +1007,7 @@ def upload_local_model( repo_name: Union[str, dict] = None, version: str = "latest", ): - """A barebones function to upload a model and any associated files to the model repository. + """A function to upload a model and any associated files to the model repository. Parameters ---------- path : Union[str, Path] @@ -1017,6 +1016,10 @@ def upload_local_model( The name of the model. project_name : str The name of the project to which the model will be uploaded. + repo_name : Union[str, dict], optional + repository in which to create the project + version: str, optional + The version of the model being uploaded """ # Use default repository if not specified try: @@ -1036,16 +1039,21 @@ def upload_local_model( # Unable to find or create the repo. if not repository and not repo_name: raise ValueError("Unable to find a default repository") - elif not repository: + if not repository: raise ValueError(f"Unable to find repository '{repo_name}'") + + # Get project from repo if it exists; if it doesn't, create a new one p = mr.get_project(project_name) if p is None: mr.create_project(project_name, repository) + + # zip up all files in directory (except any previous zip files) zip_name = str(Path(path) / (model_name + ".zip")) file_names = sorted(Path(path).glob("*[!zip]")) with zipfile.ZipFile(str(zip_name), mode="w") as zFile: for file in file_names: zFile.write(str(file), arcname=file.name) + # upload zipped model with open(zip_name, "rb") as zip_file: model = mr.import_model_from_zip( model_name, project_name, zip_file, version=version From 953126b6f55937ce2c5fa234c4197de0dab66637 Mon Sep 17 00:00:00 2001 From: djm21 Date: Tue, 29 Apr 2025 23:13:36 -0700 Subject: [PATCH 07/12] update testing to use newer version of ubuntu --- .github/workflows/build-test-deploy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index a0072348..87fac0e2 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -22,8 +22,7 @@ jobs: strategy: matrix: python-version: ['3.8', '3.9', '3.10', '3.11'] - os-version: ['ubuntu-20.04', 'windows-latest', 'macos-latest'] -# Pinned Ubuntu version to 20.04 since no Python 3.6 builds available on ubuntu-latest (22.04) as of 2022-12-7. + os-version: ['ubuntu-latest', 'windows-latest', 'macos-latest'] # os-version: [ubuntu-latest, windows-latest, macos-latest] steps: From 5b3e1f8a0f01698c840b89f78c2bd4c86fd88f31 Mon Sep 17 00:00:00 2001 From: djm21 Date: Tue, 29 Apr 2025 23:24:48 -0700 Subject: [PATCH 08/12] prep for release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e0f6724..64f51f56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ v1.11.3 (2025-04-29) -------------------- **Improvements** -- Added `upload_local_model` to `tasks.py`, which can be used to upload local directories to SAS Model Manager without any extra file generation. +- Added `upload_local_model` to `tasks.py`, which can be used to upload local directories to SAS Model Manager without any file generation. v1.11.2 (2025-04-08) -------------------- From 70af19773828a918f93798c77adf819c33166578 Mon Sep 17 00:00:00 2001 From: djm21 Date: Wed, 30 Apr 2025 13:00:54 -0700 Subject: [PATCH 09/12] fix: upload .sasast models correctly (PMMODEL-682) --- src/sasctl/tasks.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/sasctl/tasks.py b/src/sasctl/tasks.py index 364042e3..5cc0d192 100644 --- a/src/sasctl/tasks.py +++ b/src/sasctl/tasks.py @@ -1045,17 +1045,33 @@ def upload_local_model( # Get project from repo if it exists; if it doesn't, create a new one p = mr.get_project(project_name) if p is None: - mr.create_project(project_name, repository) + p = mr.create_project(project_name, repository) # zip up all files in directory (except any previous zip files) zip_name = str(Path(path) / (model_name + ".zip")) - file_names = sorted(Path(path).glob("*[!zip]")) - with zipfile.ZipFile(str(zip_name), mode="w") as zFile: + file_names = sorted(Path(path).glob("*[!(zip|sasast)]")) + sasast_file = next(Path(path).glob("*.sasast"), None) + if sasast_file: + # If a sasast file is present, upload it as well + with open(sasast_file, "rb") as sasast: + sasast_model = sasast.read() + data = { + "name": model_name, + "projectId": p.id, + "type": "ASTORE", + } + files = {"files": (sasast_file.name, sasast_model)} + model = mr.post("/models", files=files, data=data) for file in file_names: - zFile.write(str(file), arcname=file.name) - # upload zipped model - with open(zip_name, "rb") as zip_file: - model = mr.import_model_from_zip( - model_name, project_name, zip_file, version=version - ) + with open(file, "r") as f: + mr.add_model_content(model, f, file.name) + else: + with zipfile.ZipFile(str(zip_name), mode="w") as zFile: + for file in file_names: + zFile.write(str(file), arcname=file.name) + # upload zipped model + with open(zip_name, "rb") as zip_file: + model = mr.import_model_from_zip( + model_name, project_name, zip_file, version=version + ) return model From a194b7bbcdb6f1b3bba0665288032acdb9ba5395 Mon Sep 17 00:00:00 2001 From: djm21 Date: Wed, 30 Apr 2025 13:49:25 -0700 Subject: [PATCH 10/12] fix: update upload_local_model to work with model versioning (PMMODEL-682) --- src/sasctl/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sasctl/tasks.py b/src/sasctl/tasks.py index 5cc0d192..539a39e4 100644 --- a/src/sasctl/tasks.py +++ b/src/sasctl/tasks.py @@ -1019,7 +1019,7 @@ def upload_local_model( repo_name : Union[str, dict], optional repository in which to create the project version: str, optional - The version of the model being uploaded + The version of the model being uploaded. Defaults to 'latest'. For new model version, use 'new'. """ # Use default repository if not specified try: @@ -1058,7 +1058,8 @@ def upload_local_model( data = { "name": model_name, "projectId": p.id, - "type": "ASTORE", + "type": "ASTORE", + "versionOption": version } files = {"files": (sasast_file.name, sasast_model)} model = mr.post("/models", files=files, data=data) From f9198e1921921b19bf64c5f208a61e08c16da3c4 Mon Sep 17 00:00:00 2001 From: djm21 Date: Fri, 2 May 2025 11:00:47 -0700 Subject: [PATCH 11/12] black reformatting --- src/sasctl/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sasctl/tasks.py b/src/sasctl/tasks.py index 539a39e4..6bf5c67b 100644 --- a/src/sasctl/tasks.py +++ b/src/sasctl/tasks.py @@ -1059,7 +1059,7 @@ def upload_local_model( "name": model_name, "projectId": p.id, "type": "ASTORE", - "versionOption": version + "versionOption": version, } files = {"files": (sasast_file.name, sasast_model)} model = mr.post("/models", files=files, data=data) From e0934185b9a91e3f77260c64910fa30f1fced7d8 Mon Sep 17 00:00:00 2001 From: djm21 Date: Fri, 2 May 2025 13:07:27 -0700 Subject: [PATCH 12/12] prep for release --- CHANGELOG.md | 5 +++++ src/sasctl/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64f51f56..75252845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v1.11.4 (2025-05-02) +-------------------- +**Improvements** +- Improved `upload_local_model` to allow for SAS Model Manager to properly intake local ASTORE models. + v1.11.3 (2025-04-29) -------------------- **Improvements** diff --git a/src/sasctl/__init__.py b/src/sasctl/__init__.py index aa942471..18da61b8 100644 --- a/src/sasctl/__init__.py +++ b/src/sasctl/__init__.py @@ -4,7 +4,7 @@ # Copyright © 2019, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -__version__ = "1.11.3" +__version__ = "1.11.4" __author__ = "SAS" __credits__ = [ "Yi Jian Ching",