From 0f368fdf5868aa30e754a386b64419b78f811cad Mon Sep 17 00:00:00 2001 From: jwiltse Date: Fri, 14 Feb 2025 12:53:23 -0500 Subject: [PATCH 01/11] Add configuration support for custom HTTP Headers with env var interpolation for authentication to custom URLs --- portable-python.yml | 4 ++++ src/portable_python/__init__.py | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/portable-python.yml b/portable-python.yml index a3e59fd..9c1dfef 100644 --- a/portable-python.yml +++ b/portable-python.yml @@ -29,7 +29,11 @@ cpython-additional-packages: # Uncomment to override cpython or a dependency source URL # Note: string "$version" will be replaced with version string (e.g. 1.2.3) # cpython-url: https://my-cpython-mirror/cpython-$version.tar.gz +# cpython-http-headers: +# - Authorization: Bearer ${GITHUB_TOKEN} # zlib-url: https://my-zlib-mirror/zlib-$version.tar.gz +# zlib-http-headers: +# - Authorization: Bearer ${GITHUB_TOKEN} # Uncomment to override the ./configure arguments for a dependency # Note: this will replace the default arguments, not extend them diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index 411aac9..3ac135a 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -492,6 +492,10 @@ def is_usable_module(self, name): def cfg_version(self, default): return PPG.config.get_value("%s-version" % self.m_name) or default + def cfg_http_headers(self): + if config_http_headers := PPG.config.get_value("%s-http-headers" % self.m_name): + return config_http_headers + def cfg_url(self, version): if config_url := PPG.config.get_value("%s-url" % self.m_name): url_template = Template(config_url) @@ -643,7 +647,14 @@ def compile(self): https_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy") if https_proxy: proxies["https"] = https_proxy - RestClient().download(self.url, path, proxies=proxies) + + expanded_url = os.path.expandvars(self.url) + expanded_http_headers = {} + if headers := self.cfg_http_headers(): + for header_dict in headers: + for key, value in header_dict.items(): + expanded_http_headers[os.path.expandvars(key)] = os.path.expandvars(value) + RestClient().download(expanded_url, path, proxies=proxies, headers=expanded_http_headers) runez.decompress(path, self.m_src_build, simplify=True) From 42d9cb460ddeae8948d098ef654ffcc4a39252b5 Mon Sep 17 00:00:00 2001 From: jwiltse Date: Sat, 15 Feb 2025 13:05:16 -0500 Subject: [PATCH 02/11] Add missed call to self.cfg_url for GettextTiny --- src/portable_python/external/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/portable_python/external/__init__.py b/src/portable_python/external/__init__.py index 10f3fd9..34a443a 100644 --- a/src/portable_python/external/__init__.py +++ b/src/portable_python/external/__init__.py @@ -6,7 +6,7 @@ class GettextTiny(ModuleBuilder): @property def url(self): - return f"https://github.com/sabotage-linux/gettext-tiny/archive/refs/tags/v{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://github.com/sabotage-linux/gettext-tiny/archive/refs/tags/v{self.version}.tar.gz" @property def version(self): From 54e6449b3d094a4ee0fb2c332e9b69a78f37a420 Mon Sep 17 00:00:00 2001 From: jwiltse Date: Sat, 15 Feb 2025 13:06:26 -0500 Subject: [PATCH 03/11] Add config/comment with example of using github release artifact instead of --- portable-python.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/portable-python.yml b/portable-python.yml index 9c1dfef..2d51688 100644 --- a/portable-python.yml +++ b/portable-python.yml @@ -34,6 +34,12 @@ cpython-additional-packages: # zlib-url: https://my-zlib-mirror/zlib-$version.tar.gz # zlib-http-headers: # - Authorization: Bearer ${GITHUB_TOKEN} +# +# The .tar.gz in projects public releases has additional files not present the tarball of the git tag +# uuid-url: https://my-github-enterprise/api/v3/repos/opensource/libuuid/releases/assets/48151 +# uuid-http-headers: +# - Authorization: Bearer ${GITHUB_TOKEN} +# Accept: application/octet-stream # Uncomment to override the ./configure arguments for a dependency # Note: this will replace the default arguments, not extend them From 6775b16ffa987ec98b9a7a1d3ae66f6de3de5a58 Mon Sep 17 00:00:00 2001 From: jwiltse Date: Sat, 15 Feb 2025 13:11:58 -0500 Subject: [PATCH 04/11] - Add condition for handling any source URL which redirects and doesn't end in .tar.gz or .zip, such as Github enterprise releases endpoint - Move url and header env var expansion into cfg_http_headers cfg_url functions --- src/portable_python/__init__.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index 3ac135a..b35d3ce 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -494,12 +494,18 @@ def cfg_version(self, default): def cfg_http_headers(self): if config_http_headers := PPG.config.get_value("%s-http-headers" % self.m_name): - return config_http_headers + expanded_http_headers = {} + for header_dict in config_http_headers: + for key, value in header_dict.items(): + expanded_http_headers[os.path.expandvars(key)] = os.path.expandvars(value) + return expanded_http_headers def cfg_url(self, version): if config_url := PPG.config.get_value("%s-url" % self.m_name): url_template = Template(config_url) - return url_template.substitute(version=version) + url_subbed = url_template.substitute(version=version) + url_expanded = os.path.expandvars(url_subbed) + return url_expanded def cfg_configure(self, deps_lib_dir, deps_lib64_dir): if configure := PPG.config.get_value("%s-configure" % self.m_name): @@ -636,8 +642,13 @@ def compile(self): self._finalize() return - # Split on '#' for urls that include a checksum, such as #sha256=... fragment - basename = runez.basename(self.url, extension_marker="#") + if ".tar.gz" not in self.url and ".zip" not in self.url: + # TODO: add self.cfg_src_extension to remove assumption of .tar.gz + basename = f"{self.m_name}-{self.version}.tar.gz" + else: + # Split on '#' for urls that include a checksum, such as #sha256=... fragment + basename = runez.basename(self.url, extension_marker="#") + path = self.setup.folders.sources / basename if not path.exists(): proxies = {} @@ -648,13 +659,7 @@ def compile(self): if https_proxy: proxies["https"] = https_proxy - expanded_url = os.path.expandvars(self.url) - expanded_http_headers = {} - if headers := self.cfg_http_headers(): - for header_dict in headers: - for key, value in header_dict.items(): - expanded_http_headers[os.path.expandvars(key)] = os.path.expandvars(value) - RestClient().download(expanded_url, path, proxies=proxies, headers=expanded_http_headers) + RestClient().download(self.url, path, proxies=proxies, headers=self.headers) runez.decompress(path, self.m_src_build, simplify=True) From 4868efea814803bd37c5793dd4743defb429fb65 Mon Sep 17 00:00:00 2001 From: jwiltse Date: Sat, 15 Feb 2025 13:23:08 -0500 Subject: [PATCH 05/11] - Add cfg_src_suffix to handle github releases asset endpoints --- src/portable_python/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index b35d3ce..c5b74f6 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -507,6 +507,9 @@ def cfg_url(self, version): url_expanded = os.path.expandvars(url_subbed) return url_expanded + def cfg_src_suffix(self): + return PPG.config.get_value("%s-src-suffix" % self.m_name) + def cfg_configure(self, deps_lib_dir, deps_lib64_dir): if configure := PPG.config.get_value("%s-configure" % self.m_name): configure_template = Template(configure) @@ -642,9 +645,11 @@ def compile(self): self._finalize() return - if ".tar.gz" not in self.url and ".zip" not in self.url: - # TODO: add self.cfg_src_extension to remove assumption of .tar.gz - basename = f"{self.m_name}-{self.version}.tar.gz" + # Some src_url don't end in file extension, such as with redirects + # Github releases asset endpoint is this way .../releases/assets/48151 + if self.url.endswith(".zip", ".tar.gz"): + suffix = self.cfg_src_suffix() or ".tar.gz" + basename = f"{self.m_name}-{self.version}.{suffix}" else: # Split on '#' for urls that include a checksum, such as #sha256=... fragment basename = runez.basename(self.url, extension_marker="#") From d159123f0d9ff5d69a7ebd99797dcc25e5390d51 Mon Sep 17 00:00:00 2001 From: jwiltse Date: Sat, 15 Feb 2025 13:24:25 -0500 Subject: [PATCH 06/11] - Update comment --- portable-python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/portable-python.yml b/portable-python.yml index 2d51688..c48c4fa 100644 --- a/portable-python.yml +++ b/portable-python.yml @@ -37,6 +37,7 @@ cpython-additional-packages: # # The .tar.gz in projects public releases has additional files not present the tarball of the git tag # uuid-url: https://my-github-enterprise/api/v3/repos/opensource/libuuid/releases/assets/48151 +# uuid-src-suffix: .tar.gz # uuid-http-headers: # - Authorization: Bearer ${GITHUB_TOKEN} # Accept: application/octet-stream From a3ae2a3cf6b3f46bdb07b0edeff209ef6336f678 Mon Sep 17 00:00:00 2001 From: jwiltse Date: Sat, 15 Feb 2025 13:27:49 -0500 Subject: [PATCH 07/11] - Fix boolean --- src/portable_python/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index c5b74f6..134a082 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -647,7 +647,7 @@ def compile(self): # Some src_url don't end in file extension, such as with redirects # Github releases asset endpoint is this way .../releases/assets/48151 - if self.url.endswith(".zip", ".tar.gz"): + if not self.url.endswith(".zip", ".tar.gz"): suffix = self.cfg_src_suffix() or ".tar.gz" basename = f"{self.m_name}-{self.version}.{suffix}" else: From e43c659bc2c8aed69126b5aae4e3e0e78da0a91a Mon Sep 17 00:00:00 2001 From: jwiltse Date: Sat, 15 Feb 2025 13:49:17 -0500 Subject: [PATCH 08/11] - Add to @property for suffix and headers - Satisfy linter by eliminating temporary --- src/portable_python/__init__.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index 134a082..26ddbcd 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -504,8 +504,7 @@ def cfg_url(self, version): if config_url := PPG.config.get_value("%s-url" % self.m_name): url_template = Template(config_url) url_subbed = url_template.substitute(version=version) - url_expanded = os.path.expandvars(url_subbed) - return url_expanded + return os.path.expandvars(url_subbed) def cfg_src_suffix(self): return PPG.config.get_value("%s-src-suffix" % self.m_name) @@ -523,6 +522,16 @@ def url(self): """Url of source tarball, if any""" return "" + @property + def headers(self): + """Headers for connecting to source url, if any""" + return self.cfg_http_headers() + + @property + def src_suffix(self): + """Suffix of src archive for when URL doesn't end in the file extension""" + return self.cfg_src_suffix() + @property def version(self): """Version to use""" @@ -645,10 +654,10 @@ def compile(self): self._finalize() return - # Some src_url don't end in file extension, such as with redirects + # Some URL's may not end in file extension, such as with redirects. # Github releases asset endpoint is this way .../releases/assets/48151 - if not self.url.endswith(".zip", ".tar.gz"): - suffix = self.cfg_src_suffix() or ".tar.gz" + if not self.url.endswith((".zip", ".tar.gz")): + suffix = self.src_suffix or ".tar.gz" basename = f"{self.m_name}-{self.version}.{suffix}" else: # Split on '#' for urls that include a checksum, such as #sha256=... fragment From 586c5b704a30aea5551711885c5ff7724462e90e Mon Sep 17 00:00:00 2001 From: jwiltse Date: Sat, 15 Feb 2025 13:49:46 -0500 Subject: [PATCH 09/11] - Per @zsimic, update mccabe max-complexity to 18 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6864983..7513e0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ ignore = [ order-by-type = false [tool.ruff.lint.mccabe] -max-complexity = 14 +max-complexity = 18 [tool.ruff.lint.pydocstyle] convention = "numpy" From 99fa3432ddee2b99c45810639681ac885a9f8ab8 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Mon, 17 Feb 2025 18:13:12 -0800 Subject: [PATCH 10/11] Do not consider optional sha checksum in url as basename, added test --- src/portable_python/__init__.py | 12 +++++++----- tests/sample-config1.yml | 8 ++++++++ tests/test_setup.py | 7 +++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index 26ddbcd..3de7038 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -498,6 +498,7 @@ def cfg_http_headers(self): for header_dict in config_http_headers: for key, value in header_dict.items(): expanded_http_headers[os.path.expandvars(key)] = os.path.expandvars(value) + return expanded_http_headers def cfg_url(self, version): @@ -656,12 +657,13 @@ def compile(self): # Some URL's may not end in file extension, such as with redirects. # Github releases asset endpoint is this way .../releases/assets/48151 - if not self.url.endswith((".zip", ".tar.gz")): + + # Split on '#' for urls that include a checksum, such as #sha256=... fragment + basename = runez.basename(self.url, extension_marker="#") + if not basename.endswith((".zip", ".tar.gz")): suffix = self.src_suffix or ".tar.gz" - basename = f"{self.m_name}-{self.version}.{suffix}" - else: - # Split on '#' for urls that include a checksum, such as #sha256=... fragment - basename = runez.basename(self.url, extension_marker="#") + suffix = ".%s" % (suffix.strip(".")) # Ensure it starts with a dot (in case config forgot leading dot) + basename = f"{self.m_name}-{self.version}{suffix}" path = self.setup.folders.sources / basename if not path.exists(): diff --git a/tests/sample-config1.yml b/tests/sample-config1.yml index b33f231..cfbb9d1 100644 --- a/tests/sample-config1.yml +++ b/tests/sample-config1.yml @@ -40,3 +40,11 @@ cpython-pep668-externally-managed: cpython-configure: - --enable-shared + +# Exercise custom url +bzip2-url: https://my-enterprise/.../assets/bzip2/123 +bzip2-version: 1.2.3 +bzip2-src-suffix: tar.gz # Forgot leading dot on purpose +bzip2-http-headers: + - Authorization: Bearer foo + Accept: application/octet-stream diff --git a/tests/test_setup.py b/tests/test_setup.py index 7640ad1..7f09064 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -9,8 +9,15 @@ def test_config(cli): with pytest.raises(runez.system.AbortException): PPG.config.parsed_yaml("a: b\ninvalid line", "testing") + # Exercise custom url in config + cli.run("-ntmacos-arm64", "-c", cli.tests_path("sample-config1.yml"), "build", "3.9.7", "-mbzip2") + assert cli.succeeded + assert "Would download https://my-enterprise/.../assets/bzip2/123" in cli.logged + assert "Would untar build/sources/bzip2-1.2.3.tar.gz -> build/components/bzip2" in cli.logged + cli.run("-ntmacos-arm64", "-c", cli.tests_path("sample-config1.yml"), "build", "3.9.7", "-mnone") assert cli.succeeded + assert " -mpip install --no-cache-dir --upgrade my-additional-package" in cli.logged assert "env MACOSX_DEPLOYMENT_TARGET=12" in cli.logged # Comes from more specific macos-arm64.yml assert " -> dist/cpython-3.9.7-macos-arm64.tar.xz" in cli.logged # Comes from macos.yml (not defined in macos-arm64.yml) From d4db990a8eb2c6017b3058e67b7fdda9ddf559b2 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Mon, 17 Feb 2025 18:21:00 -0800 Subject: [PATCH 11/11] Fine-tuned test (added sha checksum example in url) --- tests/sample-config1.yml | 2 +- tests/test_setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/sample-config1.yml b/tests/sample-config1.yml index cfbb9d1..5199071 100644 --- a/tests/sample-config1.yml +++ b/tests/sample-config1.yml @@ -42,7 +42,7 @@ cpython-configure: - --enable-shared # Exercise custom url -bzip2-url: https://my-enterprise/.../assets/bzip2/123 +bzip2-url: https://my-enterprise/.../assets/bzip2/123#sha256=123...def bzip2-version: 1.2.3 bzip2-src-suffix: tar.gz # Forgot leading dot on purpose bzip2-http-headers: diff --git a/tests/test_setup.py b/tests/test_setup.py index 7f09064..42fcb5c 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -12,8 +12,8 @@ def test_config(cli): # Exercise custom url in config cli.run("-ntmacos-arm64", "-c", cli.tests_path("sample-config1.yml"), "build", "3.9.7", "-mbzip2") assert cli.succeeded - assert "Would download https://my-enterprise/.../assets/bzip2/123" in cli.logged - assert "Would untar build/sources/bzip2-1.2.3.tar.gz -> build/components/bzip2" in cli.logged + assert "Would download https://my-enterprise/.../assets/bzip2/123#sha256=123...def\n" in cli.logged + assert "Would untar build/sources/bzip2-1.2.3.tar.gz -> build/components/bzip2\n" in cli.logged cli.run("-ntmacos-arm64", "-c", cli.tests_path("sample-config1.yml"), "build", "3.9.7", "-mnone") assert cli.succeeded