OpenSearch: HTTP proxy support for plugin installation #11723
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| """ | ||
| Utilities related to Java runtime. | ||
| """ | ||
|
|
||
| import logging | ||
| from os import environ | ||
| from urllib.parse import urlparse | ||
|
|
||
| from localstack import config | ||
| from localstack.utils.files import new_tmp_file, rm_rf | ||
| from localstack.utils.run import run | ||
|
|
||
| LOG = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| # | ||
| # Network | ||
| # | ||
|
|
||
|
|
||
| def java_system_properties_proxy() -> dict[str, str]: | ||
| """ | ||
| Returns Java system properties for network proxy settings as per LocalStack configuration. | ||
|
|
||
| See: https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html | ||
| """ | ||
| props = {} | ||
|
|
||
| for scheme, default_port, var in [ | ||
| ("http", "80", config.OUTBOUND_HTTP_PROXY), | ||
| ("https", "443", config.OUTBOUND_HTTPS_PROXY), | ||
| ]: | ||
| if var: | ||
| netloc = urlparse(var).netloc | ||
| url = netloc.split(":") | ||
| if len(url) == 2: | ||
| hostname, port = url | ||
| else: | ||
| hostname, port = url[0], default_port | ||
|
|
||
| props[f"{scheme}.proxyHost"] = hostname | ||
| props[f"{scheme}.proxyPort"] = port | ||
|
|
||
| return props | ||
|
|
||
|
|
||
| # | ||
| # SSL | ||
| # | ||
|
|
||
|
|
||
| def build_trust_store( | ||
| keytool_path: str, pem_bundle_path: str, env_vars: dict[str, str], store_passwd: str | ||
| ) -> str: | ||
| """ | ||
| Build a TrustStore in JKS format from a PEM certificate bundle. | ||
|
|
||
| :param keytool_path: path to the `keytool` binary. | ||
| :param pem_bundle_path: path to the PEM bundle. | ||
| :param env_vars: environment variables passed during `keytool` execution. This should contain JAVA_HOME and other relevant variables. | ||
| :param store_passwd: store password to use. | ||
| :return: path to the truststore file. | ||
| """ | ||
| store_path = new_tmp_file(suffix=".jks") | ||
|
There was a problem hiding this comment. Q: I don't know how expensive these operations are, and if this is worth it, but could we cache this in There was a problem hiding this comment. Yes, currently it's one trust store used for a given version of Opensearch/Elasticsearch installation, reused for all its plugins. The JRE and thus |
||
| rm_rf(store_path) | ||
|
|
||
| LOG.debug("Building JKS trust store for %s at %s", pem_bundle_path, store_path) | ||
| cmd = f"{keytool_path} -importcert -trustcacerts -alias localstack -file {pem_bundle_path} -keystore {store_path} -storepass {store_passwd} -noprompt" | ||
| run(cmd, env_vars=env_vars) | ||
|
|
||
| return store_path | ||
|
|
||
|
|
||
| def java_system_properties_ssl(keytool_path: str, env_vars: dict[str, str]) -> dict[str, str]: | ||
| """ | ||
| Returns Java system properties for SSL settings as per LocalStack configuration. | ||
|
|
||
| See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores | ||
| """ | ||
| props = {} | ||
|
|
||
| if ca_bundle := environ.get("REQUESTS_CA_BUNDLE"): | ||
| store_passwd = "localstack" | ||
| store_path = build_trust_store(keytool_path, ca_bundle, env_vars, store_passwd) | ||
| props["javax.net.ssl.trustStore"] = store_path | ||
| props["javax.net.ssl.trustStorePassword"] = store_passwd | ||
| props["javax.net.ssl.trustStoreType"] = "jks" | ||
|
|
||
| return props | ||
|
|
||
|
|
||
| # | ||
| # Other | ||
| # | ||
|
|
||
|
|
||
| def system_properties_to_cli_args(properties: dict[str, str]) -> list[str]: | ||
|
There was a problem hiding this comment. Q: It's a pity you can't set system properties via environment variables, otherwise this could be easily integrated generally for our Java packages by adding the properties to There was a problem hiding this comment. We have the package/installer abstractions which handle the installation aspect. Personally I think we should look into introducing some abstractions for the execution/runtime aspect. Because strictly speaking, determining and setting the proxy config falls in that area. We have the Already there are some common patterns that have emerged, e.g. |
||
| """ | ||
| Convert a dict of Java system properties to a list of CLI arguments. | ||
|
|
||
| e.g.:: | ||
|
|
||
| { | ||
| 'java.sys.foo': 'bar', | ||
| 'java.sys.lorem': 'ipsum' | ||
| } | ||
|
|
||
| returns:: | ||
|
|
||
| [ | ||
| '-Djava.sys.foo=bar', | ||
| '-Djava.sys.lorem=ipsum', | ||
| ] | ||
| """ | ||
| args = [] | ||
|
|
||
| for arg_name, arg_value in properties.items(): | ||
| args.append(f"-D{arg_name}={arg_value}") | ||
|
|
||
| return args | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| from unittest.mock import MagicMock | ||
|
|
||
| from localstack import config | ||
| from localstack.utils import java | ||
|
|
||
|
|
||
| def test_java_system_properties_proxy(monkeypatch): | ||
| # Ensure various combinations of env config options are properly converted into expected sys props | ||
|
|
||
| monkeypatch.setattr(config, "OUTBOUND_HTTP_PROXY", "http://lorem.com:69") | ||
| monkeypatch.setattr(config, "OUTBOUND_HTTPS_PROXY", "") | ||
| output = java.java_system_properties_proxy() | ||
| assert len(output) == 2 | ||
| assert output["http.proxyHost"] == "lorem.com" | ||
| assert output["http.proxyPort"] == "69" | ||
|
|
||
| monkeypatch.setattr(config, "OUTBOUND_HTTP_PROXY", "") | ||
| monkeypatch.setattr(config, "OUTBOUND_HTTPS_PROXY", "http://ipsum.com") | ||
| output = java.java_system_properties_proxy() | ||
| assert len(output) == 2 | ||
| assert output["https.proxyHost"] == "ipsum.com" | ||
| assert output["https.proxyPort"] == "443" | ||
|
|
||
| # Ensure no explicit port defaults to 80 | ||
| monkeypatch.setattr(config, "OUTBOUND_HTTP_PROXY", "http://baz.com") | ||
| monkeypatch.setattr(config, "OUTBOUND_HTTPS_PROXY", "http://qux.com:42") | ||
| output = java.java_system_properties_proxy() | ||
| assert len(output) == 4 | ||
| assert output["http.proxyHost"] == "baz.com" | ||
| assert output["http.proxyPort"] == "80" | ||
| assert output["https.proxyHost"] == "qux.com" | ||
| assert output["https.proxyPort"] == "42" | ||
|
|
||
|
|
||
| def test_java_system_properties_ssl(monkeypatch): | ||
| mock = MagicMock() | ||
| mock.return_value = "/baz/qux" | ||
| monkeypatch.setattr(java, "build_trust_store", mock) | ||
|
|
||
| # Ensure that no sys props are returned if CA bundle is not set | ||
| monkeypatch.delenv("REQUESTS_CA_BUNDLE", raising=False) | ||
|
|
||
| output = java.java_system_properties_ssl("/path/keytool", {"enable_this": "true"}) | ||
| assert output == {} | ||
| mock.assert_not_called() | ||
|
|
||
| # Ensure that expected sys props are returned when CA bundle is set | ||
| mock.reset_mock() | ||
| monkeypatch.setenv("REQUESTS_CA_BUNDLE", "/foo/bar") | ||
|
|
||
| output = java.java_system_properties_ssl("/path/to/keytool", {"disable_this": "true"}) | ||
| assert len(output) == 3 | ||
| assert output["javax.net.ssl.trustStore"] == "/baz/qux" | ||
| assert output["javax.net.ssl.trustStorePassword"] == "localstack" | ||
| assert output["javax.net.ssl.trustStoreType"] == "jks" | ||
| mock.assert_called_with("/path/to/keytool", "/foo/bar", {"disable_this": "true"}, "localstack") | ||
|
|
||
|
|
||
| def test_system_properties_to_cli_args(): | ||
| assert java.system_properties_to_cli_args({}) == [] | ||
| assert java.system_properties_to_cli_args({"foo": "bar"}) == ["-Dfoo=bar"] | ||
| assert java.system_properties_to_cli_args({"foo": "bar", "baz": "qux"}) == [ | ||
| "-Dfoo=bar", | ||
| "-Dbaz=qux", | ||
| ] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I would suggest renaming
varto what it really is (proxy_url?).And parsing the urls is always a bit complicated (I guess because the URL spec is complicated?), but
urlparseshould do the hostname and port parsing for you already:If the URL does not have a port,
parsed_proxy_url.portwill beNoneThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks, it's slightly simpler now with 6eeb953