diff --git a/Dockerfile b/Dockerfile index 6ba5c727b2524..ce97491fda763 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # -# base: Stage which installs necessary runtime dependencies (OS packages, java,...) +# base: Stage which installs necessary runtime dependencies (OS packages, etc.) # FROM python:3.11.10-slim-bookworm@sha256:5501a4fe605abe24de87c2f3d6cf9fd760354416a0cad0296cf284fddcdca9e2 AS base ARG TARGETARCH @@ -91,7 +91,6 @@ ADD bin/hosts /etc/hosts # expose default environment # Set edge bind host so localstack can be reached by other containers # set library path and default LocalStack hostname -ENV LD_LIBRARY_PATH=$JAVA_HOME/lib:$JAVA_HOME/lib/server ENV USER=localstack ENV PYTHONUNBUFFERED=1 @@ -157,18 +156,12 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LOCALSTACK_CORE=${LOCALSTACK_BUILD_VERSIO RUN --mount=type=cache,target=/root/.cache \ --mount=type=cache,target=/var/lib/localstack/cache \ source .venv/bin/activate && \ - python -m localstack.cli.lpm install java --version 11 && \ python -m localstack.cli.lpm install \ lambda-runtime \ dynamodb-local && \ chown -R localstack:localstack /usr/lib/localstack && \ chmod -R 777 /usr/lib/localstack -# Set up Java -ENV JAVA_HOME /usr/lib/localstack/java/11 -RUN ln -s $JAVA_HOME/bin/java /usr/bin/java -ENV PATH="${PATH}:${JAVA_HOME}/bin" - # link the python package installer virtual environments into the localstack venv RUN echo /var/lib/localstack/lib/python-packages/lib/python3.11/site-packages > localstack-var-python-packages-venv.pth && \ mv localstack-var-python-packages-venv.pth .venv/lib/python*/site-packages/ diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 2bd280d8fbee4..258c49bad4ca8 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -25,6 +25,41 @@ } +class JavaInstallerMixin: + """ + Mixin class for packages that depend on Java. It introduces methods that install Java and help build environment. + """ + + def _prepare_installation(self, target: InstallTarget) -> None: + java_package.install(target=target) + + def get_java_home(self) -> str | None: + """ + Returns path to JRE installation. + """ + return java_package.get_installer().get_java_home() + + def get_java_env_vars(self, path: str = None) -> dict[str, str]: + """ + Returns environment variables pointing to the Java installation. This is useful to build the environment where + the application will run. + + :param path: If not specified, the value of PATH will be obtained from the environment + :return: dict consisting of two items: + - JAVA_HOME: path to JRE installation + - PATH: the env path variable updated with JRE bin path + """ + java_home = self.get_java_home() + java_bin = f"{java_home}/bin" + + path = path or os.environ["PATH"] + + return { + "JAVA_HOME": java_home, + "PATH": f"{java_bin}:{path}", + } + + class JavaPackageInstaller(ArchiveDownloadAndExtractInstaller): def __init__(self, version: str): super().__init__("java", version, extract_single_directory=True) @@ -81,7 +116,7 @@ def _post_process(self, target: InstallTarget) -> None: rm_rf(target_directory) os.rename(minimal_jre_path, target_directory) - def get_java_home(self) -> str: + def get_java_home(self) -> str | None: """ Get JAVA_HOME for this installation of Java. """ diff --git a/localstack-core/localstack/services/dynamodb/packages.py b/localstack-core/localstack/services/dynamodb/packages.py index 52e76e761ae41..2c23079966b35 100644 --- a/localstack-core/localstack/services/dynamodb/packages.py +++ b/localstack-core/localstack/services/dynamodb/packages.py @@ -4,6 +4,7 @@ from localstack import config from localstack.constants import ARTIFACTS_REPO, MAVEN_REPO_URL from localstack.packages import InstallTarget, Package, PackageInstaller +from localstack.packages.java import JavaInstallerMixin from localstack.utils.archives import ( download_and_extract_with_retry, update_jar_manifest, @@ -37,7 +38,7 @@ def get_versions(self) -> List[str]: return ["latest"] -class DynamoDBLocalPackageInstaller(PackageInstaller): +class DynamoDBLocalPackageInstaller(JavaInstallerMixin, PackageInstaller): def __init__(self): super().__init__("dynamodb-local", "latest") diff --git a/localstack-core/localstack/services/dynamodb/server.py b/localstack-core/localstack/services/dynamodb/server.py index d250b47c195b6..d85d87e1c0ce4 100644 --- a/localstack-core/localstack/services/dynamodb/server.py +++ b/localstack-core/localstack/services/dynamodb/server.py @@ -150,9 +150,15 @@ def _create_shell_command(self) -> list[str]: return cmd + parameters def do_start_thread(self) -> FuncThread: - dynamodblocal_package.install() + dynamodblocal_installer = dynamodblocal_package.get_installer() + dynamodblocal_installer.install() cmd = self._create_shell_command() + env_vars = { + "DDB_LOCAL_TELEMETRY": "0", + **dynamodblocal_installer.get_java_env_vars(), + } + LOG.debug("Starting DynamoDB Local: %s", cmd) t = ShellCommandThread( cmd, @@ -160,7 +166,7 @@ def do_start_thread(self) -> FuncThread: log_listener=_log_listener, auto_restart=True, name="dynamodb-local", - env_vars={"DDB_LOCAL_TELEMETRY": "0"}, + env_vars=env_vars, ) TMP_THREADS.append(t) t.start() diff --git a/localstack-core/localstack/services/events/event_ruler.py b/localstack-core/localstack/services/events/event_ruler.py index e48712687bbce..4a1c164e14bac 100644 --- a/localstack-core/localstack/services/events/event_ruler.py +++ b/localstack-core/localstack/services/events/event_ruler.py @@ -2,6 +2,7 @@ import os from functools import cache from pathlib import Path +from typing import Tuple from localstack.services.events.models import InvalidEventPatternException from localstack.services.events.packages import event_ruler_package @@ -25,16 +26,27 @@ def start_jvm() -> None: jpype_config.destroy_jvm = False if not jpype.isJVMStarted(): - event_ruler_libs_path = get_event_ruler_libs_path() + jvm_lib, event_ruler_libs_path = get_jpype_lib_paths() event_ruler_libs_pattern = event_ruler_libs_path.joinpath("*") - jpype.startJVM(classpath=[event_ruler_libs_pattern]) + + jpype.startJVM(str(jvm_lib), classpath=[event_ruler_libs_pattern]) @cache -def get_event_ruler_libs_path() -> Path: +def get_jpype_lib_paths() -> Tuple[Path, Path]: + """ + Downloads Event Ruler, its dependencies and returns a tuple of: + - Path to libjvm.so to be used by JPype as jvmpath. JPype requires this to start the JVM. + See https://jpype.readthedocs.io/en/latest/userguide.html#path-to-the-jvm + - Path to Event Ruler libraries to be used by JPype as classpath + """ installer = event_ruler_package.get_installer() installer.install() - return Path(installer.get_installed_dir()) + + java_home = installer.get_java_home() + jvm_lib = Path(java_home) / "lib" / "server" / "libjvm.so" + + return jvm_lib, Path(installer.get_installed_dir()) def matches_rule(event: str, rule: str) -> bool: diff --git a/localstack-core/localstack/services/events/packages.py b/localstack-core/localstack/services/events/packages.py index 5686b6844d454..7e5d8237ecb5d 100644 --- a/localstack-core/localstack/services/events/packages.py +++ b/localstack-core/localstack/services/events/packages.py @@ -1,25 +1,37 @@ from localstack.packages import Package, PackageInstaller from localstack.packages.core import MavenPackageInstaller +from localstack.packages.java import JavaInstallerMixin +# Map of Event Ruler version to Jackson version # https://central.sonatype.com/artifact/software.amazon.event.ruler/event-ruler -EVENT_RULER_VERSION = "1.7.3" # The dependent jackson.version is defined in the Maven POM File of event-ruler -JACKSON_VERSION = "2.16.2" +EVENT_RULER_VERSIONS = { + "1.7.3": "2.16.2", +} + +EVENT_RULER_DEFAULT_VERSION = "1.7.3" class EventRulerPackage(Package): def __init__(self): - super().__init__("EventRulerLibs", EVENT_RULER_VERSION) + super().__init__("EventRulerLibs", EVENT_RULER_DEFAULT_VERSION) def get_versions(self) -> list[str]: - return [EVENT_RULER_VERSION] + return list(EVENT_RULER_VERSIONS.keys()) def _get_installer(self, version: str) -> PackageInstaller: - return MavenPackageInstaller( - f"pkg:maven/software.amazon.event.ruler/event-ruler@{EVENT_RULER_VERSION}", - f"pkg:maven/com.fasterxml.jackson.core/jackson-annotations@{JACKSON_VERSION}", - f"pkg:maven/com.fasterxml.jackson.core/jackson-core@{JACKSON_VERSION}", - f"pkg:maven/com.fasterxml.jackson.core/jackson-databind@{JACKSON_VERSION}", + return EventRulerPackageInstaller(version) + + +class EventRulerPackageInstaller(JavaInstallerMixin, MavenPackageInstaller): + def __init__(self, version: str): + jackson_version = EVENT_RULER_VERSIONS[version] + + super().__init__( + f"pkg:maven/software.amazon.event.ruler/event-ruler@{version}", + f"pkg:maven/com.fasterxml.jackson.core/jackson-annotations@{jackson_version}", + f"pkg:maven/com.fasterxml.jackson.core/jackson-core@{jackson_version}", + f"pkg:maven/com.fasterxml.jackson.core/jackson-databind@{jackson_version}", ) diff --git a/localstack-core/localstack/services/opensearch/cluster.py b/localstack-core/localstack/services/opensearch/cluster.py index c50e676c887b2..dcaa966f279a2 100644 --- a/localstack-core/localstack/services/opensearch/cluster.py +++ b/localstack-core/localstack/services/opensearch/cluster.py @@ -466,6 +466,7 @@ def _create_run_command( def _create_env_vars(self, directories: Directories) -> Dict: env_vars = { + "JAVA_HOME": os.path.join(directories.install, "jdk"), "OPENSEARCH_JAVA_OPTS": os.environ.get("OPENSEARCH_JAVA_OPTS", "-Xms200m -Xmx600m"), "OPENSEARCH_TMPDIR": directories.tmp, } @@ -689,6 +690,7 @@ def _base_settings(self, dirs) -> CommandSettings: def _create_env_vars(self, directories: Directories) -> Dict: return { + "JAVA_HOME": os.path.join(directories.install, "jdk"), "ES_JAVA_OPTS": os.environ.get("ES_JAVA_OPTS", "-Xms200m -Xmx600m"), "ES_TMPDIR": directories.tmp, } diff --git a/localstack-core/localstack/services/stepfunctions/legacy/stepfunctions_starter.py b/localstack-core/localstack/services/stepfunctions/legacy/stepfunctions_starter.py index 9a456bf50905d..7cbe903b1ea38 100644 --- a/localstack-core/localstack/services/stepfunctions/legacy/stepfunctions_starter.py +++ b/localstack-core/localstack/services/stepfunctions/legacy/stepfunctions_starter.py @@ -42,7 +42,10 @@ def do_start_thread(self) -> FuncThread: return t def generate_env_vars(self) -> Dict[str, Any]: + sfn_local_installer = stepfunctions_local_package.get_installer() + return { + **sfn_local_installer.get_java_env_vars(), "EDGE_PORT": config.GATEWAY_LISTEN[0].port, "EDGE_PORT_HTTP": config.GATEWAY_LISTEN[0].port, "DATA_DIR": config.dirs.data, diff --git a/localstack-core/localstack/services/stepfunctions/packages.py b/localstack-core/localstack/services/stepfunctions/packages.py index 81f0699b90696..8bb2a6e8a1dbc 100644 --- a/localstack-core/localstack/services/stepfunctions/packages.py +++ b/localstack-core/localstack/services/stepfunctions/packages.py @@ -9,6 +9,7 @@ from localstack.constants import ARTIFACTS_REPO, MAVEN_REPO_URL from localstack.packages import InstallTarget, Package, PackageInstaller from localstack.packages.core import ExecutableInstaller +from localstack.packages.java import JavaInstallerMixin from localstack.utils.archives import add_file_to_jar, untar, update_jar_manifest from localstack.utils.files import file_exists_not_empty, mkdir, new_tmp_file, rm_rf from localstack.utils.http import download @@ -73,7 +74,7 @@ def _get_installer(self, version: str) -> PackageInstaller: return StepFunctionsLocalPackageInstaller("stepfunctions-local", version) -class StepFunctionsLocalPackageInstaller(ExecutableInstaller): +class StepFunctionsLocalPackageInstaller(JavaInstallerMixin, ExecutableInstaller): def _get_install_marker_path(self, install_dir: str) -> str: return os.path.join(install_dir, "StepFunctionsLocal.jar") diff --git a/localstack-core/localstack/utils/kinesis/kinesis_connector.py b/localstack-core/localstack/utils/kinesis/kinesis_connector.py index 334a1f0573475..f68e79a379638 100644 --- a/localstack-core/localstack/utils/kinesis/kinesis_connector.py +++ b/localstack-core/localstack/utils/kinesis/kinesis_connector.py @@ -13,6 +13,7 @@ from localstack import config from localstack.constants import LOCALSTACK_ROOT_FOLDER, LOCALSTACK_VENV_FOLDER +from localstack.packages.java import java_package from localstack.utils.aws import arns from localstack.utils.files import TMP_FILES, chmod_r, save_file from localstack.utils.kinesis import kclipy_helper @@ -240,12 +241,20 @@ def _start_kcl_client_process( ): # make sure to convert stream ARN to stream name stream_name = arns.kinesis_stream_name(stream_name) + + # install Java + java_installer = java_package.get_installer() + java_installer.install() + java_home = java_installer.get_java_home() + # disable CBOR protocol, enforce use of plain JSON # TODO evaluate why? env_vars = { "AWS_CBOR_DISABLE": "true", "AWS_ACCESS_KEY_ID": account_id, "AWS_SECRET_ACCESS_KEY": account_id, + "JAVA_HOME": java_home, + "PATH": f"{java_home}/bin:{os.getenv('PATH')}", } events_file = os.path.join(tempfile.gettempdir(), f"kclipy.{short_uid()}.fifo")