localstack · viren-nadkarni · Oct 9, 2024 · Sep 4, 2024 · Sep 4, 2024 · Sep 4, 2024
@@ -1,5 +1,5 @@
 #
-# base: Stage which installs necessary runtime dependencies (OS packages, java,...)
+# base: Stage which installs necessary runtime dependencies (OS packages, etc.)
 #
 FROM python:3.11.10-slim-bookworm@sha256:5501a4fe605abe24de87c2f3d6cf9fd760354416a0cad0296cf284fddcdca9e2 AS base
 ARG TARGETARCH
@@ -91,7 +91,6 @@ ADD bin/hosts /etc/hosts
 # expose default environment
 # Set edge bind host so localstack can be reached by other containers
 # set library path and default LocalStack hostname
-ENV LD_LIBRARY_PATH=$JAVA_HOME/lib:$JAVA_HOME/lib/server
 ENV USER=localstack
 ENV PYTHONUNBUFFERED=1
 
@@ -157,18 +156,12 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LOCALSTACK_CORE=${LOCALSTACK_BUILD_VERSIO
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/var/lib/localstack/cache \
     source .venv/bin/activate && \
-    python -m localstack.cli.lpm install java --version 11 && \
     python -m localstack.cli.lpm install \
       lambda-runtime \
       dynamodb-local && \
     chown -R localstack:localstack /usr/lib/localstack && \
     chmod -R 777 /usr/lib/localstack
 
-# Set up Java
-ENV JAVA_HOME /usr/lib/localstack/java/11
-RUN ln -s $JAVA_HOME/bin/java /usr/bin/java
-ENV PATH="${PATH}:${JAVA_HOME}/bin"
-
 # link the python package installer virtual environments into the localstack venv
 RUN echo /var/lib/localstack/lib/python-packages/lib/python3.11/site-packages > localstack-var-python-packages-venv.pth && \
     mv localstack-var-python-packages-venv.pth .venv/lib/python*/site-packages/

@@ -25,6 +25,41 @@
 }
 
 
+class JavaInstallerMixin:
+    """
+    Mixin class for packages that depend on Java. It introduces methods that install Java and help build environment.
+    """
+
+    def _prepare_installation(self, target: InstallTarget) -> None:
+        java_package.install(target=target)
+
+    def get_java_home(self) -> str | None:
+        """
+        Returns path to JRE installation.
+        """
+        return java_package.get_installer().get_java_home()
+
+    def get_java_env_vars(self, path: str = None) -> dict[str, str]:
+        """
+        Returns environment variables pointing to the Java installation. This is useful to build the environment where
+        the application will run.
+
+        :param path: If not specified, the value of PATH will be obtained from the environment
+        :return: dict consisting of two items:
+            - JAVA_HOME: path to JRE installation
+            - PATH: the env path variable updated with JRE bin path
+        """
+        java_home = self.get_java_home()
+        java_bin = f"{java_home}/bin"
+
+        path = path or os.environ["PATH"]
+
+        return {
+            "JAVA_HOME": java_home,
+            "PATH": f"{java_bin}:{path}",
+        }
+
+
 class JavaPackageInstaller(ArchiveDownloadAndExtractInstaller):
     def __init__(self, version: str):
         super().__init__("java", version, extract_single_directory=True)
@@ -81,7 +116,7 @@ def _post_process(self, target: InstallTarget) -> None:
         rm_rf(target_directory)
         os.rename(minimal_jre_path, target_directory)
 
-    def get_java_home(self) -> str:
+    def get_java_home(self) -> str | None:
         """
         Get JAVA_HOME for this installation of Java.
         """

@@ -4,6 +4,7 @@
 from localstack import config
 from localstack.constants import ARTIFACTS_REPO, MAVEN_REPO_URL
 from localstack.packages import InstallTarget, Package, PackageInstaller
+from localstack.packages.java import JavaInstallerMixin
 from localstack.utils.archives import (
     download_and_extract_with_retry,
     update_jar_manifest,
@@ -37,7 +38,7 @@ def get_versions(self) -> List[str]:
         return ["latest"]
 
 
-class DynamoDBLocalPackageInstaller(PackageInstaller):
+class DynamoDBLocalPackageInstaller(JavaInstallerMixin, PackageInstaller):
     def __init__(self):
         super().__init__("dynamodb-local", "latest")
 

@@ -150,17 +150,23 @@ def _create_shell_command(self) -> list[str]:
         return cmd + parameters
 
     def do_start_thread(self) -> FuncThread:
-        dynamodblocal_package.install()
+        dynamodblocal_installer = dynamodblocal_package.get_installer()
+        dynamodblocal_installer.install()
 
         cmd = self._create_shell_command()
+        env_vars = {
+            "DDB_LOCAL_TELEMETRY": "0",
+            **dynamodblocal_installer.get_java_env_vars(),
+        }
+
         LOG.debug("Starting DynamoDB Local: %s", cmd)
         t = ShellCommandThread(
             cmd,
             strip_color=True,
             log_listener=_log_listener,
             auto_restart=True,
             name="dynamodb-local",
-            env_vars={"DDB_LOCAL_TELEMETRY": "0"},
+            env_vars=env_vars,
         )
         TMP_THREADS.append(t)
         t.start()

@@ -2,6 +2,7 @@
 import os
 from functools import cache
 from pathlib import Path
+from typing import Tuple
 
 from localstack.services.events.models import InvalidEventPatternException
 from localstack.services.events.packages import event_ruler_package
@@ -25,16 +26,27 @@ def start_jvm() -> None:
     jpype_config.destroy_jvm = False
 
     if not jpype.isJVMStarted():
-        event_ruler_libs_path = get_event_ruler_libs_path()
+        jvm_lib, event_ruler_libs_path = get_jpype_lib_paths()
         event_ruler_libs_pattern = event_ruler_libs_path.joinpath("*")
-        jpype.startJVM(classpath=[event_ruler_libs_pattern])
+
+        jpype.startJVM(str(jvm_lib), classpath=[event_ruler_libs_pattern])
 
 
 @cache
-def get_event_ruler_libs_path() -> Path:
+def get_jpype_lib_paths() -> Tuple[Path, Path]:
+    """
+    Downloads Event Ruler, its dependencies and returns a tuple of:
+    - Path to libjvm.so to be used by JPype as jvmpath. JPype requires this to start the JVM.
+        See https://jpype.readthedocs.io/en/latest/userguide.html#path-to-the-jvm
+    - Path to Event Ruler libraries to be used by JPype as classpath
+    """
     installer = event_ruler_package.get_installer()
     installer.install()
-    return Path(installer.get_installed_dir())
+
+    java_home = installer.get_java_home()
+    jvm_lib = Path(java_home) / "lib" / "server" / "libjvm.so"
+
+    return jvm_lib, Path(installer.get_installed_dir())
 
 
 def matches_rule(event: str, rule: str) -> bool:

@@ -1,25 +1,37 @@
 from localstack.packages import Package, PackageInstaller
 from localstack.packages.core import MavenPackageInstaller
+from localstack.packages.java import JavaInstallerMixin
 
+# Map of Event Ruler version to Jackson version
 # https://central.sonatype.com/artifact/software.amazon.event.ruler/event-ruler
-EVENT_RULER_VERSION = "1.7.3"
 # The dependent jackson.version is defined in the Maven POM File of event-ruler
-JACKSON_VERSION = "2.16.2"
+EVENT_RULER_VERSIONS = {
+    "1.7.3": "2.16.2",
+}
+
+EVENT_RULER_DEFAULT_VERSION = "1.7.3"
 
 
 class EventRulerPackage(Package):
     def __init__(self):
-        super().__init__("EventRulerLibs", EVENT_RULER_VERSION)
+        super().__init__("EventRulerLibs", EVENT_RULER_DEFAULT_VERSION)
 
     def get_versions(self) -> list[str]:
-        return [EVENT_RULER_VERSION]
+        return list(EVENT_RULER_VERSIONS.keys())
 
     def _get_installer(self, version: str) -> PackageInstaller:
-        return MavenPackageInstaller(
-            f"pkg:maven/software.amazon.event.ruler/event-ruler@{EVENT_RULER_VERSION}",
-            f"pkg:maven/com.fasterxml.jackson.core/jackson-annotations@{JACKSON_VERSION}",
-            f"pkg:maven/com.fasterxml.jackson.core/jackson-core@{JACKSON_VERSION}",
-            f"pkg:maven/com.fasterxml.jackson.core/jackson-databind@{JACKSON_VERSION}",
+        return EventRulerPackageInstaller(version)
+
+
+class EventRulerPackageInstaller(JavaInstallerMixin, MavenPackageInstaller):
+    def __init__(self, version: str):
+        jackson_version = EVENT_RULER_VERSIONS[version]
+
+        super().__init__(
+            f"pkg:maven/software.amazon.event.ruler/event-ruler@{version}",
+            f"pkg:maven/com.fasterxml.jackson.core/jackson-annotations@{jackson_version}",
+            f"pkg:maven/com.fasterxml.jackson.core/jackson-core@{jackson_version}",
+            f"pkg:maven/com.fasterxml.jackson.core/jackson-databind@{jackson_version}",
         )
 
 

@@ -466,6 +466,7 @@ def _create_run_command(
 
     def _create_env_vars(self, directories: Directories) -> Dict:
         env_vars = {
+            "JAVA_HOME": os.path.join(directories.install, "jdk"),
             "OPENSEARCH_JAVA_OPTS": os.environ.get("OPENSEARCH_JAVA_OPTS", "-Xms200m -Xmx600m"),
             "OPENSEARCH_TMPDIR": directories.tmp,
         }
@@ -689,6 +690,7 @@ def _base_settings(self, dirs) -> CommandSettings:
 
     def _create_env_vars(self, directories: Directories) -> Dict:
         return {
+            "JAVA_HOME": os.path.join(directories.install, "jdk"),
             "ES_JAVA_OPTS": os.environ.get("ES_JAVA_OPTS", "-Xms200m -Xmx600m"),
             "ES_TMPDIR": directories.tmp,
         }

@@ -42,7 +42,10 @@ def do_start_thread(self) -> FuncThread:
         return t
 
     def generate_env_vars(self) -> Dict[str, Any]:
+        sfn_local_installer = stepfunctions_local_package.get_installer()
+
         return {
+            **sfn_local_installer.get_java_env_vars(),
             "EDGE_PORT": config.GATEWAY_LISTEN[0].port,
             "EDGE_PORT_HTTP": config.GATEWAY_LISTEN[0].port,
             "DATA_DIR": config.dirs.data,

@@ -9,6 +9,7 @@
 from localstack.constants import ARTIFACTS_REPO, MAVEN_REPO_URL
 from localstack.packages import InstallTarget, Package, PackageInstaller
 from localstack.packages.core import ExecutableInstaller
+from localstack.packages.java import JavaInstallerMixin
 from localstack.utils.archives import add_file_to_jar, untar, update_jar_manifest
 from localstack.utils.files import file_exists_not_empty, mkdir, new_tmp_file, rm_rf
 from localstack.utils.http import download
@@ -73,7 +74,7 @@ def _get_installer(self, version: str) -> PackageInstaller:
         return StepFunctionsLocalPackageInstaller("stepfunctions-local", version)
 
 
-class StepFunctionsLocalPackageInstaller(ExecutableInstaller):
+class StepFunctionsLocalPackageInstaller(JavaInstallerMixin, ExecutableInstaller):
     def _get_install_marker_path(self, install_dir: str) -> str:
         return os.path.join(install_dir, "StepFunctionsLocal.jar")
 

diff --git a/localstack-core/localstack/utils/kinesis/kinesis_connector.py b/localstack-core/localstack/utils/kinesis/kinesis_connector.py
@@ -13,6 +13,7 @@
 
 from localstack import config
 from localstack.constants import LOCALSTACK_ROOT_FOLDER, LOCALSTACK_VENV_FOLDER
+from localstack.packages.java import java_package
 from localstack.utils.aws import arns
 from localstack.utils.files import TMP_FILES, chmod_r, save_file
 from localstack.utils.kinesis import kclipy_helper
@@ -240,12 +241,20 @@ def _start_kcl_client_process(
 ):
     # make sure to convert stream ARN to stream name
     stream_name = arns.kinesis_stream_name(stream_name)
+
+    # install Java
+    java_installer = java_package.get_installer()
+    java_installer.install()
+    java_home = java_installer.get_java_home()
+
     # disable CBOR protocol, enforce use of plain JSON
     # TODO evaluate why?
     env_vars = {
         "AWS_CBOR_DISABLE": "true",
         "AWS_ACCESS_KEY_ID": account_id,
         "AWS_SECRET_ACCESS_KEY": account_id,
+        "JAVA_HOME": java_home,
+        "PATH": f"{java_home}/bin:{os.getenv('PATH')}",
     }
 
     events_file = os.path.join(tempfile.gettempdir(), f"kclipy.{short_uid()}.fifo")