From 439aedbf843390c51f9f185f5d0cf8b5afe139c0 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Thu, 4 Jul 2024 19:50:34 +0530 Subject: [PATCH 01/19] Use Java 21 --- Dockerfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index fb69b5614d764..5cdaa9a3dc998 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # java-builder: Stage to build a custom JRE (with jlink) -FROM eclipse-temurin:11@sha256:b26d53b2b30673492992d56fa720d7d8b2720f85c3f4702206ac8d1104284cea as java-builder +FROM eclipse-temurin:21@sha256:3f9bfce63186b9ded168250c8e350631fd643ad00afab5986cf8a7cf79f3b043 as java-builder # create a custom, minimized JRE via jlink RUN jlink --add-modules \ @@ -19,13 +19,13 @@ jdk.management.agent,\ java.security.jgss,jdk.security.auth,\ # Elasticsearch 7+ crashes without Thai Segmentation support jdk.localedata --include-locales en,th \ - --compress 2 --strip-debug --no-header-files --no-man-pages --output /usr/lib/jvm/java-11 && \ - cp ${JAVA_HOME}/bin/javac /usr/lib/jvm/java-11/bin/javac && \ - cp -r ${JAVA_HOME}/include /usr/lib/jvm/java-11/include && \ - mv /usr/lib/jvm/java-11/lib/modules /usr/lib/jvm/java-11/lib/modules.bk; \ - cp -r ${JAVA_HOME}/lib/* /usr/lib/jvm/java-11/lib/; \ - mv /usr/lib/jvm/java-11/lib/modules.bk /usr/lib/jvm/java-11/lib/modules; \ - rm -rf /usr/bin/java ${JAVA_HOME} && ln -s /usr/lib/jvm/java-11/bin/java /usr/bin/java + --compress 2 --strip-debug --no-header-files --no-man-pages --output /usr/lib/jvm/java-21 && \ + cp ${JAVA_HOME}/bin/javac /usr/lib/jvm/java-21/bin/javac && \ + cp -r ${JAVA_HOME}/include /usr/lib/jvm/java-21/include && \ + mv /usr/lib/jvm/java-21/lib/modules /usr/lib/jvm/java-21/lib/modules.bk; \ + cp -r ${JAVA_HOME}/lib/* /usr/lib/jvm/java-21/lib/; \ + mv /usr/lib/jvm/java-21/lib/modules.bk /usr/lib/jvm/java-21/lib/modules; \ + rm -rf /usr/bin/java ${JAVA_HOME} && ln -s /usr/lib/jvm/java-21/bin/java /usr/bin/java # base: Stage which installs necessary runtime dependencies (OS packages, java,...) @@ -85,15 +85,15 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ SHELL [ "/bin/bash", "-c" ] -# Install Java 11 +# Install Java 21 ENV LANG C.UTF-8 RUN { \ echo '#!/bin/sh'; echo 'set -e'; echo; \ echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \ } > /usr/local/bin/docker-java-home \ && chmod +x /usr/local/bin/docker-java-home -ENV JAVA_HOME /usr/lib/jvm/java-11 -COPY --from=java-builder /usr/lib/jvm/java-11 $JAVA_HOME +ENV JAVA_HOME /usr/lib/jvm/java-21 +COPY --from=java-builder /usr/lib/jvm/java-21 $JAVA_HOME RUN ln -s $JAVA_HOME/bin/java /usr/bin/java ENV PATH "${PATH}:${JAVA_HOME}/bin" From c3e7b6ebcb0cb6e713ae11a9d75d617cd2588890 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Wed, 31 Jul 2024 12:03:42 +0530 Subject: [PATCH 02/19] Add Java package --- localstack-core/localstack/packages/java.py | 111 ++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 localstack-core/localstack/packages/java.py diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py new file mode 100644 index 0000000000000..42625ae0ef733 --- /dev/null +++ b/localstack-core/localstack/packages/java.py @@ -0,0 +1,111 @@ +import glob +import logging +import os +from typing import List + +import distro + +from localstack.packages import InstallTarget, Package +from localstack.pro.core.packages import OSPackageInstaller # FIXME@viren +from localstack.utils.files import save_file +from localstack.utils.http import download +from localstack.utils.run import run + +LOG = logging.getLogger(__name__) + +# Version that is preinstalled during Docker build +PREINSTALLED_JAVA_VERSION = "21" + +# Versions supported by this package +JAVA_VERSIONS = ["8", "11", "21"] + +# Java home dirs inside main container +# Note: configure $JAVA_8_HOME / $JAVA_11_HOME for testing in host mode +JAVA_8_HOME = os.environ.get("JAVA_8_HOME") or "/usr/lib/jvm/java-8" +JAVA_11_HOME = os.environ.get("JAVA_11_HOME") or "/usr/lib/jvm/java-11" +JAVA_21_HOME = os.environ.get("JAVA_21_HOME") or "/usr/lib/jvm/java-21" + +ADOPTIUM_DNS_SKIP = "jfrog-prod-.*.s3.amazonaws.com" + + +class JavaPackageInstaller(OSPackageInstaller): + """ + Installer to install custom Java OS packages, required, e.g., for older versions of Spark. + Note: Currently only supported for Debian (not for Redhat) + """ + + def __init__(self, version: str): + super().__init__("java", version) + + def is_installed(self) -> bool: + java_home = self.get_java_home() + return java_home and os.path.exists(os.path.join(java_home, "bin", "java")) + + def _debian_get_install_dir(self, target: InstallTarget): + return self._get_jvm_install_dir() + + def _debian_get_install_marker_path(self, install_dir: str) -> str: + return os.path.join(install_dir, "bin", "java") + + def _debian_packages(self) -> List[str]: + if self.version == PREINSTALLED_JAVA_VERSION: + return [] + return [f"temurin-{self.version}-jdk"] + + def _debian_prepare_install(self, target: InstallTarget): + sources_file = "/etc/apt/sources.list.d/adoptium.list" + key_file = "/etc/apt/trusted.gpg.d/adoptium.asc" + openjdk_repo = "https://packages.adoptium.net/artifactory" + if self.version != PREINSTALLED_JAVA_VERSION and not os.path.exists(sources_file): + # update package index + download(f"{openjdk_repo}/api/gpg/key/public", key_file) + save_file( + sources_file, + f"deb https://packages.adoptium.net/artifactory/deb {distro.codename()} main", + ) + # the new adoptium repository uses s3, so we need to exclude the buckets from transparent endpoint injection + try: + from localstack.dns import server as dns_server + + dns_server.exclude_from_resolution(ADOPTIUM_DNS_SKIP) + except ImportError: + LOG.debug("Cannot import DNS server - skipping modification to allow apt download") + super()._debian_prepare_install(target) + + def _post_process(self, target: InstallTarget) -> None: + try: + from localstack.dns import server as dns_server + + dns_server.revert_exclude_from_resolution(ADOPTIUM_DNS_SKIP) + except ImportError: + LOG.debug("Cannot import DNS server - skipping revert of skip") + target_dir = self._get_jvm_install_dir() + install_dir = glob.glob(f"/usr/lib/jvm/*-{self.version}-jdk-*")[0] + if not os.path.exists(target_dir): + run(["ln", "-s", install_dir, target_dir]) + + def _get_jvm_install_dir(self) -> str: + return self.get_java_home() + + def get_java_home(self) -> str: + if self.version == "8": + return JAVA_8_HOME + if self.version == "11": + return JAVA_11_HOME + if self.version == "21": + return JAVA_21_HOME + return f"/usr/lib/jvm/java-{self.version}" + + +class JavaPackage(Package): + def __init__(self, default_version: str = PREINSTALLED_JAVA_VERSION): + super().__init__(name="Java", default_version=default_version) + + def get_versions(self) -> List[str]: + return JAVA_VERSIONS + + def _get_installer(self, version): + return JavaPackageInstaller(version) + + +java_package = JavaPackage() From 8fbe1694282e270fccc0697a3ae98f4cac853281 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Wed, 31 Jul 2024 13:40:25 +0530 Subject: [PATCH 03/19] Fix comments --- localstack-core/localstack/packages/java.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 42625ae0ef733..5e58da7b61670 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -13,10 +13,10 @@ LOG = logging.getLogger(__name__) -# Version that is preinstalled during Docker build -PREINSTALLED_JAVA_VERSION = "21" +# Default version +DEFAULT_JAVA_VERSION = "11" -# Versions supported by this package +# Supported versions JAVA_VERSIONS = ["8", "11", "21"] # Java home dirs inside main container @@ -48,7 +48,7 @@ def _debian_get_install_marker_path(self, install_dir: str) -> str: return os.path.join(install_dir, "bin", "java") def _debian_packages(self) -> List[str]: - if self.version == PREINSTALLED_JAVA_VERSION: + if self.version == DEFAULT_JAVA_VERSION: return [] return [f"temurin-{self.version}-jdk"] @@ -56,7 +56,7 @@ def _debian_prepare_install(self, target: InstallTarget): sources_file = "/etc/apt/sources.list.d/adoptium.list" key_file = "/etc/apt/trusted.gpg.d/adoptium.asc" openjdk_repo = "https://packages.adoptium.net/artifactory" - if self.version != PREINSTALLED_JAVA_VERSION and not os.path.exists(sources_file): + if self.version != DEFAULT_JAVA_VERSION and not os.path.exists(sources_file): # update package index download(f"{openjdk_repo}/api/gpg/key/public", key_file) save_file( @@ -98,7 +98,7 @@ def get_java_home(self) -> str: class JavaPackage(Package): - def __init__(self, default_version: str = PREINSTALLED_JAVA_VERSION): + def __init__(self, default_version: str = DEFAULT_JAVA_VERSION): super().__init__(name="Java", default_version=default_version) def get_versions(self) -> List[str]: From 52d9c72351bfd2d7243b7c6cb35e99938b7cc2d9 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Wed, 31 Jul 2024 15:17:22 +0530 Subject: [PATCH 04/19] Refactor Java installer to remove OSPackageInstaller --- localstack-core/localstack/packages/java.py | 112 ++++++------------ .../localstack/packages/plugins.py | 7 ++ 2 files changed, 42 insertions(+), 77 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 5e58da7b61670..bdcbbd581e20d 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -1,100 +1,58 @@ -import glob import logging import os from typing import List -import distro - from localstack.packages import InstallTarget, Package -from localstack.pro.core.packages import OSPackageInstaller # FIXME@viren -from localstack.utils.files import save_file -from localstack.utils.http import download -from localstack.utils.run import run +from localstack.packages.core import ArchiveDownloadAndExtractInstaller +from localstack.utils.platform import Arch, get_arch, is_linux, is_mac_os LOG = logging.getLogger(__name__) -# Default version +# Default version if not specified DEFAULT_JAVA_VERSION = "11" -# Supported versions -JAVA_VERSIONS = ["8", "11", "21"] - -# Java home dirs inside main container -# Note: configure $JAVA_8_HOME / $JAVA_11_HOME for testing in host mode -JAVA_8_HOME = os.environ.get("JAVA_8_HOME") or "/usr/lib/jvm/java-8" -JAVA_11_HOME = os.environ.get("JAVA_11_HOME") or "/usr/lib/jvm/java-11" -JAVA_21_HOME = os.environ.get("JAVA_21_HOME") or "/usr/lib/jvm/java-21" +# Supported Java LTS versions mapped with Eclipse Temurin build semvers +JAVA_VERSIONS = { + "8": "8.0.422+5", + "11": "11.0.24+8", + "17": "17.0.12+7", + "21": "21.0.4+7", +} -ADOPTIUM_DNS_SKIP = "jfrog-prod-.*.s3.amazonaws.com" +JRE_DISTRIB_URL = "https://github.com/adoptium/temurin{version}-binaries/releases/download/jdk-{semver}/OpenJDK{version}U-jre_{arch}_{os}_hotspot_{semver_safe}.tar.gz" -class JavaPackageInstaller(OSPackageInstaller): - """ - Installer to install custom Java OS packages, required, e.g., for older versions of Spark. - Note: Currently only supported for Debian (not for Redhat) - """ - +class JavaPackageInstaller(ArchiveDownloadAndExtractInstaller): def __init__(self, version: str): super().__init__("java", version) - def is_installed(self) -> bool: - java_home = self.get_java_home() - return java_home and os.path.exists(os.path.join(java_home, "bin", "java")) - - def _debian_get_install_dir(self, target: InstallTarget): - return self._get_jvm_install_dir() - - def _debian_get_install_marker_path(self, install_dir: str) -> str: - return os.path.join(install_dir, "bin", "java") - - def _debian_packages(self) -> List[str]: - if self.version == DEFAULT_JAVA_VERSION: - return [] - return [f"temurin-{self.version}-jdk"] - - def _debian_prepare_install(self, target: InstallTarget): - sources_file = "/etc/apt/sources.list.d/adoptium.list" - key_file = "/etc/apt/trusted.gpg.d/adoptium.asc" - openjdk_repo = "https://packages.adoptium.net/artifactory" - if self.version != DEFAULT_JAVA_VERSION and not os.path.exists(sources_file): - # update package index - download(f"{openjdk_repo}/api/gpg/key/public", key_file) - save_file( - sources_file, - f"deb https://packages.adoptium.net/artifactory/deb {distro.codename()} main", - ) - # the new adoptium repository uses s3, so we need to exclude the buckets from transparent endpoint injection - try: - from localstack.dns import server as dns_server - - dns_server.exclude_from_resolution(ADOPTIUM_DNS_SKIP) - except ImportError: - LOG.debug("Cannot import DNS server - skipping modification to allow apt download") - super()._debian_prepare_install(target) + self.semver = JAVA_VERSIONS[version] - def _post_process(self, target: InstallTarget) -> None: - try: - from localstack.dns import server as dns_server + def _get_install_marker_path(self, install_dir: str) -> str: + return os.path.join(install_dir, self._get_archive_subdir()) - dns_server.revert_exclude_from_resolution(ADOPTIUM_DNS_SKIP) - except ImportError: - LOG.debug("Cannot import DNS server - skipping revert of skip") - target_dir = self._get_jvm_install_dir() - install_dir = glob.glob(f"/usr/lib/jvm/*-{self.version}-jdk-*")[0] - if not os.path.exists(target_dir): - run(["ln", "-s", install_dir, target_dir]) + def _get_download_url(self) -> str: + os = "linux" if is_linux() else "mac" if is_mac_os() else None + arch = ( + "x64" if get_arch() == Arch.amd64 else "aarch64" if get_arch() == Arch.arm64 else None + ) - def _get_jvm_install_dir(self) -> str: - return self.get_java_home() + semver_safe = self.semver.replace("+", "_") + + return JRE_DISTRIB_URL.format( + version=self.version, semver=self.semver, os=os, arch=arch, semver_safe=semver_safe + ) + + def _get_archive_subdir(self) -> str | None: + return f"jdk-{self.semver}-jre" + + def _post_process(self, target: InstallTarget) -> None: + env_var = f"JAVA_{self.version}_HOME" + if not os.environ.get(env_var): + os.environ[env_var] = self.get_installed_dir() def get_java_home(self) -> str: - if self.version == "8": - return JAVA_8_HOME - if self.version == "11": - return JAVA_11_HOME - if self.version == "21": - return JAVA_21_HOME - return f"/usr/lib/jvm/java-{self.version}" + return self.get_installed_dir() class JavaPackage(Package): @@ -102,7 +60,7 @@ def __init__(self, default_version: str = DEFAULT_JAVA_VERSION): super().__init__(name="Java", default_version=default_version) def get_versions(self) -> List[str]: - return JAVA_VERSIONS + return list(JAVA_VERSIONS.keys()) def _get_installer(self, version): return JavaPackageInstaller(version) diff --git a/localstack-core/localstack/packages/plugins.py b/localstack-core/localstack/packages/plugins.py index a9147b8bd7faa..4b4b200af8e0c 100644 --- a/localstack-core/localstack/packages/plugins.py +++ b/localstack-core/localstack/packages/plugins.py @@ -13,3 +13,10 @@ def ffmpeg_package() -> Package: from localstack.packages.ffmpeg import ffmpeg_package return ffmpeg_package + + +@package(name="java") +def java_package() -> Package: + from localstack.packages.java import java_package + + return java_package From 316b49ff7a4cbaa2db2ff48578896ca25f49c1e5 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Wed, 31 Jul 2024 15:23:33 +0530 Subject: [PATCH 05/19] Explicitly specify Java 11 as DDBLocal dependency --- localstack-core/localstack/services/dynamodb/packages.py | 4 ++++ localstack-core/localstack/services/dynamodb/server.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/localstack-core/localstack/services/dynamodb/packages.py b/localstack-core/localstack/services/dynamodb/packages.py index 52e76e761ae41..2550e12f7725b 100644 --- a/localstack-core/localstack/services/dynamodb/packages.py +++ b/localstack-core/localstack/services/dynamodb/packages.py @@ -4,6 +4,7 @@ from localstack import config from localstack.constants import ARTIFACTS_REPO, MAVEN_REPO_URL from localstack.packages import InstallTarget, Package, PackageInstaller +from localstack.packages.java import java_package from localstack.utils.archives import ( download_and_extract_with_retry, update_jar_manifest, @@ -41,6 +42,9 @@ class DynamoDBLocalPackageInstaller(PackageInstaller): def __init__(self): super().__init__("dynamodb-local", "latest") + def _prepare_installation(self, target: InstallTarget) -> None: + java_package.install(version="11", target=target) + def _install(self, target: InstallTarget): # download and extract archive tmp_archive = os.path.join(config.dirs.cache, "localstack.ddb.zip") diff --git a/localstack-core/localstack/services/dynamodb/server.py b/localstack-core/localstack/services/dynamodb/server.py index d250b47c195b6..b311443379711 100644 --- a/localstack-core/localstack/services/dynamodb/server.py +++ b/localstack-core/localstack/services/dynamodb/server.py @@ -7,6 +7,7 @@ from localstack.aws.forwarder import AwsRequestProxy from localstack.config import is_env_true from localstack.constants import DEFAULT_AWS_ACCOUNT_ID +from localstack.packages.java import java_package from localstack.services.dynamodb.packages import dynamodblocal_package from localstack.utils.common import TMP_THREADS, ShellCommandThread, get_free_tcp_port, mkdir from localstack.utils.functions import run_safe @@ -160,7 +161,10 @@ def do_start_thread(self) -> FuncThread: log_listener=_log_listener, auto_restart=True, name="dynamodb-local", - env_vars={"DDB_LOCAL_TELEMETRY": "0"}, + env_vars={ + "JAVA_HOME": java_package.get_installer("11").get_java_home(), + "DDB_LOCAL_TELEMETRY": "0", + }, ) TMP_THREADS.append(t) t.start() From 378a7a2353092abcbabf8dd47b744e00171715e0 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Wed, 31 Jul 2024 16:13:38 +0530 Subject: [PATCH 06/19] Install JRE via LPM during Docker build --- Dockerfile | 60 ++++++--------------- localstack-core/localstack/packages/java.py | 20 +++---- 2 files changed, 27 insertions(+), 53 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7780f117ad465..5513e9360f4cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,6 @@ -# java-builder: Stage to build a custom JRE (with jlink) -FROM eclipse-temurin:21@sha256:3f9bfce63186b9ded168250c8e350631fd643ad00afab5986cf8a7cf79f3b043 as java-builder - -# create a custom, minimized JRE via jlink -RUN jlink --add-modules \ -# include required modules -java.base,java.desktop,java.instrument,java.management,java.naming,java.scripting,java.sql,java.xml,jdk.compiler,\ -# jdk.unsupported contains sun.misc.Unsafe which is required by certain dependencies -jdk.unsupported,\ -# add additional cipher suites -jdk.crypto.cryptoki,\ -# add ability to open ZIP/JAR files -jdk.zipfs,\ -# OpenSearch needs some jdk modules -jdk.httpserver,jdk.management,\ -# MQ Broker requires management agent -jdk.management.agent,\ -# required for Spark/Hadoop -java.security.jgss,jdk.security.auth,\ -# Elasticsearch 7+ crashes without Thai Segmentation support -jdk.localedata --include-locales en,th \ - --compress 2 --strip-debug --no-header-files --no-man-pages --output /usr/lib/jvm/java-21 && \ - cp ${JAVA_HOME}/bin/javac /usr/lib/jvm/java-21/bin/javac && \ - cp -r ${JAVA_HOME}/include /usr/lib/jvm/java-21/include && \ - mv /usr/lib/jvm/java-21/lib/modules /usr/lib/jvm/java-21/lib/modules.bk; \ - cp -r ${JAVA_HOME}/lib/* /usr/lib/jvm/java-21/lib/; \ - mv /usr/lib/jvm/java-21/lib/modules.bk /usr/lib/jvm/java-21/lib/modules; \ - rm -rf /usr/bin/java ${JAVA_HOME} && ln -s /usr/lib/jvm/java-21/bin/java /usr/bin/java - - +# # base: Stage which installs necessary runtime dependencies (OS packages, java,...) +# FROM python:3.11.9-slim-bookworm@sha256:3f3c35617e79276c5f6a2e6a13cdbabdd10257332df963c90c986858b26fad5e AS base ARG TARGETARCH @@ -84,18 +56,7 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && test ! $(which python3.9) SHELL [ "/bin/bash", "-c" ] - -# Install Java 21 ENV LANG C.UTF-8 -RUN { \ - echo '#!/bin/sh'; echo 'set -e'; echo; \ - echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \ - } > /usr/local/bin/docker-java-home \ - && chmod +x /usr/local/bin/docker-java-home -ENV JAVA_HOME /usr/lib/jvm/java-21 -COPY --from=java-builder /usr/lib/jvm/java-21 $JAVA_HOME -RUN ln -s $JAVA_HOME/bin/java /usr/bin/java -ENV PATH="${PATH}:${JAVA_HOME}/bin" # set workdir RUN mkdir -p /opt/code/localstack @@ -137,8 +98,9 @@ RUN --mount=type=cache,target=/root/.cache \ pip3 install --upgrade awscli awscli-local requests - +# # builder: Stage which installs the dependencies of LocalStack Community +# FROM base AS builder ARG TARGETARCH @@ -166,8 +128,9 @@ RUN --mount=type=cache,target=/root/.cache \ RUN . .venv/bin/activate && pip3 freeze -l > requirements-runtime.txt - +# # final stage: Builds upon base stage and copies resources from builder stages +# FROM base COPY --from=builder /opt/code/localstack/.venv /opt/code/localstack/.venv @@ -186,12 +149,23 @@ RUN make entrypoints RUN --mount=type=cache,target=/root/.cache \ --mount=type=cache,target=/var/lib/localstack/cache \ source .venv/bin/activate && \ + python -m localstack.cli.lpm install java --version 11 && \ python -m localstack.cli.lpm install \ lambda-runtime \ dynamodb-local && \ chown -R localstack:localstack /usr/lib/localstack && \ chmod -R 777 /usr/lib/localstack +# Set up Java +ENV JAVA_HOME /usr/lib/localstack/java/11 +RUN { \ + echo '#!/bin/sh'; echo 'set -e'; echo; \ + echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \ + } > /usr/local/bin/docker-java-home \ + && chmod +x /usr/local/bin/docker-java-home +RUN ln -s $JAVA_HOME/bin/java /usr/bin/java +ENV PATH="${PATH}:${JAVA_HOME}/bin" + # link the python package installer virtual environments into the localstack venv RUN echo /var/lib/localstack/lib/python-packages/lib/python3.11/site-packages > localstack-var-python-packages-venv.pth && \ mv localstack-var-python-packages-venv.pth .venv/lib/python*/site-packages/ diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index bdcbbd581e20d..48df0723ca361 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -2,7 +2,7 @@ import os from typing import List -from localstack.packages import InstallTarget, Package +from localstack.packages import Package from localstack.packages.core import ArchiveDownloadAndExtractInstaller from localstack.utils.platform import Arch, get_arch, is_linux, is_mac_os @@ -13,18 +13,18 @@ # Supported Java LTS versions mapped with Eclipse Temurin build semvers JAVA_VERSIONS = { - "8": "8.0.422+5", + "8": "8u422-b05", "11": "11.0.24+8", "17": "17.0.12+7", "21": "21.0.4+7", } -JRE_DISTRIB_URL = "https://github.com/adoptium/temurin{version}-binaries/releases/download/jdk-{semver}/OpenJDK{version}U-jre_{arch}_{os}_hotspot_{semver_safe}.tar.gz" +JRE_DISTRIB_URL = "https://github.com/adoptium/temurin{version}-binaries/releases/download/{tag_slug}/OpenJDK{version}U-jre_{arch}_{os}_hotspot_{semver_safe}.tar.gz" class JavaPackageInstaller(ArchiveDownloadAndExtractInstaller): def __init__(self, version: str): - super().__init__("java", version) + super().__init__("java", version, extract_single_directory=True) self.semver = JAVA_VERSIONS[version] @@ -37,20 +37,20 @@ def _get_download_url(self) -> str: "x64" if get_arch() == Arch.amd64 else "aarch64" if get_arch() == Arch.arm64 else None ) + tag_slug = f"jdk-{self.semver}" semver_safe = self.semver.replace("+", "_") + if self.version == "8": + semver_safe = semver_safe.replace("-", "") + tag_slug = f"jdk{self.semver}" + return JRE_DISTRIB_URL.format( - version=self.version, semver=self.semver, os=os, arch=arch, semver_safe=semver_safe + version=self.version, tag_slug=tag_slug, os=os, arch=arch, semver_safe=semver_safe ) def _get_archive_subdir(self) -> str | None: return f"jdk-{self.semver}-jre" - def _post_process(self, target: InstallTarget) -> None: - env_var = f"JAVA_{self.version}_HOME" - if not os.environ.get(env_var): - os.environ[env_var] = self.get_installed_dir() - def get_java_home(self) -> str: return self.get_installed_dir() From f1f8f93ab408da0626bfb0ab0b0701d278a0d41c Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Wed, 31 Jul 2024 17:04:13 +0530 Subject: [PATCH 07/19] Revert "Explicitly specify Java 11 as DDBLocal dependency" This reverts commit 316b49ff7a4cbaa2db2ff48578896ca25f49c1e5. --- localstack-core/localstack/services/dynamodb/packages.py | 4 ---- localstack-core/localstack/services/dynamodb/server.py | 6 +----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/localstack-core/localstack/services/dynamodb/packages.py b/localstack-core/localstack/services/dynamodb/packages.py index 2550e12f7725b..52e76e761ae41 100644 --- a/localstack-core/localstack/services/dynamodb/packages.py +++ b/localstack-core/localstack/services/dynamodb/packages.py @@ -4,7 +4,6 @@ from localstack import config from localstack.constants import ARTIFACTS_REPO, MAVEN_REPO_URL from localstack.packages import InstallTarget, Package, PackageInstaller -from localstack.packages.java import java_package from localstack.utils.archives import ( download_and_extract_with_retry, update_jar_manifest, @@ -42,9 +41,6 @@ class DynamoDBLocalPackageInstaller(PackageInstaller): def __init__(self): super().__init__("dynamodb-local", "latest") - def _prepare_installation(self, target: InstallTarget) -> None: - java_package.install(version="11", target=target) - def _install(self, target: InstallTarget): # download and extract archive tmp_archive = os.path.join(config.dirs.cache, "localstack.ddb.zip") diff --git a/localstack-core/localstack/services/dynamodb/server.py b/localstack-core/localstack/services/dynamodb/server.py index b311443379711..d250b47c195b6 100644 --- a/localstack-core/localstack/services/dynamodb/server.py +++ b/localstack-core/localstack/services/dynamodb/server.py @@ -7,7 +7,6 @@ from localstack.aws.forwarder import AwsRequestProxy from localstack.config import is_env_true from localstack.constants import DEFAULT_AWS_ACCOUNT_ID -from localstack.packages.java import java_package from localstack.services.dynamodb.packages import dynamodblocal_package from localstack.utils.common import TMP_THREADS, ShellCommandThread, get_free_tcp_port, mkdir from localstack.utils.functions import run_safe @@ -161,10 +160,7 @@ def do_start_thread(self) -> FuncThread: log_listener=_log_listener, auto_restart=True, name="dynamodb-local", - env_vars={ - "JAVA_HOME": java_package.get_installer("11").get_java_home(), - "DDB_LOCAL_TELEMETRY": "0", - }, + env_vars={"DDB_LOCAL_TELEMETRY": "0"}, ) TMP_THREADS.append(t) t.start() From 00e64333d125f1d2534d9ab96f59eb87dfc11d94 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Wed, 31 Jul 2024 17:12:00 +0530 Subject: [PATCH 08/19] Add comment --- localstack-core/localstack/packages/java.py | 1 + 1 file changed, 1 insertion(+) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 48df0723ca361..4f02bfdef1fe9 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -40,6 +40,7 @@ def _get_download_url(self) -> str: tag_slug = f"jdk-{self.semver}" semver_safe = self.semver.replace("+", "_") + # v8 uses a different tag and version scheme if self.version == "8": semver_safe = semver_safe.replace("-", "") tag_slug = f"jdk{self.semver}" From c108cb5c227993033e9f7a98f4212f5a82179b4b Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Tue, 3 Sep 2024 12:59:52 +0530 Subject: [PATCH 09/19] Remove docker-java-home executable --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e3e6f3e888f97..663794e7a3fd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,11 +164,6 @@ RUN --mount=type=cache,target=/root/.cache \ # Set up Java ENV JAVA_HOME /usr/lib/localstack/java/11 -RUN { \ - echo '#!/bin/sh'; echo 'set -e'; echo; \ - echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \ - } > /usr/local/bin/docker-java-home \ - && chmod +x /usr/local/bin/docker-java-home RUN ln -s $JAVA_HOME/bin/java /usr/bin/java ENV PATH="${PATH}:${JAVA_HOME}/bin" From a325dfc7906d7206f1c86ebfb9259c8e137772b9 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Tue, 3 Sep 2024 16:38:26 +0530 Subject: [PATCH 10/19] Optimise JRE for minimise disk footprint --- localstack-core/localstack/packages/java.py | 48 +++++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 4f02bfdef1fe9..682b614e032e7 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -2,9 +2,11 @@ import os from typing import List -from localstack.packages import Package +from localstack.packages import InstallTarget, Package from localstack.packages.core import ArchiveDownloadAndExtractInstaller +from localstack.utils.files import rm_rf from localstack.utils.platform import Arch, get_arch, is_linux, is_mac_os +from localstack.utils.run import run LOG = logging.getLogger(__name__) @@ -19,7 +21,7 @@ "21": "21.0.4+7", } -JRE_DISTRIB_URL = "https://github.com/adoptium/temurin{version}-binaries/releases/download/{tag_slug}/OpenJDK{version}U-jre_{arch}_{os}_hotspot_{semver_safe}.tar.gz" +JDK_DOWNLOAD_URL = "https://github.com/adoptium/temurin{version}-binaries/releases/download/{tag_slug}/OpenJDK{version}U-jdk_{arch}_{os}_hotspot_{semver_safe}.tar.gz" class JavaPackageInstaller(ArchiveDownloadAndExtractInstaller): @@ -45,12 +47,50 @@ def _get_download_url(self) -> str: semver_safe = semver_safe.replace("-", "") tag_slug = f"jdk{self.semver}" - return JRE_DISTRIB_URL.format( + return JDK_DOWNLOAD_URL.format( version=self.version, tag_slug=tag_slug, os=os, arch=arch, semver_safe=semver_safe ) def _get_archive_subdir(self) -> str | None: - return f"jdk-{self.semver}-jre" + return "" + + def _post_process(self, target: InstallTarget) -> None: + target_directory = self._get_install_dir(target) + minimal_jre_path = os.path.join(target.value, self.name, "jre-{self.version}-minimal") + + # If jlink is not available, use the environment as is + if not os.path.exists(os.path.join(target_directory, "bin/jlink")): + LOG.warning("Skipping JRE optimisation because jlink is not available") + return + + # Build a custom JRE with only the necessary bits to minimise disk footprint + LOG.debug("Optimising JRE installation") + cmd = ( + "bin/jlink --add-modules " + # Required modules + "java.base,java.desktop,java.instrument,java.management," + "java.naming,java.scripting,java.sql,java.xml,jdk.compiler," + # jdk.unsupported contains sun.misc.Unsafe which is required by some dependencies + "jdk.unsupported," + # Additional cipher suites + "jdk.crypto.cryptoki," + # Archive support + "jdk.zipfs," + # Required by MQ broker + "jdk.httpserver,jdk.management,jdk.management.agent," + # Required by Spark and Hadoop + "java.security.jgss,jdk.security.auth," + # Locales + "jdk.localedata --include-locales en " + # Supplementary args + "--compress 2 --strip-debug --no-header-files --no-man-pages " + # Output directory + "--output " + minimal_jre_path + ) + run(cmd, cwd=target_directory) + + rm_rf(target_directory) + os.rename(minimal_jre_path, target_directory) def get_java_home(self) -> str: return self.get_installed_dir() From 88c8736edc90e4a254c43d46efbb96ea74715523 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Tue, 3 Sep 2024 17:21:39 +0530 Subject: [PATCH 11/19] Determine the latest stable build --- localstack-core/localstack/packages/java.py | 56 ++++++++++++++------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 682b614e032e7..cc28bac613a56 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -2,6 +2,8 @@ import os from typing import List +import requests + from localstack.packages import InstallTarget, Package from localstack.packages.core import ArchiveDownloadAndExtractInstaller from localstack.utils.files import rm_rf @@ -28,28 +30,16 @@ class JavaPackageInstaller(ArchiveDownloadAndExtractInstaller): def __init__(self, version: str): super().__init__("java", version, extract_single_directory=True) - self.semver = JAVA_VERSIONS[version] - def _get_install_marker_path(self, install_dir: str) -> str: return os.path.join(install_dir, self._get_archive_subdir()) def _get_download_url(self) -> str: - os = "linux" if is_linux() else "mac" if is_mac_os() else None - arch = ( - "x64" if get_arch() == Arch.amd64 else "aarch64" if get_arch() == Arch.arm64 else None - ) - - tag_slug = f"jdk-{self.semver}" - semver_safe = self.semver.replace("+", "_") - - # v8 uses a different tag and version scheme - if self.version == "8": - semver_safe = semver_safe.replace("-", "") - tag_slug = f"jdk{self.semver}" - - return JDK_DOWNLOAD_URL.format( - version=self.version, tag_slug=tag_slug, os=os, arch=arch, semver_safe=semver_safe - ) + try: + LOG.debug("Determining the latest Java version") + return self.download_url_latest_release() + except Exception as exc: # noqa + LOG.debug("Unable to determine the latest Java version. Using pinned versions: %s", exc) + return self.download_url_fallback() def _get_archive_subdir(self) -> str | None: return "" @@ -95,6 +85,36 @@ def _post_process(self, target: InstallTarget) -> None: def get_java_home(self) -> str: return self.get_installed_dir() + def download_url_latest_release(self) -> str: + """ + Return the download URL for latest stable JDK build. + """ + endpoint = f"https://api.adoptium.net/v3/assets/latest/{self.version}/hotspot?os=linux&architecture=x64&image_type=jdk" + response = requests.get(endpoint, headers={"user-agent": "example/0.0.0"}).json() + return response[0]["binary"]["package"]["link"] + + def download_url_fallback(self) -> str: + """ + Return the download URL for pinned JDK build. + """ + os = "linux" if is_linux() else "mac" if is_mac_os() else None + arch = ( + "x64" if get_arch() == Arch.amd64 else "aarch64" if get_arch() == Arch.arm64 else None + ) + + semver = JAVA_VERSIONS[self.version] + tag_slug = f"jdk-{semver}" + semver_safe = semver.replace("+", "_") + + # v8 uses a different tag and version scheme + if self.version == "8": + semver_safe = semver_safe.replace("-", "") + tag_slug = f"jdk{semver}" + + return JDK_DOWNLOAD_URL.format( + version=self.version, tag_slug=tag_slug, os=os, arch=arch, semver_safe=semver_safe + ) + class JavaPackage(Package): def __init__(self, default_version: str = DEFAULT_JAVA_VERSION): From f1062cfd46462b1897fc2cb82e945484c0403aa1 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Tue, 3 Sep 2024 17:26:18 +0530 Subject: [PATCH 12/19] Improve already-installed detection --- localstack-core/localstack/packages/java.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index cc28bac613a56..441569d328acb 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -31,7 +31,7 @@ def __init__(self, version: str): super().__init__("java", version, extract_single_directory=True) def _get_install_marker_path(self, install_dir: str) -> str: - return os.path.join(install_dir, self._get_archive_subdir()) + return os.path.join(install_dir, "bin/java") def _get_download_url(self) -> str: try: @@ -41,9 +41,6 @@ def _get_download_url(self) -> str: LOG.debug("Unable to determine the latest Java version. Using pinned versions: %s", exc) return self.download_url_fallback() - def _get_archive_subdir(self) -> str | None: - return "" - def _post_process(self, target: InstallTarget) -> None: target_directory = self._get_install_dir(target) minimal_jre_path = os.path.join(target.value, self.name, "jre-{self.version}-minimal") From 1ac97fcd57b1523298cf9c8beb6ea312ffce68ae Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Tue, 3 Sep 2024 17:45:48 +0530 Subject: [PATCH 13/19] Improve cross-arch compatibility --- localstack-core/localstack/packages/java.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 441569d328acb..30c54d262c69f 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -35,10 +35,12 @@ def _get_install_marker_path(self, install_dir: str) -> str: def _get_download_url(self) -> str: try: - LOG.debug("Determining the latest Java version") + LOG.debug("Determining the latest Java build version") return self.download_url_latest_release() except Exception as exc: # noqa - LOG.debug("Unable to determine the latest Java version. Using pinned versions: %s", exc) + LOG.debug( + "Unable to determine the latest Java build version. Using pinned versions: %s", exc + ) return self.download_url_fallback() def _post_process(self, target: InstallTarget) -> None: @@ -82,11 +84,17 @@ def _post_process(self, target: InstallTarget) -> None: def get_java_home(self) -> str: return self.get_installed_dir() + @property + def arch(self) -> str: + return ( + "x64" if get_arch() == Arch.amd64 else "aarch64" if get_arch() == Arch.arm64 else None + ) + def download_url_latest_release(self) -> str: """ Return the download URL for latest stable JDK build. """ - endpoint = f"https://api.adoptium.net/v3/assets/latest/{self.version}/hotspot?os=linux&architecture=x64&image_type=jdk" + endpoint = f"https://api.adoptium.net/v3/assets/latest/{self.version}/hotspot?os=linux&architecture={self.arch}&image_type=jdk" response = requests.get(endpoint, headers={"user-agent": "example/0.0.0"}).json() return response[0]["binary"]["package"]["link"] @@ -95,9 +103,6 @@ def download_url_fallback(self) -> str: Return the download URL for pinned JDK build. """ os = "linux" if is_linux() else "mac" if is_mac_os() else None - arch = ( - "x64" if get_arch() == Arch.amd64 else "aarch64" if get_arch() == Arch.arm64 else None - ) semver = JAVA_VERSIONS[self.version] tag_slug = f"jdk-{semver}" @@ -109,7 +114,7 @@ def download_url_fallback(self) -> str: tag_slug = f"jdk{semver}" return JDK_DOWNLOAD_URL.format( - version=self.version, tag_slug=tag_slug, os=os, arch=arch, semver_safe=semver_safe + version=self.version, tag_slug=tag_slug, os=os, arch=self.arch, semver_safe=semver_safe ) From c76b4297c2e01b79addf83eb87dab68027e95fae Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Tue, 3 Sep 2024 18:47:31 +0530 Subject: [PATCH 14/19] Add Thai locale --- localstack-core/localstack/packages/java.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 30c54d262c69f..e5db46c1c2514 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -69,8 +69,8 @@ def _post_process(self, target: InstallTarget) -> None: "jdk.httpserver,jdk.management,jdk.management.agent," # Required by Spark and Hadoop "java.security.jgss,jdk.security.auth," - # Locales - "jdk.localedata --include-locales en " + # OpenSearch requires Thai locale for segmentation support + "jdk.localedata --include-locales en,th " # Supplementary args "--compress 2 --strip-debug --no-header-files --no-man-pages " # Output directory From 56570585ceb1b25d509398c21aa2620922961f9d Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Wed, 4 Sep 2024 17:41:06 +0530 Subject: [PATCH 15/19] Minor rearrangement --- localstack-core/localstack/packages/java.py | 26 +++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index e5db46c1c2514..ad5f19b3700c0 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -23,8 +23,6 @@ "21": "21.0.4+7", } -JDK_DOWNLOAD_URL = "https://github.com/adoptium/temurin{version}-binaries/releases/download/{tag_slug}/OpenJDK{version}U-jdk_{arch}_{os}_hotspot_{semver_safe}.tar.gz" - class JavaPackageInstaller(ArchiveDownloadAndExtractInstaller): def __init__(self, version: str): @@ -36,16 +34,17 @@ def _get_install_marker_path(self, install_dir: str) -> str: def _get_download_url(self) -> str: try: LOG.debug("Determining the latest Java build version") - return self.download_url_latest_release() + return self._download_url_latest_release() except Exception as exc: # noqa LOG.debug( "Unable to determine the latest Java build version. Using pinned versions: %s", exc ) - return self.download_url_fallback() + return self._download_url_fallback() def _post_process(self, target: InstallTarget) -> None: target_directory = self._get_install_dir(target) - minimal_jre_path = os.path.join(target.value, self.name, "jre-{self.version}-minimal") + minimal_jre_path = os.path.join(target.value, self.name, "{self.version}.minimal") + rm_rf(minimal_jre_path) # If jlink is not available, use the environment as is if not os.path.exists(os.path.join(target_directory, "bin/jlink")): @@ -82,6 +81,9 @@ def _post_process(self, target: InstallTarget) -> None: os.rename(minimal_jre_path, target_directory) def get_java_home(self) -> str: + """ + Get JAVA_HOME for this installation of Java. + """ return self.get_installed_dir() @property @@ -90,15 +92,18 @@ def arch(self) -> str: "x64" if get_arch() == Arch.amd64 else "aarch64" if get_arch() == Arch.arm64 else None ) - def download_url_latest_release(self) -> str: + def _download_url_latest_release(self) -> str: """ Return the download URL for latest stable JDK build. """ - endpoint = f"https://api.adoptium.net/v3/assets/latest/{self.version}/hotspot?os=linux&architecture={self.arch}&image_type=jdk" + endpoint = ( + f"https://api.adoptium.net/v3/assets/latest/{self.version}/hotspot?" + f"os=linux&architecture={self.arch}&image_type=jdk" + ) response = requests.get(endpoint, headers={"user-agent": "example/0.0.0"}).json() return response[0]["binary"]["package"]["link"] - def download_url_fallback(self) -> str: + def _download_url_fallback(self) -> str: """ Return the download URL for pinned JDK build. """ @@ -113,8 +118,9 @@ def download_url_fallback(self) -> str: semver_safe = semver_safe.replace("-", "") tag_slug = f"jdk{semver}" - return JDK_DOWNLOAD_URL.format( - version=self.version, tag_slug=tag_slug, os=os, arch=self.arch, semver_safe=semver_safe + return ( + f"https://github.com/adoptium/temurin{self.version}-binaries/releases/download/{tag_slug}/" + f"OpenJDK{self.version}U-jdk_{self.arch}_{os}_hotspot_{semver_safe}.tar.gz" ) From a286af0826a138c41d4e34e59cf437d70db0918e Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Thu, 5 Sep 2024 18:24:43 +0530 Subject: [PATCH 16/19] Add binutils package which provides objcopy command --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 663794e7a3fd1..5e4205ebf38a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN --mount=type=cache,target=/var/cache/apt \ # Install dependencies to add additional repos apt-get install -y --no-install-recommends \ # Runtime packages (groff-base is necessary for AWS CLI help) - ca-certificates curl gnupg git make openssl tar pixz zip unzip groff-base iputils-ping nss-passwords procps iproute2 xz-utils libatomic1 + ca-certificates curl gnupg git make openssl tar pixz zip unzip groff-base iputils-ping nss-passwords procps iproute2 xz-utils libatomic1 binutils # FIXME Node 18 actually shouldn't be necessary in Community, but we assume its presence in lots of tests # Install nodejs package from the dist release server. Note: we're installing from dist binaries, and not via From f0d6bf737378c8b2d22ae42414d4b678cc2d64fd Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Tue, 10 Sep 2024 14:56:03 +0530 Subject: [PATCH 17/19] Use distinct levels when joining paths --- localstack-core/localstack/packages/java.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index ad5f19b3700c0..ec97770cd1205 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -29,7 +29,7 @@ def __init__(self, version: str): super().__init__("java", version, extract_single_directory=True) def _get_install_marker_path(self, install_dir: str) -> str: - return os.path.join(install_dir, "bin/java") + return os.path.join(install_dir, "bin", "java") def _get_download_url(self) -> str: try: @@ -47,7 +47,7 @@ def _post_process(self, target: InstallTarget) -> None: rm_rf(minimal_jre_path) # If jlink is not available, use the environment as is - if not os.path.exists(os.path.join(target_directory, "bin/jlink")): + if not os.path.exists(os.path.join(target_directory, "bin", "jlink")): LOG.warning("Skipping JRE optimisation because jlink is not available") return From 4b8f57fa691cfa593a5a4b8bef51d5cdd309d15d Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Tue, 10 Sep 2024 15:13:35 +0530 Subject: [PATCH 18/19] Fix missing f-string --- localstack-core/localstack/packages/java.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index ec97770cd1205..372ae9cf50196 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -43,7 +43,7 @@ def _get_download_url(self) -> str: def _post_process(self, target: InstallTarget) -> None: target_directory = self._get_install_dir(target) - minimal_jre_path = os.path.join(target.value, self.name, "{self.version}.minimal") + minimal_jre_path = os.path.join(target.value, self.name, f"{self.version}.minimal") rm_rf(minimal_jre_path) # If jlink is not available, use the environment as is From 680a58a65f83bb797686955e07056eef9d5d04d4 Mon Sep 17 00:00:00 2001 From: Viren Nadkarni Date: Thu, 12 Sep 2024 13:16:29 +0530 Subject: [PATCH 19/19] Define and use LocalStack user-agent string --- localstack-core/localstack/constants.py | 3 +++ localstack-core/localstack/packages/java.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/localstack-core/localstack/constants.py b/localstack-core/localstack/constants.py index b884d5d5ef4ec..abd60c8f2b199 100644 --- a/localstack-core/localstack/constants.py +++ b/localstack-core/localstack/constants.py @@ -20,6 +20,9 @@ LOCALHOST_IP = "127.0.0.1" LOCALHOST_HOSTNAME = "localhost.localstack.cloud" +# User-agent string used in outgoing HTTP requests made by LocalStack +USER_AGENT_STRING = f"localstack/{VERSION}" + # version of the Maven dependency with Java utility code LOCALSTACK_MAVEN_VERSION = "0.2.21" MAVEN_REPO_URL = "https://repo1.maven.org/maven2" diff --git a/localstack-core/localstack/packages/java.py b/localstack-core/localstack/packages/java.py index 372ae9cf50196..2bd280d8fbee4 100644 --- a/localstack-core/localstack/packages/java.py +++ b/localstack-core/localstack/packages/java.py @@ -4,6 +4,7 @@ import requests +from localstack.constants import USER_AGENT_STRING from localstack.packages import InstallTarget, Package from localstack.packages.core import ArchiveDownloadAndExtractInstaller from localstack.utils.files import rm_rf @@ -100,7 +101,8 @@ def _download_url_latest_release(self) -> str: f"https://api.adoptium.net/v3/assets/latest/{self.version}/hotspot?" f"os=linux&architecture={self.arch}&image_type=jdk" ) - response = requests.get(endpoint, headers={"user-agent": "example/0.0.0"}).json() + # Override user-agent because Adoptium API denies service to `requests` library + response = requests.get(endpoint, headers={"user-agent": USER_AGENT_STRING}).json() return response[0]["binary"]["package"]["link"] def _download_url_fallback(self) -> str: