From b95b158b5b17db7843b3ab8048c489fbe2247dc2 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Mon, 13 May 2024 12:54:12 +0100 Subject: [PATCH 1/7] Introduce LOGGING_OVERRIDE config var Co-Authored-By: Daniel Fangl --- localstack-core/localstack/config.py | 3 +++ localstack-core/localstack/logging/setup.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/localstack-core/localstack/config.py b/localstack-core/localstack/config.py index b46b86e7ee92f..42b29b322c615 100644 --- a/localstack-core/localstack/config.py +++ b/localstack-core/localstack/config.py @@ -442,6 +442,9 @@ def in_docker(): # path to the lambda debug mode configuration file. LAMBDA_DEBUG_MODE_CONFIG_PATH = os.environ.get("LAMBDA_DEBUG_MODE_CONFIG_PATH") +# allow setting custom log levels for individual loggers +LOGGING_OVERRIDE = os.environ.get("LOGGING_OVERRIDE", "{}") + # whether to enable debugpy DEVELOP = is_env_true("DEVELOP") diff --git a/localstack-core/localstack/logging/setup.py b/localstack-core/localstack/logging/setup.py index 444742083e687..390551c9c3e57 100644 --- a/localstack-core/localstack/logging/setup.py +++ b/localstack-core/localstack/logging/setup.py @@ -1,3 +1,4 @@ +import json import logging import sys import warnings @@ -81,6 +82,23 @@ def setup_logging_from_config(): for name, level in trace_internal_log_levels.items(): logging.getLogger(name).setLevel(level) + if raw_value := config.LOGGING_OVERRIDE: + try: + logging_overrides = json.loads(raw_value) + for logger, level_name in logging_overrides.items(): + level = getattr(logging, level_name, None) + if not level: + raise RuntimeError( + f"Failed to configure logging overrides ({raw_value}): '{level_name}' is not a valid log level" + ) + logging.getLogger(logger).setLevel(level) + except RuntimeError: + raise + except Exception as e: + raise RuntimeError( + f"Failed to configure logging overrides ({raw_value}): Malformed value. ({e})" + ) from e + def create_default_handler(log_level: int): log_handler = logging.StreamHandler(stream=sys.stderr) From 6bb6abb3a22ce2bbe9f206dcaa9423e53674fb76 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Mon, 13 May 2024 16:15:25 +0100 Subject: [PATCH 2/7] Do not use walrus operator for python version compatibility --- localstack-core/localstack/logging/setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/localstack-core/localstack/logging/setup.py b/localstack-core/localstack/logging/setup.py index 390551c9c3e57..955561a6331d8 100644 --- a/localstack-core/localstack/logging/setup.py +++ b/localstack-core/localstack/logging/setup.py @@ -82,21 +82,22 @@ def setup_logging_from_config(): for name, level in trace_internal_log_levels.items(): logging.getLogger(name).setLevel(level) - if raw_value := config.LOGGING_OVERRIDE: + raw_logging_override = config.LOGGING_OVERRIDE + if raw_logging_override: try: - logging_overrides = json.loads(raw_value) + logging_overrides = json.loads(raw_logging_override) for logger, level_name in logging_overrides.items(): level = getattr(logging, level_name, None) if not level: raise RuntimeError( - f"Failed to configure logging overrides ({raw_value}): '{level_name}' is not a valid log level" + f"Failed to configure logging overrides ({raw_logging_override}): '{level_name}' is not a valid log level" ) logging.getLogger(logger).setLevel(level) except RuntimeError: raise except Exception as e: raise RuntimeError( - f"Failed to configure logging overrides ({raw_value}): Malformed value. ({e})" + f"Failed to configure logging overrides ({raw_logging_override}): Malformed value. ({e})" ) from e From d158ab96e4693f9003fdf33db001e99d3a8d60ef Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Mon, 13 May 2024 16:34:29 +0100 Subject: [PATCH 3/7] Convert envar to key-value pairs --- localstack-core/localstack/config.py | 2 +- localstack-core/localstack/logging/setup.py | 4 +-- .../localstack/utils/collections.py | 15 +++++++++++ tests/unit/utils/test_collections.py | 26 +++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/localstack-core/localstack/config.py b/localstack-core/localstack/config.py index 42b29b322c615..772a5533e2e74 100644 --- a/localstack-core/localstack/config.py +++ b/localstack-core/localstack/config.py @@ -443,7 +443,7 @@ def in_docker(): LAMBDA_DEBUG_MODE_CONFIG_PATH = os.environ.get("LAMBDA_DEBUG_MODE_CONFIG_PATH") # allow setting custom log levels for individual loggers -LOGGING_OVERRIDE = os.environ.get("LOGGING_OVERRIDE", "{}") +LOGGING_OVERRIDE = os.environ.get("LOGGING_OVERRIDE", "") # whether to enable debugpy DEVELOP = is_env_true("DEVELOP") diff --git a/localstack-core/localstack/logging/setup.py b/localstack-core/localstack/logging/setup.py index 955561a6331d8..9c49c2d4b0192 100644 --- a/localstack-core/localstack/logging/setup.py +++ b/localstack-core/localstack/logging/setup.py @@ -1,10 +1,10 @@ -import json import logging import sys import warnings from localstack import config, constants +from ..utils.collections import parse_key_value_pairs from .format import AddFormattedAttributes, DefaultFormatter # The log levels for modules are evaluated incrementally for logging granularity, @@ -85,7 +85,7 @@ def setup_logging_from_config(): raw_logging_override = config.LOGGING_OVERRIDE if raw_logging_override: try: - logging_overrides = json.loads(raw_logging_override) + logging_overrides = parse_key_value_pairs(raw_logging_override) for logger, level_name in logging_overrides.items(): level = getattr(logging, level_name, None) if not level: diff --git a/localstack-core/localstack/utils/collections.py b/localstack-core/localstack/utils/collections.py index 41860cd9a190c..e4015c1a260b3 100644 --- a/localstack-core/localstack/utils/collections.py +++ b/localstack-core/localstack/utils/collections.py @@ -534,3 +534,18 @@ def is_comma_delimited_list(string: str, item_regex: Optional[str] = None) -> bo if pattern.match(string) is None: return False return True + + +def parse_key_value_pairs(raw_text: str) -> dict: + result = {} + for pair in raw_text.split(","): + items = pair.split("=", maxsplit=2) + if len(items) != 2: + raise ValueError(f"invalid key/value pair: '{pair}'") + raw_key, raw_value = items[0].strip(), items[1].strip() + if not raw_key: + raise ValueError(f"missing key: '{pair}'") + if not raw_value: + raise ValueError(f"missing value: '{pair}'") + result[raw_key] = raw_value + return result diff --git a/tests/unit/utils/test_collections.py b/tests/unit/utils/test_collections.py index adb2581e77460..2a92e12689a70 100644 --- a/tests/unit/utils/test_collections.py +++ b/tests/unit/utils/test_collections.py @@ -9,6 +9,7 @@ ImmutableList, convert_to_typed_dict, is_comma_delimited_list, + parse_key_value_pairs, select_from_typed_dict, ) @@ -193,3 +194,28 @@ def test_is_comma_limited_list(): assert not is_comma_delimited_list("foo, bar baz") assert not is_comma_delimited_list("foo,") assert not is_comma_delimited_list("") + + +@pytest.mark.parametrize( + "input_text,expected", + [ + ("a=b", {"a": "b"}), + ("a=b,c=d", {"a": "b", "c": "d"}), + ], +) +def test_parse_key_value_pairs(input_text, expected): + assert parse_key_value_pairs(input_text) == expected + + +@pytest.mark.parametrize( + "input_text,message", + [ + ("a=b,", "invalid key/value pair: ''"), + ("a=b,c=", "missing value: 'c='"), + ], +) +def test_parse_key_value_pairs_error_messages(input_text, message): + with pytest.raises(ValueError) as exc_info: + parse_key_value_pairs(input_text) + + assert str(exc_info.value) == message From d9621ceb9df2e16fef4a0b5764645f958d38cf5d Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Wed, 15 May 2024 16:47:05 +0100 Subject: [PATCH 4/7] Address review comments --- localstack-core/localstack/logging/setup.py | 2 +- localstack-core/localstack/utils/collections.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/localstack-core/localstack/logging/setup.py b/localstack-core/localstack/logging/setup.py index 9c49c2d4b0192..db77075f836af 100644 --- a/localstack-core/localstack/logging/setup.py +++ b/localstack-core/localstack/logging/setup.py @@ -97,7 +97,7 @@ def setup_logging_from_config(): raise except Exception as e: raise RuntimeError( - f"Failed to configure logging overrides ({raw_logging_override}): Malformed value. ({e})" + f"Failed to configure logging overrides ({raw_logging_override})" ) from e diff --git a/localstack-core/localstack/utils/collections.py b/localstack-core/localstack/utils/collections.py index e4015c1a260b3..adf0cfa8113d7 100644 --- a/localstack-core/localstack/utils/collections.py +++ b/localstack-core/localstack/utils/collections.py @@ -537,9 +537,15 @@ def is_comma_delimited_list(string: str, item_regex: Optional[str] = None) -> bo def parse_key_value_pairs(raw_text: str) -> dict: + """ + Parse a series of key-value pairs, in an environment variable format into a dictionary + + >>> input = "a=b,c=d" + >>> assert parse_key_value_pairs(input) == {"a": "b", "c": "d"} + """ result = {} for pair in raw_text.split(","): - items = pair.split("=", maxsplit=2) + items = pair.split("=") if len(items) != 2: raise ValueError(f"invalid key/value pair: '{pair}'") raw_key, raw_value = items[0].strip(), items[1].strip() From 682315b549a510fa33b2f3916c916d40390e6859 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Sun, 2 Mar 2025 06:25:06 +0000 Subject: [PATCH 5/7] Add experimental prefix to documentation comment as per review --- localstack-core/localstack/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localstack-core/localstack/config.py b/localstack-core/localstack/config.py index 772a5533e2e74..ca72319f0a500 100644 --- a/localstack-core/localstack/config.py +++ b/localstack-core/localstack/config.py @@ -442,7 +442,7 @@ def in_docker(): # path to the lambda debug mode configuration file. LAMBDA_DEBUG_MODE_CONFIG_PATH = os.environ.get("LAMBDA_DEBUG_MODE_CONFIG_PATH") -# allow setting custom log levels for individual loggers +# EXPERIMENTAL: allow setting custom log levels for individual loggers LOGGING_OVERRIDE = os.environ.get("LOGGING_OVERRIDE", "") # whether to enable debugpy From b7fdaa3f24e90b22c5fdb7aa69342e78f3850fc0 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Wed, 7 May 2025 17:00:26 +0200 Subject: [PATCH 6/7] Use new utility, rename config variable --- localstack-core/localstack/config.py | 2 +- localstack-core/localstack/logging/setup.py | 10 +++---- .../localstack/utils/collections.py | 21 --------------- tests/unit/utils/test_collections.py | 26 ------------------- 4 files changed, 6 insertions(+), 53 deletions(-) diff --git a/localstack-core/localstack/config.py b/localstack-core/localstack/config.py index ca72319f0a500..5c2af11762fb4 100644 --- a/localstack-core/localstack/config.py +++ b/localstack-core/localstack/config.py @@ -443,7 +443,7 @@ def in_docker(): LAMBDA_DEBUG_MODE_CONFIG_PATH = os.environ.get("LAMBDA_DEBUG_MODE_CONFIG_PATH") # EXPERIMENTAL: allow setting custom log levels for individual loggers -LOGGING_OVERRIDE = os.environ.get("LOGGING_OVERRIDE", "") +LOG_LEVEL_OVERRIDES = os.environ.get("LOG_LEVEL_OVERRIDES", "") # whether to enable debugpy DEVELOP = is_env_true("DEVELOP") diff --git a/localstack-core/localstack/logging/setup.py b/localstack-core/localstack/logging/setup.py index db77075f836af..f714bfb1bf359 100644 --- a/localstack-core/localstack/logging/setup.py +++ b/localstack-core/localstack/logging/setup.py @@ -4,7 +4,7 @@ from localstack import config, constants -from ..utils.collections import parse_key_value_pairs +from ..utils.strings import key_value_pairs_to_dict from .format import AddFormattedAttributes, DefaultFormatter # The log levels for modules are evaluated incrementally for logging granularity, @@ -82,18 +82,18 @@ def setup_logging_from_config(): for name, level in trace_internal_log_levels.items(): logging.getLogger(name).setLevel(level) - raw_logging_override = config.LOGGING_OVERRIDE + raw_logging_override = config.LOG_LEVEL_OVERRIDES if raw_logging_override: try: - logging_overrides = parse_key_value_pairs(raw_logging_override) + logging_overrides = key_value_pairs_to_dict(raw_logging_override) for logger, level_name in logging_overrides.items(): level = getattr(logging, level_name, None) if not level: - raise RuntimeError( + raise ValueError( f"Failed to configure logging overrides ({raw_logging_override}): '{level_name}' is not a valid log level" ) logging.getLogger(logger).setLevel(level) - except RuntimeError: + except ValueError: raise except Exception as e: raise RuntimeError( diff --git a/localstack-core/localstack/utils/collections.py b/localstack-core/localstack/utils/collections.py index adf0cfa8113d7..41860cd9a190c 100644 --- a/localstack-core/localstack/utils/collections.py +++ b/localstack-core/localstack/utils/collections.py @@ -534,24 +534,3 @@ def is_comma_delimited_list(string: str, item_regex: Optional[str] = None) -> bo if pattern.match(string) is None: return False return True - - -def parse_key_value_pairs(raw_text: str) -> dict: - """ - Parse a series of key-value pairs, in an environment variable format into a dictionary - - >>> input = "a=b,c=d" - >>> assert parse_key_value_pairs(input) == {"a": "b", "c": "d"} - """ - result = {} - for pair in raw_text.split(","): - items = pair.split("=") - if len(items) != 2: - raise ValueError(f"invalid key/value pair: '{pair}'") - raw_key, raw_value = items[0].strip(), items[1].strip() - if not raw_key: - raise ValueError(f"missing key: '{pair}'") - if not raw_value: - raise ValueError(f"missing value: '{pair}'") - result[raw_key] = raw_value - return result diff --git a/tests/unit/utils/test_collections.py b/tests/unit/utils/test_collections.py index 2a92e12689a70..adb2581e77460 100644 --- a/tests/unit/utils/test_collections.py +++ b/tests/unit/utils/test_collections.py @@ -9,7 +9,6 @@ ImmutableList, convert_to_typed_dict, is_comma_delimited_list, - parse_key_value_pairs, select_from_typed_dict, ) @@ -194,28 +193,3 @@ def test_is_comma_limited_list(): assert not is_comma_delimited_list("foo, bar baz") assert not is_comma_delimited_list("foo,") assert not is_comma_delimited_list("") - - -@pytest.mark.parametrize( - "input_text,expected", - [ - ("a=b", {"a": "b"}), - ("a=b,c=d", {"a": "b", "c": "d"}), - ], -) -def test_parse_key_value_pairs(input_text, expected): - assert parse_key_value_pairs(input_text) == expected - - -@pytest.mark.parametrize( - "input_text,message", - [ - ("a=b,", "invalid key/value pair: ''"), - ("a=b,c=", "missing value: 'c='"), - ], -) -def test_parse_key_value_pairs_error_messages(input_text, message): - with pytest.raises(ValueError) as exc_info: - parse_key_value_pairs(input_text) - - assert str(exc_info.value) == message From 6ed2dd64761225e44c8b3223a6f86e29a2e8f380 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Wed, 7 May 2025 17:15:46 +0200 Subject: [PATCH 7/7] Remove try catch --- localstack-core/localstack/logging/setup.py | 23 +++++++-------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/localstack-core/localstack/logging/setup.py b/localstack-core/localstack/logging/setup.py index f714bfb1bf359..4a10d7cb7452d 100644 --- a/localstack-core/localstack/logging/setup.py +++ b/localstack-core/localstack/logging/setup.py @@ -84,21 +84,14 @@ def setup_logging_from_config(): raw_logging_override = config.LOG_LEVEL_OVERRIDES if raw_logging_override: - try: - logging_overrides = key_value_pairs_to_dict(raw_logging_override) - for logger, level_name in logging_overrides.items(): - level = getattr(logging, level_name, None) - if not level: - raise ValueError( - f"Failed to configure logging overrides ({raw_logging_override}): '{level_name}' is not a valid log level" - ) - logging.getLogger(logger).setLevel(level) - except ValueError: - raise - except Exception as e: - raise RuntimeError( - f"Failed to configure logging overrides ({raw_logging_override})" - ) from e + logging_overrides = key_value_pairs_to_dict(raw_logging_override) + for logger, level_name in logging_overrides.items(): + level = getattr(logging, level_name, None) + if not level: + raise ValueError( + f"Failed to configure logging overrides ({raw_logging_override}): '{level_name}' is not a valid log level" + ) + logging.getLogger(logger).setLevel(level) def create_default_handler(log_level: int):