8000 [Prometheus] Add & instrument Lambda environment metrics (#94) · localstack/localstack-extensions@34b22f5 · GitHub
[go: up one dir, main page]
Skip to content

    • Commit 34b22f5

    Browse files
    gregfurmangregfurman
    authored
    [Prometheus] Add & instrument Lambda environment metrics (#94)
    1 parent 67ad5d4 commit 34b22f5

    File tree

    8 files changed

    +144
    -22
    lines changed

    8 files changed

    +144
    -22
    lines changed

    ‎prometheus/localstack_prometheus/expose.py

    Lines changed: 1 addition & 3 deletions
    Original file line numberDiff line numberDiff line change
    @@ -4,8 +4,6 @@
    44

    55
    def retrieve_metrics(request: http.Request):
    66
    """Expose the Prometheus metrics"""
    7-
    _generate_latest_metrics, content_type = choose_encoder(
    8-
    request.headers.get("Content-Type", "")
    9-
    )
    7+
    _generate_latest_metrics, content_type = choose_encoder(request.headers.get("Content-Type", ""))
    108
    data = _generate_latest_metrics()
    119
    return http.Response(response=data, status=200, mimetype=content_type)

    ‎prometheus/localstack_prometheus/extension.py

    Lines changed: 5 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -9,7 +9,10 @@
    99

    1010
    from localstack_prometheus.expose import retrieve_metrics
    1111
    from localstack_prometheus.handler import RequestMetricsHandler, ResponseMetricsHandler
    12-
    from localstack_prometheus.instruments.patch import apply_poller_tracking_patches
    12+
    from localstack_prometheus.instruments.patch import (
    13+
    apply_lambda_tracking_patches,
    14+
    apply_poller_tracking_patches,
    15+
    )
    1316

    1417
    LOG = logging.getLogger(__name__)
    1518

    @@ -18,6 +21,7 @@ class PrometheusMetricsExtension(Extension):
    1821
    name = "prometheus"
    1922

    2023
    def on_extension_load(self):
    24+
    apply_lambda_tracking_patches()
    2125
    apply_poller_tracking_patches()
    2226
    LOG.debug("PrometheusMetricsExtension: extension is loaded")
    2327

    ‎prometheus/localstack_prometheus/handler.py

    Lines changed: 5 additions & 13 deletions
    Original file line numberDiff line numberDiff line change
    @@ -6,7 +6,7 @@
    66
    from localstack.http import Response
    77

    88
    from localstack_prometheus.metrics.core import (
    9-
    LOCALSTACK_IN_FLIGHT_REQUESTS_GAUGE,
    9+
    LOCALSTACK_IN_FLIGHT_REQUESTS,
    1010
    LOCALSTACK_REQUEST_PROCESSING_DURATION_SECONDS,
    1111
    )
    1212

    @@ -22,9 +22,7 @@ class RequestMetricsHandler(Handler):
    2222
    Handler that records the start time of incoming requests
    2323
    """
    2424

    25-
    def __call__(
    26-
    self, chain: HandlerChain, context: TimedRequestContext, response: Response
    27-
    ):
    25+
    def __call__(self, chain: HandlerChain, context: TimedRequestContext, response: Response):
    2826
    # Record the start time
    2927
    context.start_time = time.perf_counter()
    3028

    @@ -33,27 +31,21 @@ def __call__(
    3331
    return
    3432

    3533
    service, operation = context.service_operation
    36-
    LOCALSTACK_IN_FLIGHT_REQUESTS_GAUGE.labels(
    37-
    service=service, operation=operation
    38-
    ).inc()
    34+
    LOCALSTACK_IN_FLIGHT_REQUESTS.labels(service=service, operation=operation).inc()
    3935

    4036

    4137
    class ResponseMetricsHandler(Handler):
    4238
    """
    4339
    Handler that records metrics when a response is ready
    4440
    """
    4541

    46-
    def __call__(
    47-
    self, chain: HandlerChain, context: TimedRequestContext, response: Response
    48-
    ):
    42+
    def __call__(self, chain: HandlerChain, context: TimedRequestContext, response: Response):
    4943
    # Do not record metrics if no service operation information is found
    5044
    if not context.service_operation:
    5145
    return
    5246

    5347
    service, operation = context.service_operation
    54-
    LOCALSTACK_IN_FLIGHT_REQUESTS_GAUGE.labels(
    55-
    service=service, operation=operation
    56-
    ).dec()
    48+
    LOCALSTACK_IN_FLIGHT_REQUESTS.labels(service=service, operation=operation).dec()
    5749

    5850
    # Do not record if response is None
    5951
    if response is None:

    ‎prometheus/localstack_prometheus/instruments/lambda_.py

    Lines changed: 81 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,81 @@
    1+
    import contextlib
    2+
    from typing import ContextManager
    3+
    4+
    from localstack.services.lambda_.invocation.assignment import AssignmentService
    5+
    from localstack.services.lambda_.invocation.docker_runtime_executor import (
    6+
    DockerRuntimeExecutor,
    7+
    )
    8+
    from localstack.services.lambda_.invocation.execution_environment import (
    9+
    ExecutionEnvironment,
    10+
    )
    11+
    from localstack.services.lambda_.invocation.lambda_models import (
    12+
    FunctionVersion,
    13+
    InitializationType,
    14+
    )
    15+
    16+
    from localstack_prometheus.metrics.lambda_ import (
    17+
    LOCALSTACK_LAMBDA_ENVIRONMENT_ACTIVE,
    18+
    LOCALSTACK_LAMBDA_ENVIRONMENT_CONTAINERS_RUNNING,
    19+
    LOCALSTACK_LAMBDA_ENVIRONMENT_START_TOTAL,
    20+
    )
    21+
    22+
    23+
    def count_version_environments(
    24+
    assignment_service: AssignmentService, version_manager_id: str, prov_type: InitializationType
    25+
    ):
    26+
    """Count environments of a specific provisioning type for a specific version manager"""
    27+
    return sum(
    28+
    env.initialization_type == prov_type
    29+
    for env in assignment_service.environments.get(version_manager_id, {}).values()
    30+
    )
    31+
    32+
    33+
    def count_service_environments(
    34+
    assignment_service: AssignmentService, prov_type: InitializationType
    35+
    ):
    36+
    """Count environments of a specific provisioning type across all function versions"""
    37+
    return sum(
    38+
    count_version_environments(assignment_service, version_manager_id, prov_type)
    39+
    for version_manager_id in assignment_service.environments
    40+
    )
    41+
    42+
    43+
    def init_assignment_service_with_metrics(fn, self: AssignmentService):
    44+
    fn(self)
    45+
    # Initialise these once, with all subsequent calls being evaluated at collection time.
    46+
    LOCALSTACK_LAMBDA_ENVIRONMENT_ACTIVE.labels(
    47+
    provisioning_type="provisioned-concurrency"
    48+
    ).set_function(lambda: count_service_environments(self, "provisioned-concurrency"))
    49+
    50+
    LOCALSTACK_LAMBDA_ENVIRONMENT_ACTIVE.labels(provisioning_type="on-demand").set_function(
    51+
    lambda: count_service_environments(self, "on-demand")
    52+
    )
    53+
    54+
    55+
    def tracked_docker_start(fn, self: DockerRuntimeExecutor, env_vars: dict[str, str]):
    56+
    fn(self, env_vars)
    57+
    LOCALSTACK_LAMBDA_ENVIRONMENT_CONTAINERS_RUNNING.inc()
    58+
    59+
    60+
    def tracked_docker_stop(fn, self: DockerRuntimeExecutor):
    61+
    fn(self)
    62+
    LOCALSTACK_LAMBDA_ENVIRONMENT_CONTAINERS_RUNNING.dec()
    63+
    64+
    65+
    @contextlib.contextmanager
    66+
    def tracked_get_environment(
    67+ fn,
    68+
    self: AssignmentService,
    69+
    version_manager_id: str,
    70+
    function_version: FunctionVersion,
    71+
    provisioning_type: InitializationType,
    72+
    ) -> ContextManager[ExecutionEnvironment]:
    73+
    applicable_env_count = count_version_environments(self, version_manager_id, provisioning_type)
    74+
    # If there are no applicable environments, this will be a cold start.
    75+
    # Otherwise, it'll be warm.
    76+
    start_type = "warm" if applicable_env_count > 0 else "cold"
    77+
    LOCALSTACK_LAMBDA_ENVIRONMENT_START_TOTAL.labels(
    78+
    start_type=start_type, provisioning_type=provisioning_type
    79+
    ).inc()
    80+
    with fn(self, version_manager_id, function_version, provisioning_type) as execution_env:
    81+
    yield execution_env

    ‎prometheus/localstack_prometheus/instruments/patch.py

    Lines changed: 32 additions & 3 deletions
    Original file line numberDiff line numberDiff line change
    @@ -12,8 +12,18 @@
    1212
    from localstack.services.lambda_.event_source_mapping.senders.lambda_sender import (
    1313
    LambdaSender,
    1414
    )
    15+
    from localstack.services.lambda_.invocation.assignment import AssignmentService
    16+
    from localstack.services.lambda_.invocation.docker_runtime_executor import (
    17+
    DockerRuntimeExecutor,
    18+
    )
    1519
    from localstack.utils.patch import Patch, Patches
    1620

    21+
    from localstack_prometheus.instruments.lambda_ import (
    22+
    init_assignment_service_with_metrics,
    23+
    tracked_docker_start,
    24+
    tracked_docker_stop,
    25+
    tracked_get_environment,
    26+
    )
    1727
    from localstack_prometheus.instruments.poller import tracked_poll_events
    1828
    from localstack_prometheus.instruments.sender import tracked_send_events
    1929
    from localstack_prometheus.instruments.sqs_poller import tracked_sqs_handle_messages
    @@ -22,6 +32,27 @@
    2232
    LOG = logging.getLogger(__name__)
    2333

    2434

    35+
    def apply_lambda_tracking_patches():
    36+
    """Apply all Lambda environment metrics tracking patches in one call"""
    37+
    patches = Patches(
    38+
    [
    39+
    # Track starting and stopping of containers function
    40+
    Patch.function(target=DockerRuntimeExecutor.start, fn=tracked_docker_start),
    41+
    Patch.function(target=DockerRuntimeExecutor.stop, fn=tracked_docker_stop),
    42+
    # Track cold and warm starts
    43+
    Patch.function(target=AssignmentService.get_environment, fn=tracked_get_environment),
    44+
    # Track and collect all environment
    45+
    Patch.function(
    46+
    target=AssignmentService.__init__, fn=init_assignment_service_with_metrics
    47+
    ),
    48+
    ]
    49+
    )
    50+
    51+
    patches.apply()
    52+
    LOG.debug("Applied all Lambda environment tracking patches")
    53+
    return patches
    54+
    55+
    2556
    def apply_poller_tracking_patches():
    2657
    """Apply all poller metrics tracking patches in one call"""
    2758
    patches = Patches(
    @@ -34,9 +65,7 @@ def apply_poller_tracking_patches():
    3465
    Patch.function(target=LambdaSender.send_events, fn=tracked_send_events),
    3566
    # TODO: Standardise a single abstract method that all Poller subclasses can use to fetch records
    3667
    # SQS-specific patches
    37-
    Patch.function(
    38-
    target=SqsPoller.handle_messages, fn=tracked_sqs_handle_messages
    39-
    ),
    68+
    Patch.function(target=SqsPoller.handle_messages, fn=tracked_sqs_handle_messages),
    4069
    # Stream-specific patches
    4170
    Patch.function(target=KinesisPoller.get_records, fn=tracked_get_records),
    4271
    Patch.function(target=DynamoDBPoller.get_records, fn=tracked_get_records),

    ‎prometheus/localstack_prometheus/metrics/__init__.py

    Lines changed: 0 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -1 +0,0 @@
    1-

    ‎prometheus/localstack_prometheus/metrics/core.py

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -8,7 +8,7 @@
    88
    buckets=[0.005, 0.05, 0.5, 5, 30, 60, 300, 900, 3600],
    99
    )
    1010

    11-
    LOCALSTACK_IN_FLIGHT_REQUESTS_GAUGE = Gauge(
    11+
    LOCALSTACK_IN_FLIGHT_REQUESTS = Gauge(
    1212
    "localstack_in_flight_requests",
    1313
    "Total number of currently in-flight requests",
    1414
    ["service", "operation"],

    ‎prometheus/localstack_prometheus/metrics/lambda_.py

    Lines changed: 19 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,19 @@
    1+
    from prometheus_client import Counter, Gauge
    2+
    3+
    # Lambda environment metrics
    4+
    LOCALSTACK_LAMBDA_ENVIRONMENT_START_TOTAL = Counter(
    5+
    "localstack_lambda_environment_start_total",
    6+
    "Total count of all Lambda environment starts.",
    7+
    ["start_type", "provisioning_type"],
    8+
    )
    9+
    10+
    LOCALSTACK_LAMBDA_ENVIRONMENT_CONTAINERS_RUNNING = Gauge(
    11+
    "localstack_lambda_environment_containers_running",
    12+
    "Number of LocalStack Lambda Docker containers currently running.",
    13+
    )
    14+
    15+
    LOCALSTACK_LAMBDA_ENVIRONMENT_ACTIVE = Gauge(
    16+
    "localstack_lambda_environments_active",
    17+
    "Number of currently active LocalStack Lambda environments.",
    18+
    ["provisioning_type"],
    19+
    )

    0 commit comments

    Comments
     (0)
    0