8000 implement text based native histograms · D1CED/prometheus_client_python@4665eed · GitHub
[go: up one dir, main page]
8000
Skip to content

Commit 4665eed

Browse files
D1CEDD1CED
committed
implement text based native histograms
1 parent e3902ea commit 4665eed

File tree

16 files changed

+550
-51
lines changed

16 files changed

+550
-51
lines changed

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ dist
77
.coverage
88
.tox
99
.*cache
10-
htmlcov
10+
htmlcov

‎docs/content/instrumenting/histogram.md

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,40 @@ def f():
2424

2525
with h.time():
2626
pass
27-
```
27+
```
28+
29+
## Native Histograms Text Format
30+
31+
You can enable the collection of observations into native histograms by setting the `native`
32+
parameter to `True` when constructing a `Histogram`.
33+
34+
Native histograms and classic histograms can be used simultaneously.
35+
36+
```python
37+
from prometheus_client import Histogram
38+
h = Histogram('request_latency_seconds', 'Description of histogram', native=True)
39+
h.observe(4.7) # Observe 4.7 (seconds in this case)
40+
```
41+
42+
Native histograms can be configured by four parameters:
43+
44+
1. The `nh_bucket_factor`,
45+
2. the `nh_max_populated_buckets`,
46+
3. the `nh_min_reset_duration`,
47+
4. the `nh_zero_threshold`.
48+
49+
It is only presented to a client that advertises support for the
50+
`application/openmetrics-text;version=1.1.0-nativehistogram.*` version.
51+
52+
### Limitations of the Current Native Histogram Implementation
53+
54+
- Only the OTel format with counter-integer type and positive observations are supported.
55+
- Exemplars are not supported, but can be exposed in a hybrid configuration on the classic histogram.
56+
- Floating point calculations around 0 or close to ±infinity is likely wonky.
57+
- Multiprocessing is not supported.
58+
59+
Sources:
60+
61+
https://github.com/prometheus/proposals/blob/main/proposals/2024-01-29_native_histograms_text_format.md
62+
https://prometheus.io/docs/specs/native_histograms/
63+
https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exponentialhistogram

‎prometheus_client/bridge/graphite.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ def push(self, prefix: str = '') -> None:
8282
for k, v in sorted(s.labels.items())])
8383
else:
8484
labelstr = ''
85-
output.append(f'{prefixstr}{_sanitize(s.name)}{labelstr} {float(s.value)} {now}\n')
85+
if s.value is not None:
86+
output.append(f'{prefixstr}{_sanitize(s.name)}{labelstr} {float(s.value)} {now}\n')
8687

8788
conn = socket.create_connection(self._address, self._timeout)
8889
conn.sendall(''.join(output).encode('ascii'))

‎prometheus_client/exposition.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -317,9 +317,12 @@ def sample_line(samples):
317317
def choose_encoder(accept_header: str) -> Tuple[Callable[[CollectorRegistry], bytes], str]:
318318
accept_header = accept_header or ''
319319
for accepted in accept_header.split(','):
320-
if accepted.split(';')[0].strip() == 'application/openmetrics-text':
321-
return (openmetrics.generate_latest,
322-
openmetrics.CONTENT_TYPE_LATEST)
320+
mime_type, *specifiers = map(str.strip, accepted.split(';'))
321+
if mime_type == 'application/openmetrics-text':
322+
if 'version=1.1.0-nativehistogram.*' in specifiers:
323+
return openmetrics.generate_nh, openmetrics.CONTENT_TYPE_NH
324+
else:
325+
return openmetrics.generate_latest, openmetrics.CONTENT_TYPE_LATEST
323326
return generate_latest, CONTENT_TYPE_LATEST
324327

325328

‎prometheus_client/metrics.py

Lines changed: 118 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from math import floor, log2
12
import os
23
from threading import Lock
34
import time
@@ -555,12 +556,32 @@ def create_response(request):
555556
with REQUEST_TIME.time():
556557
pass # Logic to be timed
557558
559+
There are two kinds of histograms: classic and native. A Histogram object can be both.
560+
561+
For classic histograms you can configure buckets.
562+
558563
The default buckets are intended to cover a typical web/rpc request from milliseconds to seconds.
559564
They can be overridden by passing `buckets` keyword argument to `Histogram`.
565+
566+
For native histograms you can set a schema, the maximum count of populated
567+
buckets, a reset timer and a zero threshold.
568+
569+
The schema is set indirectly by the `nh_bucket_factor` which determines how much larger the
570+
next higher bucket is compared to a chosen one. It must be a float greater than one.
571+
572+
If one more than the maximum number of populated buckets is filled excluding the zero bucket
573+
the histogram is reset except if the duration since the last reset is less than the reset time.
574+
In this case the resolution of the histogram is reduced.
575+
In case the resultion was reduced as soon as the reset time is reached since the last reset
576+
the histogram is reset to its initial schema.
560577
"""
561578
_type = 'histogram'
562579
_reserved_labelnames = ['le']
563580
DEFAULT_BUCKETS = (.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, INF)
581+
DEFAULT_BUCKET_FACTOR = 1.1
582+
DEFAULT_MIN_RESET_DURATION_SECONDS = 10 * 60
583+
DEFAULT_MAX_POPULATED_BUCKETS = 100
584+
DEFAULT_ZERO_THRESHOLD = 1 / 2**2**4
564585

565586
def __init__(self,
566587
name: str,
@@ -572,8 +593,38 @@ def __init__(self,
572593
registry: Optional[CollectorRegistry] = REGISTRY,
573594
_labelvalues: Optional[Sequence[str]] = None,
574595
buckets: Sequence[Union[float, str]] = DEFAULT_BUCKETS,
596+
*,
597+
classic: bool = True,
598+
native: bool = False,
599+
nh_bucket_factor: float = DEFAULT_BUCKET_FACTOR,
600+
nh_max_populated_buckets: int = DEFAULT_MAX_POPULATED_BUCKETS,
601+
nh_min_reset_duration_seconds: int = DEFAULT_MIN_RESET_DURATION_SECONDS,
602+
nh_zero_threshold: float = DEFAULT_ZERO_THRESHOLD,
575603
):
576-
self._prepare_buckets(buckets)
604+
if not (classic or native):
605+
raise ValueError('Histogram must be classic or native or both')
606+
607+
if classic:
608+
self._upper_bounds = self._prepare_buckets(buckets)
609+
610+
if native:
611+
if nh_bucket_factor <= 1:
612+
raise ValueError('native_histogram_bucket_factor must be greater than one')
613+
if nh_min_reset_duration_seconds <= 0:
614+
raise ValueError('min_reset_duration_seconds must be positive')
615+
if nh_zero_threshold is not None and nh_zero_threshold < 0:
616+
raise ValueError('zero_threshold must be non-negative or None')
617+
if values.ValueClass._multiprocess:
618+
raise ValueError('native histograms are only supported in threaded mode')
619+
620+
self._nh_bucket_factor = nh_bucket_factor
621+
self._nh_max_populated_buckets = nh_max_populated_buckets
622+
self._nh_min_reset_duration_seconds = nh_min_reset_duration_seconds
623+
self._nh_zero_threshold = nh_zero_threshold
624+
625+
self._is_classic_histogram = classic
626+
self._is_native_histogram = native
627+
577628
super().__init__(
578629
name=name,
579630
documentation=documentation,
@@ -584,9 +635,17 @@ def __init__(self,
584635
registry=registry,
585636
_labelvalues=_labelvalues,
586637
)
587-
self._kwargs['buckets'] = buckets
588638

589-
def _prepare_buckets(self, source_buckets: Sequence[Union[float, str]]) -> None:
639+
self._kwargs['buckets'] = buckets
640+
self._kwargs['classic'] = classic
641+
self._kwargs['native'] = native
642+
self._kwargs['nh_bucket_factor'] = nh_bucket_factor
643+
self._kwargs['nh_max_populated_buckets'] = nh_max_populated_buckets
644+
self._kwargs['nh_min_reset_duration_seconds'] = nh_min_reset_duration_seconds
645+
self._kwargs['nh_zero_threshold'] = nh_zero_threshold
646+
647+
@staticmethod
648+
def _prepare_buckets(source_buckets: Sequence[Union[float, str]]) -> Sequence[float]:
590649
buckets = [float(b) for b in source_buckets]
591650
if buckets != sorted(buckets):
592651
# This is probably an error on the part of the user,
@@ -596,21 +655,37 @@ def _prepare_buckets(self, source_buckets: Sequence[Union[float, str]]) -> None:
596655
buckets.append(INF)
597656
if len(buckets) < 2:
598657
raise ValueError('Must have at least two buckets')
599-
self._upper_bounds = buckets
658+
return buckets
659+
660+
@staticmethod
661+
def _choose_schema_from_bucket_factor(bucket_factor: float) -> int:
662+
schema = -floor(log2(log2(bucket_factor)))
663+
return max(min(schema, 8), -4)
600664

601665
def _metric_init(self) -> None:
602-
self._buckets: List[values.ValueClass] = []
603666
self._created = time.time()
604-
bucket_labelnames = self._labelnames + ('le',)
605-
self._sum = values.ValueClass(self._type, self._name, self._name + '_sum', self._labelnames, self._labelvalues, self._documentation)
606-
for b in self._upper_bounds:
607-
self._buckets.append(values.ValueClass(
608-
self._type,
609-
self._name,
610-
self._name + '_bucket',
611-
bucket_labelnames,
612-
self._labelvalues + (floatToGoString(b),),
613-
self._documentation)
667+
668+
if self._is_classic_histogram:
669+
self._buckets: List[values.ValueClass] = []
670+
bucket_labelnames = self._labelnames + ('le',)
671+
self._sum = values.ValueClass(self._type, self._name, self._name + '_sum', self._labelnames, self._labelvalues, self._documentation)
672+
for b in self._upper_bounds:
673+
self._buckets.append(values.ValueClass(
674+
self._type,
675+
self._name,
676+
self._name + '_bucket',
677+
bucket_labelnames,
678+
self._labelvalues + (floatToGoString(b),),
679+
self._documentation)
680+
)
681+
682+
if self._is_native_histogram:
683+
schema = self._choose_schema_from_bucket_factor(self._nh_bucket_factor)
684+
self._native_histogram = values.ThreadSafeNativeHistogram(
685+
schema=schema,
686+
zero_threshold=self._nh_zero_threshold,
687+
max_populated_buckets=self._nh_max_populated_buckets,
688+
min_reset_duration_seconds=self._nh_min_reset_duration_seconds,
614689
)
615690

616691
def observe(self, amount: float, exemplar: Optional[Dict[str, str]] = None) -> None:
@@ -624,14 +699,19 @@ def observe(self, amount: float, exemplar: Optional[Dict[str, str]] = None) -> N
624699
for details.
625700
"""
626701
self._raise_if_not_observable()
627-
self._sum.inc(amount)
628-
for i, bound in enumerate(self._upper_bounds):
629-
if amount <= bound:
630-
self._buckets[i].inc(1)
631-
if exemplar:
632-
_validate_exemplar(exemplar)
633-
self._buckets[i].set_exemplar(Exemplar(exemplar, amount, time.time()))
634-
break
702+
703+
if self._is_classic_histogram:
704+
self._sum.inc(amount)
705+
for i, bound in enumerate(self._upper_bounds):
706+
if amount <= bound:
707+
self._buckets[i].inc(1)
708+
if exemplar:
709+
_validate_exemplar(exemplar)
710+
self._buckets[i].set_exemplar(Exemplar(exemplar, amount, time.time()))
711+
break
712+
713+
if self._is_native_histogram:
714+
self._native_histogram.add_observation(amount)
635715

636716
def time(self) -> Timer:
637717
"""Time a block of code or function, and observe the duration in seconds.
@@ -642,15 +722,23 @@ def time(self) -> Timer:
642722

643723
def _child_samples(self) -> Iterable[Sample]:
644724
samples = []
645-
acc = 0.0
646-
for i, bound in enumerate(self._upper_bounds):
647-
acc += self._buckets[i].get()
648-
samples.append(Sample('_bucket', {'le': floatToGoString(bound)}, acc, None, self._buckets[i].get_exemplar()))
649-
samples.append(Sample('_count', {}, acc, None, None))
650-
if self._upper_bounds[0] >= 0:
651-
samples.append(Sample('_sum', {}, self._sum.get(), None, None))
725+
726+
# must come first
727+
if self._is_native_histogram:
728+
samples.append(Sample('', {}, native_histogram=self._native_histogram.extract()))
729+
730+
if self._is_classic_histogram:
731+
acc = 0.0
732+
for i, bound in enumerate(self._upper_bounds):
733+
acc += self._buckets[i].get()
734+
samples.append(Sample('_bucket', {'le': floatToGoString(bound)}, acc, None, self._buckets[i].get_exemplar()))
735+
samples.append(Sample('_count', {}, acc, None, None))
736+
if self._upper_bounds[0] >= 0:
737+
samples.append(Sample('_sum', {}, self._sum.get(), None, None))
738+
652739
if _use_created:
653740
samples.append(Sample('_created', {}, self._created, None, None))
741+
654742
return tuple(samples)
655743

656744

‎prometheus_client/metrics_core.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ def __init__(self, name: str, documentation: str, typ: str, unit: str = ''):
3232
self.type: str = typ
3333
self.samples: List[Sample] = []
3434

35-
def add_sample(self, name: str, labels: Dict[str, str], value: float, timestamp: Optional[Union[Timestamp, float]] = None, exemplar: Optional[Exemplar] = None, native_histogram: Optional[NativeHistogram] = None) -> None:
35+
def add_sample(self, name: str, labels: Dict[str, str], value: Optional[float] = None, timestamp: Optional[Union[Timestamp, float]] = None, exemplar: Optional[Exemplar] = None, native_histogram: Optional[NativeHistogram] = None) -> None:
3636
"""Add a sample to the metric.
3737
3838
Internal-only, do not use."""

‎prometheus_client/openmetrics/exposition.py

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/usr/bin/env python
22

33

4+
from ..samples import NativeHistogram
45
from ..utils import floatToGoString
56
from ..validation import (
67
_is_valid_legacy_labelname, _is_valid_legacy_metric_name,
@@ -9,6 +10,8 @@
910
CONTENT_TYPE_LATEST = 'application/openmetrics-text; version=1.0.0; charset=utf-8'
1011
"""Content type of the latest OpenMetrics text format"""
1112

13+
CONTENT_TYPE_NH = 'application/openmetrics-text; version=1.1.0-nativehistogram.*; charset=utf-8'
14+
1215

1316
def _is_valid_exemplar_metric(metric, sample):
1417
if metric.type == 'counter' and sample.name.endswith('_total'):
@@ -20,7 +23,7 @@ def _is_valid_exemplar_metric(metric, sample):
2023
return False
2124

2225

23-
def generate_latest(registry):
26+
def generate_latest(registry, allow_native_histograms=False):
2427
'''Returns the metrics from the registry in latest text format as a string.'''
2528
output = []
2629
for metric in registry.collect():
@@ -32,6 +35,8 @@ def generate_latest(registry):
3235
if metric.unit:
3336
output.append(f'# UNIT {escape_metric_name(mname)} {metric.unit}\n')
3437
for s in metric.samples:
38+
if s.native_histogram is not None and not allow_native_histograms:
39+
continue
3540
if not _is_valid_legacy_metric_name(s.name):
3641
labelstr = escape_metric_name(s.name)
3742
if s.labels:
@@ -71,18 +76,27 @@ def generate_latest(registry):
7176
timestamp = ''
7277
if s.timestamp is not None:
7378
timestamp = f' {s.timestamp}'
79+
80+
value = None
81+
if s.value is not None:
82+
value = floatToGoString(s.value)
83+
elif s.native_histogram is not None:
84+
value = native_histogram_as_str(s.native_histogram)
85+
else:
86+
raise ValueError('sample must hold float or native_histogram')
87+
7488
if _is_valid_legacy_metric_name(s.name):
7589
output.append('{}{} {}{}{}\n'.format(
7690
s.name,
7791
labelstr,
78-
floatToGoString(s.value),
92+
value,
7993
timestamp,
8094
exemplarstr,
8195
))
8296
else:
8397
output.append('{} {}{}{}\n'.format(
8498
labelstr,
85-
floatToGoString(s.value),
99+
value,
86100
timestamp,
87101
exemplarstr,
88102
))
@@ -94,6 +108,28 @@ def generate_latest(registry):
94108
return ''.join(output).encode('utf-8')
95109

96110

111+
def generate_nh(registry):
112+
return generate_latest(registry, allow_native_histograms=True)
113+
114+
115+
def native_histogram_as_str(native_histogram: NativeHistogram) -> str:
116+
nh_sum = floatToGoString(native_histogram.sum_value)
117+
nh_count = int(native_histogram.count_value)
118+
nh_schema = native_histogram.schema
119+
nh_zero_threshold = floatToGoString(native_histogram.zero_threshold)
120+
nh_zero_count = int(native_histogram.zero_count)
121+
122+
nh_pos_spans = ''
123+
if native_histogram.pos_spans:
124+
nh_pos_spans = ','.join(f'{span.offset}:{span.length}' for span in native_histogram.pos_spans)
125+
126+
nh_pos_deltas = ''
127+
if native_histogram.pos_deltas:
128+
nh_pos_deltas = ','.join(map(str, native_histogram.pos_deltas))
129+
130+
return f'{{sum:{nh_sum},count:{nh_count},schema:{nh_schema},zero_threshold:{nh_zero_threshold},zero_count:{nh_zero_count},positive_spans:[{nh_pos_spans}],positive_deltas:[{nh_pos_deltas}]}}'
131+
132+
97133
def escape_metric_name(s: str) -> str:
98134
"""Escapes the metric name and puts it in quotes iff the name does not
99135
conform to the legacy Prometheus character set.

‎prometheus_client/openmetrics/parser.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,7 @@ def _parse_nh_struct(text):
315315
deltas = dict(re_deltas.findall(text))
316316

317317
count_value = int(items['count'])
318-
sum_value = int(items['sum'])
318+
sum_value = float(items['sum'])
319319
schema = int(items['schema'])
320320
zero_threshold = float(items['zero_threshold'])
321321
zero_count = int(items['zero_count'])

0 commit comments

Comments
 (0)
0