8000 Add metrics when using the requests client (#226) · python-microservices/pyms@c098a09 · GitHub
[go: up one dir, main page]
Skip to content

Commit c098a09

Browse files
alexppgalexppg
authored
Add metrics when using the requests client (#226)
* feat: add metrics when using the requests client * feat: Update pyms/flask/services/requests.py * fix: check if metrics are enabled before using them * fix: change metrics name * fix: move metrics checks to outside class
1 parent 9854c50 commit c098a09

File tree

4 files changed

+97
-26
lines changed

4 files changed

+97
-26
lines changed

‎pyms/flask/services/metrics.py

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,23 @@
44
from flask import Blueprint, Flask, Response, request
55
from prometheus_client import REGISTRY, CollectorRegistry, Counter, Histogram, generate_latest, multiprocess
66

7-
from pyms.flask.services.driver import DriverService
7+
from pyms.flask.services.driver import DriverService, get_service_name
8+
from pyms.config.conf import get_conf
89

910
# Based on https://github.com/sbarratt/flask-prometheus
1011
# and https://github.com/korfuri/python-logging-prometheus/
1112

12-
FLASK_REQUEST_LATENCY = Histogram(
13-
"http_server_requests_seconds", "Flask Request Latency", ["service", "method", "uri", "status"]
14-
)
13+
METRICS_CONFIG = get_conf(service=get_service_name(
14+
service="metrics"), empty_init=True)
15+
1516
FLASK_REQUEST_COUNT = Counter(
16-
"http_server_requests_count", "Flask Request Count", ["service", "method", "uri", "status"]
17+
"http_server_requests_count", "Flask Request Count", [
18+
"service", "method", "uri", "status"]
19+
)
20+
21+
FLASK_REQUEST_LATENCY = Histogram(
22+
"http_server_requests_seconds", "Flask Request Latency", [
23+
"service", "method", "uri", "status"]
1724
)
1825

1926
LOGGER_TOTAL_MESSAGES = Counter(
@@ -36,8 +43,10 @@ def after_request(self, response: Response) -> Response:
3643
else:
3744
path = request.path
3845
request_latency = time.time() - request.start_time
39-
FLASK_REQUEST_LATENCY.labels(self.app_name, request.method, path, response.status_code).observe(request_latency)
40-
FLASK_REQUEST_COUNT.labels(self.app_name, request.method, path, response.status_code).inc()
46+
FLASK_REQUEST_COUNT.labels(
47+
self.app_name, request.method, path, response.status_code).inc()
48+
FLASK_REQUEST_LATENCY.labels(
49+
self.app_name, request.method, path, response.status_code).observe(request_latency)
4150

4251
return response
4352

@@ -54,11 +63,14 @@ def __init__(self, *args, **kwargs):
5463
self.serve_metrics()
5564

5665
def init_action(self, microservice_instance):
57-
microservice_instance.application.register_blueprint(microservice_instance.metrics.metrics_blueprint)
66+
microservice_instance.application.register_blueprint(
67+
microservice_instance.metrics.metrics_blueprint)
5868
self.add_logger_handler(
59-
microservice_instance.application.logger, microservice_instance.application.config["APP_NAME"]
69+
microservice_instance.application.logger, microservice_instance.application.config[
70+
"APP_NAME"]
6071
)
61-
self.monitor(microservice_instance.application.config["APP_NAME"], microservice_instance.application)
72+
self.monitor(
73+
microservice_instance.application.config["APP_NAME"], microservice_instance.application)
6274

6375
def init_registry(self) -> None:
6476
try:

‎pyms/flask/services/requests.py

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,21 @@
99
from urllib3.util.retry import Retry
1010

1111
from pyms.constants import LOGGER_NAME
12-
from pyms.flask.services.driver import DriverService
12+
from pyms.flask.services.driver import DriverService, get_service_name
1313
from pyms.flask.services.tracer import inject_span_in_headers
14+
from pyms.config.conf import get_conf
15+
16+
try:
17+
from prometheus_client import Counter, Histogram
18+
19+
REQUESTS_COUNT = Counter(
20+
"http_client_requests_count", "Python requests count", ["service", "method", "uri", "status"]
21+
)
22+
REQUESTS_LATENCY = Histogram(
23+
"http_client_requests_seconds", "Python requests latency", ["service", "method", "uri", "status"]
24+
)
25+
except ModuleNotFoundError: # pragma: no cover
26+
pass
1427

1528
logger = logging.getLogger(LOGGER_NAME)
1629

@@ -55,6 +68,9 @@ class Service(DriverService):
5568
}
5669
tracer = None
5770

71+
def init_action(self, microservice_instance):
72+
self.app_name = microservice_instance.application.config["APP_NAME"] # pylint: disable=W0201
73+
5874
def requests(self, session: requests.Session) -> requests.Session:
5975
"""
6076
A backoff factor to apply between attempts after the second try (most errors are resolved immediately by a
@@ -75,6 +91,10 @@ def requests(self, session: requests.Session) -> requests.Session:
7591
adapter = HTTPAdapter(max_retries=max_retries)
7692
session_r.mount("http://", adapter)
7793
session_r.mount("https://", adapter)
94+
95+
metrics_enabled = get_conf(service=get_service_name(service="metrics"), empty_init=True)
96+
if metrics_enabled:
97+
session_r.hooks["response"] = [self.observe_requests]
7898
return session_r
7999

80100
@staticmethod
@@ -355,3 +375,9 @@ def delete(self, url: str, path_params: dict = None, headers: dict = None, **kwa
355375
logger.debug("Response {}".format(response))
356376

357377
return response
378+
379+
def observe_requests(self, response, *args, **kwargs):
380+
REQUESTS_COUNT.labels(self.app_name, response.request.method, response.url, response.status_code).inc()
381+
REQUESTS_LATENCY.labels(self.app_name, response.request.method, response.url, response.status_code).observe(
382+
float(response.elapsed.total_seconds())
383+
)

‎tests/config-tests-metrics.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
pyms:
33
services:
44
metrics: true
5+
requests:
6+
data: /
57
tracer:
68
client: "jaeger"
79
component_name: "Python Microservice with Jaeger"

‎tests/test_metrics.py

Lines changed: 46 additions & 15 deletions
< 10000 td data-grid-cell-id="diff-c013839ae9f8d11da7ba9ed964de9d4b9ef733c1dd3d1a5487f32a5b2062e1cf-46-48-0" data-selected="false" role="gridcell" style="background-color:var(--bgColor-default);text-align:center" tabindex="-1" valign="top" class="focusable-grid-cell diff-line-number position-relative diff-line-number-neutral left-side">46
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from pathlib import Path
44
from tempfile import TemporaryDirectory
55

6+
import requests_mock
67
from opentracing import global_tracer
78
from prometheus_client import generate_latest, values
89

@@ -36,23 +37,46 @@ def setUp(self):
3637
ms.reload_conf()
3738
self.app = ms.create_app()
3839
self.client = self.app.test_client()
40+
self.request = ms.requests
3941

40-
def test_metrics_latency(self):
42+
def test_metrics_requests_latency(self):
4143
self.client.get("/")
4244
self.client.get("/metrics")
4345
generated_latency_root = b'http_server_requests_seconds_bucket{le="0.005",method="GET",service="Python Microservice with Jaeger",status="404",uri="/"}'
4446
generated_latency_metrics = b'http_server_requests_seconds_bucket{le="0.005",method="GET",service="Python Microservice with Jaeger",status="200",uri="/metrics"}'
4547
assert generated_latency_root in generate_latest()
48
assert generated_latency_metrics in generate_latest()
4749

48-
def test_metrics_count(self):
50+
def test_metrics_requests_count(self):
4951
self.client.get("/")
5052
self.client.get("/metrics")
5153
generated_count_root = b'http_server_requests_count_total{method="GET",service="Python Microservice with Jaeger",status="404",uri="/"}'
5254
generated_count_metrics = b'http_server_requests_count_total{method="GET",service="Python Microservice with Jaeger",status="200",uri="/metrics"}'
5355
assert generated_count_root in generate_latest()
5456
assert generated_count_metrics in generate_latest()
5557

58+
@requests_mock.Mocker()
59+
def test_metrics_responses_latency(self, mock_request):
60+
url = "http://www.my-site.com/users"
61+
full_url = url
62+
with self.app.app_context():
63+
mock_request.get(full_url)
64+
self.request.get(url)
65+
self.client.get("/metrics")
66+
generated_latency_url = b'http_client_requests_seconds_bucket{le="0.005",method="GET",service="Python Microservice with Jaeger",status="200",uri="http://www.my-site.com/users"}'
67+
assert generated_latency_url in generate_latest()
68+
69+
@requests_mock.Mocker()
70+
def test_metrics_responses_count(self, mock_request):
71+
url = "http://www.my-site.com/users"
72+
full_url = url
73+
with self.app.app_context():
74+
mock_request.get(full_url)
75+
self.request.get(url)
76+
self.client.get("/metrics")
77+
generated_count_url = b'http_client_requests_count_total{method="GET",service="Python Microservice with Jaeger",status="200",uri="http://www.my-site.com/users"}'
78+
assert generated_count_url in generate_latest()
79+
5680
def test_metrics_logger(self):
5781
self.client.get("/")
5882
self.client.get("/metrics")
@@ -94,6 +118,7 @@ def setUp(self):
94118
for path in Path(self.temp_dir.name).iterdir():
95119
if self._testMethodName not in path.name:
96120
path.unlink()
121+
self.request = ms.requests
97122

98123
@classmethod
99124
def tearDownClass(cls):
@@ -110,21 +135,27 @@ def test_metrics_stored_in_directory(self):
110135
assert f"counter_{self._testMethodName}.db" in metrics
111136
assert f"histogram_{self._testMethodName}.db" in metrics
112137

113-
def test_metrics_latency(self):
114-
self.client.get("/")
138+
@requests_mock.Mocker()
139+
def test_metrics_responses_latency(self, mock_request):
140+
url = "http://www.my-site.com/users"
141+
full_url = url
142+
with self.app.app_context():
143+
mock_request.get(full_url)
144+
self.request.get(url)
115145
self.client.get("/metrics")
116-
generated_latency_root = b'http_server_requests_seconds_bucket{le="0.005",method="GET",service="Python Microservice with Jaeger",status="404",uri="/"}'
117-
generated_latency_metrics = b'http_server_requests_seconds_bucket{le="0.005",method="GET",service="Python Microservice with Jaeger",status="200",uri="/metrics"}'
118-
assert generated_latency_root in generate_latest(self.app.ms.metrics.registry)
119-
assert generated_latency_metrics in generate_latest(self.app.ms.metrics.registry)
120-
121-
def test_metrics_count(self):
122-
self.client.get("/")
146+
generated_latency_url = b'http_client_requests_seconds_bucket{le="0.005",method="GET",service="Python Microservice with Jaeger",status="200",uri="http://www.my-site.com/users"}'
147+
assert generated_latency_url in generate_latest()
148+
149+
@requests_mock.Mocker()
150+
def test_metrics_responses_count(self, mock_request):
151+
url = "http://www.my-site.com/users"
152+
full_url = url
153+
with self.app.app_context():
154+
mock_request.get(full_url)
155+
self.request.get(url)
123156
self.client.get("/metrics")
124-
generated_count_root = b'http_server_requests_count_total{method="GET",service="Python Microservice with Jaeger",status="404",uri="/"}'
125-
generated_count_metrics = b'http_server_requests_count_total{method="GET",service="Python Microservice with Jaeger",status="200",uri="/metrics"}'
126-
assert generated_count_root in generate_latest(self.app.ms.metrics.registry)
127-
assert generated_count_metrics in generate_latest(self.app.ms.metrics.registry)
157+
generated_count_url = b'http_client_requests_count_total{method="GET",service="Python Microservice with Jaeger",status="200",uri="http://www.my-site.com/users"}'
158+
assert generated_count_url in generate_latest()
128159

129160
def test_metrics_logger(self):
130161
self.client.get("/")

0 commit comments

Comments
 (0)
0