From a008c2846892ac3a594fe8caaaa7f387481ca11f Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 2 Nov 2020 21:01:06 +0100 Subject: [PATCH 01/27] feat: added basic consul register --- .../config.yml | 10 +++ .../main.py | 15 ++++ pyms/flask/app/create_app.py | 36 +++------- pyms/flask/services/service_discovery.py | 71 +++++++++++++++++++ 4 files changed, 106 insertions(+), 26 deletions(-) create mode 100644 examples/microservice_service_discovery_consul/config.yml create mode 100644 examples/microservice_service_discovery_consul/main.py create mode 100644 pyms/flask/services/service_discovery.py diff --git a/examples/microservice_service_discovery_consul/config.yml b/examples/microservice_service_discovery_consul/config.yml new file mode 100644 index 0000000..7081292 --- /dev/null +++ b/examples/microservice_service_discovery_consul/config.yml @@ -0,0 +1,10 @@ +pyms: + services: + service_discovery: + client: "consul" + host: "http://localhost:8500" + config: + DEBUG: true + TESTING: false + APP_NAME: "Python Microservice" + APPLICATION_ROOT: "" \ No newline at end of file diff --git a/examples/microservice_service_discovery_consul/main.py b/examples/microservice_service_discovery_consul/main.py new file mode 100644 index 0000000..9c1e940 --- /dev/null +++ b/examples/microservice_service_discovery_consul/main.py @@ -0,0 +1,15 @@ +from flask import jsonify + +from pyms.flask.app import Microservice + +ms = Microservice(path=__file__) +app = ms.create_app() + + +@app.route("/") +def example(): + return jsonify({"main": "hello world"}) + + +if __name__ == '__main__': + app.run() diff --git a/pyms/flask/app/create_app.py b/pyms/flask/app/create_app.py index eac66b2..ccafbd8 100644 --- a/pyms/flask/app/create_app.py +++ b/pyms/flask/app/create_app.py @@ -108,6 +108,15 @@ def init_services(self) -> None: if service_name not in self.services or not getattr(self, service_name, False): self.services.append(service_name) setattr(self, service_name, service) + # if getattr(service, "init_action"): + # service.init_action(self) + + def init_services_actions(self): + for service_name in self.services: + srv_action = getattr(getattr(self, service_name), "init_action") + if srv_action: + srv_action(self) + def init_crypt(self, *args, **kwargs) -> None: """ @@ -135,15 +144,6 @@ def init_libs(self) -> Flask: """ return self.application - def init_tracer(self) -> None: - """Set attribute in flask `tracer`. See in `pyms.flask.services.tracer` how it works - :return: None - """ - if self.tracer: - FlaskTracing = import_from("flask_opentracing", "FlaskTracing") - client = self.tracer.get_client() - self.application.tracer = FlaskTracing(client, True, self.application) - def init_logger(self) -> None: """ Set a logger and return in JSON format. @@ -185,18 +185,6 @@ def init_app(self) -> Flask: return application - def init_metrics(self) -> None: - """Set attribute in flask `metrics`. See in `pyms.flask.services.metrics` how it works - :return: None - """ - if self.metrics: - self.application.register_blueprint(self.metrics.metrics_blueprint) - self.metrics.add_logger_handler( - self.application.logger, - self.application.config["APP_NAME"] - ) - self.metrics.monitor(self.application.config["APP_NAME"], self.application) - def reload_conf(self): self.delete_services() self.config.reload() @@ -214,7 +202,6 @@ def create_app(self) -> Flask: """ self.application = self.init_app() self.application.config.from_object(self.config.to_flask()) - self.application.tracer = None self.application.ms = self # Initialize Blueprints @@ -223,12 +210,9 @@ def create_app(self) -> Flask: self.init_libs() self.add_error_handlers() - - self.init_tracer() - self.init_logger() - self.init_metrics() + self.init_services_actions() logger.debug("Started app with PyMS and this services: {}".format(self.services)) diff --git a/pyms/flask/services/service_discovery.py b/pyms/flask/services/service_discovery.py new file mode 100644 index 0000000..dbdc524 --- /dev/null +++ b/pyms/flask/services/service_discovery.py @@ -0,0 +1,71 @@ +import json +import logging +import uuid + +import requests + +from pyms.constants import LOGGER_NAME +from pyms.flask.services.driver import DriverService + +logger = logging.getLogger(LOGGER_NAME) + +CONSUL_SERVICE_DISCOVERY = "consul" + +DEFAULT_SERVICE_DISCOVERY = CONSUL_SERVICE_DISCOVERY + + +class ServiceDiscoveryBase: + def register_service(self, id_app, host, port, healtcheck_url, app_name): + pass + + +class ServiceDiscoveryConsul(ServiceDiscoveryBase): + def register_service(self, id_app, host, port, healtcheck_url, app_name): + headers = { + 'Content-Type': 'application/json; charset=utf-8' + } + data = { + "id": app_name + "-" + id_app, + "name": app_name, + "port": port, + "check": { + "name": "ping check", + "http": healtcheck_url, + "interval": "30s", + "status": "passing" + } + } + response = requests.put("{host}/v1/agent/service/register".format(host=host), data=json.dumps(data), + headers=headers) + assert response.status_code == 200 + + +class Service(DriverService): + id_app = str(uuid.uuid1()) + config_resource = "service_discovery" + default_values = { + "client": DEFAULT_SERVICE_DISCOVERY, + "host": "http://localhost:8500", + "port": 5000, + "healtcheck_url": "http://127.0.0.1.nip.io:5000/healthcheck", + } + + def init_action(self, microservice_instance): + self.client.register_service( + id_app=self.id_app, + host=self.host, + healtcheck_url=self.healtcheck_url, + port=self.port, + app_name=microservice_instance.application.config["APP_NAME"]) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.client = self.get_client() + + def get_client(self) -> ServiceDiscoveryBase: + client = False + if self.config.client == CONSUL_SERVICE_DISCOVERY: + client = ServiceDiscoveryConsul() + + logger.debug("Init %s as service discovery", client) + return client From 3088274315404706b16e45d475d3d882ff57af5b Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 2 Nov 2020 21:01:28 +0100 Subject: [PATCH 02/27] refactor!: init actions in services, not in create_app --- pyms/flask/services/driver.py | 2 ++ pyms/flask/services/metrics.py | 9 ++++++++- pyms/flask/services/tracer.py | 5 +++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pyms/flask/services/driver.py b/pyms/flask/services/driver.py index 1b118c1..8aaf78a 100644 --- a/pyms/flask/services/driver.py +++ b/pyms/flask/services/driver.py @@ -39,6 +39,8 @@ class DriverService(ConfigResource): """ enabled = True + init_action = False + def __init__(self, *args, **kwargs): self.config_resource = get_service_name(service=self.config_resource) super().__init__(*args, **kwargs) diff --git a/pyms/flask/services/metrics.py b/pyms/flask/services/metrics.py index 744da4b..5d469e4 100644 --- a/pyms/flask/services/metrics.py +++ b/pyms/flask/services/metrics.py @@ -47,7 +47,6 @@ class Service(DriverService): """ Adds [Prometheus](https://prometheus.io/) metrics using the [Prometheus Client Library](https://github.com/prometheus/client_python). """ - config_resource: Text = "metrics" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -55,6 +54,14 @@ def __init__(self, *args, **kwargs): self.init_registry() self.serve_metrics() + def init_action(self, microservice_instance): + microservice_instance.application.register_blueprint(microservice_instance.metrics.metrics_blueprint) + self.add_logger_handler( + microservice_instance.application.logger, + microservice_instance.application.config["APP_NAME"] + ) + self.monitor(microservice_instance.application.config["APP_NAME"], microservice_instance.application) + def init_registry(self) -> None: try: multiprocess_registry = CollectorRegistry() diff --git a/pyms/flask/services/tracer.py b/pyms/flask/services/tracer.py index 4a5dbd4..dde7f52 100644 --- a/pyms/flask/services/tracer.py +++ b/pyms/flask/services/tracer.py @@ -58,6 +58,11 @@ class Service(DriverService): "client": DEFAULT_CLIENT, } + def init_action(self, microservice_instance): + FlaskTracing = import_from("flask_opentracing", "FlaskTracing") + client = self.get_client() + microservice_instance.application.tracer = FlaskTracing(client, True, microservice_instance.application) + def get_client(self) -> Union[bool, type]: opentracing_tracer = False if self.config.client == JAEGER_CLIENT: From 1f9b19deac030f68860f5b564bb8f8ba2386c7fc Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 2 Nov 2020 21:08:22 +0100 Subject: [PATCH 03/27] fix: linters errors --- .github/workflows/python-package.yml | 2 +- pyms/flask/app/create_app.py | 5 ++--- pyms/flask/services/metrics.py | 17 ++++++++--------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 80dba14..4824d8f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -39,7 +39,7 @@ jobs: - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 pyms --show-source --statistics --statistics + flake8 pyms --show-source --statistics - name: Lint with pylint run: | pylint --rcfile=pylintrc pyms diff --git a/pyms/flask/app/create_app.py b/pyms/flask/app/create_app.py index ccafbd8..9cb3504 100644 --- a/pyms/flask/app/create_app.py +++ b/pyms/flask/app/create_app.py @@ -9,11 +9,11 @@ from pyms.constants import LOGGER_NAME, CONFIG_BASE from pyms.crypt.driver import CryptResource from pyms.flask.app.utils import SingletonMeta, ReverseProxied -from pyms.flask.healthcheck import healthcheck_blueprint from pyms.flask.configreload import configreload_blueprint +from pyms.flask.healthcheck import healthcheck_blueprint from pyms.flask.services.driver import ServicesResource, DriverService from pyms.logger import CustomJsonFormatter -from pyms.utils import check_package_exists, import_from +from pyms.utils import check_package_exists logger = logging.getLogger(LOGGER_NAME) @@ -117,7 +117,6 @@ def init_services_actions(self): if srv_action: srv_action(self) - def init_crypt(self, *args, **kwargs) -> None: """ Set the Attributes of all service defined in config.yml and exists in `pyms.flask.service` module diff --git a/pyms/flask/services/metrics.py b/pyms/flask/services/metrics.py index 5d469e4..2ee99d9 100644 --- a/pyms/flask/services/metrics.py +++ b/pyms/flask/services/metrics.py @@ -1,10 +1,9 @@ -import time import logging - -from typing import Text +import time from flask import Blueprint, Response, request, Flask from prometheus_client import multiprocess, Counter, Histogram, generate_latest, CollectorRegistry, REGISTRY + from pyms.flask.services.driver import DriverService # Based on https://github.com/sbarratt/flask-prometheus @@ -55,12 +54,12 @@ def __init__(self, *args, **kwargs): self.serve_metrics() def init_action(self, microservice_instance): - microservice_instance.application.register_blueprint(microservice_instance.metrics.metrics_blueprint) - self.add_logger_handler( - microservice_instance.application.logger, - microservice_instance.application.config["APP_NAME"] - ) - self.monitor(microservice_instance.application.config["APP_NAME"], microservice_instance.application) + microservice_instance.application.register_blueprint(microservice_instance.metrics.metrics_blueprint) + self.add_logger_handler( + microservice_instance.application.logger, + microservice_instance.application.config["APP_NAME"] + ) + self.monitor(microservice_instance.application.config["APP_NAME"], microservice_instance.application) def init_registry(self) -> None: try: From 90c33bd99c175f68e651ad6674d611297b7564cf Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 2 Nov 2020 21:27:09 +0100 Subject: [PATCH 04/27] fix: removed assert to testing --- pyms/exceptions.py | 4 ++++ pyms/flask/services/service_discovery.py | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pyms/exceptions.py b/pyms/exceptions.py index 2a2a5f0..3258c9b 100644 --- a/pyms/exceptions.py +++ b/pyms/exceptions.py @@ -23,3 +23,7 @@ class ConfigErrorException(Exception): class PackageNotExists(Exception): pass + + +class ServiceDiscoveryConnectionException(Exception): + pass diff --git a/pyms/flask/services/service_discovery.py b/pyms/flask/services/service_discovery.py index dbdc524..45383e5 100644 --- a/pyms/flask/services/service_discovery.py +++ b/pyms/flask/services/service_discovery.py @@ -3,8 +3,9 @@ import uuid import requests - +from requests.exceptions import ConnectionError from pyms.constants import LOGGER_NAME +from pyms.exceptions import ServiceDiscoveryConnectionException from pyms.flask.services.driver import DriverService logger = logging.getLogger(LOGGER_NAME) @@ -35,9 +36,19 @@ def register_service(self, id_app, host, port, healtcheck_url, app_name): "status": "passing" } } - response = requests.put("{host}/v1/agent/service/register".format(host=host), data=json.dumps(data), - headers=headers) - assert response.status_code == 200 + error = False + msg_error = "Failed to establish a new connection" + try: + response = requests.put("{host}/v1/agent/service/register".format(host=host), data=json.dumps(data), + headers=headers) + if response.status_code != 200: + msg_error = response.content + error = True + except ConnectionError: + error = True + + if error: + raise ServiceDiscoveryConnectionException("Host %s raise an error: %s" % (host, msg_error)) class Service(DriverService): From 9012f3840cd41a8beccc5f0faf7cc4b0ec0c2a2c Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 2 Nov 2020 21:31:05 +0100 Subject: [PATCH 05/27] fix: linters errors --- pyms/flask/services/service_discovery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyms/flask/services/service_discovery.py b/pyms/flask/services/service_discovery.py index 45383e5..98d9437 100644 --- a/pyms/flask/services/service_discovery.py +++ b/pyms/flask/services/service_discovery.py @@ -3,7 +3,7 @@ import uuid import requests -from requests.exceptions import ConnectionError +from requests.exceptions import RequestException from pyms.constants import LOGGER_NAME from pyms.exceptions import ServiceDiscoveryConnectionException from pyms.flask.services.driver import DriverService @@ -44,7 +44,7 @@ def register_service(self, id_app, host, port, healtcheck_url, app_name): if response.status_code != 200: msg_error = response.content error = True - except ConnectionError: + except RequestException: error = True if error: From 60d4227592474d50ed84126ded91ee398fd7fcca Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 13:51:24 +0100 Subject: [PATCH 06/27] feature: addec consulate fork Added a fork from https://github.com/gmr/consulate --- .coveragerc | 3 +- Pipfile.lock | 128 ++-- .../config.yml | 5 +- .../main.py | 3 +- pyms/flask/services/service_discovery.py | 81 +-- pyms/services_discovery/consulate/__init__.py | 34 + pyms/services_discovery/consulate/adapters.py | 140 ++++ .../consulate/api/__init__.py | 28 + pyms/services_discovery/consulate/api/acl.py | 442 ++++++++++++ .../services_discovery/consulate/api/agent.py | 400 +++++++++++ pyms/services_discovery/consulate/api/base.py | 190 ++++++ .../consulate/api/catalog.py | 181 +++++ .../consulate/api/coordinate.py | 63 ++ .../services_discovery/consulate/api/event.py | 55 ++ .../consulate/api/health.py | 65 ++ pyms/services_discovery/consulate/api/kv.py | 407 +++++++++++ pyms/services_discovery/consulate/api/lock.py | 101 +++ .../consulate/api/session.py | 115 ++++ .../consulate/api/status.py | 31 + pyms/services_discovery/consulate/cli.py | 637 ++++++++++++++++++ pyms/services_discovery/consulate/client.py | 200 ++++++ .../consulate/exceptions.py | 49 ++ .../consulate/models/__init__.py | 2 + .../consulate/models/acl.py | 168 +++++ .../consulate/models/agent.py | 246 +++++++ .../consulate/models/base.py | 167 +++++ pyms/services_discovery/consulate/utils.py | 111 +++ setup.py | 3 +- tests/config-tests-service-discovery.yml | 9 + tests/test_metrics.py | 20 +- tests/test_service_discovery.py | 37 + 31 files changed, 4007 insertions(+), 114 deletions(-) create mode 100644 pyms/services_discovery/consulate/__init__.py create mode 100644 pyms/services_discovery/consulate/adapters.py create mode 100644 pyms/services_discovery/consulate/api/__init__.py create mode 100644 pyms/services_discovery/consulate/api/acl.py create mode 100644 pyms/services_discovery/consulate/api/agent.py create mode 100644 pyms/services_discovery/consulate/api/base.py create mode 100644 pyms/services_discovery/consulate/api/catalog.py create mode 100644 pyms/services_discovery/consulate/api/coordinate.py create mode 100644 pyms/services_discovery/consulate/api/event.py create mode 100644 pyms/services_discovery/consulate/api/health.py create mode 100644 pyms/services_discovery/consulate/api/kv.py create mode 100644 pyms/services_discovery/consulate/api/lock.py create mode 100644 pyms/services_discovery/consulate/api/session.py create mode 100644 pyms/services_discovery/consulate/api/status.py create mode 100644 pyms/services_discovery/consulate/cli.py create mode 100644 pyms/services_discovery/consulate/client.py create mode 100644 pyms/services_discovery/consulate/exceptions.py create mode 100644 pyms/services_discovery/consulate/models/__init__.py create mode 100644 pyms/services_discovery/consulate/models/acl.py create mode 100644 pyms/services_discovery/consulate/models/agent.py create mode 100644 pyms/services_discovery/consulate/models/base.py create mode 100644 pyms/services_discovery/consulate/utils.py create mode 100644 tests/config-tests-service-discovery.yml create mode 100644 tests/test_service_discovery.py diff --git a/.coveragerc b/.coveragerc index dbda08f..2fcf985 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,4 +5,5 @@ include= omit = *examples/* .tox/* - venv/* \ No newline at end of file + venv/* + pyms/services_discovery/consulate/* \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index a233362..b216970 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -23,31 +23,31 @@ }, "attrs": { "hashes": [ - "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", - "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], - "version": "==20.2.0" + "version": "==20.3.0" }, "boto3": { "hashes": [ - "sha256:f0b0a3611f9471137d3714b5dd8a8d0911f2b035f3de2d71ebe3e830e4e6d724", - "sha256:fb94f75d394fc97e5328f9b1a5badcfbd79e0ab5596254f8c49cfd40c69bb436" + "sha256:23d2575b7bd01c4e7153f283c1d1c12d329dabf78a6279d4192f2e752bb67b1a", + "sha256:cb3f4c2f2576153b845e5b4f325d54a04f4ca00c156f2063965432bfa5d714dd" ], - "version": "==1.16.6" + "version": "==1.16.13" }, "botocore": { "hashes": [ - "sha256:66e1d9f3ef22480e679fcfb1bdeeb2ef1cbe671278fd27464ab3b594a928e0bf", - "sha256:97d03523324dfff078aac12f88304f73038d449e10e05d378993fb3a22727c42" + "sha256:1b1b4cf5efd552ecc7f1ce0fc674d5fba56857db5ffe6394ee76edd1a568d7a6", + "sha256:b3b4b8fa33620f015c52e426a92e7db21b5e667ed4785c5fbc484ebfdb2b5153" ], - "version": "==1.19.6" + "version": "==1.19.13" }, "certifi": { "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", + "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" ], - "version": "==2020.6.20" + "version": "==2020.11.8" }, "cffi": { "hashes": [ @@ -465,10 +465,10 @@ }, "attrs": { "hashes": [ - "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", - "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], - "version": "==20.2.0" + "version": "==20.3.0" }, "bandit": { "hashes": [ @@ -485,10 +485,10 @@ }, "certifi": { "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", + "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" ], - "version": "==2020.6.20" + "version": "==2020.11.8" }, "cffi": { "hashes": [ @@ -840,10 +840,10 @@ }, "mkdocs-material": { "hashes": [ - "sha256:81095982fc4d634ee4b16abb91fa7b2e1f50ab7e39df67aafa8f5d4fd7b28e24", - "sha256:a4bfd736898e6bfeb2c0d00ab448f464a389fb6a7dd2a30a90343756fd8aec83" + "sha256:c0591b806e85302314bf308d61745e2cbb6b3caa3cec3d5fcb8d70a81076e7cb", + "sha256:e15b016bea35300956b66d553043331cbe1793ca53946180424b8bca21bbb781" ], - "version": "==6.1.0" + "version": "==6.1.4" }, "mkdocs-material-extensions": { "hashes": [ @@ -999,10 +999,10 @@ }, "pytest": { "hashes": [ - "sha256:7a8190790c17d79a11f847fba0b004ee9a8122582ebff4729a082c109e81a4c9", - "sha256:8f593023c1a0f916110285b6efd7f99db07d59546e3d8c36fc60e2ab05d3be92" + "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe", + "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e" ], - "version": "==6.1.1" + "version": "==6.1.2" }, "pytest-cov": { "hashes": [ @@ -1035,35 +1035,51 @@ }, "regex": { "hashes": [ - "sha256:0cb23ed0e327c18fb7eac61ebbb3180ebafed5b9b86ca2e15438201e5903b5dd", - "sha256:1a065e7a6a1b4aa851a0efa1a2579eabc765246b8b3a5fd74000aaa3134b8b4e", - "sha256:1a511470db3aa97432ac8c1bf014fcc6c9fbfd0f4b1313024d342549cf86bcd6", - "sha256:1c447b0d108cddc69036b1b3910fac159f2b51fdeec7f13872e059b7bc932be1", - "sha256:2278453c6a76280b38855a263198961938108ea2333ee145c5168c36b8e2b376", - "sha256:240509721a663836b611fa13ca1843079fc52d0b91ef3f92d9bba8da12e768a0", - "sha256:4e21340c07090ddc8c16deebfd82eb9c9e1ec5e62f57bb86194a2595fd7b46e0", - "sha256:570e916a44a361d4e85f355aacd90e9113319c78ce3c2d098d2ddf9631b34505", - "sha256:59d5c6302d22c16d59611a9fd53556554010db1d47e9df5df37be05007bebe75", - "sha256:6a46eba253cedcbe8a6469f881f014f0a98819d99d341461630885139850e281", - "sha256:6f567df0601e9c7434958143aebea47a9c4b45434ea0ae0286a4ec19e9877169", - "sha256:781906e45ef1d10a0ed9ec8ab83a09b5e0d742de70e627b20d61ccb1b1d3964d", - "sha256:8469377a437dbc31e480993399fd1fd15fe26f382dc04c51c9cb73e42965cc06", - "sha256:8cd0d587aaac74194ad3e68029124c06245acaeddaae14cb45844e5c9bebeea4", - "sha256:97a023f97cddf00831ba04886d1596ef10f59b93df7f855856f037190936e868", - "sha256:a973d5a7a324e2a5230ad7c43f5e1383cac51ef4903bf274936a5634b724b531", - "sha256:af360e62a9790e0a96bc9ac845d87bfa0e4ee0ee68547ae8b5a9c1030517dbef", - "sha256:b706c70070eea03411b1761fff3a2675da28d042a1ab7d0863b3efe1faa125c9", - "sha256:bfd7a9fddd11d116a58b62ee6c502fd24cfe22a4792261f258f886aa41c2a899", - "sha256:c30d8766a055c22e39dd7e1a4f98f6266169f2de05db737efe509c2fb9c8a3c8", - "sha256:c53dc8ee3bb7b7e28ee9feb996a0c999137be6c1d3b02cb6b3c4cba4f9e5ed09", - "sha256:c95d514093b80e5309bdca5dd99e51bcf82c44043b57c34594d9d7556bd04d05", - "sha256:d43cf21df524283daa80ecad551c306b7f52881c8d0fe4e3e76a96b626b6d8d8", - "sha256:d62205f00f461fe8b24ade07499454a3b7adf3def1225e258b994e2215fd15c5", - "sha256:e289a857dca3b35d3615c3a6a438622e20d1bf0abcb82c57d866c8d0be3f44c4", - "sha256:e5f6aa56dda92472e9d6f7b1e6331f4e2d51a67caafff4d4c5121cadac03941e", - "sha256:f4b1c65ee86bfbf7d0c3dfd90592a9e3d6e9ecd36c367c884094c050d4c35d04" - ], - "version": "==2020.10.23" + "sha256:03855ee22980c3e4863dc84c42d6d2901133362db5daf4c36b710dd895d78f0a", + "sha256:06b52815d4ad38d6524666e0d50fe9173533c9cc145a5779b89733284e6f688f", + "sha256:11116d424734fe356d8777f89d625f0df783251ada95d6261b4c36ad27a394bb", + "sha256:119e0355dbdd4cf593b17f2fc5dbd4aec2b8899d0057e4957ba92f941f704bf5", + "sha256:127a9e0c0d91af572fbb9e56d00a504dbd4c65e574ddda3d45b55722462210de", + "sha256:1ec66700a10e3c75f1f92cbde36cca0d3aaee4c73dfa26699495a3a30b09093c", + "sha256:227a8d2e5282c2b8346e7f68aa759e0331a0b4a890b55a5cfbb28bd0261b84c0", + "sha256:2564def9ce0710d510b1fc7e5178ce2d20f75571f788b5197b3c8134c366f50c", + "sha256:297116e79074ec2a2f885d22db00ce6e88b15f75162c5e8b38f66ea734e73c64", + "sha256:2dc522e25e57e88b4980d2bdd334825dbf6fa55f28a922fc3bfa60cc09e5ef53", + "sha256:3a5f08039eee9ea195a89e180c5762bfb55258bfb9abb61a20d3abee3b37fd12", + "sha256:3dfca201fa6b326239e1bccb00b915e058707028809b8ecc0cf6819ad233a740", + "sha256:49461446b783945597c4076aea3f49aee4b4ce922bd241e4fcf62a3e7c61794c", + "sha256:4afa350f162551cf402bfa3cd8302165c8e03e689c897d185f16a167328cc6dd", + "sha256:4b5a9bcb56cc146c3932c648603b24514447eafa6ce9295234767bf92f69b504", + "sha256:52e83a5f28acd621ba8e71c2b816f6541af7144b69cc5859d17da76c436a5427", + "sha256:625116aca6c4b57c56ea3d70369cacc4d62fead4930f8329d242e4fe7a58ce4b", + "sha256:654c1635f2313d0843028487db2191530bca45af61ca85d0b16555c399625b0e", + "sha256:8092a5a06ad9a7a247f2a76ace121183dc4e1a84c259cf9c2ce3bbb69fac3582", + "sha256:832339223b9ce56b7b15168e691ae654d345ac1635eeb367ade9ecfe0e66bee0", + "sha256:8ca9dca965bd86ea3631b975d63b0693566d3cc347e55786d5514988b6f5b84c", + "sha256:96f99219dddb33e235a37283306834700b63170d7bb2a1ee17e41c6d589c8eb9", + "sha256:9b6305295b6591e45f069d3553c54d50cc47629eb5c218aac99e0f7fafbf90a1", + "sha256:a62162be05edf64f819925ea88d09d18b09bebf20971b363ce0c24e8b4aa14c0", + "sha256:aacc8623ffe7999a97935eeabbd24b1ae701d08ea8f874a6ff050e93c3e658cf", + "sha256:b45bab9f224de276b7bc916f6306b86283f6aa8afe7ed4133423efb42015a898", + "sha256:b88fa3b8a3469f22b4f13d045d9bd3eda797aa4e406fde0a2644bc92bbdd4bdd", + "sha256:b8a686a6c98872007aa41fdbb2e86dc03b287d951ff4a7f1da77fb7f14113e4d", + "sha256:bd904c0dec29bbd0769887a816657491721d5f545c29e30fd9d7a1a275dc80ab", + "sha256:bf4f896c42c63d1f22039ad57de2644c72587756c0cfb3cc3b7530cfe228277f", + "sha256:c13d311a4c4a8d671f5860317eb5f09591fbe8259676b86a85769423b544451e", + "sha256:c2c6c56ee97485a127555c9595c069201b5161de9d05495fbe2132b5ac104786", + "sha256:c32c91a0f1ac779cbd73e62430de3d3502bbc45ffe5bb6c376015acfa848144b", + "sha256:c3466a84fce42c2016113101018a9981804097bacbab029c2d5b4fcb224b89de", + "sha256:c454ad88e56e80e44f824ef8366bb7e4c3def12999151fd5c0ea76a18fe9aa3e", + "sha256:c8a2b7ccff330ae4c460aff36626f911f918555660cc28163417cb84ffb25789", + "sha256:cb905f3d2e290a8b8f1579d3984f2cfa7c3a29cc7cba608540ceeed18513f520", + "sha256:cfcf28ed4ce9ced47b9b9670a4f0d3d3c0e4d4779ad4dadb1ad468b097f808aa", + "sha256:dd3e6547ecf842a29cf25123fbf8d2461c53c8d37aa20d87ecee130c89b7079b", + "sha256:de7fd57765398d141949946c84f3590a68cf5887dac3fc52388df0639b01eda4", + "sha256:ea37320877d56a7f0a1e6a625d892cf963aa7f570013499f5b8d5ab8402b5625", + "sha256:f1fce1e4929157b2afeb4bb7069204d4370bab9f4fc03ca1fbec8bd601f8c87d", + "sha256:f43109822df2d3faac7aad79613f5f02e4eab0fc8ad7932d2e70e2a83bd49c26" + ], + "version": "==2020.10.28" }, "requests": { "hashes": [ @@ -1115,10 +1131,10 @@ }, "toml": { "hashes": [ - "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", - "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "version": "==0.10.1" + "version": "==0.10.2" }, "tornado": { "hashes": [ diff --git a/examples/microservice_service_discovery_consul/config.yml b/examples/microservice_service_discovery_consul/config.yml index 7081292..8356707 100644 --- a/examples/microservice_service_discovery_consul/config.yml +++ b/examples/microservice_service_discovery_consul/config.yml @@ -1,8 +1,9 @@ pyms: services: service_discovery: - client: "consul" - host: "http://localhost:8500" + service: "consul" + host: "localhost" + autoregister: true config: DEBUG: true TESTING: false diff --git a/examples/microservice_service_discovery_consul/main.py b/examples/microservice_service_discovery_consul/main.py index 9c1e940..22eb25e 100644 --- a/examples/microservice_service_discovery_consul/main.py +++ b/examples/microservice_service_discovery_consul/main.py @@ -8,7 +8,8 @@ @app.route("/") def example(): - return jsonify({"main": "hello world"}) + checks = ms.service_discovery.client.agent.checks() + return jsonify({"main": checks}) if __name__ == '__main__': diff --git a/pyms/flask/services/service_discovery.py b/pyms/flask/services/service_discovery.py index 98d9437..ae3dd8f 100644 --- a/pyms/flask/services/service_discovery.py +++ b/pyms/flask/services/service_discovery.py @@ -1,82 +1,71 @@ -import json import logging import uuid -import requests -from requests.exceptions import RequestException from pyms.constants import LOGGER_NAME -from pyms.exceptions import ServiceDiscoveryConnectionException from pyms.flask.services.driver import DriverService +from pyms.services_discovery import consulate +from pyms.utils import import_from logger = logging.getLogger(LOGGER_NAME) CONSUL_SERVICE_DISCOVERY = "consul" +EUREKA_SERVICE_DISCOVERY = "eureka" + DEFAULT_SERVICE_DISCOVERY = CONSUL_SERVICE_DISCOVERY class ServiceDiscoveryBase: - def register_service(self, id_app, host, port, healtcheck_url, app_name): + client = None + + def register_service(self, *args, **kwargs): pass class ServiceDiscoveryConsul(ServiceDiscoveryBase): - def register_service(self, id_app, host, port, healtcheck_url, app_name): - headers = { - 'Content-Type': 'application/json; charset=utf-8' - } - data = { - "id": app_name + "-" + id_app, - "name": app_name, - "port": port, - "check": { - "name": "ping check", - "http": healtcheck_url, - "interval": "30s", - "status": "passing" - } - } - error = False - msg_error = "Failed to establish a new connection" - try: - response = requests.put("{host}/v1/agent/service/register".format(host=host), data=json.dumps(data), - headers=headers) - if response.status_code != 200: - msg_error = response.content - error = True - except RequestException: - error = True - - if error: - raise ServiceDiscoveryConnectionException("Host %s raise an error: %s" % (host, msg_error)) + def __init__(self, config): + self.client = consulate.Consul(host=config.host, port=config.port, token=config.token, scheme=config.scheme, + adapter=config.adapter) + + def register_service(self, *args, **kwargs): + self.client.agent.check.register(kwargs["app_name"], + http=kwargs["healtcheck_url"], + interval=kwargs.get("interval", "10s")) class Service(DriverService): id_app = str(uuid.uuid1()) config_resource = "service_discovery" default_values = { - "client": DEFAULT_SERVICE_DISCOVERY, - "host": "http://localhost:8500", - "port": 5000, + "service": DEFAULT_SERVICE_DISCOVERY, + "host": "localhost", + "scheme": "http", + "port": 8500, "healtcheck_url": "http://127.0.0.1.nip.io:5000/healthcheck", + "autoregister": False } def init_action(self, microservice_instance): - self.client.register_service( - id_app=self.id_app, - host=self.host, - healtcheck_url=self.healtcheck_url, - port=self.port, - app_name=microservice_instance.application.config["APP_NAME"]) + if self.autoregister: + self._client.register_service( + id_app=self.id_app, + host=self.host, + healtcheck_url=self.healtcheck_url, + port=self.port, + app_name=microservice_instance.application.config["APP_NAME"]) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.client = self.get_client() + self._client = self.get_client() + self.client = self._client.client def get_client(self) -> ServiceDiscoveryBase: - client = False - if self.config.client == CONSUL_SERVICE_DISCOVERY: - client = ServiceDiscoveryConsul() + if self.service == CONSUL_SERVICE_DISCOVERY: + client = ServiceDiscoveryConsul(self) + else: + service_paths = self.service.split(".") + package = ".".join(service_paths[:-1]) + client = import_from(package, service_paths[-1])() logger.debug("Init %s as service discovery", client) return client diff --git a/pyms/services_discovery/consulate/__init__.py b/pyms/services_discovery/consulate/__init__.py new file mode 100644 index 0000000..c954123 --- /dev/null +++ b/pyms/services_discovery/consulate/__init__.py @@ -0,0 +1,34 @@ +""" +Consulate: A client library for Consul + +""" +import logging +from logging import NullHandler + +from pyms.services_discovery.consulate.client import Consul +from pyms.services_discovery.consulate.exceptions import (ConsulateException, + ClientError, + ServerError, + ACLDisabled, + Forbidden, + NotFound, + LockFailure, + RequestError) + +__version__ = '1.0.0' + +# Prevent undesired log output to the root logger +logging.getLogger('consulate').addHandler(NullHandler()) + +__all__ = [ + __version__, + Consul, + ConsulateException, + ClientError, + ServerError, + ACLDisabled, + Forbidden, + NotFound, + LockFailure, + RequestError +] diff --git a/pyms/services_discovery/consulate/adapters.py b/pyms/services_discovery/consulate/adapters.py new file mode 100644 index 0000000..43bfc12 --- /dev/null +++ b/pyms/services_discovery/consulate/adapters.py @@ -0,0 +1,140 @@ +# coding=utf-8 +""" +HTTP Client Library Adapters + +""" +import json +import logging +import socket + +import requests +import requests.exceptions +try: + import requests_unixsocket +except ImportError: # pragma: no cover + requests_unixsocket = None + +from pyms.services_discovery.consulate import api, exceptions, utils + +LOGGER = logging.getLogger(__name__) + +CONTENT_FORM = 'application/x-www-form-urlencoded; charset=utf-8' +CONTENT_JSON = 'application/json; charset=utf-8' + + +def prepare_data(fun): + """Decorator for transforming the data being submitted to Consul + + :param function fun: The decorated function + + """ + + def inner(*args, **kwargs): + """Inner wrapper function for the decorator + + :param list args: positional arguments + :param dict kwargs: keyword arguments + + """ + if kwargs.get('data'): + if not utils.is_string(kwargs.get('data')): + kwargs['data'] = json.dumps(kwargs['data']) + elif len(args) == 3 and \ + not (utils.is_string(args[2]) or args[2] is None): + args = args[0], args[1], json.dumps(args[2]) + return fun(*args, **kwargs) + + return inner + + +class Request(object): + """The Request adapter class""" + + def __init__(self, timeout=None, verify=True, cert=None): + """ + Create a new request adapter instance. + + :param int timeout: [optional] timeout to use while sending requests + to consul. + """ + self.session = requests.Session() + self.session.verify = verify + self.session.cert = cert + self.timeout = timeout + + def delete(self, uri): + """Perform a HTTP delete + + :param src uri: The URL to send the DELETE to + :rtype: consulate.api.Response + + """ + LOGGER.debug("DELETE %s", uri) + return api.Response(self.session.delete(uri, timeout=self.timeout)) + + def get(self, uri, timeout=None): + """Perform a HTTP get + + :param src uri: The URL to send the DELETE to + :param timeout: How long to wait on the response + :type timeout: int or float or None + :rtype: consulate.api.Response + + """ + LOGGER.debug("GET %s", uri) + try: + return api.Response(self.session.get( + uri, timeout=timeout or self.timeout)) + except (requests.exceptions.RequestException, + OSError, socket.error) as err: + raise exceptions.RequestError(str(err)) + + def get_stream(self, uri): + """Perform a HTTP get that returns the response as a stream. + + :param src uri: The URL to send the DELETE to + :rtype: iterator + + """ + LOGGER.debug("GET Stream from %s", uri) + try: + response = self.session.get(uri, stream=True) + except (requests.exceptions.RequestException, + OSError, socket.error) as err: + raise exceptions.RequestError(str(err)) + if utils.response_ok(response): + for line in response.iter_lines(): # pragma: no cover + yield line.decode('utf-8') + + @prepare_data + def put(self, uri, data=None, timeout=None): + """Perform a HTTP put + + :param src uri: The URL to send the DELETE to + :param str data: The PUT data + :param timeout: How long to wait on the response + :type timeout: int or float or None + :rtype: consulate.api.Response + + """ + LOGGER.debug("PUT %s with %r", uri, data) + headers = { + 'Content-Type': CONTENT_FORM + if utils.is_string(data) else CONTENT_JSON + } + try: + return api.Response( + self.session.put( + uri, data=data, headers=headers, + timeout=timeout or self.timeout)) + except (requests.exceptions.RequestException, + OSError, socket.error) as err: + raise exceptions.RequestError(str(err)) + + +class UnixSocketRequest(Request): # pragma: no cover + """Use to communicate with Consul over a Unix socket""" + + def __init__(self, timeout=None): + super(UnixSocketRequest, self).__init__(timeout) + self.session = requests_unixsocket.Session() diff --git a/pyms/services_discovery/consulate/api/__init__.py b/pyms/services_discovery/consulate/api/__init__.py new file mode 100644 index 0000000..178a068 --- /dev/null +++ b/pyms/services_discovery/consulate/api/__init__.py @@ -0,0 +1,28 @@ +""" +Consul API Endpoints + +""" +from pyms.services_discovery.consulate.api.acl import ACL +from pyms.services_discovery.consulate.api.agent import Agent +from pyms.services_discovery.consulate.api.catalog import Catalog +from pyms.services_discovery.consulate.api.event import Event +from pyms.services_discovery.consulate.api.health import Health +from pyms.services_discovery.consulate.api.coordinate import Coordinate +from pyms.services_discovery.consulate.api.kv import KV +from pyms.services_discovery.consulate.api.lock import Lock +from pyms.services_discovery.consulate.api.session import Session +from pyms.services_discovery.consulate.api.status import Status +from pyms.services_discovery.consulate.api.base import Response + +__all__ = [ + ACL, + Agent, + Catalog, + Event, + Health, + KV, + Lock, + Session, + Status, + Response +] diff --git a/pyms/services_discovery/consulate/api/acl.py b/pyms/services_discovery/consulate/api/acl.py new file mode 100644 index 0000000..b876e85 --- /dev/null +++ b/pyms/services_discovery/consulate/api/acl.py @@ -0,0 +1,442 @@ +""" +Consul ACL Endpoint Access + +""" +import logging + +from pyms.services_discovery.consulate.models import acl as model +from pyms.services_discovery.consulate.api import base +from pyms.services_discovery.consulate import exceptions +# from typing import List, Dict, Union + +LOGGER = logging.getLogger(__name__) + +# ServiceIdentity = Dict[str, Union[str, List[str]]] +# ServiceIdentities = List[ServiceIdentity] +# PolicyLink = Dict[str, str] +# PolicyLinks = List[PolicyLink] +# RoleLink = Dict[str, str] +# RoleLinks = List[RoleLink] + + +class ACL(base.Endpoint): + """The ACL endpoints are used to create, update, destroy, and query ACL + tokens. + + """ + def list_policies(self): + """List all ACL policies available in cluster. + + :param rtype: list + + """ + return self._get(["policies"]) + + def read_policy(self, id): + """Read an existing policy with the given ID. + + :param str id: The ID of the policy. + :param rtype: dict + + """ + return self._get(["policy", id]) + + def create_policy(self, + name, + datacenters=None, + description=None, + rules=None): + """Create policy with name given and rules. + + :param str name: name of the policy + :param list() datacenters: A list of datacenters to filter on policy. + :param str description: Human readable description of the policy. + :param str rules: A json serializable string for ACL rules. + :param rtype: dict + + """ + return self._put_response_body(["policy"], {}, + dict( + model.ACLPolicy( + name=name, + datacenters=datacenters, + description=description, + rules=rules))) + + def update_policy(self, + id, + name, + datacenters=None, + description=None, + rules=None): + """Update policy with id given. + + :param str id: A UUID for the policy to update. + :param str name: name of the policy + :param list() datacenters: A list of datacenters to filter on policy. + :param str description: Human readable description of the policy. + :param str rules: A json serializable string for ACL rules. + :param rtype: dict + + """ + return self._put_response_body(["policy", id], {}, + dict( + model.ACLPolicy( + name=name, + datacenters=datacenters, + description=description, + rules=rules))) + + def delete_policy(self, id): + """Delete an existing policy with the given ID. + + :param str id: The ID of the policy. + :param rtype: bool + + """ + return self._delete(["policy", id]) + + def list_roles(self): + """List all ACL roles available in cluster + :param rtype: list + + """ + return self._get(["roles"]) + + def read_role(self, id=None, name=None): + """Read an existing role with the given ID or Name. + + :param str id: The ID of the role. + :param str name: The name of the role. + :param rtype: dict + + """ + if id is not None: + return self._get(["role", id]) + elif name is not None: + return self._get(["role", "name", name]) + else: + raise exceptions.NotFound("Either id or name must be specified") + + def create_role(self, + name, + description=None, + policies=None, + service_identities=None): + """Create an ACL role from a list of policies or service identities. + + :param str name: The name of the ACL role. Must be unique. + :param str description: The description of the ACL role. + :param PolicyLinks policies: An array of PolicyLink. + :param ServiceIdentities service_identities: A ServiceIdentity array. + :param rtype: dict + + """ + return self._put_response_body( + ["role"], {}, + dict( + model.ACLRole(name=name, + description=description, + policies=policies, + service_identities=service_identities))) + + def update_role(self, + id, + name, + description=None, + policies=None, + service_identities=None): + """Update role with id given. + + :param str id: A UUID for the policy to update. + :param str name: name of the policy + :param list() datacenters: A list of datacenters to filter on policy. + :param str description: Human readable description of the policy. + :param str rules: A json serializable string for ACL rules. + :param rtype: dict + + """ + return self._put_response_body( + ["role", id], {}, + dict( + model.ACLRole(name=name, + description=description, + policies=policies, + service_identities=service_identities))) + + def delete_role(self, id): + """Delete an existing role with the given ID. + + :param str id: The ID of the role. + :param rtype: bool + + """ + return self._delete(["policy", id]) + + def list_tokens(self): + """List all ACL tokens available in cluster. + + :param rtype: list + + """ + return self._get(["tokens"]) + + def read_token(self, accessor_id): + """Read an existing token with the given ID. + + :param str id: The ID of the role. + :param rtype: dict + + """ + return self._get(["token", accessor_id]) + + def read_self_token(self): + """Retrieve the currently used token. + + :param rtype: dict + + """ + return self._get(["token", "self"]) + + def create_token(self, + accessor_id=None, + description=None, + expiration_time=None, + expiration_ttl=None, + local=False, + policies=None, + roles=None, + secret_id=None, + service_identities=None): + """Create a token from the roles, policies, and service identities + provided. + + :param str accessor_id: A UUID for accessing the token. + :param str description: A human-readable description of the token. + :param str expiration_time: The amount of time till the token expires. + :param str expiration_ttl: Sets expiration_time to creation time + + expiration_ttl value. + :param bool local: Whether the token is only locally available in the + current datacenter or to all datacenters defined. + :param PolicyLinks policies: A PolicyLink array. + :param RoleLinks roles: A RoleLink array. + :param str secret_id: A UUID for making requests to consul. + :param ServiceIdentities service_identities: A ServiceIdentity array. + :param rtype: dict + + """ + return self._put_response_body( + ["token"], {}, + dict( + model.ACLToken(accessor_id=accessor_id, + description=description, + expiration_time=expiration_time, + expiration_ttl=expiration_ttl, + local=local, + policies=policies, + roles=roles, + secret_id=secret_id, + service_identities=service_identities))) + + def update_token(self, + accessor_id, + description=None, + expiration_time=None, + expiration_ttl=None, + local=False, + policies=None, + roles=None, + secret_id=None, + service_identities=None): + """Create a token from the roles, policies, and service identities + provided. + + :param str accessor_id: A UUID for accessing the token. + :param str description: A human-readable description of the token. + :param str expiration_time: The amount of time till the token expires. + :param str expiration_ttl: Sets expiration_time to creation time + + expiration_ttl value. + :param bool local: Whether the token is only locally available in the + current datacenter or to all datacenters defined. + :param PolicyLinks policies: A PolicyLink array. + :param RoleLinks roles: A RoleLink array. + :param str secret_id: A UUID for making requests to consul. + :param ServiceIdentities service_identities: A ServiceIdentity array. + :param rtype: dict + + """ + return self._put_response_body( + ["token", accessor_id], {}, + dict( + model.ACLToken(accessor_id=accessor_id, + description=description, + expiration_time=expiration_time, + expiration_ttl=expiration_ttl, + local=local, + policies=policies, + roles=roles, + secret_id=secret_id, + service_identities=service_identities))) + + def clone_token(self, accessor_id, description=None): + """Clone a token by the accessor_id. + + :param str accessor_id: A UUID for accessing the token. + :param str description: A human-readable description of the token. + :param rtype: dict + + """ + return self._put_response_body( + ["token", accessor_id, "clone"], {}, + dict(model.ACLToken(description=description))) + + def delete_token(self, accessor_id): + """Delete an existing token with the given AcccessorID. + + :param str id: The AccessorID of the token. + :param rtype: bool + + """ + return self._delete(["token", accessor_id]) + + # NOTE: Everything below here is deprecated post consul-1.4.0. + + def bootstrap(self): + """This endpoint does a special one-time bootstrap of the ACL system, + making the first management token if the acl_master_token is not + specified in the Consul server configuration, and if the cluster has + not been bootstrapped previously. + + This is available in Consul 0.9.1 and later, and requires all Consul + servers to be upgraded in order to operate. + + You can detect if something has interfered with the ACL bootstrapping + by the response of this method. If you get a string response with the + ``ID``, the bootstrap was a success. If the method raises a + :exc:`~consulate.exceptions.Forbidden` exception, the cluster has + already been bootstrapped, at which point you should consider the + cluster in a potentially compromised state. + + .. versionadded: 1.0.0 + + :rtype: str + :raises: :exc:`~consulate.exceptions.Forbidden` + + """ + return self._put_response_body(['bootstrap'])['ID'] + + def create(self, name, acl_type='client', rules=None): + """The create endpoint is used to make a new token. A token has a name, + a type, and a set of ACL rules. + + The ``name`` property is opaque to Consul. To aid human operators, it + should be a meaningful indicator of the ACL's purpose. + + ``acl_type`` is either client or management. A management token is + comparable to a root user and has the ability to perform any action + including creating, modifying, and deleting ACLs. + + By contrast, a client token can only perform actions as permitted by + the rules associated. Client tokens can never manage ACLs. Given this + limitation, only a management token can be used to make requests to + the create endpoint. + + ``rules`` is a HCL string defining the rule policy. See + `Internals on `_ ACL + for more information on defining rules. + + The call to create will return the ID of the new ACL. + + :param str name: The name of the ACL to create + :param str acl_type: One of "client" or "management" + :param str rules: The rules HCL string + :rtype: str + :raises: consulate.exceptions.Forbidden + + """ + return self._put_response_body(['create'], {}, + dict( + model.ACL(name=name, + type=acl_type, + rules=rules)))['ID'] + + def clone(self, acl_id): + """Clone an existing ACL returning the new ACL ID + + :param str acl_id: The ACL id + :rtype: bool + :raises: consulate.exceptions.Forbidden + + """ + return self._put_response_body(['clone', acl_id])['ID'] + + def destroy(self, acl_id): + """Delete the specified ACL + + :param str acl_id: The ACL id + :rtype: bool + :raises: consulate.exceptions.Forbidden + + """ + response = self._adapter.put(self._build_uri(['destroy', acl_id])) + if response.status_code == 403: + raise exceptions.Forbidden(response.body) + return response.status_code == 200 + + def info(self, acl_id): + """Return a dict of information about the ACL + + :param str acl_id: The ACL id + :rtype: dict + :raises: consulate.exceptions.Forbidden + :raises: consulate.exceptions.NotFound + + """ + response = self._get(['info', acl_id], raise_on_404=True) + if not response: + raise exceptions.NotFound('ACL not found') + return response + + def list(self): + """Return a list of all ACLs + + :rtype: list([dict]) + :raises: consulate.exceptions.Forbidden + + """ + return self._get(['list']) + + def replication(self): + """Return the status of the ACL replication process in the datacenter. + + This is intended to be used by operators, or by automation checking the + health of ACL replication. + + .. versionadded: 1.0.0 + + :rtype: dict + :raises: consulate.exceptions.Forbidden + + """ + return self._get(['replication']) + + def update(self, acl_id, name, acl_type='client', rules=None): + """Update an existing ACL, updating its values or add a new ACL if + the ACL ID specified is not found. + + The call will return the ID of the ACL. + + :param str acl_id: The ACL id + :param str name: The name of the ACL + :param str acl_type: The ACL type + :param str rules: The ACL rules document + :rtype: str + :raises: consulate.exceptions.Forbidden + + """ + return self._put_response_body(['update'], {}, + dict( + model.ACL(id=acl_id, + name=name, + type=acl_type, + rules=rules)))['ID'] diff --git a/pyms/services_discovery/consulate/api/agent.py b/pyms/services_discovery/consulate/api/agent.py new file mode 100644 index 0000000..993c4f3 --- /dev/null +++ b/pyms/services_discovery/consulate/api/agent.py @@ -0,0 +1,400 @@ +""" +Consul Agent Endpoint Access + +""" +from pyms.services_discovery.consulate.api import base +from pyms.services_discovery.consulate.models import agent as models + +_TOKENS = [ + 'acl_token', + 'acl_agent_token', + 'acl_agent_master_token', + 'acl_replication_token' +] + + +class Agent(base.Endpoint): + """The Consul agent is the core process of Consul. The agent maintains + membership information, registers services, runs checks, responds to + queries and more. The agent must run on every node that is part of a + Consul cluster. + + """ + + def __init__(self, uri, adapter, datacenter=None, token=None): + """Create a new instance of the Agent class + + :param str uri: Base URI + :param consul.adapters.Request adapter: Request adapter + :param str datacenter: datacenter + :param str token: Access Token + + """ + super(Agent, self).__init__(uri, adapter, datacenter, token) + self.check = Agent.Check(self._base_uri, adapter, datacenter, token) + self.service = Agent.Service( + self._base_uri, adapter, datacenter, token) + + class Check(base.Endpoint): + """One of the primary roles of the agent is the management of system + and application level health checks. A health check is considered to be + application level if it associated with a service. A check is defined + in a configuration file, or added at runtime over the HTTP interface. + + There are two different kinds of checks: + + - Script + Interval: These checks depend on invoking an external + application which does the health check and + exits with an appropriate exit code, + potentially generating some output. A script + is paired with an invocation interval + (e.g. every 30 seconds). This is similar to + the Nagios plugin system. + + - TTL: These checks retain their last known state for a given TTL. + The state of the check must be updated periodically + over the HTTP interface. If an external system fails to + update the status within a given TTL, the check is set to + the failed state. This mechanism is used to allow an + application to directly report it's health. For example, + a web app can periodically curl the endpoint, and if the + app fails, then the TTL will expire and the health check + enters a critical state. This is conceptually similar to a + dead man's switch. + + """ + + def register(self, + name, + check_id=None, + interval=None, + notes=None, + deregister_critical_service_after=None, + args=None, + docker_container_id=None, + grpc=None, + grpc_use_tls=None, + http=None, + http_method=None, + header=None, + timeout=None, + tls_skip_verify=None, + tcp=None, + ttl=None, + service_id=None, + status=None): + """Add a new check to the local agent. Checks are either a script + or TTL type. The agent is responsible for managing the status of + the check and keeping the Catalog in sync. + + :param str name: + :param str check_id: + :param str interval: + :param str notes: + :param str deregister_critical_service_after: + :param str args: + :param str docker_container_id: + :param str grpc: + :param str grpc_use_tls: + :param str http: + :param str http_method: + :param str header: + :param str timeout: + :param str tls_skip_verify: + :param str tcp: + :param str ttl: + :param str service_id: + :param str status: + + :rtype: bool + :raises: ValueError + + """ + return self._put_no_response_body( + ['register'], None, dict( + models.Check( + name=name, + id=check_id, + interval=interval, + notes=notes, + deregister_critical_service_after= + deregister_critical_service_after, + args=args, + docker_container_id=docker_container_id, + grpc=grpc, + grpc_use_tls=grpc_use_tls, + http=http, + method=http_method, + header=header, + timeout=timeout, + tls_skip_verify=tls_skip_verify, + tcp=tcp, + ttl=ttl, + service_id=service_id, + status=status))) + + def deregister(self, check_id): + """Remove a check from the local agent. The agent will take care + of deregistering the check with the Catalog. + + :param str check_id: The check id + :rtype: bool + + """ + return self._put_no_response_body(['deregister', check_id]) + + def ttl_pass(self, check_id, note=None): + """This endpoint is used with a check that is of the TTL type. + When this endpoint is accessed, the status of the check is set to + "passing", and the TTL clock is reset. + + :param str check_id: The check id + :param str note: Note to include with the check pass + :rtype: bool + + """ + return self._put_no_response_body( + ['pass', check_id], {'note': note} if note else None) + + def ttl_warn(self, check_id, note=None): + """This endpoint is used with a check that is of the TTL type. + When this endpoint is accessed, the status of the check is set + to "warning", and the TTL clock is reset. + + :param str check_id: The check id + :param str note: Note to include with the check warning + :rtype: bool + + """ + return self._put_no_response_body( + ['warn', check_id], {'note': note} if note else None) + + def ttl_fail(self, check_id, note=None): + """This endpoint is used with a check that is of the TTL type. + When this endpoint is accessed, the status of the check is set + to "critical", and the TTL clock is reset. + + :param str check_id: The check id + :param str note: Note to include with the check failure + :rtype: bool + + """ + return self._put_no_response_body( + ['fail', check_id], {'note': note} if note else None) + + class Service(base.Endpoint): + """One of the main goals of service discovery is to provide a catalog + of available services. To that end, the agent provides a simple + service definition format to declare the availability of a service, a + nd to potentially associate it with a health check. A health check is + considered to be application level if it associated with a service. A + service is defined in a configuration file, or added at runtime over + the HTTP interface. + + """ + def register(self, + name, + service_id=None, + address=None, + port=None, + tags=None, + meta=None, + check=None, + checks=None, + enable_tag_override=None): + """Add a new service to the local agent. + + :param str name: The name of the service + :param str service_id: The id for the service (optional) + :param str address: The service IP address + :param int port: The service port + :param list tags: A list of tags for the service + :param list meta: A list of KV pairs for the service + :param check: An optional check definition for the service + :type check: :class:`consulate.models.agent.Check` + :param checks: A list of check definitions for the service + :type checks: list([:class:`consulate.models.agent.Check`]) + :param bool enable_tag_override: Toggle the tag override feature + :rtype: bool + :raises: ValueError + + """ + return self._put_no_response_body( + ['register'], None, + dict(models.Service( + name=name, id=service_id, address=address, port=port, + tags=tags, meta=meta, check=check, checks=checks, + enable_tag_override=enable_tag_override))) + + def deregister(self, service_id): + """Deregister the service from the local agent. The agent will + take care of deregistering the service with the Catalog. If there + is an associated check, that is also deregistered. + + :param str service_id: The service id to deregister + :rtype: bool + + """ + return self._put_no_response_body(['deregister', service_id]) + + def maintenance(self, service_id, enable=True, reason=None): + """Place given service into "maintenance mode". + + :param str service_id: The id for the service + :param bool enable: Enable maintenance mode + :param str reason: Reason for putting node in maintenance + :rtype: bool + + """ + query_params = {'enable': enable} + if reason: + query_params['reason'] = reason + return self._put_no_response_body(['maintenance', service_id], + query_params) + + def checks(self): + """Return the all the checks that are registered with the local agent. + These checks were either provided through configuration files, or + added dynamically using the HTTP API. It is important to note that + the checks known by the agent may be different than those reported + by the Catalog. This is usually due to changes being made while there + is no leader elected. The agent performs active anti-entropy, so in + most situations everything will be in sync within a few seconds. + + :rtype: dict + + """ + return self._get(['checks']) + + def force_leave(self, node): + """Instructs the agent to force a node into the left state. If a node + fails unexpectedly, then it will be in a "failed" state. Once in this + state, Consul will attempt to reconnect, and additionally the services + and checks belonging to that node will not be cleaned up. Forcing a + node into the left state allows its old entries to be removed. + + """ + return self._put_no_response_body(['force-leave', node]) + + def join(self, address, wan=False): + """This endpoint is hit with a GET and is used to instruct the agent + to attempt to connect to a given address. For agents running in + server mode, setting wan=True causes the agent to attempt to join + using the WAN pool. + + :param str address: The address to join + :param bool wan: Join a WAN pool as a server + :rtype: bool + + """ + query_params = {'wan': 1} if wan else None + return self._put_no_response_body(['join', address], query_params) + + def maintenance(self, enable=True, reason=None): + """Places the agent into or removes the agent from "maintenance mode". + + .. versionadded:: 1.0.0 + + :param bool enable: Enable or disable maintenance. Default: `True` + :param str reason: The reason for the maintenance + :rtype: bool + + """ + query_params = {'enable': enable} + if reason: + query_params['reason'] = reason + return self._put_no_response_body(['maintenance'], query_params) + + def members(self): + """Returns the members the agent sees in the cluster gossip pool. + Due to the nature of gossip, this is eventually consistent and the + results may differ by agent. The strongly consistent view of nodes + is instead provided by ``Consulate.catalog.nodes``. + + :rtype: list + + """ + return self._get_list(['members']) + + def metrics(self): + """Returns agent's metrics for the most recent finished interval + + .. versionadded:: 1.0.0 + + :rtype: dict + + """ + return self._get(['metrics']) + + def monitor(self): + """Iterator over logs from the local agent. + + .. versionadded:: 1.0.0 + + :rtype: iterator + + """ + for line in self._get_stream(['monitor']): + yield line + + def reload(self): + """This endpoint instructs the agent to reload its configuration. + Any errors encountered during this process are returned. + + .. versionadded:: 1.0.0 + + :rtype: list + + """ + return self._put_response_body(['reload']) or None + + def services(self): + """return the all the services that are registered with the local + agent. These services were either provided through configuration + files, or added dynamically using the HTTP API. It is important to + note that the services known by the agent may be different than those + ]reported by the Catalog. This is usually due to changes being made + while there is no leader elected. The agent performs active + anti-entropy, so in most situations everything will be in sync + within a few seconds. + + :rtype: dict + + """ + return self._get(['services']) + + def self(self): + """ This endpoint is used to return the configuration and member + information of the local agent under the Config key. + + :rtype: dict + + """ + return self._get(['self']) + + def token(self, name, value): + """Update the ACL tokens currently in use by the agent. It can be used + to introduce ACL tokens to the agent for the first time, or to update + tokens that were initially loaded from the agent's configuration. + Tokens are not persisted, so will need to be updated again if the agent + is restarted. + + Valid names: + + - ``acl_token`` + - ``acl_agent_token`` + - ``acl_agent_master_token`` + - ``acl_replication_token`` + + .. versionadded:: 1.0.0 + + :param str name: One of the valid token names. + :param str value: The new token value + :rtype: bool + :raises: ValueError + + """ + if name not in _TOKENS: + raise ValueError('Invalid token name: {}'.format(name)) + return self._put_no_response_body( + ['token', name], {}, {'Token': value}) diff --git a/pyms/services_discovery/consulate/api/base.py b/pyms/services_discovery/consulate/api/base.py new file mode 100644 index 0000000..44910b0 --- /dev/null +++ b/pyms/services_discovery/consulate/api/base.py @@ -0,0 +1,190 @@ +""" +Base Endpoint class used by all endpoint classes + +""" +import base64 +import json +try: + from urllib.parse import urlencode # Python 3 +except ImportError: + from urllib import urlencode # Python 2 + +from pyms.services_discovery.consulate import utils + + +class Endpoint(object): + """Base class for API endpoints""" + + KEYWORD = '' + + def __init__(self, uri, adapter, datacenter=None, token=None): + """Create a new instance of the Endpoint class + + :param str uri: Base URI + :param consul.adapters.Request adapter: Request adapter + :param str datacenter: datacenter + :param str token: Access Token + + """ + self._adapter = adapter + self._base_uri = '{0}/{1}'.format(uri, self.__class__.__name__.lower()) + self._dc = datacenter + self._token = token + + def _build_uri(self, params, query_params=None): + """Build the request URI + + :param list params: List of path parts + :param dict query_params: Build query parameters + + """ + if not query_params: + query_params = dict() + if self._dc: + query_params['dc'] = self._dc + if self._token: + query_params['token'] = self._token + path = '/'.join(params) + if query_params: + return '{0}/{1}?{2}'.format(self._base_uri, path, + urlencode(query_params)) + return '{0}/{1}'.format(self._base_uri, path) + + def _get(self, params, query_params=None, raise_on_404=False, + timeout=None): + """Perform a GET request + + :param list params: List of path parts + :param dict query_params: Build query parameters + :param timeout: How long to wait on the request for + :type timeout: int or float or None + :rtype: dict or list or None + + """ + response = self._adapter.get(self._build_uri(params, query_params), + timeout=timeout) + if utils.response_ok(response, raise_on_404): + return response.body + return [] + + def _delete( + self, + params, + raise_on_404=False, + ): + """Perform a DELETE request + + :param list params: List of path parts + :rtype: bool + + """ + response = self._adapter.delete(self._build_uri(params)) + if utils.response_ok(response, raise_on_404): + return response.body + return False + + def _get_list(self, params, query_params=None): + """Return a list queried from Consul + + :param list params: List of path parts + :param dict query_params: Build query parameters + + """ + result = self._get(params, query_params) + if isinstance(result, dict): + return [result] + return result + + def _get_stream(self, params, query_params=None): + """Return a list queried from Consul + + :param list params: List of path parts + :param dict query_params: Build query parameters + :rtype: iterator + + """ + for line in self._adapter.get_stream( + self._build_uri(params, query_params)): + yield line + + def _get_no_response_body(self, url_parts, query=None): + return utils.response_ok( + self._adapter.get(self._build_uri(url_parts, query))) + + def _get_response_body(self, url_parts, query=None): + response = self._adapter.get(self._build_uri(url_parts, query)) + if utils.response_ok(response): + return response.body + + def _put_no_response_body(self, url_parts, query=None, payload=None): + return utils.response_ok( + self._adapter.put(self._build_uri(url_parts, query), payload)) + + def _put_response_body(self, url_parts, query=None, payload=None): + response = self._adapter.put(self._build_uri(url_parts, query), + data=payload) + if utils.response_ok(response): + return response.body + + +class Response(object): + """Used to process and wrap the responses from Consul. + + :param int status_code: HTTP Status code + :param str body: The response body + :param dict headers: Response headers + + """ + status_code = None + body = None + headers = None + + def __init__(self, response): + """Create a new instance of the Response class. + + :param requests.response response: The requests response + + """ + self.status_code = response.status_code + self.body = self._demarshal(response.content) + self.headers = response.headers + + def _demarshal(self, body): + """Demarshal the request payload. + + :param str body: The string response body + :rtype: dict or str + + """ + if body is None: + return None + if self.status_code == 200: + try: + if utils.PYTHON3 and isinstance(body, bytes): + try: + body = body.decode('utf-8') + except UnicodeDecodeError: + pass + value = json.loads(body) + except (TypeError, ValueError): + return body + if value is None: + return None + if isinstance(value, bool): + return value + if 'error' not in value: + for row in value: + if 'Value' in row: + try: + row['Value'] = base64.b64decode(row['Value']) + if isinstance(row['Value'], bytes): + try: + row['Value'] = row['Value'].decode('utf-8') + except UnicodeDecodeError: + pass + except TypeError: + pass + if isinstance(value, list) and len(value) == 1: + return value[0] + return value + return body diff --git a/pyms/services_discovery/consulate/api/catalog.py b/pyms/services_discovery/consulate/api/catalog.py new file mode 100644 index 0000000..055a065 --- /dev/null +++ b/pyms/services_discovery/consulate/api/catalog.py @@ -0,0 +1,181 @@ +""" +Consul Catalog Endpoint Access + +""" +from pyms.services_discovery.consulate.api import base + + +class Catalog(base.Endpoint): + """The Consul agent is the core process of Consul. The agent maintains + membership information, registers services, runs checks, responds to + queries and more. The agent must run on every node that is part of a + Consul cluster. + + """ + + def __init__(self, uri, adapter, dc=None, token=None): + super(Catalog, self).__init__(uri, adapter, dc, token) + + def register(self, node, address, + datacenter=None, + service=None, + check=None, + node_meta=None): + """A a low level mechanism for directly registering or updating + entries in the catalog. It is usually recommended to use the agent + local endpoints, as they are simpler and perform anti-entropy. + + The behavior of the endpoint depends on what keys are provided. The + endpoint requires Node and Address to be provided, while Datacenter + will be defaulted to match that of the agent. If only those are + provided, the endpoint will register the node with the catalog. + + If the Service key is provided, then the service will also be + registered. If ID is not provided, it will be defaulted to Service. + It is mandated that the ID be node-unique. Both Tags and Port can + be omitted. + + If the Check key is provided, then a health check will also be + registered. It is important to remember that this register API is + very low level. This manipulates the health check entry, but does + not setup a script or TTL to actually update the status. For that + behavior, an agent local check should be setup. + + The CheckID can be omitted, and will default to the Name. Like + before, the CheckID must be node-unique. The Notes is an opaque + field that is meant to hold human readable text. If a ServiceID is + provided that matches the ID of a service on that node, then the + check is treated as a service level health check, instead of a node + level health check. Lastly, the status must be one of "unknown", + "passing", "warning", or "critical". The "unknown" status is used to + indicate that the initial check has not been performed yet. + + It is important to note that Check does not have to be provided + with Service and visa-versa. They can be provided or omitted at will. + + Example service dict: + + .. code:: python + + 'Service': { + 'ID': 'redis1', + 'Service': 'redis', + 'Tags': ['master', 'v1'], + 'Port': 8000, + } + + Example check dict: + + .. code:: python + + 'Check': { + 'Node': 'foobar', + 'CheckID': 'service:redis1', + 'Name': 'Redis health check', + 'Notes': 'Script based health check', + 'Status': 'passing', + 'ServiceID': 'redis1' + } + + Example node_meta dict: + + .. code:: python + + 'NodeMeta': { + 'somekey': 'somevalue' + } + + :param str node: The node name + :param str address: The node address + :param str datacenter: The optional node datacenter + :param dict service: An optional node service + :param dict check: An optional node check + :param dict node_meta: Optional node metadata + :rtype: bool + + """ + payload = {'Node': node, 'Address': address} + if datacenter: + payload['Datacenter'] = datacenter + if service: + payload['Service'] = service + if check: + payload['Check'] = check + if node_meta: + payload['NodeMeta'] = node_meta + + return self._put_response_body(['register'], None, payload) + + def deregister(self, node, datacenter=None, + check_id=None, service_id=None): + """Directly remove entries in the catalog. It is usually recommended + to use the agent local endpoints, as they are simpler and perform + anti-entropy. + + The behavior of the endpoint depends on what keys are provided. The + endpoint requires ``node`` to be provided, while ``datacenter`` will + be defaulted to match that of the agent. If only ``node`` is provided, + then the node, and all associated services and checks are deleted. If + ``check_id`` is provided, only that check belonging to the node is + removed. If ``service_id`` is provided, then the service along with + it's associated health check (if any) is removed. + + :param str node: The node for the action + :param str datacenter: The optional datacenter for the node + :param str check_id: The optional check_id to remove + :param str service_id: The optional service_id to remove + :rtype: bool + + """ + payload = {'Node': node} + if datacenter: + payload['Datacenter'] = datacenter + if check_id: + payload['CheckID'] = check_id + if service_id: + payload['ServiceID'] = service_id + return self._put_response_body(['deregister'], None, payload) + + def datacenters(self): + """Return all the datacenters that are known by the Consul server. + + :rtype: list + + """ + return self._get_list(['datacenters']) + + def node(self, node_id): + """Return the node data for the specified node + + :param str node_id: The node ID + :rtype: dict + + """ + return self._get(['node', node_id]) + + def nodes(self, node_meta=None): + """Return all of the nodes for the current datacenter. + + :param str node_meta: Desired node metadata + :rtype: list + + """ + query_params = {'node-meta': node_meta} if node_meta else {} + return self._get_list(['nodes'], query_params) + + def service(self, service_id): + """Return the service details for the given service + + :param str service_id: The service id + :rtype: list + + """ + return self._get_list(['service', service_id]) + + def services(self): + """Return a list of all of the services for the current datacenter. + + :rtype: list + + """ + return self._get_list(['services']) diff --git a/pyms/services_discovery/consulate/api/coordinate.py b/pyms/services_discovery/consulate/api/coordinate.py new file mode 100644 index 0000000..a2a6a81 --- /dev/null +++ b/pyms/services_discovery/consulate/api/coordinate.py @@ -0,0 +1,63 @@ +""" +Consul Coordinate Endpoint Access + +""" +from pyms.services_discovery.consulate.api import base +from math import sqrt + +class Coordinate(base.Endpoint): + """Used to query node coordinates. + """ + + def node(self, node_id): + """Return coordinates for the given node. + + :param str node_id: The node ID + :rtype: dict + + """ + return self._get(['node', node_id]) + + def nodes(self): + """Return coordinates for the current datacenter. + + :rtype: list + + """ + return self._get_list(['nodes']) + + def rtt(self, src, dst): + """Calculated RTT between two node coordinates. + + :param dict src + :param dict dst + :rtype float + + """ + + if not isinstance(src, (dict)): + raise ValueError('coordinate object must be a dictionary') + if not isinstance(dst, (dict)): + raise ValueError('coordinate object must be a dictionary') + if 'Coord' not in src: + raise ValueError('coordinate object has no Coord key') + if 'Coord' not in dst: + raise ValueError('coordinate object has no Coord key') + + src_coord = src['Coord'] + dst_coord = dst['Coord'] + + if len(src_coord.get('Vec')) != len(dst_coord.get('Vec')): + raise ValueError('coordinate objects are not compatible due to different length') + + sumsq = 0.0 + for i in xrange(len(src_coord.get('Vec'))): + diff = src_coord.get('Vec')[i] - dst_coord.get('Vec')[i] + sumsq += diff * diff + + rtt = sqrt(sumsq) + src_coord.get('Height') + dst_coord.get('Height') + adjusted = rtt + src_coord.get('Adjustment') + dst_coord.get('Adjustment') + if adjusted > 0.0: + rtt = adjusted + + return rtt * 1000 diff --git a/pyms/services_discovery/consulate/api/event.py b/pyms/services_discovery/consulate/api/event.py new file mode 100644 index 0000000..3c215b9 --- /dev/null +++ b/pyms/services_discovery/consulate/api/event.py @@ -0,0 +1,55 @@ +""" +Consul Event Endpoint Access + +""" +from pyms.services_discovery.consulate.api import base + + +class Event(base.Endpoint): + """The Event endpoints are used to fire a new event and list recent events. + + """ + + def fire(self, name, + payload=None, + datacenter=None, + node=None, + service=None, + tag=None): + """Trigger a new user Event + + :param str name: The name of the event + :param str payload: The opaque event payload + :param str datacenter: Optional datacenter to fire the event in + :param str node: Optional node to fire the event for + :param str service: Optional service to fire the event for + :param str tag: Option tag to fire the event for + :return str: the new event ID + + """ + query_args = {} + if datacenter: + query_args['dc'] = datacenter + if node: + query_args['node'] = node + if service: + query_args['service'] = service + if tag: + query_args['tag'] = tag + response = self._adapter.put(self._build_uri(['fire', name], + query_args), payload) + return response.body.get('ID') + + def list(self, name=None): + """Returns the most recent events known by the agent. As a consequence + of how the event command works, each agent may have a different view of + the events. Events are broadcast using the gossip protocol, so they + have no global ordering nor do they make a promise of delivery. + + :return: list + + """ + query_args = {} + if name: + query_args['name'] = name + return self._get(['list'], query_args) diff --git a/pyms/services_discovery/consulate/api/health.py b/pyms/services_discovery/consulate/api/health.py new file mode 100644 index 0000000..f2ca951 --- /dev/null +++ b/pyms/services_discovery/consulate/api/health.py @@ -0,0 +1,65 @@ +""" +Consul Health Endpoint Access + +""" +from pyms.services_discovery.consulate.api import base + + +class Health(base.Endpoint): + """Used to query health related information. It is provided separately + from the Catalog, since users may prefer to not use the health checking + mechanisms as they are totally optional. Additionally, some of the query + results from the Health system are filtered, while the Catalog endpoints + provide the raw entries. + + """ + + def checks(self, service_id, node_meta=None): + """Return checks for the given service. + + :param str service_id: The service ID + :param str node_meta: Filter checks using node metadata + :rtype: list + + """ + query_params = {'node-meta': node_meta} if node_meta else {} + return self._get_list(['checks', service_id], query_params) + + def node(self, node_id): + """Return the health info for a given node. + + :param str node_id: The node ID + :rtype: list + + """ + return self._get_list(['node', node_id]) + + def service(self, service_id, tag=None, passing=None, node_meta=None): + """Returns the nodes and health info of a service + + :param str service_id: The service ID + :param str node_meta: Filter services using node metadata + :rtype: list + + """ + + query_params = {} + if tag: + query_params['tag'] = tag + if passing: + query_params['passing'] = '' + if node_meta: + query_params['node-meta'] = node_meta + + return self._get_list(['service', service_id], + query_params=query_params) + + def state(self, state): + """Returns the checks in a given state where state is one of + "unknown", "passing", "warning", or "critical". + + :param str state: The state to get checks for + :rtype: list + + """ + return self._get_list(['state', state]) diff --git a/pyms/services_discovery/consulate/api/kv.py b/pyms/services_discovery/consulate/api/kv.py new file mode 100644 index 0000000..b3596eb --- /dev/null +++ b/pyms/services_discovery/consulate/api/kv.py @@ -0,0 +1,407 @@ +""" +Consul KV Endpoint Access + +""" +from pyms.services_discovery.consulate.api import base +from pyms.services_discovery.consulate import utils, exceptions + + +class KV(base.Endpoint): + """The :py:class:`consul.api.KV` class implements a :py:class:`dict` like + interface for working with the Key/Value service. Simply use items on the + :py:class:`consulate.Session` like you would with a :py:class:`dict` to + :py:meth:`get `, + :py:meth:`set `, or + :py:meth:`delete ` values in the key/value store. + + Additionally, :py:class:`KV ` acts as an + :py:meth:`iterator `, providing methods to + iterate over :py:meth:`keys `, + :py:meth:`values `, + :py:meth:`keys and values `, etc. + + Should you need access to get or set the flag value, the + :py:meth:`get_record `, + :py:meth:`set_record `, + and :py:meth:`records ` provide a way to access + the additional fields exposed by the KV service. + + """ + + def __contains__(self, item): + """Return True if there is a value set in the Key/Value service for the + given key. + + :param str item: The key to check for + :rtype: bool + + """ + item = item.lstrip('/') + return self._get_no_response_body([item]) + + def __delitem__(self, item): + """Delete an item from the Key/Value service + + :param str item: The key name + + """ + self._delete_item(item) + + def __getitem__(self, item): + """Get a value from the Key/Value service, returning it fully + decoded if possible. + + :param str item: The item name + :rtype: mixed + :raises: KeyError + + """ + value = self._get_item(item) + if not value: + raise KeyError('Key not found ({0})'.format(item)) + return value.get('Value') + + def __iter__(self): + """Iterate over all the keys in the Key/Value service + + :rtype: iterator + + """ + for key in self.keys(): + yield key + + def __len__(self): + """Return the number if items in the Key/Value service + + :return: int + + """ + return len(self._get_all_items()) + + def __setitem__(self, item, value): + """Set a value in the Key/Value service, using the CAS mechanism + to ensure that the set is atomic. If the value passed in is not a + string, an attempt will be made to JSON encode the value prior to + setting it. + + :param str item: The key to set + :param mixed value: The value to set + :raises: KeyError + + """ + self._set_item(item, value) + + def acquire_lock(self, item, session, value=None, cas=None, flags=None): + """Use Consul for locking by specifying the item/key to lock with + and a session value for removing the lock. + + :param str item: The item in the Consul KV database + :param str session: The session value for the lock + :param mixed value: An optional value to set for the lock + :param int cas: Optional Check-And-Set index value + :param int flags: User defined flags to set + :return: bool + + """ + query_params = {'acquire': session} + if cas is not None: + query_params['cas'] = cas + if flags is not None: + query_params['flags'] = flags + return self._put_response_body([item], query_params, value) + + def delete(self, item, recurse=False): + """Delete an item from the Key/Value service + + :param str item: The item key + :param bool recurse: Remove keys prefixed with the item pattern + :raises: KeyError + + """ + return self._delete_item(item, recurse) + + def get(self, item, default=None, raw=False): + """Get a value from the Key/Value service, returning it fully + decoded if possible. + + :param str item: The item key + :param mixed default: A default value to return if the get fails + :param bool raw: Return the raw value from Consul + :rtype: mixed + :raises: KeyError + + """ + response = self._get_item(item, raw) + if isinstance(response, dict): + return response.get('Value', default) + return response or default + + def get_record(self, item): + """Get the full record from the Key/Value service, returning + all fields including the flag. + + :param str item: The item key + :rtype: dict + :raises: KeyError + + """ + return self._get_item(item) + + def find(self, prefix, separator=None): + """Find all keys with the specified prefix, returning a dict of + matches. + + *Example:* + + .. code:: python + + >>> consul.kv.find('b') + {'baz': 'qux', 'bar': 'baz'} + + :param str prefix: The prefix to search with + :rtype: dict + + """ + query_params = {'recurse': None} + if separator: + query_params['keys'] = prefix + query_params['separator'] = separator + response = self._get_list([prefix.lstrip('/')], query_params) + if separator: + results = response + else: + results = {} + for row in response: + results[row['Key']] = row['Value'] + return results + + def items(self): + """Return a dict of all of the key/value pairs in the Key/Value service + + *Example:* + + .. code:: python + + >>> consul.kv.items() + {'foo': 'bar', 'bar': 'baz', 'quz': True, 'corgie': 'dog'} + + :rtype: dict + + """ + return [{item['Key']: item['Value']} for item in self._get_all_items()] + + def iteritems(self): + """Iterate over the dict of key/value pairs in the Key/Value service + + *Example:* + + .. code:: python + + >>> for key, value in consul.kv.iteritems(): + ... print(key, value) + ... + (u'bar', 'baz') + (u'foo', 'bar') + (u'quz', True) + + :rtype: iterator + + """ + for item in self._get_all_items(): + yield item['Key'], item['Value'] + + def keys(self): + """Return a list of all of the keys in the Key/Value service + + *Example:* + + .. code:: python + + >>> consul.kv.keys() + [u'bar', u'foo', u'quz'] + + :rtype: list + + """ + return sorted([row['Key'] for row in self._get_all_items()]) + + def records(self, key=None): + """Return a list of tuples for all of the records in the Key/Value + service + + *Example:* + + .. code:: python + + >>> consul.kv.records() + [(u'bar', 0, 'baz'), + (u'corgie', 128, 'dog'), + (u'foo', 0, 'bar'), + (u'quz', 0, True)] + + :rtype: list of (Key, Flags, Value) + + """ + if key: + return [(item['Key'], item['Flags'], item['Value']) + for item in self._get_list([key], {'recurse': None})] + else: + return [(item['Key'], item['Flags'], item['Value']) + for item in self._get_all_items()] + + def release_lock(self, item, session): + """Release an existing lock from the Consul KV database. + + :param str item: The item in the Consul KV database + :param str session: The session value for the lock + :return: bool + + """ + return self._put_response_body([item], {'release': session}) + + def set(self, item, value): + """Set a value in the Key/Value service, using the CAS mechanism + to ensure that the set is atomic. If the value passed in is not a + string, an attempt will be made to JSON encode the value prior to + setting it. + + :param str item: The key to set + :param mixed value: The value to set + :raises: KeyError + + """ + return self.__setitem__(item, value) + + def set_record(self, item, flags=0, value=None, replace=True): + """Set a full record, including the item flag + + :param str item: The key to set + :param mixed value: The value to set + :param replace: If True existing value will be overwritten: + + """ + self._set_item(item, value, flags, replace) + + def values(self): + """Return a list of all of the values in the Key/Value service + + *Example:* + + .. code:: python + + >>> consul.kv.values() + [True, 'bar', 'baz'] + + :rtype: list + + """ + return [row['Value'] for row in self._get_all_items()] + + def _delete_item(self, item, recurse=False): + """Remove an item from the Consul database + + :param str item: + :param recurse: + :return: + """ + query_params = {'recurse': True} if recurse else {} + return self._adapter.delete(self._build_uri([item], query_params)) + + def _get_all_items(self): + """Internal method to return a list of all items in the Key/Value + service + + :rtype: list + + """ + return self._get_list([''], {'recurse': None}) + + def _get_item(self, item, raw=False): + """Internal method to get the full item record from the Key/Value + service + + :param str item: The item to get + :param bool raw: Return only the raw body + :rtype: mixed + + """ + item = item.lstrip('/') + query_params = {'raw': True} if raw else {} + response = self._adapter.get(self._build_uri([item], query_params)) + if response.status_code == 200: + return response.body + return None + + def _get_modify_index(self, item, value, replace): + """Get the modify index of the specified item. If replace is False + and an item is found, return ``None``. If the existing value + and the passed in value match, return ``None``. If no item exists in + the KV database, return ``0``, otherwise return the ``ModifyIndex``. + + :param str item: The item to get the index for + :param str value: The item to evaluate for equality + :param bool replace: Should the item be replaced + :rtype: int|None + + """ + response = self._adapter.get(self._build_uri([item])) + index = 0 + if response.status_code == 200: + index = response.body.get('ModifyIndex') + rvalue = response.body.get('Value') + if rvalue == value: + return None + if not replace: + return None + return index + + @staticmethod + def _prepare_value(value): + """Prepare the value passed in and ensure that it is properly encoded + + :param mixed value: The value to prepare + :rtype: bytes + + """ + if not utils.is_string(value) or isinstance(value, bytes): + return value + try: + if utils.PYTHON3: + return value.encode('utf-8') + elif isinstance(value, unicode): + return value.encode('utf-8') + except UnicodeDecodeError: + return value + return value + + def _set_item(self, item, value, flags=None, replace=True, + query_params=None): + """Internal method for setting a key/value pair with flags in the + Key/Value service + + :param str item: The key to set + :param mixed value: The value to set + :param int flags: User defined flags to set + :param bool replace: Overwrite existing values + :raises: KeyError + + """ + value = self._prepare_value(value) + if value and item.endswith('/'): + item = item.rstrip('/') + + index = self._get_modify_index(item, value, replace) + if index is None: + return True + query_params = query_params or {} + query_params.update({'cas': index}) + if flags is not None: + query_params['flags'] = flags + response = self._adapter.put(self._build_uri([item], query_params), + value) + if not response.status_code == 200 or not response.body: + if response.status_code == 500: + raise exceptions.ServerError( + response.body or 'Internal Consul server error') + raise KeyError( + 'Error setting "{0}" ({1})'.format(item, response.status_code)) diff --git a/pyms/services_discovery/consulate/api/lock.py b/pyms/services_discovery/consulate/api/lock.py new file mode 100644 index 0000000..3738f9b --- /dev/null +++ b/pyms/services_discovery/consulate/api/lock.py @@ -0,0 +1,101 @@ +""" +Lock Object for easy locking + +""" +import contextlib +import logging +import uuid + +from pyms.services_discovery.consulate.api import base +from pyms.services_discovery.consulate import exceptions + +LOGGER = logging.getLogger(__name__) + + +class Lock(base.Endpoint): + """Wrapper for easy :class:`~consulate.api.kv.KV` locks. Keys are + automatically prefixed with ``consulate/locks/``. To change the prefix or + remove it invoke the :meth:~consulate.api.lock.Lock.prefix` method. + + Example: + + .. code:: python + + from pyms.services_discovery import consulate + + consul = consulate.Consul() + with consul.lock.acquire('my-key'): + print('Locked: {}'.format(consul.lock.key)) + # Do stuff + + :raises: :exc:`~consulate.exception.LockError` + + """ + DEFAULT_PREFIX = 'consulate/locks' + + def __init__(self, uri, adapter, session, datacenter=None, token=None): + """Create a new instance of the Lock + + :param str uri: Base URI + :param consul.adapters.Request adapter: Request adapter + :param consul.api.session.Session session: Session endpoint instance + :param str datacenter: datacenter + :param str token: Access Token + + """ + super(Lock, self).__init__(uri, adapter, datacenter, token) + self._base_uri = '{0}/kv'.format(uri) + self._session = session + self._session_id = None + self._item = str(uuid.uuid4()) + self._prefix = self.DEFAULT_PREFIX + + @contextlib.contextmanager + def acquire(self, key=None, value=None): + """A context manager that allows you to acquire the lock, optionally + passing in a key and/or value. + + :param str key: The key to lock + :param str value: The value to set in the lock + :raises: :exc:`~consulate.exception.LockError` + + """ + self._acquire(key, value) + yield + self._release() + + @property + def key(self): + """Return the lock key + + :rtype: str + + """ + return self._item + + def prefix(self, value): + """Override the path prefix for the lock key + + :param str value: The value to set the path prefix to + + """ + self._prefix = value or '' + + def _acquire(self, key=None, value=None): + self._session_id = self._session.create() + self._item = '/'.join([self._prefix, (key or str(uuid.uuid4()))]) + LOGGER.debug('Acquiring a lock of %s for session %s', + self._item, self._session_id) + response = self._put_response_body([self._item], + {'acquire': self._session_id}, + value) + if not response: + self._session.destroy(self._session_id) + raise exceptions.LockFailure() + + def _release(self): + """Release the lock""" + self._put_response_body([self._item], {'release': self._session_id}) + self._adapter.delete(self._build_uri([self._item])) + self._session.destroy(self._session_id) + self._item, self._session_id = None, None diff --git a/pyms/services_discovery/consulate/api/session.py b/pyms/services_discovery/consulate/api/session.py new file mode 100644 index 0000000..43fe850 --- /dev/null +++ b/pyms/services_discovery/consulate/api/session.py @@ -0,0 +1,115 @@ +""" +Consul Session Endpoint Access + +""" +from pyms.services_discovery.consulate.api import base + + +class Session(base.Endpoint): + """Create, destroy, and query Consul sessions.""" + + def create(self, + name=None, + behavior='release', + node=None, + delay=None, + ttl=None, + checks=None): + """Initialize a new session. + + None of the fields are mandatory, and in fact no body needs to be PUT + if the defaults are to be used. + + Name can be used to provide a human-readable name for the Session. + + Behavior can be set to either ``release`` or ``delete``. This controls + the behavior when a session is invalidated. By default, this is + release, causing any locks that are held to be released. Changing this + to delete causes any locks that are held to be deleted. delete is + useful for creating ephemeral key/value entries. + + Node must refer to a node that is already registered, if specified. + By default, the agent's own node name is used. + + LockDelay (``delay``) can be specified as a duration string using a + "s" suffix for seconds. The default is 15s. + + The TTL field is a duration string, and like LockDelay it can use "s" + as a suffix for seconds. If specified, it must be between 10s and + 3600s currently. When provided, the session is invalidated if it is + not renewed before the TTL expires. See the session internals page + for more documentation of this feature. + + Checks is used to provide a list of associated health checks. It is + highly recommended that, if you override this list, you include the + default "serfHealth". + + :param str name: A human readable session name + :param str behavior: One of ``release`` or ``delete`` + :param str node: A node to create the session on + :param str delay: A lock delay for the session + :param str ttl: The time to live for the session + :param lists checks: A list of associated health checks + :return str: session ID + + """ + payload = {'name': name} if name else {} + if node: + payload['Node'] = node + if behavior: + payload['Behavior'] = behavior + if delay: + payload['LockDelay'] = delay + if ttl: + payload['TTL'] = ttl + if checks: + payload['Checks'] = checks + return self._put_response_body(['create'], None, payload).get('ID') + + def destroy(self, session_id): + """Destroy an existing session + + :param str session_id: The session to destroy + :return: bool + + """ + return self._put_no_response_body(['destroy', session_id]) + + def info(self, session_id): + """Returns the requested session information within a given dc. + By default, the dc of the agent is queried. + + :param str session_id: The session to get info about + :return: dict + + """ + return self._get_response_body(['info', session_id]) + + def list(self): + """Returns the active sessions for a given dc. + + :return: list + + """ + return self._get_response_body(['list']) + + def node(self, node): + """Returns the active sessions for a given node and dc. + By default, the dc of the agent is queried. + + :param str node: The node to get active sessions for + :return: list + + """ + return self._get_response_body(['node', node]) + + def renew(self, session_id): + """Renew the given session. This is used with sessions that have a TTL, + and it extends the expiration by the TTL. By default, the dc + of the agent is queried. + + :param str session_id: The session to renew + :return: dict + + """ + return self._put_response_body(['renew', session_id]) diff --git a/pyms/services_discovery/consulate/api/status.py b/pyms/services_discovery/consulate/api/status.py new file mode 100644 index 0000000..336be36 --- /dev/null +++ b/pyms/services_discovery/consulate/api/status.py @@ -0,0 +1,31 @@ +""" +Consul Status Endpoint Access + +""" +from pyms.services_discovery.consulate.api import base + + +class Status(base.Endpoint): + """Get information about the status of the Consul cluster. This are + generally very low level, and not really useful for clients. + + """ + + def leader(self): + """Get the Raft leader for the datacenter the agent is running in. + + :rtype: str + + """ + return self._get(['leader']) + + def peers(self): + """Get the Raft peers for the datacenter the agent is running in. + + :rtype: list + + """ + value = self._get(['peers']) + if not isinstance(value, list): + return [value] + return value diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py new file mode 100644 index 0000000..96b5853 --- /dev/null +++ b/pyms/services_discovery/consulate/cli.py @@ -0,0 +1,637 @@ +"""Consulate CLI commands""" +# pragma: no cover +import argparse +import base64 +import json +import sys +import os +try: + import urlparse +except ImportError: + import urllib.parse as urlparse + +from requests import exceptions + +from pyms.services_discovery from pyms.services_discovery import consulate +from pyms.services_discovery.consulate import adapters +from pyms.services_discovery.consulate import utils + +CONSUL_ENV_VAR = 'CONSUL_RPC_ADDR' +EPILOG = ('If the CONSUL_RPC_ADDR environment variable is set, it will be ' + 'parsed and used for default values when connecting.') + + +def on_error(message, exit_code=2): + """Write out the specified message to stderr and exit the specified + exit code, defaulting to ``2``. + + :param str message: The exit message + :param int exit_code: The numeric exit code + + """ + sys.stderr.write(message + '\n') + sys.exit(exit_code) + + +def connection_error(): + """Common exit routine when consulate can't connect to Consul""" + on_error('Could not connect to consul', 1) + + +ACL_PARSERS = [ + ('backup', 'Backup to stdout or a JSON file', [ + [['-f', '--file'], {'help': 'JSON file to write instead of stdout', + 'nargs': '?'}], + [['-p', '--pretty'], {'help': 'pretty-print JSON output', + 'action': 'store_true'}]]), + ('restore', 'Restore from stdin or a JSON file', [ + [['-f', '--file'], + {'help': 'JSON file to read instead of stdin', + 'nargs': '?'}], + [['-n', '--no-replace'], + {'help': 'Do not replace existing entries', + 'action': 'store_true'}]]) + ] + +KV_PARSERS = [ + ('backup', 'Backup to stdout or a JSON file', [ + [['key'], {'help': 'The key to use as target to backup a ' + 'specific key or folder.', + 'nargs': '?'}], + [['-b', '--base64'], {'help': 'Base64 encode values', + 'action': 'store_true'}], + [['-f', '--file'], {'help': 'JSON file to write instead of stdout', + 'nargs': '?'}], + [['-p', '--pretty'], {'help': 'pretty-print JSON output', + 'action': 'store_true'}]]), + ('restore', 'Restore from stdin or a JSON file', [ + [['key'], {'help': 'The key as target to restore to a specific key ' + 'or folder.', + 'nargs': '?'}], + [['-p', '--prune'], {'help': 'Remove entries from consul tree that ' + 'are not in restore file.', + 'action': 'store_true'}], + [['-b', '--base64'], {'help': 'Restore from Base64 encode values', + 'action': 'store_true'}], + [['-f', '--file'], + {'help': 'JSON file to read instead of stdin', + 'nargs': '?'}], + [['-n', '--no-replace'], + {'help': 'Do not replace existing entries', + 'action': 'store_true'}]]), + ('ls', 'List all of the keys', [ + [['key'], {'help': 'The key to use as target to list contents of ' + 'specific key or folder', + 'nargs': '?'}], + [['-l', '--long'], + {'help': 'Long format', + 'action': 'store_true'}]]), + ('mkdir', 'Create a folder', [ + [['path'], + {'help': 'The path to create'}]]), + ('get', 'Get a key from the database', [ + [['key'], {'help': 'The key to get'}], + [['-r', '--recurse'], + {'help': 'Get all keys prefixed with the specified key', + 'action': 'store_true'}], + [['-t', '--trim'], + {'help': 'Number of levels of prefix to trim from returned key', + 'type': int, + 'default': 0}]]), + ('set', 'Set a key in the database', [ + [['key'], {'help': 'The key to set'}], + [['value'], {'help': 'The value of the key'}]]), + ('rm', 'Remove a key from the database', [ + [['key'], {'help': 'The key to remove'}], + [['-r', '--recurse'], + {'help': 'Delete all keys prefixed with the specified key', + 'action': 'store_true'}]])] + + +def add_acl_args(parser): + """Add the acl command and arguments. + + :param argparse.Subparser parser: parser + + """ + kv_parser = parser.add_parser('acl', help='ACL Utilities') + + subparsers = kv_parser.add_subparsers(dest='action', + title='ACL Database Utilities') + + for (name, help_text, arguments) in ACL_PARSERS: + parser = subparsers.add_parser(name, help=help_text) + for (args, kwargs) in arguments: + parser.add_argument(*args, **kwargs) + + +def add_kv_args(parser): + """Add the kv command and arguments. + + :param argparse.Subparser parser: parser + + """ + kv_parser = parser.add_parser('kv', help='Key/Value Database Utilities') + + subparsers = kv_parser.add_subparsers(dest='action', + title='Key/Value Database Utilities') + + for (name, help_text, arguments) in KV_PARSERS: + parser = subparsers.add_parser(name, help=help_text) + for (args, kwargs) in arguments: + parser.add_argument(*args, **kwargs) + + +def add_register_args(parser): + """Add the register command and arguments. + + :param argparse.Subparser parser: parser + + """ + # Service registration + registerp = parser.add_parser('register', + help='Register a service for this node') + registerp.add_argument('name', help='The service name') + registerp.add_argument('-a', '--address', default=None, + help='Specify an address') + registerp.add_argument('-p', '--port', default=None, type=int, + help='Specify a port') + registerp.add_argument('-s', '--service-id', default=None, + help='Specify a service ID') + registerp.add_argument('-t', '--tags', default=[], + help='Specify a comma delimited list of tags') + rsparsers = registerp.add_subparsers(dest='ctype', + title='Service Check Options') + check = rsparsers.add_parser('check', + help='Define an external script-based check') + check.add_argument('interval', default=10, type=int, + help='How often to run the check script') + check.add_argument('path', default=None, + help='Path to the script invoked by Consul') + httpcheck = rsparsers.add_parser('httpcheck', + help='Define an HTTP-based check') + httpcheck.add_argument('interval', default=10, type=int, + help='How often to run the check script') + httpcheck.add_argument('url', default=None, + help='HTTP URL to be polled by Consul') + rsparsers.add_parser('no-check', help='Do not enable service monitoring') + ttl = rsparsers.add_parser('ttl', help='Define a duration based TTL check') + ttl.add_argument('duration', type=int, default=10, + help='TTL duration for a service with missing check data') + + +def add_run_once_args(parser): + """Add the run_once command and arguments. + + :param argparse.Subparser parser: parser + + """ + run_oncep = parser.add_parser('run_once', + help='Run a command locked to a single ' + 'execution') + run_oncep.add_argument('lock', + help='The name of the lock which will be ' + 'held in Consul.') + run_oncep.add_argument('command_to_run', nargs=argparse.REMAINDER, + help='The command to lock') + run_oncep.add_argument('-i', '--interval', default=None, + help='Hold the lock for X seconds') + + +def add_deregister_args(parser): + """Add the deregister command and arguments. + + :param argparse.Subparser parser: parser + + """ + # Service registration + registerp = parser.add_parser('deregister', + help='Deregister a service for this node') + registerp.add_argument('service_id', help='The service registration id') + + +def add_services_args(parser): + """Add the services command and arguments. + + :param argparse.Subparser parser: parser + + """ + # Service registration + registerp = parser.add_parser('services', + help='List services for this node') + + registerp.add_argument('-i', '--indent', type=int, default=None, help='The indent level for output') + +def parse_cli_args(): + """Create the argument parser and add the arguments""" + parser = argparse.ArgumentParser(description='CLI utilities for Consul', + epilog=EPILOG) + + env_var = os.environ.get(CONSUL_ENV_VAR, '') + parsed_defaults = urlparse.urlparse(env_var) + + parser.add_argument('--api-scheme', + default=parsed_defaults.scheme or 'http', + help='The scheme to use for connecting to Consul with') + parser.add_argument('--api-host', + default=parsed_defaults.hostname or 'localhost', + help='The consul host to connect on') + parser.add_argument('--api-port', + default=parsed_defaults.port or 8500, + help='The consul API port to connect to') + parser.add_argument('--datacenter', + dest='dc', + default=None, + help='The datacenter to specify for the connection') + parser.add_argument('--token', default=None, help='ACL token') + parser.add_argument('--version', action='version', + version=consulate.__version__, + help='Current consulate version') + + sparser = parser.add_subparsers(title='Commands', dest='command') + add_acl_args(sparser) + add_kv_args(sparser) + add_register_args(sparser) + add_deregister_args(sparser) + add_run_once_args(sparser) + add_services_args(sparser) + return parser.parse_args() + + +def acl_backup(consul, args): + """Dump the ACLs from Consul to JSON + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + handle = open(args.file, 'w') if args.file else sys.stdout + acls = consul.acl.list() + try: + if args.pretty: + handle.write(json.dumps(acls, sort_keys=True, indent=2, + separators=(',', ': ')) + '\n') + else: + handle.write(json.dumps(acls, sort_keys=True) + '\n') + except exceptions.ConnectionError: + connection_error() + + +def acl_restore(consul, args): + """Restore the Consul KV store + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + handle = open(args.file, 'r') if args.file else sys.stdin + data = json.load(handle) + for row in data: + consul.acl.update(row['ID'], row['Name'], row['Type'], row['Rules']) + print('{0} ACLs written'.format(len(data))) + + +ACL_ACTIONS = { + 'backup': acl_backup, + 'restore': acl_restore +} + + +def kv_backup(consul, args): + """Backup the Consul KV database + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + handle = open(args.file, 'w') if args.file else sys.stdout + if args.key: + args.key = args.key.strip('/') + prefixlen = len(args.key.split('/')) + records = [('/'.join(k.split('/')[prefixlen:]), f, v) + for k, f, v in consul.kv.records(args.key)] + else: + records = consul.kv.records() + if args.base64: + if utils.PYTHON3: + records = [(k, f, str(base64.b64encode(utils.maybe_encode(v)), + 'ascii') if v else v) + for k, f, v in records] + else: + records = [(k, f, base64.b64encode(v) if v else v) + for k, f, v in records] + try: + if args.pretty: + handle.write(json.dumps(records, sort_keys=True, indent=2, + separators=(',', ': ')) + '\n') + else: + handle.write(json.dumps(records) + '\n') + except exceptions.ConnectionError: + connection_error() + + +def kv_delete(consul, args): + """Remove a key from the Consulate database + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + try: + del consul.kv[args.key] + except exceptions.ConnectionError: + connection_error() + + +def kv_get(consul, args): + """Get the value of a key from the Consul database + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + try: + if args.recurse: + for key in sorted(consul.kv.find(args.key)): + displaykey = key + if args.trim: + keyparts = displaykey.split('/') + if (args.trim >= len(keyparts)): + displaykey = keyparts[-1] + else: + displaykey = '/'.join(keyparts[args.trim:]) + sys.stdout.write('%s\t%s\n' % (displaykey, consul.kv.get(key))) + else: + sys.stdout.write('%s\n' % consul.kv.get(args.key)) + except exceptions.ConnectionError: + connection_error() + + +def kv_ls(consul, args): + """List out the keys from the Consul KV database + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + try: + if args.key: + args.key = args.key.lstrip('/') + keylist = sorted(consul.kv.find(args.key)) + else: + keylist = consul.kv.keys() + for key in keylist: + if args.long: + keylen = 0 + if consul.kv[key]: + keylen = len(consul.kv[key]) + print('{0:>14} {1}'.format(keylen, key)) + else: + print(key) + except exceptions.ConnectionError: + connection_error() + + +def kv_mkdir(consul, args): + """Make a key based path/directory in the KV database + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + if not args.path[:-1] == '/': + args.path += '/' + try: + consul.kv.set(args.path, None) + except exceptions.ConnectionError: + connection_error() + + +def kv_restore(consul, args): + """Restore the Consul KV store + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + if args.prune: + if args.key: + args.key = args.key.strip('/') + keylist = consul.kv.find(args.key) + else: + keylist = consul.kv.find('') + handle = open(args.file, 'r') if args.file else sys.stdin + data = json.load(handle) + for row in data: + if isinstance(row, dict): + # translate raw api export to internal representation + if row['Value'] is not None: + row['Value'] = base64.b64decode(row['Value']) + row = [row['Key'], row['Flags'], row['Value']] + + if args.base64 and row[2] is not None: + row[2] = base64.b64decode(row[2]) + + # Here's an awesome thing to make things work + if not utils.PYTHON3 and isinstance(row[2], unicode): + row[2] = row[2].encode('utf-8') + if args.key: + if row[0] == "": + rowkey = args.key + else: + rowkey = args.key + '/' + row[0] + else: + rowkey = row[0] + if args.prune: + if rowkey in keylist: + del keylist[rowkey] + try: + consul.kv.set_record(rowkey, row[1], row[2], not args.no_replace) + except exceptions.ConnectionError: + connection_error() + if args.prune: + for key in keylist: + print("Pruning {0}".format(key)) + try: + consul.kv.delete(key) + except exceptions.ConnectionError: + connection_error() + + +def kv_rm(consul, args): + """Remove a key from the Consulate database + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + try: + consul.kv.delete(args.key, args.recurse) + except exceptions.ConnectionError: + connection_error() + + +def kv_set(consul, args): + """Set a value of a key int the Consul database + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + try: + consul.kv[args.key] = args.value + except exceptions.ConnectionError: + connection_error() + + +# Mapping dict to simplify the code in main() +KV_ACTIONS = { + 'backup': kv_backup, + 'del': kv_delete, + 'get': kv_get, + 'ls': kv_ls, + 'mkdir': kv_mkdir, + 'restore': kv_restore, + 'rm': kv_rm, + 'set': kv_set} + + +def register(consul, args): + """Handle service registration. + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + check = args.path if args.ctype == 'check' else None + httpcheck = args.url if args.ctype == 'httpcheck' else None + interval = '%ss' % args.interval if args.ctype in ['check', + 'httpcheck'] else None + ttl = '%ss' % args.duration if args.ctype == 'ttl' else None + tags = args.tags.split(',') if args.tags else None + try: + consul.agent.service.register(args.name, args.service_id, args.address, + int(args.port), tags, check, interval, + ttl, httpcheck) + except exceptions.ConnectionError: + connection_error() + + +def deregister(consul, args): + """Handle service deregistration. + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + try: + consul.agent.service.deregister(args.service_id) + except exceptions.ConnectionError: + connection_error() + + +def run_once(consul, args): + """Ensure only one process can run a command at a time + + :param consulate.api_old.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + import time + import subprocess + + error_msg, error_code = None, None + try: + consul.lock.prefix('') + with consul.lock.acquire(args.lock): + if args.interval: + now = int(time.time()) + last_run = consul.kv.get("{0}_last_run".format(args.lock)) + if str(last_run) not in ['null', 'None'] and \ + int(last_run) + int(args.interval) > now: + sys.stdout.write( + 'Last run happened fewer than {0} seconds ago. ' + 'Exiting\n'.format(args.interval)) + return + consul.kv["{0}_last_run".format(args.lock)] = now + + # Should the subprocess return an error code, release the lock + try: + print(subprocess.check_output(args.command_to_run[0].strip(), + stderr=subprocess.STDOUT, + shell=True)) + # If the subprocess fails + except subprocess.CalledProcessError as err: + error_code = 1 + error_msg = ('"{0}" exited with return code "{1}" and ' + 'output {2}'.format(args.command_to_run, + err.returncode, + err.output)) + except OSError as err: + error_code = 1 + error_msg = '"{0}" command does not exist'.format( + args.command_to_run, err) + except Exception as err: + error_code = 1 + error_msg = '"{0}" exited with error "{1}"'.format( + args.command_to_run, err) + + except consulate.LockFailure: + on_error('Cannot obtain the required lock. Exiting') + + except exceptions.ConnectionError: + connection_error() + + if error_msg: + on_error(error_msg, error_code) + + +def services(consul, args): + """Dump the list of services registered with Consul + + :param consulate.api.Consul consul: The Consul instance + :param argparser.namespace args: The cli args + + """ + + svcs = consul.agent.services() + print(json.dumps(svcs, + sort_keys=True, + indent=args.indent, + separators=(',', ': ')) + '\n') + + +def main(): + """Entrypoint for the consulate cli application""" + args = parse_cli_args() + + if args.api_scheme == 'http+unix': + adapter = adapters.UnixSocketRequest + port = None + + api_host = os.environ.get('CONSUL_HTTP_ADDR').replace('unix://', '') + if args.api_host: + api_host = args.api_host + else: + adapter = None + port = args.api_port + + api_host = 'localhost' + if args.api_host: + api_host = args.api_host + + consul = consulate.Consul(api_host, port, args.dc, + args.token, args.api_scheme, adapter) + + if args.command == 'acl': + ACL_ACTIONS[args.action](consul, args) + elif args.command == 'kv': + KV_ACTIONS[args.action](consul, args) + elif args.command == 'register': + register(consul, args) + elif args.command == 'deregister': + deregister(consul, args) + elif args.command == 'services': + services(consul, args) + elif args.command == 'run_once': + run_once(consul, args) diff --git a/pyms/services_discovery/consulate/client.py b/pyms/services_discovery/consulate/client.py new file mode 100644 index 0000000..2d3b926 --- /dev/null +++ b/pyms/services_discovery/consulate/client.py @@ -0,0 +1,200 @@ +""" +Consul client object + +""" +import os +from pyms.services_discovery.consulate import adapters, api, utils + +DEFAULT_HOST = os.environ.get('CONSUL_HOST') or 'localhost' +DEFAULT_PORT = os.environ.get('CONSUL_PORT') or 8500 +DEFAULT_ADDR = os.environ.get('CONSUL_HTTP_ADDR') +DEFAULT_SCHEME = 'http' +DEFAULT_TOKEN = os.environ.get('CONSUL_HTTP_TOKEN') +API_VERSION = 'v1' + + +class Consul(object): + """Access the Consul HTTP API via Python. + + The default values connect to Consul via ``localhost:8500`` via http. If + you want to connect to Consul via a local UNIX socket, you'll need to + override both the ``scheme``, ``port`` and the ``adapter`` like so: + + .. code:: python + + consul = consulate.Consul('/path/to/socket', None, scheme='http+unix', + adapter=consulate.adapters.UnixSocketRequest) + services = consul.agent.services() + + :param str addr: The CONSUL_HTTP_ADDR if available (Default: None) + :param str host: The host name to connect to (Default: localhost) + :param int port: The port to connect on (Default: 8500) + :param str datacenter: Specify a specific data center + :param str token: Specify a ACL token to use + :param str scheme: Specify the scheme (Default: http) + :param class adapter: Specify to override the request adapter + (Default: :py:class:`consulate.adapters.Request`) + :param bool/str verify: Specify how to verify TLS certificates + :param tuple cert: Specify client TLS certificate and key files + :param float timeout: Timeout in seconds for API requests (Default: None) + + """ + def __init__(self, + addr=DEFAULT_ADDR, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + datacenter=None, + token=DEFAULT_TOKEN, + scheme=DEFAULT_SCHEME, + adapter=None, + verify=True, + cert=None, + timeout=None): + """Create a new instance of the Consul class""" + base_uri = self._base_uri(addr=addr, + scheme=scheme, + host=host, + port=port) + self._adapter = adapter() if adapter else adapters.Request( + timeout=timeout, verify=verify, cert=cert) + self._acl = api.ACL(base_uri, self._adapter, datacenter, token) + self._agent = api.Agent(base_uri, self._adapter, datacenter, token) + self._catalog = api.Catalog(base_uri, self._adapter, datacenter, token) + self._event = api.Event(base_uri, self._adapter, datacenter, token) + self._health = api.Health(base_uri, self._adapter, datacenter, token) + self._coordinate = api.Coordinate(base_uri, self._adapter, datacenter, + token) + self._kv = api.KV(base_uri, self._adapter, datacenter, token) + self._session = api.Session(base_uri, self._adapter, datacenter, token) + self._status = api.Status(base_uri, self._adapter, datacenter, token) + self._lock = api.Lock(base_uri, self._adapter, self._session, + datacenter, token) + + @property + def acl(self): + """Access the Consul + `ACL `_ API + + :rtype: :py:class:`consulate.api.acl.ACL` + + """ + return self._acl + + @property + def agent(self): + """Access the Consul + `Agent `_ API + + :rtype: :py:class:`consulate.api.agent.Agent` + + """ + return self._agent + + @property + def catalog(self): + """Access the Consul + `Catalog `_ API + + :rtype: :py:class:`consulate.api.catalog.Catalog` + + """ + return self._catalog + + @property + def event(self): + """Access the Consul + `Events `_ API + + :rtype: :py:class:`consulate.api.event.Event` + + """ + return self._event + + @property + def health(self): + """Access the Consul + `Health `_ API + + :rtype: :py:class:`consulate.api.health.Health` + + """ + return self._health + + @property + def coordinate(self): + """Access the Consul + `Coordinate `_ API + + :rtype: :py:class:`consulate.api.coordinate.Coordinate` + + """ + return self._coordinate + + @property + def kv(self): + """Access the Consul + `KV `_ API + + :rtype: :py:class:`consulate.api.kv.KV` + + """ + return self._kv + + @property + def lock(self): + """Wrapper for easy :class:`~consulate.api.kv.KV` locks. + `Semaphore ` _Guide + Example: + + .. code:: python + + from pyms.services_discovery import consulate + + consul = consulate.Consul() + with consul.lock.acquire('my-key'): + print('Locked: {}'.format(consul.lock.key)) + # Do stuff + + :rtype: :class:`~consulate.api.lock.Lock` + + """ + return self._lock + + @property + def session(self): + """Access the Consul + `Session `_ API + + :rtype: :py:class:`consulate.api.session.Session` + + """ + return self._session + + @property + def status(self): + """Access the Consul + `Status `_ API + + :rtype: :py:class:`consulate.api.status.Status` + + """ + return self._status + + @staticmethod + def _base_uri(scheme, host, port, addr=None): + """Return the base URI to use for API requests. Set ``port`` to None + when creating a UNIX Socket URL. + + :param str scheme: The scheme to use (Default: http) + :param str host: The host name to connect to (Default: localhost) + :param int|None port: The port to connect on (Default: 8500) + :rtype: str + + """ + if addr is None: + if port: + return '{0}://{1}:{2}/{3}'.format(scheme, host, port, + API_VERSION) + return '{0}://{1}/{2}'.format(scheme, utils.quote(host, ''), + API_VERSION) + return '{0}/{1}'.format(addr, API_VERSION) diff --git a/pyms/services_discovery/consulate/exceptions.py b/pyms/services_discovery/consulate/exceptions.py new file mode 100644 index 0000000..591a3ca --- /dev/null +++ b/pyms/services_discovery/consulate/exceptions.py @@ -0,0 +1,49 @@ +""" +Consulate Exceptions + +""" + + +class ConsulateException(Exception): + """Base Consul exception""" + + +class RequestError(ConsulateException): + """There was an error making the request to the consul server""" + + +class ClientError(ConsulateException): + """There was an error in the request that was made to consul""" + + +class ServerError(ConsulateException): + """An internal Consul server error occurred""" + + +class ACLDisabled(ConsulateException): + """Raised when ACL related calls are made while ACLs are disabled""" + + +class ACLFormatError(ConsulateException): + """Raised when PolicyLinks is missing 'ID' and 'Name' in a PolicyLink or + when ServiceIdentities is missing 'ServiceName' field in a ServiceIdentity. + + """ + + +class Forbidden(ConsulateException): + """Raised when ACLs are enabled and the token does not validate""" + + +class NotFound(ConsulateException): + """Raised when an operation is attempted with a value that can not be + found. + + """ + + +class LockFailure(ConsulateException): + """Raised by :class:`~consulate.api.lock.Lock` if the lock can not be + acquired. + + """ diff --git a/pyms/services_discovery/consulate/models/__init__.py b/pyms/services_discovery/consulate/models/__init__.py new file mode 100644 index 0000000..35c7e78 --- /dev/null +++ b/pyms/services_discovery/consulate/models/__init__.py @@ -0,0 +1,2 @@ +# coding=utf-8 +"""Consulate Data Models""" diff --git a/pyms/services_discovery/consulate/models/acl.py b/pyms/services_discovery/consulate/models/acl.py new file mode 100644 index 0000000..734ca07 --- /dev/null +++ b/pyms/services_discovery/consulate/models/acl.py @@ -0,0 +1,168 @@ +# coding=utf-8 +"""Models for the ACL endpoints""" +import uuid + +from pyms.services_discovery.consulate.models import base + + +def _validate_link_array(value, model): + """ Validate the policies or roles links are formatted correctly. + + :param list value: An array of PolicyLink or RoleLink. + :param rtype: bool + + """ + return all(['ID' in link or 'Name' in link for link in value]) + + +def _validate_service_identities(value, model): + """ Validate service_identities is formatted correctly. + + :param ServiceIdentities value: A ServiceIdentity list + :param rtype: bool + + """ + return all( + ['ServiceName' in service_identity for service_identity in value]) + + +class ACLPolicy(base.Model): + """Defines the model used for an ACL policy.""" + __slots__ = ['datacenters', 'description', 'id', 'name', 'rules'] + + __attributes__ = { + 'datacenters': { + 'key': 'Datacenters', + 'type': list, + }, + 'description': { + 'key': 'Description', + 'type': str, + }, + 'id': { + 'key': 'ID', + 'type': uuid.UUID, + 'cast_from': str, + 'cast_to': str, + }, + 'name': { + 'key': 'Name', + 'type': str, + }, + 'rules': { + 'key': 'Rules', + 'type': str, + } + } + + +class ACLRole(base.Model): + """Defines the model used for an ACL role.""" + __slots__ = ['description', 'name', 'policies', 'service_identities'] + + __attributes__ = { + 'description': { + 'key': 'Description', + 'type': str, + }, + 'name': { + 'key': 'Name', + 'type': str, + 'required': True, + }, + 'policies': { + 'key': 'Policies', + 'type': list, + 'validator': _validate_link_array, + }, + "service_identities": { + 'key': 'ServiceIdentities', + 'type': list, + 'validator': _validate_service_identities, + } + } + + +class ACLToken(base.Model): + """Defines the model used for an ACL token.""" + __slots__ = [ + 'accessor_id', 'description', 'expiration_time', 'expiration_ttl', + 'local', 'policies', 'roles', 'secret_id', 'service_identities' + ] + + __attributes__ = { + 'accessor_id': { + 'key': 'AccessorID', + 'type': uuid.UUID, + 'cast_from': str, + 'cast_to': str, + }, + 'description': { + 'key': 'Description', + 'type': str, + }, + 'expiration_time': { + 'key': 'ExpirationTime', + 'type': str, + }, + 'expiration_ttl': { + 'key': 'ExpirationTTL', + 'type': str, + }, + 'local': { + 'key': 'Local', + 'type': bool, + }, + 'policies': { + 'key': 'Policies', + 'type': list, + 'validator': _validate_link_array, + }, + 'roles': { + 'key': 'Roles', + 'type': list, + 'validator': _validate_link_array, + }, + 'secret_id': { + 'key': 'SecretID', + 'type': uuid.UUID, + 'cast_from': str, + 'cast_to': str, + }, + "service_identities": { + 'key': 'ServiceIdentities', + 'type': list, + 'validator': _validate_service_identities, + } + } + + +# NOTE: Everything below here is deprecated post consul-1.4.0. + + +class ACL(base.Model): + """Defines the model used for an individual ACL token.""" + __slots__ = ['id', 'name', 'type', 'rules'] + + __attributes__ = { + 'id': { + 'key': 'ID', + 'type': uuid.UUID, + 'cast_from': str, + 'cast_to': str + }, + 'name': { + 'key': 'Name', + 'type': str + }, + 'type': { + 'key': 'Type', + 'type': str, + 'enum': {'client', 'management'}, + 'required': True + }, + 'rules': { + 'key': 'Rules', + 'type': str + } + } diff --git a/pyms/services_discovery/consulate/models/agent.py b/pyms/services_discovery/consulate/models/agent.py new file mode 100644 index 0000000..d5862d2 --- /dev/null +++ b/pyms/services_discovery/consulate/models/agent.py @@ -0,0 +1,246 @@ +# coding=utf-8 +"""Models for the Agent endpoints""" +from pyms.services_discovery.consulate.models import base +from pyms.services_discovery.consulate import utils + + +def _validate_args(value, model): + """Validate that the args values are all strings and that it does not + conflict with other attributes. + + :param list([str]) value: The args value + :param consulate.models.agent.Check model: The model instance. + :rtype: bool + + """ + return all([isinstance(v, str) for v in value]) \ + and not model.args and not model.grpc and not model.http \ + and not model.ttl + + +def _validate_grpc(value, model): + """Validate that the HTTP value is a URL and that it does not conflict + with other attributes. + + :param str value: The URL value + :param consulate.models.agent.Check model: The model instance. + :rtype: bool + + """ + return utils.validate_url(value) \ + and not model.args and not model.http \ + and not model.tcp and not model.ttl + + +def _validate_http(value, model): + """Validate that the HTTP value is a URL and that it does not conflict + with other attributes. + + :param str value: The URL value + :param consulate.models.agent.Check model: The model instance. + :rtype: bool + + """ + return utils.validate_url(value) \ + and not model.args and not model.grpc and not model.tcp \ + and not model.ttl + + +def _validate_interval(value, model): + """Validate that interval does not conflict with other attributes. + + :param str value: The interval value + :param consulate.models.agent.Check model: The model instance. + :rtype: bool + + """ + return utils.validate_go_interval(value) and not model.ttl + + +def _validate_tcp(_value, model): + """Validate that the TCP does not conflict with other attributes. + + :param str _value: The TCP value + :param consulate.models.agent.Check model: The model instance. + :rtype: bool + + """ + return not model.args and not model.grpc \ + and not model.http and not model.ttl + + +def _validate_ttl(value, model): + """Validate that the TTL does not conflict with other attributes. + + :param str value: The TTL value + :param consulate.models.agent.Check model: The model instance. + :rtype: bool + + """ + return utils.validate_go_interval(value) and not model.args \ + and not model.grpc and not model.http \ + and not model.tcp and not model.interval + + +class Check(base.Model): + """Model for making Check API requests to Consul.""" + + __slots__ = ['id', 'name', 'interval', 'notes', + 'deregister_critical_service_after', 'args', + 'docker_container_id', 'grpc', 'grpc_use_tls', + 'http', 'method', 'header', 'timeout', 'tls_skip_verify', + 'tcp', 'ttl', 'service_id', 'status'] + + __attributes__ = { + 'id': { + 'key': 'ID', + 'type': str + }, + 'name': { + 'key': 'Name', + 'type': str, + 'required': True + }, + 'interval': { + 'key': 'Interval', + 'type': str, + 'validator': _validate_interval + }, + 'notes': { + 'key': 'Notes', + 'type': str + }, + 'deregister_critical_service_after': { + 'key': 'DeregisterCriticalServiceAfter', + 'type': str, + 'validator': utils.validate_go_interval + }, + 'args': { + 'key': 'Args', + 'type': list, + 'validator': _validate_args + }, + 'docker_container_id': { + 'key': 'DockerContainerID', + 'type': str + }, + 'grpc': { + 'key': 'GRPC', + 'type': str, + 'validator': _validate_grpc + }, + 'grpc_use_tls': { + 'key': 'GRPCUseTLS', + 'type': bool + }, + 'http': { + 'key': 'HTTP', + 'type': str, + 'validator': _validate_http + }, + 'method': { + 'key': 'Method', + 'type': str, + 'enum': { + 'HEAD', 'GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'TRACE' + } + }, + 'header': { + 'key': 'Header', + 'type': dict, + 'validator': lambda h, _m: all( + [(isinstance(k, str) and isinstance(v, str)) + for k, v in h.items()]), + 'cast_to': lambda h: {k: [v] for k, v in h.items()} + }, + 'timeout': { + 'key': 'Timeout', + 'type': str, + 'validator': utils.validate_go_interval + }, + 'tls_skip_verify': { + 'key': 'TLSSkipVerify', + 'type': bool + }, + 'tcp': { + 'key': 'TCP', + 'type': str, + 'validator': _validate_tcp + }, + 'ttl': { + 'key': 'TTL', + 'type': str, + 'validator': _validate_ttl + }, + 'service_id': { + 'key': 'ServiceID', + 'type': str + }, + 'status': { + 'key': 'Status', + 'type': str, + 'enum': {'passing', 'warning', 'critical', 'maintenance'} + } + } + + def __init__(self, **kwargs): + super(Check, self).__init__(**kwargs) + if (self.args or self.grpc or self.http or self.tcp) \ + and not self.interval: + raise ValueError('"interval" must be specified when specifying ' + 'args, grpc, http, or tcp.') + + +class Service(base.Model): + """Model for making Check API requests to Consul.""" + + __slots__ = ['id', 'name', 'tags', 'meta', 'address', 'port', 'check', + 'checks', 'enable_tag_override'] + + __attributes__ = { + 'id': { + 'key': 'ID', + 'type': str + }, + 'name': { + 'key': 'Name', + 'type': str, + 'required': True + }, + 'tags': { + 'key': 'Tags', + 'type': list, + 'validator': lambda t, _m: all([isinstance(v, str) for v in t]) + }, + 'meta': { + 'key': 'Meta', + 'type': dict, + 'validator': lambda h, _m: all( + [(isinstance(k, str) and isinstance(v, str)) + for k, v in h.items()]), + }, + 'address': { + 'key': 'Address', + 'type': str + }, + 'port': { + 'key': 'Port', + 'type': int + }, + 'check': { + 'key': 'Check', + 'type': Check, + 'cast_to': dict + }, + 'checks': { + 'key': 'Checks', + 'type': list, + 'validator': lambda c, _m: all([isinstance(v, Check) for v in c]), + 'cast_to': lambda c: [dict(check) for check in c] + }, + 'enable_tag_override': { + 'Key': 'EnableTagOverride', + 'type': bool + } + } + diff --git a/pyms/services_discovery/consulate/models/base.py b/pyms/services_discovery/consulate/models/base.py new file mode 100644 index 0000000..50802bf --- /dev/null +++ b/pyms/services_discovery/consulate/models/base.py @@ -0,0 +1,167 @@ +# coding=utf-8 +""" +Base Model + +""" +import collections + + +class Model(collections.Iterable): + """A model contains an __attribute__ map that defines the name, + its type for type validation, an optional validation method, a method + used to + + .. python:: + + class MyModel(Model): + + __attributes__ = { + 'ID': { + 'type': uuid.UUID, + 'required': False, + 'default': None, + 'cast_from': str, + 'cast_to': str + }, + 'Serial': { + 'type': int + 'required': True, + 'default': 0, + 'validator': lambda v: v >= 0 end, + } + } + + """ + + __attributes__ = {} + """The attributes that define the data elements of the model""" + + def __init__(self, **kwargs): + super(Model, self).__init__() + [setattr(self, name, value) for name, value in kwargs.items()] + [self._set_default(name) for name in self.__attributes__.keys() + if name not in kwargs.keys()] + + def __iter__(self): + """Iterate through the model's key, value pairs. + + :rtype: iterator + + """ + for name in self.__attributes__.keys(): + value = self._maybe_cast_value(name) + if value is not None: + yield self._maybe_return_key(name), value + + def __setattr__(self, name, value): + """Set the value for an attribute of the model, validating the + attribute name and its type if the type is defined in ``__types__``. + + :param str name: The attribute name + :param mixed value: The value to set + :raises: AttributeError + :raises: TypeError + :raises: ValueError + + """ + if name not in self.__attributes__: + raise AttributeError('Invalid attribute "{}"'.format(name)) + value = self._validate_value(name, value) + super(Model, self).__setattr__(name, value) + + def __getattribute__(self, name): + """Return the attribute from the model if it is set, otherwise + returning the default if one is set. + + :param str name: The attribute name + :rtype: mixed + + """ + try: + return super(Model, self).__getattribute__(name) + except AttributeError: + if name in self.__attributes__: + return self.__attributes__[name].get('default', None) + raise + + def _maybe_cast_value(self, name): + """Return the attribute value, possibly cast to a different type if + the ``cast_to`` item is set in the attribute definition. + + :param str name: The attribute name + :rtype: mixed + + """ + value = getattr(self, name) + if value is not None and self.__attributes__[name].get('cast_to'): + return self.__attributes__[name]['cast_to'](value) + return value + + def _maybe_return_key(self, name): + """Return the attribute name as specified in it's ``key`` definition, + if specified. This is to map python attribute names to their Consul + alternatives. + + :param str name: The attribute name + :rtype: mixed + + """ + if self.__attributes__[name].get('key'): + return self.__attributes__[name]['key'] + return name + + def _required_attr(self, name): + """Returns :data:`True` if the attribute is required. + + :param str name: The attribute name + :rtype: bool + + """ + return self.__attributes__[name].get('required', False) + + def _set_default(self, name): + """Set the default value for the attribute name. + + :param str name: The attribute name + + """ + setattr(self, name, self.__attributes__[name].get('default', None)) + + def _validate_value(self, name, value): + """Ensures the the value validates based upon the type or a validation + function, raising an error if it does not. + + :param str name: The attribute name + :param mixed value: The value that is being set + :rtype: mixed + :raises: TypeError + :raises: ValueError + + """ + if value is None: + if self._required_attr(name): + raise ValueError('Attribute "{}" is required'.format(name)) + return + + if not isinstance(value, self.__attributes__[name].get('type')): + cast_from = self.__attributes__[name].get('cast_from') + if cast_from and isinstance(value, cast_from): + value = self.__attributes__[name]['type'](value) + else: + raise TypeError( + 'Attribute "{}" must be of type {} not {}'.format( + name, self.__attributes__[name]['type'].__name__, + value.__class__.__name__)) + + if self.__attributes__[name].get('enum') \ + and value not in self.__attributes__[name]['enum']: + raise ValueError( + 'Attribute "{}" value {!r} not valid'.format(name, value)) + + validator = self.__attributes__[name].get('validator') + if callable(validator): + if not validator(value, self): + raise ValueError( + 'Attribute "{}" value {!r} did not validate'.format( + name, value)) + return value diff --git a/pyms/services_discovery/consulate/utils.py b/pyms/services_discovery/consulate/utils.py new file mode 100644 index 0000000..36a6224 --- /dev/null +++ b/pyms/services_discovery/consulate/utils.py @@ -0,0 +1,111 @@ +# coding=utf-8 +""" +Misc utility functions and constants + +""" +import re +import sys +try: # pylint: disable=import-error + from urllib.parse import quote +except ImportError: + from urllib import quote + +try: # pylint: disable=import-error + from urllib import parse as _urlparse +except ImportError: + import urlparse as _urlparse + + +from pyms.services_discovery.consulate import exceptions + +DURATION_PATTERN = re.compile(r'^(?:(?:-|)(?:\d+|\d+\.\d+)(?:µs|ms|s|m|h))+$') +PYTHON3 = True if sys.version_info > (3, 0, 0) else False + + +def is_string(value): + """Python 2 & 3 safe way to check if a value is either an instance of str + or unicode. + + :param mixed value: The value to evaluate + :rtype: bool + + """ + checks = [isinstance(value, t) for t in [bytes, str]] + if not PYTHON3: + checks.append(isinstance(value, unicode)) + return any(checks) + + +def maybe_encode(value): + """If the value passed in is a str, encode it as UTF-8 bytes for Python 3 + + :param str|bytes value: The value to maybe encode + :rtype: bytes + + """ + try: + return value.encode('utf-8') + except AttributeError: + return value + + +def _response_error(response): + """Return the decoded response error or status code if no content exists. + + :param requests.response response: The HTTP response + :rtype: str + + """ + return (response.body.decode('utf-8') + if hasattr(response, 'body') and response.body + else str(response.status_code)) + + +def response_ok(response, raise_on_404=False): + """Evaluate the HTTP response and raise the appropriate exception if + required. + + :param requests.response response: The HTTP response + :param bool raise_on_404: Raise an exception on 404 error + :rtype: bool + :raises: consulate.exceptions.ConsulateException + + """ + if response.status_code == 200: + return True + elif response.status_code == 400: + raise exceptions.ClientError(_response_error(response)) + elif response.status_code == 401: + raise exceptions.ACLDisabled(_response_error(response)) + elif response.status_code == 403: + raise exceptions.Forbidden(_response_error(response)) + elif response.status_code == 404 and raise_on_404: + raise exceptions.NotFound(_response_error(response)) + elif response.status_code == 500: + raise exceptions.ServerError(_response_error(response)) + return False + + +def validate_go_interval(value, _model=None): + """Validate the value passed in returning :data:`True` if it is a Go + Duration value. + + :param str value: The string to check + :param consulate.models.base.Model _model: Optional model passed in + :rtype: bool + + """ + return DURATION_PATTERN.match(value) is not None + + +def validate_url(value, _model=None): + """Validate that the value passed in is a URL, returning :data:`True` if + it is. + + :param str value: The string to check + :param consulate.models.base.Model _model: Optional model passed in + :rtype: bool + + """ + parsed = _urlparse.urlparse(value) + return parsed.scheme and parsed.netloc diff --git a/setup.py b/setup.py index 36c788b..9cbcbbd 100644 --- a/setup.py +++ b/setup.py @@ -29,10 +29,10 @@ 'boto3>=1.15.6', ] - install_request_requires = [ 'requests>=2.24.0', ] + install_swagger_requires = [ 'connexion[swagger-ui]>=2.7.0', 'swagger-ui-bundle>=0.0.6', @@ -66,6 +66,7 @@ 'lightstep>=4.4.8', 'safety==1.9.0', 'mypy>=0.782' + 'httmock>=1.4.0' ] install_all_requires = (install_request_requires + install_swagger_requires + diff --git a/tests/config-tests-service-discovery.yml b/tests/config-tests-service-discovery.yml new file mode 100644 index 0000000..3d79d58 --- /dev/null +++ b/tests/config-tests-service-discovery.yml @@ -0,0 +1,9 @@ +pyms: + services: + service_discovery: + service: "consul" + autoregister: true + config: + DEBUG: true + TESTING: true + APP_NAME: "Python Microservice Service Discovery" \ No newline at end of file diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 07abc93..c237ced 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,29 +1,33 @@ import os import unittest.mock -from tempfile import TemporaryDirectory from pathlib import Path +from tempfile import TemporaryDirectory +from opentracing import global_tracer from prometheus_client import generate_latest from prometheus_client import values -from opentracing import global_tracer + from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT from pyms.flask.services.metrics import LOGGER_TOTAL_MESSAGES, FLASK_REQUEST_COUNT, FLASK_REQUEST_LATENCY from tests.common import MyMicroserviceNoSingleton + def reset_metric(metric): - metric._metric_init() # pylint: disable=protected-access - metric._metrics = {} # pylint: disable=protected-access + metric._metric_init() # pylint: disable=protected-access + metric._metrics = {} # pylint: disable=protected-access + def reset_metrics(): reset_metric(LOGGER_TOTAL_MESSAGES) reset_metric(FLASK_REQUEST_COUNT) reset_metric(FLASK_REQUEST_LATENCY) try: - for metric in global_tracer().metrics_factory._cache.values(): # pylint: disable=protected-access + for metric in global_tracer().metrics_factory._cache.values(): # pylint: disable=protected-access reset_metric(metric) - except AttributeError: # Not a Jaeger tracer + except AttributeError: # Not a Jaeger tracer pass + class TestMetricsFlask(unittest.TestCase): BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -62,6 +66,7 @@ def test_metrics_jaeger(self): generated_logger = b'jaeger:reporter_spans_total' assert generated_logger in generate_latest() + class TestMultiprocessMetricsFlask(unittest.TestCase): BASE_DIR = os.path.dirname(os.path.abspath(__file__)) current = None @@ -74,7 +79,8 @@ def current_test(cls): def setUpClass(cls): cls.temp_dir = TemporaryDirectory() os.environ["prometheus_multiproc_dir"] = cls.temp_dir.name - cls.patch_value_class = unittest.mock.patch.object(values, "ValueClass", values.MultiProcessValue(cls.current_test)) + cls.patch_value_class = unittest.mock.patch.object(values, "ValueClass", + values.MultiProcessValue(cls.current_test)) cls.patch_value_class.start() def setUp(self): diff --git a/tests/test_service_discovery.py b/tests/test_service_discovery.py new file mode 100644 index 0000000..3b9f8a4 --- /dev/null +++ b/tests/test_service_discovery.py @@ -0,0 +1,37 @@ +"""Test common rest operations wrapper. +""" +import os +import unittest + +import requests_mock + +from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT +from pyms.flask.app import Microservice +from pyms.flask.services.service_discovery import ServiceDiscoveryConsul + + +class ServiceDiscoveryTests(unittest.TestCase): + """Test common rest operations wrapper. + """ + + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + def setUp(self): + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(self.BASE_DIR, "config-tests-service-discovery.yml") + ms = Microservice(path=__file__) + ms.reload_conf() + self.ms = ms + + @requests_mock.Mocker() + def test_init(self, mock_request): + url = "http://localhost:8500/v1/agent/check/register" + + mock_request.put(url) + self.ms.create_app() + + self.assertTrue(True) + + def test_get_client(self): + client = self.ms.service_discovery.get_client() + + self.assertTrue(isinstance(client, ServiceDiscoveryConsul)) From 2ff6c1a7294d986473ce693062a4bd0faa12a9cb Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 14:00:30 +0100 Subject: [PATCH 07/27] fix: session reload out of scope --- tests/test_service_discovery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_service_discovery.py b/tests/test_service_discovery.py index 3b9f8a4..b67ae22 100644 --- a/tests/test_service_discovery.py +++ b/tests/test_service_discovery.py @@ -19,7 +19,6 @@ class ServiceDiscoveryTests(unittest.TestCase): def setUp(self): os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(self.BASE_DIR, "config-tests-service-discovery.yml") ms = Microservice(path=__file__) - ms.reload_conf() self.ms = ms @requests_mock.Mocker() From bc1a543db73544fb5bd6d42b97d89e444f282de4 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 14:33:31 +0100 Subject: [PATCH 08/27] feat: added Black and fix pylint errors of consul --- .pre-commit-config.yaml | 7 + Pipfile | 3 + Pipfile.lock | 236 +++++---- examples/microservice_configuration/main.py | 3 +- examples/microservice_configuration/views.py | 2 +- examples/microservice_crypt_aws_kms/main.py | 2 +- .../ms1/main.py | 2 +- .../ms2/main.py | 2 +- examples/microservice_metrics/__init__.py | 2 +- examples/microservice_metrics/main.py | 2 +- examples/microservice_metrics/views.py | 2 +- examples/microservice_requests/main.py | 2 +- .../__init__.py | 0 .../microservice_service_discovery/config.yml | 11 + .../microservice_service_discovery/main.py | 15 + .../microservice_service_discovery/service.py | 22 + .../main.py | 2 +- examples/microservice_swagger/main.py | 2 +- examples/microservice_tracer/main.py | 3 +- examples/mininum_microservice/main.py | 2 +- examples/mininum_microservice_docker/main.py | 2 +- pyms/cloud/aws/kms.py | 12 +- pyms/cmd/__init__.py | 2 +- pyms/cmd/main.py | 59 +-- pyms/config/__init__.py | 2 +- pyms/config/conf.py | 42 +- pyms/config/confile.py | 17 +- pyms/crypt/driver.py | 3 +- pyms/crypt/fernet.py | 15 +- pyms/flask/app/__init__.py | 2 +- pyms/flask/app/create_app.py | 10 +- pyms/flask/app/utils.py | 15 +- pyms/flask/configreload/__init__.py | 2 +- pyms/flask/configreload/configreload.py | 4 +- pyms/flask/healthcheck/__init__.py | 2 +- pyms/flask/healthcheck/healthcheck.py | 4 +- pyms/flask/services/driver.py | 9 +- pyms/flask/services/metrics.py | 5 +- pyms/flask/services/requests.py | 55 ++- pyms/flask/services/service_discovery.py | 21 +- pyms/flask/services/swagger.py | 28 +- pyms/flask/services/tracer.py | 24 +- pyms/logger/__init__.py | 4 +- pyms/logger/logger.py | 18 +- pyms/services_discovery/__init__.py | 0 pyms/services_discovery/consulate/__init__.py | 37 +- pyms/services_discovery/consulate/adapters.py | 42 +- .../consulate/api/__init__.py | 13 +- pyms/services_discovery/consulate/api/acl.py | 236 +++++---- .../services_discovery/consulate/api/agent.py | 161 ++++--- pyms/services_discovery/consulate/api/base.py | 63 ++- .../consulate/api/catalog.py | 45 +- .../consulate/api/coordinate.py | 50 +- .../services_discovery/consulate/api/event.py | 28 +- .../consulate/api/health.py | 17 +- pyms/services_discovery/consulate/api/kv.py | 109 ++--- pyms/services_discovery/consulate/api/lock.py | 20 +- .../consulate/api/session.py | 32 +- .../consulate/api/status.py | 4 +- pyms/services_discovery/consulate/cli.py | 447 ++++++++---------- pyms/services_discovery/consulate/client.py | 61 ++- .../consulate/models/acl.py | 215 ++++----- .../consulate/models/agent.py | 248 ++++------ .../consulate/models/base.py | 54 +-- pyms/services_discovery/consulate/utils.py | 28 +- pyms/utils/__init__.py | 2 +- pyms/utils/files.py | 2 +- pyms/utils/utils.py | 3 +- pyproject.toml | 16 + setup.py | 124 ++--- tests/common.py | 1 + tests/test_cmd.py | 37 +- tests/test_config.py | 20 +- tests/test_crypt.py | 34 +- tests/test_flask.py | 69 ++- tests/test_metrics.py | 9 +- tests/test_requests.py | 195 ++++---- tests/test_service_discovery.py | 3 +- tests/test_swagger.py | 28 +- tests/test_utils.py | 7 +- 80 files changed, 1557 insertions(+), 1582 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 examples/microservice_service_discovery/__init__.py create mode 100644 examples/microservice_service_discovery/config.yml create mode 100644 examples/microservice_service_discovery/main.py create mode 100644 examples/microservice_service_discovery/service.py create mode 100644 pyms/services_discovery/__init__.py create mode 100644 pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..740d90c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/ambv/black + rev: stable + hooks: + - id: black + files: . + diff --git a/Pipfile b/Pipfile index 9eeffe2..5914d22 100644 --- a/Pipfile +++ b/Pipfile @@ -8,3 +8,6 @@ py-ms = {editable = true,extras = ["tests"],path = "."} [packages] py-ms = {editable = true,extras = ["all"],path = "."} + +[pipenv] +allow_prereleases = true \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index b216970..6b29300 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -198,10 +198,10 @@ }, "itsdangerous": { "hashes": [ - "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", - "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + "sha256:1f08ac48fe58cb99a1f58add0c90924b4267398bbd1640268d26a29f6a03eaa4", + "sha256:26ba8fb99157bbd5a1016f9d9dc5ed5ff1325f6f6f2c10b18107199470676b4d" ], - "version": "==1.1.0" + "version": "==2.0.0a1" }, "jaeger-client": { "hashes": [ @@ -211,10 +211,10 @@ }, "jinja2": { "hashes": [ - "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", - "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" + "sha256:c10142f819c2d22bdcd17548c46fa9b77cf4fda45097854c689666bf425e7484", + "sha256:c922560ac46888d47384de1dbdc3daaa2ea993af4b26a436dec31fa2c19ec668" ], - "version": "==2.11.2" + "version": "==3.0.0a1" }, "jmespath": { "hashes": [ @@ -232,41 +232,30 @@ }, "markupsafe": { "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", - "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" - ], - "version": "==1.1.1" + "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", + "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", + "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", + "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", + "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", + "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", + "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", + "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", + "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", + "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", + "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", + "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", + "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", + "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", + "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", + "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", + "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", + "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", + "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", + "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", + "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", + "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" + ], + "version": "==2.0.0a1" }, "openapi-spec-validator": { "hashes": [ @@ -483,6 +472,12 @@ ], "version": "==3.1.0" }, + "black": { + "hashes": [ + "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" + ], + "version": "==20.8b1" + }, "certifi": { "hashes": [ "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", @@ -531,6 +526,13 @@ ], "version": "==1.14.3" }, + "cfgv": { + "hashes": [ + "sha256:32e43d604bbe7896fe7c248a9c2276447dbef840feb28fe20494f62af110211d", + "sha256:cf22deb93d4bcf92f345a5c3cd39d3d41d6340adc60c78bbbd6588c384fda6a1" + ], + "version": "==3.2.0" + }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", @@ -673,6 +675,20 @@ ], "version": "==1.52.0" }, + "httmock": { + "hashes": [ + "sha256:13e6c63f135a928e15d386af789a2890efb03e0e280f29bdc9961f3f0dc34cb9", + "sha256:44eaf4bb59cc64cd6f5d8bf8700b46aa3097cc5651b9bc85c527dfbc71792f41" + ], + "version": "==1.4.0" + }, + "identify": { + "hashes": [ + "sha256:5dd84ac64a9a115b8e0b27d1756b244b882ad264c3c423f42af8235a6e71ca12", + "sha256:c9504ba6a043ee2db0a9d69e43246bc138034895f6338d5aed1b41e4a73b1513" + ], + "version": "==1.5.9" + }, "idna": { "hashes": [ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", @@ -704,17 +720,17 @@ }, "itsdangerous": { "hashes": [ - "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", - "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + "sha256:1f08ac48fe58cb99a1f58add0c90924b4267398bbd1640268d26a29f6a03eaa4", + "sha256:26ba8fb99157bbd5a1016f9d9dc5ed5ff1325f6f6f2c10b18107199470676b4d" ], - "version": "==1.1.0" + "version": "==2.0.0a1" }, "jinja2": { "hashes": [ - "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", - "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" + "sha256:c10142f819c2d22bdcd17548c46fa9b77cf4fda45097854c689666bf425e7484", + "sha256:c922560ac46888d47384de1dbdc3daaa2ea993af4b26a436dec31fa2c19ec668" ], - "version": "==2.11.2" + "version": "==3.0.0a1" }, "joblib": { "hashes": [ @@ -788,41 +804,30 @@ }, "markupsafe": { "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", - "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" - ], - "version": "==1.1.1" + "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", + "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", + "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", + "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", + "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", + "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", + "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", + "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", + "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", + "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", + "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", + "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", + "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", + "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", + "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", + "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", + "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", + "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", + "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", + "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", + "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", + "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" + ], + "version": "==2.0.0a1" }, "mccabe": { "hashes": [ @@ -884,6 +889,13 @@ ], "version": "==3.5" }, + "nodeenv": { + "hashes": [ + "sha256:5304d424c529c997bc888453aeaa6362d242b6b4631e90f3d4bf1b290f1c84a9", + "sha256:ab45090ae383b716c4ef89e690c41ff8c2b257b85b309f01f3654df3d084bd7c" + ], + "version": "==1.5.0" + }, "opentracing": { "hashes": [ "sha256:33b10634917a7496a7e3a18b18b7c055053d0a8da5101e2a95d454783cac9765" @@ -897,6 +909,13 @@ ], "version": "==20.4" }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "version": "==0.8.1" + }, "pbr": { "hashes": [ "sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9", @@ -911,28 +930,35 @@ ], "version": "==0.13.1" }, + "pre-commit": { + "hashes": [ + "sha256:22e6aa3bd571debb01eb7d34483f11c01b65237be4eebbf30c3d4fb65762d315", + "sha256:905ebc9b534b991baec87e934431f2d0606ba27f2b90f7f652985f5a5b8b6ae6" + ], + "version": "==2.8.2" + }, "protobuf": { "hashes": [ - "sha256:0bba42f439bf45c0f600c3c5993666fcb88e8441d011fad80a11df6f324eef33", - "sha256:1e834076dfef9e585815757a2c7e4560c7ccc5962b9d09f831214c693a91b463", - "sha256:339c3a003e3c797bc84499fa32e0aac83c768e67b3de4a5d7a5a9aa3b0da634c", - "sha256:361acd76f0ad38c6e38f14d08775514fbd241316cce08deb2ce914c7dfa1184a", - "sha256:3dee442884a18c16d023e52e32dd34a8930a889e511af493f6dc7d4d9bf12e4f", - "sha256:4d1174c9ed303070ad59553f435846a2f877598f59f9afc1b89757bdf846f2a7", - "sha256:5db9d3e12b6ede5e601b8d8684a7f9d90581882925c96acf8495957b4f1b204b", - "sha256:6a82e0c8bb2bf58f606040cc5814e07715b2094caeba281e2e7d0b0e2e397db5", - "sha256:8c35bcbed1c0d29b127c886790e9d37e845ffc2725cc1db4bd06d70f4e8359f4", - "sha256:91c2d897da84c62816e2f473ece60ebfeab024a16c1751aaf31100127ccd93ec", - "sha256:9c2e63c1743cba12737169c447374fab3dfeb18111a460a8c1a000e35836b18c", - "sha256:9edfdc679a3669988ec55a989ff62449f670dfa7018df6ad7f04e8dbacb10630", - "sha256:c0c5ab9c4b1eac0a9b838f1e46038c3175a95b0f2d944385884af72876bd6bc7", - "sha256:c8abd7605185836f6f11f97b21200f8a864f9cb078a193fe3c9e235711d3ff1e", - "sha256:d69697acac76d9f250ab745b46c725edf3e98ac24763990b24d58c16c642947a", - "sha256:df3932e1834a64b46ebc262e951cd82c3cf0fa936a154f0a42231140d8237060", - "sha256:e7662437ca1e0c51b93cadb988f9b353fa6b8013c0385d63a70c8a77d84da5f9", - "sha256:f68eb9d03c7d84bd01c790948320b768de8559761897763731294e3bc316decb" - ], - "version": "==3.13.0" + "sha256:0e55cf11e3cdc7af9e280539144c0896ff046144dc17ddf6344e33f7bd01c53b", + "sha256:2ceef235ec88363f0853679363e215aee79f2998931b83e5d1bf08289e3c6a6f", + "sha256:36583c483ddd3c60f25a9a1e1660baa1e033e9470a49e019d0705d6fefaea52b", + "sha256:3bb580ee33b844087e419a9302a255a956d695147a64d59e96a3e0abd78fa67d", + "sha256:4136552036dbff1e5841cd240d1be973348752cfdaa91c9088d8a32ae43f06c0", + "sha256:428930d8f9607723ab6482f07bc7a651e95f86c7f5db6347a0b1f24319cd1e3b", + "sha256:471b0cd067e1ea2c6c5cc82fc7c04990a0497914668ca0bdc7db7925e25a8045", + "sha256:4ed0b7df03cd668dbd4bf1a432e58fc9201ae4f15a9a6f837bce4c7496c431da", + "sha256:76f4a7f5c418167496b68a7310efe93a33066aa83f5fac3a9189bc5ace6ff905", + "sha256:991301d26c33cc25c8a81fd7d25ad5a31cc6eb166b96afcbce93675545e03532", + "sha256:ae99d4b1c15439586d7d3bfc3ec7d3e933419fdcff3e483ed5a14e653b45d39c", + "sha256:afef26dd04202c8f4efe8292bd9a683166e32b6004abb0302d6a92db1a994bf1", + "sha256:bb056806dee32e6f8ad47935ca2be34bed124675c0132a469e70aa6847ce2223", + "sha256:bb0b3df5c29a8dc51e6435505d42907ac3651e2b0be40c30e9f40d08aca6d47c", + "sha256:bd644bf3865a72d5a244e7193f784cc6cd51bb453d447ee73a5e05d60dc22717", + "sha256:d6bb25e26cbee3eebf460012e3406ad5d77ced6824abbea919d8c7a0466e82f1", + "sha256:ed7143575f71fed2599e1548a0378cfd6dc958920221666e3a33785af3f0d7e2", + "sha256:ff82b53153edda066698bae674d673566029e856fa126dcb95a432aed4161af2" + ], + "version": "==3.14.0rc1" }, "py": { "hashes": [ @@ -992,10 +1018,10 @@ }, "pyparsing": { "hashes": [ - "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", - "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + "sha256:13140e8d0e1edd806eb50f18535d77f2143b40771d4aaef6b4950dd93d48a7db", + "sha256:38891c1032d0c759f0fa5ed3a8f249fd992b083fa2303ead58ee48a51b269e02" ], - "version": "==2.4.7" + "version": "==3.0.0b1" }, "pytest": { "hashes": [ diff --git a/examples/microservice_configuration/main.py b/examples/microservice_configuration/main.py index f82613c..2ddb10f 100644 --- a/examples/microservice_configuration/main.py +++ b/examples/microservice_configuration/main.py @@ -1,5 +1,6 @@ from examples.microservice_configuration import ms + app = ms.create_app() -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/microservice_configuration/views.py b/examples/microservice_configuration/views.py index 5684791..3a608ed 100644 --- a/examples/microservice_configuration/views.py +++ b/examples/microservice_configuration/views.py @@ -9,5 +9,5 @@ def example(): "GLOBAL_VARIABLE": GLOBAL_VARIABLE, "GLOBAL_VARIABLE2": GLOBAL_VARIABLE2, "test1": config().test1, - "test2": config().test2 + "test2": config().test2, } diff --git a/examples/microservice_crypt_aws_kms/main.py b/examples/microservice_crypt_aws_kms/main.py index e96dd62..02da4ec 100644 --- a/examples/microservice_crypt_aws_kms/main.py +++ b/examples/microservice_crypt_aws_kms/main.py @@ -11,5 +11,5 @@ def example(): return jsonify({"main": app.ms.config.encrypted_key}) -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/microservice_distribued_tracing/ms1/main.py b/examples/microservice_distribued_tracing/ms1/main.py index 7e89643..87c7b10 100644 --- a/examples/microservice_distribued_tracing/ms1/main.py +++ b/examples/microservice_distribued_tracing/ms1/main.py @@ -12,5 +12,5 @@ def index(): return jsonify({"main": "hello world {}".format(current_app.config["APP_NAME"])}) -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/microservice_distribued_tracing/ms2/main.py b/examples/microservice_distribued_tracing/ms2/main.py index 5ffb165..8588d3a 100644 --- a/examples/microservice_distribued_tracing/ms2/main.py +++ b/examples/microservice_distribued_tracing/ms2/main.py @@ -13,5 +13,5 @@ def index(): return jsonify({"response": response.json()}) -if __name__ == '__main__': +if __name__ == "__main__": app.run(port=5001) diff --git a/examples/microservice_metrics/__init__.py b/examples/microservice_metrics/__init__.py index 35ed887..02d325e 100644 --- a/examples/microservice_metrics/__init__.py +++ b/examples/microservice_metrics/__init__.py @@ -1,3 +1,3 @@ from pyms.flask.app import Microservice -ms = Microservice() \ No newline at end of file +ms = Microservice() diff --git a/examples/microservice_metrics/main.py b/examples/microservice_metrics/main.py index 69b0447..20e9a90 100644 --- a/examples/microservice_metrics/main.py +++ b/examples/microservice_metrics/main.py @@ -2,5 +2,5 @@ app = ms.create_app() -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/microservice_metrics/views.py b/examples/microservice_metrics/views.py index fb8fdad..a06ae71 100644 --- a/examples/microservice_metrics/views.py +++ b/examples/microservice_metrics/views.py @@ -7,4 +7,4 @@ def example(): current_app.logger.info("start request") result = ms.requests.get_for_object("https://ghibliapi.herokuapp.com/films/2baf70d1-42bb-4437-b551-e5fed5a87abe") current_app.logger.info("end request") - return result \ No newline at end of file + return result diff --git a/examples/microservice_requests/main.py b/examples/microservice_requests/main.py index 3d11381..6934e31 100644 --- a/examples/microservice_requests/main.py +++ b/examples/microservice_requests/main.py @@ -2,5 +2,5 @@ app = ms.create_app() -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/microservice_service_discovery/__init__.py b/examples/microservice_service_discovery/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/microservice_service_discovery/config.yml b/examples/microservice_service_discovery/config.yml new file mode 100644 index 0000000..2237279 --- /dev/null +++ b/examples/microservice_service_discovery/config.yml @@ -0,0 +1,11 @@ +pyms: + services: + service_discovery: + service: "examples.microservice_service_discovery.service.ServiceDiscoveryConsulBasic" + host: "localhost" + autoregister: true + config: + DEBUG: true + TESTING: false + APP_NAME: "Python Microservice My personal Service Discovery" + APPLICATION_ROOT: "" \ No newline at end of file diff --git a/examples/microservice_service_discovery/main.py b/examples/microservice_service_discovery/main.py new file mode 100644 index 0000000..fdbccf2 --- /dev/null +++ b/examples/microservice_service_discovery/main.py @@ -0,0 +1,15 @@ +from flask import jsonify + +from pyms.flask.app import Microservice + +ms = Microservice(path=__file__) +app = ms.create_app() + + +@app.route("/") +def example(): + return jsonify({"main": "hello world"}) + + +if __name__ == "__main__": + app.run() diff --git a/examples/microservice_service_discovery/service.py b/examples/microservice_service_discovery/service.py new file mode 100644 index 0000000..7327021 --- /dev/null +++ b/examples/microservice_service_discovery/service.py @@ -0,0 +1,22 @@ +import json + +import requests + +from pyms.flask.services.service_discovery import ServiceDiscoveryBase + + +class ServiceDiscoveryConsulBasic(ServiceDiscoveryBase): + def register_service(self, id_app, host, port, healtcheck_url, app_name): + headers = {"Content-Type": "application/json; charset=utf-8"} + data = { + "id": app_name + "-" + id_app, + "name": app_name, + "check": {"name": "ping check", "http": healtcheck_url, "interval": "30s", "status": "passing"}, + } + response = requests.put( + "http://{host}:{port}/v1/agent/service/register".format(host=host, port=port), + data=json.dumps(data), + headers=headers, + ) + if response.status_code != 200: + raise Exception(response.content) diff --git a/examples/microservice_service_discovery_consul/main.py b/examples/microservice_service_discovery_consul/main.py index 22eb25e..5f2cf1c 100644 --- a/examples/microservice_service_discovery_consul/main.py +++ b/examples/microservice_service_discovery_consul/main.py @@ -12,5 +12,5 @@ def example(): return jsonify({"main": checks}) -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/microservice_swagger/main.py b/examples/microservice_swagger/main.py index d7c9347..c9aa259 100644 --- a/examples/microservice_swagger/main.py +++ b/examples/microservice_swagger/main.py @@ -3,5 +3,5 @@ ms = Microservice() app = ms.create_app() -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/microservice_tracer/main.py b/examples/microservice_tracer/main.py index 12c1874..6934e31 100644 --- a/examples/microservice_tracer/main.py +++ b/examples/microservice_tracer/main.py @@ -1,5 +1,6 @@ from examples.microservice_requests import ms + app = ms.create_app() -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/mininum_microservice/main.py b/examples/mininum_microservice/main.py index 9c1e940..fdbccf2 100644 --- a/examples/mininum_microservice/main.py +++ b/examples/mininum_microservice/main.py @@ -11,5 +11,5 @@ def example(): return jsonify({"main": "hello world"}) -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/examples/mininum_microservice_docker/main.py b/examples/mininum_microservice_docker/main.py index 0c7bf34..0a91fc0 100644 --- a/examples/mininum_microservice_docker/main.py +++ b/examples/mininum_microservice_docker/main.py @@ -11,5 +11,5 @@ def example(): return jsonify({"main": ms.config.environment}) -if __name__ == '__main__': +if __name__ == "__main__": app.run() diff --git a/pyms/cloud/aws/kms.py b/pyms/cloud/aws/kms.py index 640d540..fbfcfc0 100644 --- a/pyms/cloud/aws/kms.py +++ b/pyms/cloud/aws/kms.py @@ -12,7 +12,7 @@ def __init__(self, *args, **kwargs) -> None: self._init_boto() super().__init__(*args, **kwargs) - def encrypt(self, message: str) -> str: # pragma: no cover + def encrypt(self, message: str) -> str: # pragma: no cover ciphertext = self.client.encrypt( KeyId=self.config.key_id, Plaintext=bytes(message, encoding="UTF-8"), @@ -22,16 +22,14 @@ def encrypt(self, message: str) -> str: # pragma: no cover def _init_boto(self) -> None: # pragma: no cover check_package_exists("boto3") boto3 = import_package("boto3") - boto3.set_stream_logger(name='botocore') - self.client = boto3.client('kms') + boto3.set_stream_logger(name="botocore") + self.client = boto3.client("kms") def _aws_decrypt(self, blob_text: bytes) -> str: # pragma: no cover response = self.client.decrypt( - CiphertextBlob=blob_text, - KeyId=self.config.key_id, - EncryptionAlgorithm=self.encryption_algorithm + CiphertextBlob=blob_text, KeyId=self.config.key_id, EncryptionAlgorithm=self.encryption_algorithm ) - return str(response['Plaintext'], encoding="UTF-8") + return str(response["Plaintext"], encoding="UTF-8") def _parse_encrypted(self, encrypted: str) -> bytes: blob_text = base64.b64decode(encrypted) diff --git a/pyms/cmd/__init__.py b/pyms/cmd/__init__.py index c334bcc..ec80692 100755 --- a/pyms/cmd/__init__.py +++ b/pyms/cmd/__init__.py @@ -1,3 +1,3 @@ from .main import Command -__all__ = ['Command'] +__all__ = ["Command"] diff --git a/pyms/cmd/main.py b/pyms/cmd/main.py index 0802f4e..041cde6 100755 --- a/pyms/cmd/main.py +++ b/pyms/cmd/main.py @@ -27,37 +27,40 @@ def __init__(self, *args, **kwargs): if not arguments: # pragma: no cover arguments = sys.argv[1:] - parser = argparse.ArgumentParser(description='Python Microservices') + parser = argparse.ArgumentParser(description="Python Microservices") - commands = parser.add_subparsers(title="Commands", description='Available commands', dest='command_name') + commands = parser.add_subparsers(title="Commands", description="Available commands", dest="command_name") - parser_encrypt = commands.add_parser('encrypt', help='Encrypt a string') - parser_encrypt.add_argument("encrypt", default='', type=str, help='Encrypt a string') + parser_encrypt = commands.add_parser("encrypt", help="Encrypt a string") + parser_encrypt.add_argument("encrypt", default="", type=str, help="Encrypt a string") - parser_create_key = commands.add_parser('create-key', help='Generate a Key to encrypt strings in config') - parser_create_key.add_argument("create_key", action='store_true', - help='Generate a Key to encrypt strings in config') + parser_create_key = commands.add_parser("create-key", help="Generate a Key to encrypt strings in config") + parser_create_key.add_argument( + "create_key", action="store_true", help="Generate a Key to encrypt strings in config" + ) parser_startproject = commands.add_parser( - 'startproject', - help='Generate a project from https://github.com/python-microservices/microservices-template') + "startproject", + help="Generate a project from https://github.com/python-microservices/microservices-template", + ) parser_startproject.add_argument( - "startproject", action='store_true', - help='Generate a project from https://github.com/python-microservices/microservices-template') + "startproject", + action="store_true", + help="Generate a project from https://github.com/python-microservices/microservices-template", + ) parser_startproject.add_argument( - "-b", "--branch", - help='Select a branch from https://github.com/python-microservices/microservices-template') + "-b", "--branch", help="Select a branch from https://github.com/python-microservices/microservices-template" + ) - parser_merge_swagger = commands.add_parser('merge-swagger', help='Merge swagger into a single file') - parser_merge_swagger.add_argument("merge_swagger", action='store_true', - help='Merge swagger into a single file') + parser_merge_swagger = commands.add_parser("merge-swagger", help="Merge swagger into a single file") + parser_merge_swagger.add_argument("merge_swagger", action="store_true", help="Merge swagger into a single file") parser_merge_swagger.add_argument( - "-f", "--file", default=os.path.join('project', 'swagger', 'swagger.yaml'), - help='Swagger file path') + "-f", "--file", default=os.path.join("project", "swagger", "swagger.yaml"), help="Swagger file path" + ) - parser_create_config = commands.add_parser('create-config', help='Generate a config file') - parser_create_config.add_argument("create_config", action='store_true', help='Generate a config file') + parser_create_config = commands.add_parser("create-config", help="Generate a config file") + parser_create_config.add_argument("create_config", action="store_true", help="Generate a config file") parser.add_argument("-v", "--verbose", default="", type=str, help="Verbose ") @@ -100,10 +103,10 @@ def run(self): crypt = Crypt() if self.create_key: path = crypt._loader.get_path_from_env() # pylint: disable=protected-access - pwd = self.get_input('Type a password to generate the key file: ') + pwd = self.get_input("Type a password to generate the key file: ") # Should use yes_no_input insted of get input below # the result should be validated for Yes (Y|y) rather allowing anything other than 'n' - generate_file = self.get_input('Do you want to generate a file in {}? [Y/n]'.format(path)) + generate_file = self.get_input("Do you want to generate a file in {}? [Y/n]".format(path)) generate_file = generate_file.lower() != "n" key = crypt.generate_key(pwd, generate_file) if generate_file: @@ -118,7 +121,7 @@ def run(self): if self.startproject: check_package_exists("cookiecutter") cookiecutter = import_from("cookiecutter.main", "cookiecutter") - cookiecutter('gh:python-microservices/cookiecutter-pyms', checkout=self.branch) + cookiecutter("gh:python-microservices/cookiecutter-pyms", checkout=self.branch) self.print_ok("Created project OK") if self.merge_swagger: try: @@ -128,8 +131,8 @@ def run(self): self.print_error(ex.__str__()) return False if self.create_config: - use_requests = self.yes_no_input('Do you want to use request') - use_swagger = self.yes_no_input('Do you want to use swagger') + use_requests = self.yes_no_input("Do you want to use request") + use_swagger = self.yes_no_input("Do you want to use swagger") try: conf_file_path = create_conf_file(use_requests, use_swagger) self.print_ok(f'Config file "{conf_file_path}" created') @@ -140,7 +143,9 @@ def run(self): return True def yes_no_input(self, msg=""): # pragma: no cover - answer = input(utils.colored_text(f'{msg}{"?" if not msg.endswith("?") else ""} [Y/n] :', utils.Colors.BLUE, True)) # nosec + answer = input( + utils.colored_text(f'{msg}{"?" if not msg.endswith("?") else ""} [Y/n] :', utils.Colors.BLUE, True) + ) # nosec try: return strtobool(answer) except ValueError: @@ -168,6 +173,6 @@ def exit_ok(self, msg=""): # pragma: no cover sys.exit(0) -if __name__ == '__main__': # pragma: no cover +if __name__ == "__main__": # pragma: no cover cmd = Command(arguments=sys.argv[1:], autorun=False) cmd.run() diff --git a/pyms/config/__init__.py b/pyms/config/__init__.py index 763ae86..3b02d25 100644 --- a/pyms/config/__init__.py +++ b/pyms/config/__init__.py @@ -1,4 +1,4 @@ from .confile import ConfFile from .conf import get_conf, create_conf_file -__all__ = ['get_conf', 'create_conf_file', 'ConfFile'] +__all__ = ["get_conf", "create_conf_file", "ConfFile"] diff --git a/pyms/config/conf.py b/pyms/config/conf.py index 9fb55cc..4652793 100644 --- a/pyms/config/conf.py +++ b/pyms/config/conf.py @@ -4,9 +4,15 @@ import yaml from pyms.utils import utils from pyms.config.confile import ConfFile -from pyms.constants import PYMS_CONFIG_WHITELIST_KEYWORDS, CONFIGMAP_FILE_ENVIRONMENT_LEGACY, \ - CONFIGMAP_FILE_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, LOGGER_NAME, \ - DEFAULT_CONFIGMAP_FILENAME +from pyms.constants import ( + PYMS_CONFIG_WHITELIST_KEYWORDS, + CONFIGMAP_FILE_ENVIRONMENT_LEGACY, + CONFIGMAP_FILE_ENVIRONMENT, + CRYPT_FILE_KEY_ENVIRONMENT, + CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, + LOGGER_NAME, + DEFAULT_CONFIGMAP_FILENAME, +) from pyms.exceptions import ServiceDoesNotExistException, ConfigErrorException, AttrDoesNotExistException logger = logging.getLogger(LOGGER_NAME) @@ -43,7 +49,7 @@ def get_conf(*args, **kwargs): :return: """ - service = kwargs.pop('service', None) + service = kwargs.pop("service", None) if not service: raise ServiceDoesNotExistException("Service not defined") config = ConfFile(*args, **kwargs) @@ -59,7 +65,8 @@ def validate_conf(*args, **kwargs): except AttrDoesNotExistException: is_config_ok = False if not is_config_ok: - raise ConfigErrorException("""Config file must start with `pyms` keyword, for example: + raise ConfigErrorException( + """Config file must start with `pyms` keyword, for example: pyms: services: metrics: true @@ -74,13 +81,15 @@ def validate_conf(*args, **kwargs): component_name: "Python Microservice" config: DEBUG: true - TESTING: true""") + TESTING: true""" + ) try: config.pyms.config except AttrDoesNotExistException: is_config_ok = False if not is_config_ok: - raise ConfigErrorException("""`pyms` block must contain a `config` keyword in your Config file, for example: + raise ConfigErrorException( + """`pyms` block must contain a `config` keyword in your Config file, for example: pyms: services: metrics: true @@ -95,10 +104,12 @@ def validate_conf(*args, **kwargs): component_name: "Python Microservice" config: DEBUG: true - TESTING: true""") + TESTING: true""" + ) wrong_keywords = [i for i in config.pyms if i not in PYMS_CONFIG_WHITELIST_KEYWORDS] if len(wrong_keywords) > 0: - raise ConfigErrorException("""{} isn`t a valid keyword for pyms block, for example: + raise ConfigErrorException( + """{} isn`t a valid keyword for pyms block, for example: pyms: services: metrics: true @@ -113,15 +124,18 @@ def validate_conf(*args, **kwargs): component_name: "Python Microservice" config: DEBUG: true - TESTING: true""".format(wrong_keywords)) + TESTING: true""".format( + wrong_keywords + ) + ) # TODO Remove temporally deprecated warnings on future versions __verify_deprecated_env_variables(config) def __verify_deprecated_env_variables(config): - env_var_duplicated = "IMPORTANT: If you are using \"{}\" environment variable, \"{}\" value will be ignored." - env_var_deprecated = "IMPORTANT: \"{}\" environment variable is deprecated on this version, use \"{}\" instead." + env_var_duplicated = 'IMPORTANT: If you are using "{}" environment variable, "{}" value will be ignored.' + env_var_deprecated = 'IMPORTANT: "{}" environment variable is deprecated on this version, use "{}" instead.' if os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None: if os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is not None: @@ -184,10 +198,10 @@ def create_conf_file(use_requests: bool = False, use_swagger: bool = False) -> U "DEBUG": True, "TESTING": False, "APP_NAME": "Python Microservice", - "APPLICATION_ROOT": "" + "APPLICATION_ROOT": "", } try: - with open(CONFIG_FILE, 'w', encoding='utf-8') as config_file: + with open(CONFIG_FILE, "w", encoding="utf-8") as config_file: config_file.write(yaml.dump(config, default_flow_style=False, default_style=None, sort_keys=False)) except Exception as ex: raise ex diff --git a/pyms/config/confile.py b/pyms/config/confile.py index 9036297..db2bd43 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -6,8 +6,12 @@ import anyconfig -from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, LOGGER_NAME, DEFAULT_CONFIGMAP_FILENAME, \ - CONFIGMAP_FILE_ENVIRONMENT_LEGACY +from pyms.constants import ( + CONFIGMAP_FILE_ENVIRONMENT, + LOGGER_NAME, + DEFAULT_CONFIGMAP_FILENAME, + CONFIGMAP_FILE_ENVIRONMENT_LEGACY, +) from pyms.exceptions import AttrDoesNotExistException, ConfigDoesNotFoundException from pyms.utils.files import LoadFile @@ -22,6 +26,7 @@ class ConfFile(dict): * empty_init: Allow blank variables * config: Allow to pass a dictionary to ConfFile without use a file """ + _empty_init = False _crypt = None @@ -73,8 +78,8 @@ def set_config(self, config: Dict) -> Dict: add_decripted_keys = [] for k, v in config.items(): if k.lower().startswith("enc_"): - k_not_crypt = re.compile(re.escape('enc_'), re.IGNORECASE) - decrypted_key = k_not_crypt.sub('', k) + k_not_crypt = re.compile(re.escape("enc_"), re.IGNORECASE) + decrypted_key = k_not_crypt.sub("", k) decrypted_value = self._crypt.decrypt(v) if self._crypt else None setattr(self, decrypted_key, decrypted_value) add_decripted_keys.append((decrypted_key, decrypted_value)) @@ -134,6 +139,8 @@ def __setattr__(self, name, value, *args, **kwargs): @staticmethod def __get_updated_configmap_file_env() -> str: result = CONFIGMAP_FILE_ENVIRONMENT - if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None) and (os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is None): + if (os.getenv(CONFIGMAP_FILE_ENVIRONMENT_LEGACY) is not None) and ( + os.getenv(CONFIGMAP_FILE_ENVIRONMENT) is None + ): result = CONFIGMAP_FILE_ENVIRONMENT_LEGACY return result diff --git a/pyms/crypt/driver.py b/pyms/crypt/driver.py index 2234be1..5db9576 100644 --- a/pyms/crypt/driver.py +++ b/pyms/crypt/driver.py @@ -11,7 +11,6 @@ class CryptAbstract(ABC): - def __init__(self, *args, **kwargs): self.config = kwargs.get("config") @@ -25,7 +24,6 @@ def decrypt(self, encrypted): class CryptNone(CryptAbstract): - def encrypt(self, message): return message @@ -37,6 +35,7 @@ class CryptResource(ConfigResource): """This class works between `pyms.flask.create_app.Microservice` and `pyms.flask.services.[THESERVICE]`. Search for a file with the name you want to load, set the configuration and return a instance of the class you want """ + config_resource = CRYPT_BASE def get_crypt(self, *args, **kwargs) -> CryptAbstract: diff --git a/pyms/crypt/fernet.py b/pyms/crypt/fernet.py index 9e95864..602d1a7 100644 --- a/pyms/crypt/fernet.py +++ b/pyms/crypt/fernet.py @@ -24,15 +24,11 @@ def generate_key(self, password: Text, write_to_file: bool = False) -> bytes: byte_password = password.encode() # Convert to type bytes salt = os.urandom(16) kdf = PBKDF2HMAC( - algorithm=hashes.SHA512_256(), - length=32, - salt=salt, - iterations=100000, - backend=default_backend() + algorithm=hashes.SHA512_256(), length=32, salt=salt, iterations=100000, backend=default_backend() ) key = base64.urlsafe_b64encode(kdf.derive(byte_password)) # Can only use kdf once if write_to_file: - self._loader.put_file(key, 'wb') + self._loader.put_file(key, "wb") return key def read_key(self): @@ -42,7 +38,8 @@ def read_key(self): crypt_file_key_env = self.__get_updated_crypt_file_key_env() # Temporally backward compatibility raise FileDoesNotExistException( "Decrypt key {} not exists. You must set a correct env var {} " - "or run `pyms crypt create-key` command".format(self._loader.path, crypt_file_key_env)) + "or run `pyms crypt create-key` command".format(self._loader.path, crypt_file_key_env) + ) return key def encrypt(self, message): @@ -65,6 +62,8 @@ def delete_key(self): @staticmethod def __get_updated_crypt_file_key_env() -> str: result = CRYPT_FILE_KEY_ENVIRONMENT - if (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None) and (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is None): + if (os.getenv(CRYPT_FILE_KEY_ENVIRONMENT_LEGACY) is not None) and ( + os.getenv(CRYPT_FILE_KEY_ENVIRONMENT) is None + ): result = CRYPT_FILE_KEY_ENVIRONMENT_LEGACY return result diff --git a/pyms/flask/app/__init__.py b/pyms/flask/app/__init__.py index 6516942..c0b477a 100644 --- a/pyms/flask/app/__init__.py +++ b/pyms/flask/app/__init__.py @@ -2,4 +2,4 @@ from .create_config import config -__all__ = ['Microservice', 'config'] +__all__ = ["Microservice", "config"] diff --git a/pyms/flask/app/create_app.py b/pyms/flask/app/create_app.py index 9cb3504..7e8e28f 100644 --- a/pyms/flask/app/create_app.py +++ b/pyms/flask/app/create_app.py @@ -72,6 +72,7 @@ def example(): Current services are swagger, request, tracer, metrics """ + config_resource = CONFIG_BASE services: List[str] = [] application = Flask @@ -149,7 +150,7 @@ def init_logger(self) -> None: :return: """ self.application.logger = logger - os.environ['WERKZEUG_RUN_MAIN'] = "true" + os.environ["WERKZEUG_RUN_MAIN"] = "true" formatter = CustomJsonFormatter() formatter.add_service_name(self.application.config["APP_NAME"]) @@ -174,8 +175,11 @@ def init_app(self) -> Flask: application = self.swagger.init_app(config=self.config.to_flask(), path=self.path) else: check_package_exists("flask") - application = Flask(__name__, static_folder=os.path.join(self.path, 'static'), - template_folder=os.path.join(self.path, 'templates')) + application = Flask( + __name__, + static_folder=os.path.join(self.path, "static"), + template_folder=os.path.join(self.path, "templates"), + ) application.root_path = self.path diff --git a/pyms/flask/app/utils.py b/pyms/flask/app/utils.py index c16f5bd..1231ee8 100644 --- a/pyms/flask/app/utils.py +++ b/pyms/flask/app/utils.py @@ -7,6 +7,7 @@ class SingletonMeta(type): possible methods include: base class, decorator, metaclass. We will use the metaclass because it is best suited for this purpose. """ + _instances: Dict[type, type] = {} _singleton = True @@ -43,10 +44,10 @@ def _extract_prefix(environ: dict) -> str: :return: """ # Get path from Traefik, Nginx and Apache - path = environ.get('HTTP_X_SCRIPT_NAME', '') + path = environ.get("HTTP_X_SCRIPT_NAME", "") if not path: # Get path from Zuul - path = environ.get('HTTP_X_FORWARDED_PREFIX', '') + path = environ.get("HTTP_X_FORWARDED_PREFIX", "") if path and not path.startswith("/"): path = "/" + path return path @@ -54,12 +55,12 @@ def _extract_prefix(environ: dict) -> str: def __call__(self, environ, start_response): script_name = self._extract_prefix(environ) if script_name: - environ['SCRIPT_NAME'] = script_name - path_info = environ['PATH_INFO'] + environ["SCRIPT_NAME"] = script_name + path_info = environ["PATH_INFO"] if path_info.startswith(script_name): - environ['PATH_INFO'] = path_info[len(script_name):] + environ["PATH_INFO"] = path_info[len(script_name) :] - scheme = environ.get('HTTP_X_SCHEME', '') + scheme = environ.get("HTTP_X_SCHEME", "") if scheme: - environ['wsgi.url_scheme'] = scheme + environ["wsgi.url_scheme"] = scheme return self.app(environ, start_response) diff --git a/pyms/flask/configreload/__init__.py b/pyms/flask/configreload/__init__.py index 5629da3..32c702d 100644 --- a/pyms/flask/configreload/__init__.py +++ b/pyms/flask/configreload/__init__.py @@ -1,4 +1,4 @@ from .configreload import configreload_blueprint -__all__ = ['configreload_blueprint'] +__all__ = ["configreload_blueprint"] diff --git a/pyms/flask/configreload/configreload.py b/pyms/flask/configreload/configreload.py index c18e7cc..e60d3ed 100644 --- a/pyms/flask/configreload/configreload.py +++ b/pyms/flask/configreload/configreload.py @@ -2,10 +2,10 @@ from flask import Blueprint from flask import current_app -configreload_blueprint = Blueprint('configreload', __name__, static_url_path='/static') +configreload_blueprint = Blueprint("configreload", __name__, static_url_path="/static") -@configreload_blueprint.route('/reload-config', methods=['POST']) +@configreload_blueprint.route("/reload-config", methods=["POST"]) def reloadconfig(): """ Reread configuration from file. diff --git a/pyms/flask/healthcheck/__init__.py b/pyms/flask/healthcheck/__init__.py index 931a15e..7374bb2 100644 --- a/pyms/flask/healthcheck/__init__.py +++ b/pyms/flask/healthcheck/__init__.py @@ -1,4 +1,4 @@ from .healthcheck import healthcheck_blueprint -__all__ = ['healthcheck_blueprint'] +__all__ = ["healthcheck_blueprint"] diff --git a/pyms/flask/healthcheck/healthcheck.py b/pyms/flask/healthcheck/healthcheck.py index 3022740..e584272 100644 --- a/pyms/flask/healthcheck/healthcheck.py +++ b/pyms/flask/healthcheck/healthcheck.py @@ -2,10 +2,10 @@ from flask import Blueprint -healthcheck_blueprint = Blueprint('healthcheck', __name__, static_url_path='/static') +healthcheck_blueprint = Blueprint("healthcheck", __name__, static_url_path="/static") -@healthcheck_blueprint.route('/healthcheck', methods=['GET']) +@healthcheck_blueprint.route("/healthcheck", methods=["GET"]) def healthcheck(): """Set a healthcheck to help other service to discover this microservice, like Kubernetes, AWS ELB, etc. :return: diff --git a/pyms/flask/services/driver.py b/pyms/flask/services/driver.py index 8aaf78a..d0c2080 100644 --- a/pyms/flask/services/driver.py +++ b/pyms/flask/services/driver.py @@ -37,6 +37,7 @@ class DriverService(ConfigResource): * `swagger`: is set as the service `pyms.services.swagger` * `tracer`: is set as the service `pyms.services.tracer` """ + enabled = True init_action = False @@ -47,8 +48,11 @@ def __init__(self, *args, **kwargs): def __getattr__(self, attr, *args, **kwargs): config_attribute = getattr(self.config, attr) - return config_attribute if config_attribute == "" or config_attribute != {} else self.default_values.get(attr, - None) + return ( + config_attribute + if config_attribute == "" or config_attribute != {} + else self.default_values.get(attr, None) + ) def is_enabled(self) -> bool: return self.enabled @@ -61,6 +65,7 @@ class ServicesResource(ConfigResource): """This class works between `pyms.flask.create_app.Microservice` and `pyms.flask.services.[THESERVICE]`. Search for a file with the name you want to load, set the configuration and return a instance of the class you want """ + config_resource = SERVICE_BASE def get_services(self) -> Iterator[Tuple[Text, DriverService]]: diff --git a/pyms/flask/services/metrics.py b/pyms/flask/services/metrics.py index 2ee99d9..279f3fa 100644 --- a/pyms/flask/services/metrics.py +++ b/pyms/flask/services/metrics.py @@ -23,7 +23,7 @@ ) -class FlaskMetricsWrapper(): +class FlaskMetricsWrapper: def __init__(self, app_name): self.app_name = app_name @@ -56,8 +56,7 @@ def __init__(self, *args, **kwargs): def init_action(self, microservice_instance): microservice_instance.application.register_blueprint(microservice_instance.metrics.metrics_blueprint) self.add_logger_handler( - microservice_instance.application.logger, - microservice_instance.application.config["APP_NAME"] + microservice_instance.application.logger, microservice_instance.application.config["APP_NAME"] ) self.monitor(microservice_instance.application.config["APP_NAME"], microservice_instance.application) diff --git a/pyms/flask/services/requests.py b/pyms/flask/services/requests.py index 75deea8..2d34936 100644 --- a/pyms/flask/services/requests.py +++ b/pyms/flask/services/requests.py @@ -45,6 +45,7 @@ class Service(DriverService): Encapsulate common rest operations between business services propagating trace headers if set up. All default values keys are created as class attributes in `DriverService` """ + config_resource = "requests" default_values = { "data": "", @@ -72,8 +73,8 @@ def requests(self, session: requests.Session) -> requests.Session: status_forcelist=self.status_retries, ) adapter = HTTPAdapter(max_retries=max_retries) - session_r.mount('http://', adapter) - session_r.mount('https://', adapter) + session_r.mount("http://", adapter) + session_r.mount("https://", adapter) return session_r @staticmethod @@ -143,8 +144,15 @@ def parse_response(self, response: Response) -> dict: return {} @retry - def get(self, url: str, path_params: dict = None, params: dict = None, headers: dict = None, - propagate_headers: bool = False, **kwargs) -> Response: + def get( + self, + url: str, + path_params: dict = None, + params: dict = None, + headers: dict = None, + propagate_headers: bool = False, + **kwargs + ) -> Response: """Sends a GET request. :param url: URL for the new :class:`Request` object. Could contain path parameters @@ -161,16 +169,16 @@ def get(self, url: str, path_params: dict = None, params: dict = None, headers: full_url = self._build_url(url, path_params) headers = self._get_headers(headers=headers, propagate_headers=propagate_headers) headers = self.insert_trace_headers(headers) - logger.debug("Get with url {}, params {}, headers {}, kwargs {}". - format(full_url, params, headers, kwargs)) + logger.debug("Get with url {}, params {}, headers {}, kwargs {}".format(full_url, params, headers, kwargs)) session = requests.Session() response = self.requests(session=session).get(full_url, params=params, headers=headers, **kwargs) return response - def get_for_object(self, url: str, path_params: dict = None, params: dict = None, headers: dict = None, - **kwargs) -> dict: + def get_for_object( + self, url: str, path_params: dict = None, params: dict = None, headers: dict = None, **kwargs + ) -> dict: """Sends a GET request and returns the json representation found in response's content data node. :param url: URL for the new :class:`Request` object. Could contain path parameters @@ -187,8 +195,9 @@ def get_for_object(self, url: str, path_params: dict = None, params: dict = None return self.parse_response(response) @retry - def post(self, url: str, path_params: dict = None, data: dict = None, json: dict = None, headers: dict = None, - **kwargs) -> Response: + def post( + self, url: str, path_params: dict = None, data: dict = None, json: dict = None, headers: dict = None, **kwargs + ) -> Response: """Sends a POST request. :param url: URL for the new :class:`Request` object. Could contain path parameters @@ -205,8 +214,9 @@ def post(self, url: str, path_params: dict = None, data: dict = None, json: dict full_url = self._build_url(url, path_params) headers = self._get_headers(headers) headers = self.insert_trace_headers(headers) - logger.debug("Post with url {}, data {}, json {}, headers {}, kwargs {}".format(full_url, data, json, - headers, kwargs)) + logger.debug( + "Post with url {}, data {}, json {}, headers {}, kwargs {}".format(full_url, data, json, headers, kwargs) + ) session = requests.Session() response = self.requests(session=session).post(full_url, data=data, json=json, headers=headers, **kwargs) @@ -214,8 +224,9 @@ def post(self, url: str, path_params: dict = None, data: dict = None, json: dict return response - def post_for_object(self, url: str, path_params: dict = None, data: dict = None, json: dict = None, - headers: dict = None, **kwargs) -> dict: + def post_for_object( + self, url: str, path_params: dict = None, data: dict = None, json: dict = None, headers: dict = None, **kwargs + ) -> dict: """Sends a POST request and returns the json representation found in response's content data node. :param url: URL for the new :class:`Request` object. Could contain path parameters @@ -250,8 +261,7 @@ def put(self, url: str, path_params: dict = None, data: dict = None, headers: di full_url = self._build_url(url, path_params) headers = self._get_headers(headers) headers = self.insert_trace_headers(headers) - logger.debug("Put with url {}, data {}, headers {}, kwargs {}".format(full_url, data, headers, - kwargs)) + logger.debug("Put with url {}, data {}, headers {}, kwargs {}".format(full_url, data, headers, kwargs)) session = requests.Session() response = self.requests(session=session).put(full_url, data, headers=headers, **kwargs) @@ -259,8 +269,9 @@ def put(self, url: str, path_params: dict = None, data: dict = None, headers: di return response - def put_for_object(self, url: str, path_params: dict = None, data: dict = None, headers: dict = None, - **kwargs) -> dict: + def put_for_object( + self, url: str, path_params: dict = None, data: dict = None, headers: dict = None, **kwargs + ) -> dict: """Sends a PUT request and returns the json representation found in response's content data node. :param url: URL for the new :class:`Request` object. Could contain path parameters @@ -295,8 +306,7 @@ def patch(self, url: str, path_params: dict = None, data: dict = None, headers: full_url = self._build_url(url, path_params) headers = self._get_headers(headers) headers = self.insert_trace_headers(headers) - logger.debug("Patch with url {}, data {}, headers {}, kwargs {}".format(full_url, data, headers, - kwargs)) + logger.debug("Patch with url {}, data {}, headers {}, kwargs {}".format(full_url, data, headers, kwargs)) session = requests.Session() response = self.requests(session=session).patch(full_url, data, headers=headers, **kwargs) @@ -304,8 +314,9 @@ def patch(self, url: str, path_params: dict = None, data: dict = None, headers: return response - def patch_for_object(self, url: str, path_params: dict = None, data: dict = None, headers: dict = None, - **kwargs) -> dict: + def patch_for_object( + self, url: str, path_params: dict = None, data: dict = None, headers: dict = None, **kwargs + ) -> dict: """Sends a PATCH request and returns the json representation found in response's content data node. :param url: URL for the new :class:`Request` object. Could contain path parameters diff --git a/pyms/flask/services/service_discovery.py b/pyms/flask/services/service_discovery.py index ae3dd8f..7e65ea9 100644 --- a/pyms/flask/services/service_discovery.py +++ b/pyms/flask/services/service_discovery.py @@ -24,13 +24,14 @@ def register_service(self, *args, **kwargs): class ServiceDiscoveryConsul(ServiceDiscoveryBase): def __init__(self, config): - self.client = consulate.Consul(host=config.host, port=config.port, token=config.token, scheme=config.scheme, - adapter=config.adapter) + self.client = consulate.Consul( + host=config.host, port=config.port, token=config.token, scheme=config.scheme, adapter=config.adapter + ) def register_service(self, *args, **kwargs): - self.client.agent.check.register(kwargs["app_name"], - http=kwargs["healtcheck_url"], - interval=kwargs.get("interval", "10s")) + self.client.agent.check.register( + kwargs["app_name"], http=kwargs["healtcheck_url"], interval=kwargs.get("interval", "10s") + ) class Service(DriverService): @@ -42,17 +43,13 @@ class Service(DriverService): "scheme": "http", "port": 8500, "healtcheck_url": "http://127.0.0.1.nip.io:5000/healthcheck", - "autoregister": False + "autoregister": False, } def init_action(self, microservice_instance): if self.autoregister: - self._client.register_service( - id_app=self.id_app, - host=self.host, - healtcheck_url=self.healtcheck_url, - port=self.port, - app_name=microservice_instance.application.config["APP_NAME"]) + app_name = microservice_instance.application.config["APP_NAME"] + self._client.register_service(healtcheck_url=self.healtcheck_url, app_name=app_name) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/pyms/flask/services/swagger.py b/pyms/flask/services/swagger.py index 501cf9b..cd13510 100644 --- a/pyms/flask/services/swagger.py +++ b/pyms/flask/services/swagger.py @@ -29,8 +29,7 @@ def get_bundled_specs(main_file: Path) -> Dict[str, Any]: :param main_file: Swagger file path :return: """ - parser = prance.ResolvingParser(str(main_file.absolute()), - lazy=True, backend='openapi-spec-validator') + parser = prance.ResolvingParser(str(main_file.absolute()), lazy=True, backend="openapi-spec-validator") parser.parse() return parser.specification @@ -42,15 +41,13 @@ def merge_swagger_file(main_file: str) -> None: :return: """ input_file = Path(main_file) - output_file = Path(input_file.parent, 'swagger-complete.yaml') + output_file = Path(input_file.parent, "swagger-complete.yaml") contents = formats.serialize_spec( specs=get_bundled_specs(input_file), filename=output_file, ) - fs.write_file(filename=output_file, - contents=contents, - encoding='utf-8') + fs.write_file(filename=output_file, contents=contents, encoding="utf-8") class Service(DriverService): @@ -64,13 +61,9 @@ class Service(DriverService): All default values keys are created as class attributes in `DriverService` """ + config_resource = "swagger" - default_values = { - "path": SWAGGER_PATH, - "file": SWAGGER_FILE, - "url": SWAGGER_URL, - "project_dir": PROJECT_DIR - } + default_values = {"path": SWAGGER_PATH, "file": SWAGGER_FILE, "url": SWAGGER_URL, "project_dir": PROJECT_DIR} @staticmethod def _get_application_root(config) -> str: @@ -110,14 +103,13 @@ def init_app(self, config, path: Path) -> Flask: if not os.path.isabs(self.path): specification_dir = os.path.join(path, self.path) - app = connexion.App(__name__, - specification_dir=specification_dir, - resolver=RestyResolver(self.project_dir)) + app = connexion.App(__name__, specification_dir=specification_dir, resolver=RestyResolver(self.project_dir)) params = { - "specification": get_bundled_specs( - Path(os.path.join(specification_dir, self.file))) if prance else self.file, - "arguments": {'title': config.APP_NAME}, + "specification": get_bundled_specs(Path(os.path.join(specification_dir, self.file))) + if prance + else self.file, + "arguments": {"title": config.APP_NAME}, "base_path": application_root, "options": {"swagger_url": self.url}, } diff --git a/pyms/flask/services/tracer.py b/pyms/flask/services/tracer.py index dde7f52..718f54b 100644 --- a/pyms/flask/services/tracer.py +++ b/pyms/flask/services/tracer.py @@ -53,6 +53,7 @@ class Service(DriverService): Add trace to all executions with [opentracing](https://github.com/opentracing-contrib/python-flask). All default values keys are created as class attributes in `DriverService` """ + config_resource = "tracer" default_values = { "client": DEFAULT_CLIENT, @@ -83,11 +84,7 @@ def init_jaeger_tracer(self): Config = import_from("jaeger_client", "Config") host = {} if self.host: - host = { - 'local_agent': { - 'reporting_host': self.host - } - } + host = {"local_agent": {"reporting_host": self.host}} metrics_config = get_conf(service=get_service_name(service="metrics"), empty_init=True) metrics = "" if metrics_config: @@ -96,17 +93,18 @@ def init_jaeger_tracer(self): config = Config( config={ **{ - 'sampler': { - 'type': 'const', - 'param': 1, + "sampler": { + "type": "const", + "param": 1, }, - 'propagation': 'b3', - 'logging': True + "propagation": "b3", + "logging": True, }, - **host - }, service_name=self.component_name, + **host, + }, + service_name=self.component_name, metrics_factory=metrics, - validate=True + validate=True, ) return config.initialize_tracer() diff --git a/pyms/logger/__init__.py b/pyms/logger/__init__.py index 621faa3..bfcd552 100644 --- a/pyms/logger/__init__.py +++ b/pyms/logger/__init__.py @@ -2,4 +2,6 @@ """ from .logger import CustomJsonFormatter -__all__ = ['CustomJsonFormatter', ] +__all__ = [ + "CustomJsonFormatter", +] diff --git a/pyms/logger/logger.py b/pyms/logger/logger.py index 42c1e40..5e6491c 100644 --- a/pyms/logger/logger.py +++ b/pyms/logger/logger.py @@ -17,21 +17,21 @@ class CustomJsonFormatter(jsonlogger.JsonFormatter): def add_fields(self, log_record, record, message_dict): super().add_fields(log_record, record, message_dict) - if not log_record.get('timestamp'): + if not log_record.get("timestamp"): # this doesn't use record.created, so it is slightly off - now = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ') - log_record['timestamp'] = now - if log_record.get('level'): - log_record['severity'] = log_record['level'].upper() + now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + log_record["timestamp"] = now + if log_record.get("level"): + log_record["severity"] = log_record["level"].upper() else: - log_record['severity'] = record.levelname + log_record["severity"] = record.levelname log_record["service"] = self.service_name try: headers = inject_span_in_headers({}) - log_record["trace"] = headers.get('X-B3-TraceId', "") - log_record["span"] = headers.get('X-B3-SpanId', "") - log_record["parent"] = headers.get('X-B3-ParentSpanId', "") + log_record["trace"] = headers.get("X-B3-TraceId", "") + log_record["span"] = headers.get("X-B3-SpanId", "") + log_record["parent"] = headers.get("X-B3-ParentSpanId", "") except Exception as ex: # pragma: no cover logger.error("Tracer error: {}".format(ex)) diff --git a/pyms/services_discovery/__init__.py b/pyms/services_discovery/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyms/services_discovery/consulate/__init__.py b/pyms/services_discovery/consulate/__init__.py index c954123..ed06f46 100644 --- a/pyms/services_discovery/consulate/__init__.py +++ b/pyms/services_discovery/consulate/__init__.py @@ -6,23 +6,7 @@ from logging import NullHandler from pyms.services_discovery.consulate.client import Consul -from pyms.services_discovery.consulate.exceptions import (ConsulateException, - ClientError, - ServerError, - ACLDisabled, - Forbidden, - NotFound, - LockFailure, - RequestError) - -__version__ = '1.0.0' - -# Prevent undesired log output to the root logger -logging.getLogger('consulate').addHandler(NullHandler()) - -__all__ = [ - __version__, - Consul, +from pyms.services_discovery.consulate.exceptions import ( ConsulateException, ClientError, ServerError, @@ -30,5 +14,22 @@ Forbidden, NotFound, LockFailure, - RequestError + RequestError, +) + +__version__ = "1.0.0" + +# Prevent undesired log output to the root logger +logging.getLogger("consulate").addHandler(NullHandler()) + +__all__ = [ + "Consul", + "ConsulateException", + "ClientError", + "ServerError", + "ACLDisabled", + "Forbidden", + "NotFound", + "LockFailure", + "RequestError", ] diff --git a/pyms/services_discovery/consulate/adapters.py b/pyms/services_discovery/consulate/adapters.py index 43bfc12..163ae3d 100644 --- a/pyms/services_discovery/consulate/adapters.py +++ b/pyms/services_discovery/consulate/adapters.py @@ -9,6 +9,7 @@ import requests import requests.exceptions + try: import requests_unixsocket except ImportError: # pragma: no cover @@ -18,8 +19,8 @@ LOGGER = logging.getLogger(__name__) -CONTENT_FORM = 'application/x-www-form-urlencoded; charset=utf-8' -CONTENT_JSON = 'application/json; charset=utf-8' +CONTENT_FORM = "application/x-www-form-urlencoded; charset=utf-8" +CONTENT_JSON = "application/json; charset=utf-8" def prepare_data(fun): @@ -36,18 +37,17 @@ def inner(*args, **kwargs): :param dict kwargs: keyword arguments """ - if kwargs.get('data'): - if not utils.is_string(kwargs.get('data')): - kwargs['data'] = json.dumps(kwargs['data']) - elif len(args) == 3 and \ - not (utils.is_string(args[2]) or args[2] is None): + if kwargs.get("data"): + if not utils.is_string(kwargs.get("data")): + kwargs["data"] = json.dumps(kwargs["data"]) + elif len(args) == 3 and not (utils.is_string(args[2]) or args[2] is None): args = args[0], args[1], json.dumps(args[2]) return fun(*args, **kwargs) return inner -class Request(object): +class Request: """The Request adapter class""" def __init__(self, timeout=None, verify=True, cert=None): @@ -83,10 +83,8 @@ def get(self, uri, timeout=None): """ LOGGER.debug("GET %s", uri) try: - return api.Response(self.session.get( - uri, timeout=timeout or self.timeout)) - except (requests.exceptions.RequestException, - OSError, socket.error) as err: + return api.Response(self.session.get(uri, timeout=timeout or self.timeout)) + except (requests.exceptions.RequestException, OSError, socket.error) as err: raise exceptions.RequestError(str(err)) def get_stream(self, uri): @@ -99,12 +97,11 @@ def get_stream(self, uri): LOGGER.debug("GET Stream from %s", uri) try: response = self.session.get(uri, stream=True) - except (requests.exceptions.RequestException, - OSError, socket.error) as err: + except (requests.exceptions.RequestException, OSError, socket.error) as err: raise exceptions.RequestError(str(err)) if utils.response_ok(response): for line in response.iter_lines(): # pragma: no cover - yield line.decode('utf-8') + yield line.decode("utf-8") @prepare_data def put(self, uri, data=None, timeout=None): @@ -118,17 +115,10 @@ def put(self, uri, data=None, timeout=None): """ LOGGER.debug("PUT %s with %r", uri, data) - headers = { - 'Content-Type': CONTENT_FORM - if utils.is_string(data) else CONTENT_JSON - } + headers = {"Content-Type": CONTENT_FORM if utils.is_string(data) else CONTENT_JSON} try: - return api.Response( - self.session.put( - uri, data=data, headers=headers, - timeout=timeout or self.timeout)) - except (requests.exceptions.RequestException, - OSError, socket.error) as err: + return api.Response(self.session.put(uri, data=data, headers=headers, timeout=timeout or self.timeout)) + except (requests.exceptions.RequestException, OSError, socket.error) as err: raise exceptions.RequestError(str(err)) @@ -136,5 +126,5 @@ class UnixSocketRequest(Request): # pragma: no cover """Use to communicate with Consul over a Unix socket""" def __init__(self, timeout=None): - super(UnixSocketRequest, self).__init__(timeout) + super().__init__(timeout) self.session = requests_unixsocket.Session() diff --git a/pyms/services_discovery/consulate/api/__init__.py b/pyms/services_discovery/consulate/api/__init__.py index 178a068..068abc6 100644 --- a/pyms/services_discovery/consulate/api/__init__.py +++ b/pyms/services_discovery/consulate/api/__init__.py @@ -14,15 +14,4 @@ from pyms.services_discovery.consulate.api.status import Status from pyms.services_discovery.consulate.api.base import Response -__all__ = [ - ACL, - Agent, - Catalog, - Event, - Health, - KV, - Lock, - Session, - Status, - Response -] +__all__ = ["ACL", "Agent", "Catalog", "Event", "Health", "KV", "Lock", "Session", "Status", "Response"] diff --git a/pyms/services_discovery/consulate/api/acl.py b/pyms/services_discovery/consulate/api/acl.py index b876e85..f383357 100644 --- a/pyms/services_discovery/consulate/api/acl.py +++ b/pyms/services_discovery/consulate/api/acl.py @@ -4,13 +4,15 @@ """ import logging -from pyms.services_discovery.consulate.models import acl as model -from pyms.services_discovery.consulate.api import base from pyms.services_discovery.consulate import exceptions +from pyms.services_discovery.consulate.api import base +from pyms.services_discovery.consulate.models import acl as model + # from typing import List, Dict, Union LOGGER = logging.getLogger(__name__) + # ServiceIdentity = Dict[str, Union[str, List[str]]] # ServiceIdentities = List[ServiceIdentity] # PolicyLink = Dict[str, str] @@ -19,11 +21,12 @@ # RoleLinks = List[RoleLink] -class ACL(base.Endpoint): +class ACL(base.Endpoint): # pylint: disable=too-many-public-methods """The ACL endpoints are used to create, update, destroy, and query ACL tokens. """ + def list_policies(self): """List all ACL policies available in cluster. @@ -32,7 +35,7 @@ def list_policies(self): """ return self._get(["policies"]) - def read_policy(self, id): + def read_policy(self, id): # pylint: disable=redefined-builtin """Read an existing policy with the given ID. :param str id: The ID of the policy. @@ -41,11 +44,7 @@ def read_policy(self, id): """ return self._get(["policy", id]) - def create_policy(self, - name, - datacenters=None, - description=None, - rules=None): + def create_policy(self, name, datacenters=None, description=None, rules=None): """Create policy with name given and rules. :param str name: name of the policy @@ -55,20 +54,15 @@ def create_policy(self, :param rtype: dict """ - return self._put_response_body(["policy"], {}, - dict( - model.ACLPolicy( - name=name, - datacenters=datacenters, - description=description, - rules=rules))) - - def update_policy(self, - id, - name, - datacenters=None, - description=None, - rules=None): + return self._put_response_body( + ["policy"], + {}, + dict(model.ACLPolicy(name=name, datacenters=datacenters, description=description, rules=rules)), + ) + + def update_policy( + self, id, name, datacenters=None, description=None, rules=None + ): # pylint: disable=redefined-builtin """Update policy with id given. :param str id: A UUID for the policy to update. @@ -79,15 +73,13 @@ def update_policy(self, :param rtype: dict """ - return self._put_response_body(["policy", id], {}, - dict( - model.ACLPolicy( - name=name, - datacenters=datacenters, - description=description, - rules=rules))) - - def delete_policy(self, id): + return self._put_response_body( + ["policy", id], + {}, + dict(model.ACLPolicy(name=name, datacenters=datacenters, description=description, rules=rules)), + ) + + def delete_policy(self, id): # pylint: disable=redefined-builtin """Delete an existing policy with the given ID. :param str id: The ID of the policy. @@ -103,7 +95,7 @@ def list_roles(self): """ return self._get(["roles"]) - def read_role(self, id=None, name=None): + def read_role(self, id=None, name=None): # pylint: disable=redefined-builtin """Read an existing role with the given ID or Name. :param str id: The ID of the role. @@ -111,18 +103,14 @@ def read_role(self, id=None, name=None): :param rtype: dict """ - if id is not None: + if id is not None: # pylint: disable=no-else-return return self._get(["role", id]) elif name is not None: return self._get(["role", "name", name]) else: raise exceptions.NotFound("Either id or name must be specified") - def create_role(self, - name, - description=None, - policies=None, - service_identities=None): + def create_role(self, name, description=None, policies=None, service_identities=None): """Create an ACL role from a list of policies or service identities. :param str name: The name of the ACL role. Must be unique. @@ -133,19 +121,18 @@ def create_role(self, """ return self._put_response_body( - ["role"], {}, + ["role"], + {}, dict( - model.ACLRole(name=name, - description=description, - policies=policies, - service_identities=service_identities))) - - def update_role(self, - id, - name, - description=None, - policies=None, - service_identities=None): + model.ACLRole( + name=name, description=description, policies=policies, service_identities=service_identities + ) + ), + ) + + def update_role( + self, id, name, description=None, policies=None, service_identities=None + ): # pylint: disable=redefined-builtin """Update role with id given. :param str id: A UUID for the policy to update. @@ -157,14 +144,16 @@ def update_role(self, """ return self._put_response_body( - ["role", id], {}, + ["role", id], + {}, dict( - model.ACLRole(name=name, - description=description, - policies=policies, - service_identities=service_identities))) + model.ACLRole( + name=name, description=description, policies=policies, service_identities=service_identities + ) + ), + ) - def delete_role(self, id): + def delete_role(self, id): # pylint: disable=redefined-builtin """Delete an existing role with the given ID. :param str id: The ID of the role. @@ -198,16 +187,18 @@ def read_self_token(self): """ return self._get(["token", "self"]) - def create_token(self, - accessor_id=None, - description=None, - expiration_time=None, - expiration_ttl=None, - local=False, - policies=None, - roles=None, - secret_id=None, - service_identities=None): + def create_token( + self, + accessor_id=None, + description=None, + expiration_time=None, + expiration_ttl=None, + local=False, + policies=None, + roles=None, + secret_id=None, + service_identities=None, + ): # pylint: disable=too-many-arguments """Create a token from the roles, policies, and service identities provided. @@ -226,28 +217,35 @@ def create_token(self, """ return self._put_response_body( - ["token"], {}, + ["token"], + {}, dict( - model.ACLToken(accessor_id=accessor_id, - description=description, - expiration_time=expiration_time, - expiration_ttl=expiration_ttl, - local=local, - policies=policies, - roles=roles, - secret_id=secret_id, - service_identities=service_identities))) - - def update_token(self, - accessor_id, - description=None, - expiration_time=None, - expiration_ttl=None, - local=False, - policies=None, - roles=None, - secret_id=None, - service_identities=None): + model.ACLToken( + accessor_id=accessor_id, + description=description, + expiration_time=expiration_time, + expiration_ttl=expiration_ttl, + local=local, + policies=policies, + roles=roles, + secret_id=secret_id, + service_identities=service_identities, + ) + ), + ) + + def update_token( + self, + accessor_id, + description=None, + expiration_time=None, + expiration_ttl=None, + local=False, + policies=None, + roles=None, + secret_id=None, + service_identities=None, + ): # pylint: disable=too-many-arguments """Create a token from the roles, policies, and service identities provided. @@ -266,17 +264,22 @@ def update_token(self, """ return self._put_response_body( - ["token", accessor_id], {}, + ["token", accessor_id], + {}, dict( - model.ACLToken(accessor_id=accessor_id, - description=description, - expiration_time=expiration_time, - expiration_ttl=expiration_ttl, - local=local, - policies=policies, - roles=roles, - secret_id=secret_id, - service_identities=service_identities))) + model.ACLToken( + accessor_id=accessor_id, + description=description, + expiration_time=expiration_time, + expiration_ttl=expiration_ttl, + local=local, + policies=policies, + roles=roles, + secret_id=secret_id, + service_identities=service_identities, + ) + ), + ) def clone_token(self, accessor_id, description=None): """Clone a token by the accessor_id. @@ -287,8 +290,8 @@ def clone_token(self, accessor_id, description=None): """ return self._put_response_body( - ["token", accessor_id, "clone"], {}, - dict(model.ACLToken(description=description))) + ["token", accessor_id, "clone"], {}, dict(model.ACLToken(description=description)) + ) def delete_token(self, accessor_id): """Delete an existing token with the given AcccessorID. @@ -323,9 +326,9 @@ def bootstrap(self): :raises: :exc:`~consulate.exceptions.Forbidden` """ - return self._put_response_body(['bootstrap'])['ID'] + return self._put_response_body(["bootstrap"])["ID"] - def create(self, name, acl_type='client', rules=None): + def create(self, name, acl_type="client", rules=None): """The create endpoint is used to make a new token. A token has a name, a type, and a set of ACL rules. @@ -354,11 +357,7 @@ def create(self, name, acl_type='client', rules=None): :raises: consulate.exceptions.Forbidden """ - return self._put_response_body(['create'], {}, - dict( - model.ACL(name=name, - type=acl_type, - rules=rules)))['ID'] + return self._put_response_body(["create"], {}, dict(model.ACL(name=name, type=acl_type, rules=rules)))["ID"] def clone(self, acl_id): """Clone an existing ACL returning the new ACL ID @@ -368,7 +367,7 @@ def clone(self, acl_id): :raises: consulate.exceptions.Forbidden """ - return self._put_response_body(['clone', acl_id])['ID'] + return self._put_response_body(["clone", acl_id])["ID"] def destroy(self, acl_id): """Delete the specified ACL @@ -378,7 +377,7 @@ def destroy(self, acl_id): :raises: consulate.exceptions.Forbidden """ - response = self._adapter.put(self._build_uri(['destroy', acl_id])) + response = self._adapter.put(self._build_uri(["destroy", acl_id])) if response.status_code == 403: raise exceptions.Forbidden(response.body) return response.status_code == 200 @@ -392,9 +391,9 @@ def info(self, acl_id): :raises: consulate.exceptions.NotFound """ - response = self._get(['info', acl_id], raise_on_404=True) + response = self._get(["info", acl_id], raise_on_404=True) if not response: - raise exceptions.NotFound('ACL not found') + raise exceptions.NotFound("ACL not found") return response def list(self): @@ -404,7 +403,7 @@ def list(self): :raises: consulate.exceptions.Forbidden """ - return self._get(['list']) + return self._get(["list"]) def replication(self): """Return the status of the ACL replication process in the datacenter. @@ -418,9 +417,9 @@ def replication(self): :raises: consulate.exceptions.Forbidden """ - return self._get(['replication']) + return self._get(["replication"]) - def update(self, acl_id, name, acl_type='client', rules=None): + def update(self, acl_id, name, acl_type="client", rules=None): """Update an existing ACL, updating its values or add a new ACL if the ACL ID specified is not found. @@ -434,9 +433,6 @@ def update(self, acl_id, name, acl_type='client', rules=None): :raises: consulate.exceptions.Forbidden """ - return self._put_response_body(['update'], {}, - dict( - model.ACL(id=acl_id, - name=name, - type=acl_type, - rules=rules)))['ID'] + return self._put_response_body( + ["update"], {}, dict(model.ACL(id=acl_id, name=name, type=acl_type, rules=rules)) + )["ID"] diff --git a/pyms/services_discovery/consulate/api/agent.py b/pyms/services_discovery/consulate/api/agent.py index 993c4f3..9a1a8c6 100644 --- a/pyms/services_discovery/consulate/api/agent.py +++ b/pyms/services_discovery/consulate/api/agent.py @@ -5,12 +5,7 @@ from pyms.services_discovery.consulate.api import base from pyms.services_discovery.consulate.models import agent as models -_TOKENS = [ - 'acl_token', - 'acl_agent_token', - 'acl_agent_master_token', - 'acl_replication_token' -] +_TOKENS = ["acl_token", "acl_agent_token", "acl_agent_master_token", "acl_replication_token"] class Agent(base.Endpoint): @@ -30,10 +25,9 @@ def __init__(self, uri, adapter, datacenter=None, token=None): :param str token: Access Token """ - super(Agent, self).__init__(uri, adapter, datacenter, token) + super().__init__(uri, adapter, datacenter, token) self.check = Agent.Check(self._base_uri, adapter, datacenter, token) - self.service = Agent.Service( - self._base_uri, adapter, datacenter, token) + self.service = Agent.Service(self._base_uri, adapter, datacenter, token) class Check(base.Endpoint): """One of the primary roles of the agent is the management of system @@ -64,25 +58,27 @@ class Check(base.Endpoint): """ - def register(self, - name, - check_id=None, - interval=None, - notes=None, - deregister_critical_service_after=None, - args=None, - docker_container_id=None, - grpc=None, - grpc_use_tls=None, - http=None, - http_method=None, - header=None, - timeout=None, - tls_skip_verify=None, - tcp=None, - ttl=None, - service_id=None, - status=None): + def register( + self, + name, + check_id=None, + interval=None, + notes=None, + deregister_critical_service_after=None, + args=None, + docker_container_id=None, + grpc=None, + grpc_use_tls=None, + http=None, + http_method=None, + header=None, + timeout=None, + tls_skip_verify=None, + tcp=None, + ttl=None, + service_id=None, + status=None, + ): # pylint: disable=too-many-arguments,too-many-locals """Add a new check to the local agent. Checks are either a script or TTL type. The agent is responsible for managing the status of the check and keeping the Catalog in sync. @@ -111,14 +107,15 @@ def register(self, """ return self._put_no_response_body( - ['register'], None, dict( + ["register"], + None, + dict( models.Check( name=name, id=check_id, interval=interval, notes=notes, - deregister_critical_service_after= - deregister_critical_service_after, + deregister_critical_service_after=deregister_critical_service_after, args=args, docker_container_id=docker_container_id, grpc=grpc, @@ -131,7 +128,10 @@ def register(self, tcp=tcp, ttl=ttl, service_id=service_id, - status=status))) + status=status, + ) + ), + ) def deregister(self, check_id): """Remove a check from the local agent. The agent will take care @@ -141,7 +141,7 @@ def deregister(self, check_id): :rtype: bool """ - return self._put_no_response_body(['deregister', check_id]) + return self._put_no_response_body(["deregister", check_id]) def ttl_pass(self, check_id, note=None): """This endpoint is used with a check that is of the TTL type. @@ -153,8 +153,7 @@ def ttl_pass(self, check_id, note=None): :rtype: bool """ - return self._put_no_response_body( - ['pass', check_id], {'note': note} if note else None) + return self._put_no_response_body(["pass", check_id], {"note": note} if note else None) def ttl_warn(self, check_id, note=None): """This endpoint is used with a check that is of the TTL type. @@ -166,8 +165,7 @@ def ttl_warn(self, check_id, note=None): :rtype: bool """ - return self._put_no_response_body( - ['warn', check_id], {'note': note} if note else None) + return self._put_no_response_body(["warn", check_id], {"note": note} if note else None) def ttl_fail(self, check_id, note=None): """This endpoint is used with a check that is of the TTL type. @@ -179,8 +177,7 @@ def ttl_fail(self, check_id, note=None): :rtype: bool """ - return self._put_no_response_body( - ['fail', check_id], {'note': note} if note else None) + return self._put_no_response_body(["fail", check_id], {"note": note} if note else None) class Service(base.Endpoint): """One of the main goals of service discovery is to provide a catalog @@ -192,16 +189,19 @@ class Service(base.Endpoint): the HTTP interface. """ - def register(self, - name, - service_id=None, - address=None, - port=None, - tags=None, - meta=None, - check=None, - checks=None, - enable_tag_override=None): + + def register( + self, + name, + service_id=None, + address=None, + port=None, + tags=None, + meta=None, + check=None, + checks=None, + enable_tag_override=None, + ): # pylint: disable=too-many-arguments """Add a new service to the local agent. :param str name: The name of the service @@ -220,11 +220,22 @@ def register(self, """ return self._put_no_response_body( - ['register'], None, - dict(models.Service( - name=name, id=service_id, address=address, port=port, - tags=tags, meta=meta, check=check, checks=checks, - enable_tag_override=enable_tag_override))) + ["register"], + None, + dict( + models.Service( + name=name, + id=service_id, + address=address, + port=port, + tags=tags, + meta=meta, + check=check, + checks=checks, + enable_tag_override=enable_tag_override, + ) + ), + ) def deregister(self, service_id): """Deregister the service from the local agent. The agent will @@ -235,7 +246,7 @@ def deregister(self, service_id): :rtype: bool """ - return self._put_no_response_body(['deregister', service_id]) + return self._put_no_response_body(["deregister", service_id]) def maintenance(self, service_id, enable=True, reason=None): """Place given service into "maintenance mode". @@ -246,11 +257,10 @@ def maintenance(self, service_id, enable=True, reason=None): :rtype: bool """ - query_params = {'enable': enable} + query_params = {"enable": enable} if reason: - query_params['reason'] = reason - return self._put_no_response_body(['maintenance', service_id], - query_params) + query_params["reason"] = reason + return self._put_no_response_body(["maintenance", service_id], query_params) def checks(self): """Return the all the checks that are registered with the local agent. @@ -264,7 +274,7 @@ def checks(self): :rtype: dict """ - return self._get(['checks']) + return self._get(["checks"]) def force_leave(self, node): """Instructs the agent to force a node into the left state. If a node @@ -274,7 +284,7 @@ def force_leave(self, node): node into the left state allows its old entries to be removed. """ - return self._put_no_response_body(['force-leave', node]) + return self._put_no_response_body(["force-leave", node]) def join(self, address, wan=False): """This endpoint is hit with a GET and is used to instruct the agent @@ -287,8 +297,8 @@ def join(self, address, wan=False): :rtype: bool """ - query_params = {'wan': 1} if wan else None - return self._put_no_response_body(['join', address], query_params) + query_params = {"wan": 1} if wan else None + return self._put_no_response_body(["join", address], query_params) def maintenance(self, enable=True, reason=None): """Places the agent into or removes the agent from "maintenance mode". @@ -300,10 +310,10 @@ def maintenance(self, enable=True, reason=None): :rtype: bool """ - query_params = {'enable': enable} + query_params = {"enable": enable} if reason: - query_params['reason'] = reason - return self._put_no_response_body(['maintenance'], query_params) + query_params["reason"] = reason + return self._put_no_response_body(["maintenance"], query_params) def members(self): """Returns the members the agent sees in the cluster gossip pool. @@ -314,7 +324,7 @@ def members(self): :rtype: list """ - return self._get_list(['members']) + return self._get_list(["members"]) def metrics(self): """Returns agent's metrics for the most recent finished interval @@ -324,7 +334,7 @@ def metrics(self): :rtype: dict """ - return self._get(['metrics']) + return self._get(["metrics"]) def monitor(self): """Iterator over logs from the local agent. @@ -334,7 +344,7 @@ def monitor(self): :rtype: iterator """ - for line in self._get_stream(['monitor']): + for line in self._get_stream(["monitor"]): yield line def reload(self): @@ -346,7 +356,7 @@ def reload(self): :rtype: list """ - return self._put_response_body(['reload']) or None + return self._put_response_body(["reload"]) or None def services(self): """return the all the services that are registered with the local @@ -361,16 +371,16 @@ def services(self): :rtype: dict """ - return self._get(['services']) + return self._get(["services"]) def self(self): - """ This endpoint is used to return the configuration and member + """This endpoint is used to return the configuration and member information of the local agent under the Config key. :rtype: dict """ - return self._get(['self']) + return self._get(["self"]) def token(self, name, value): """Update the ACL tokens currently in use by the agent. It can be used @@ -395,6 +405,5 @@ def token(self, name, value): """ if name not in _TOKENS: - raise ValueError('Invalid token name: {}'.format(name)) - return self._put_no_response_body( - ['token', name], {}, {'Token': value}) + raise ValueError("Invalid token name: {}".format(name)) + return self._put_no_response_body(["token", name], {}, {"Token": value}) diff --git a/pyms/services_discovery/consulate/api/base.py b/pyms/services_discovery/consulate/api/base.py index 44910b0..6499b34 100644 --- a/pyms/services_discovery/consulate/api/base.py +++ b/pyms/services_discovery/consulate/api/base.py @@ -4,6 +4,7 @@ """ import base64 import json + try: from urllib.parse import urlencode # Python 3 except ImportError: @@ -12,10 +13,10 @@ from pyms.services_discovery.consulate import utils -class Endpoint(object): +class Endpoint: """Base class for API endpoints""" - KEYWORD = '' + KEYWORD = "" def __init__(self, uri, adapter, datacenter=None, token=None): """Create a new instance of the Endpoint class @@ -27,7 +28,7 @@ def __init__(self, uri, adapter, datacenter=None, token=None): """ self._adapter = adapter - self._base_uri = '{0}/{1}'.format(uri, self.__class__.__name__.lower()) + self._base_uri = "{0}/{1}".format(uri, self.__class__.__name__.lower()) self._dc = datacenter self._token = token @@ -41,17 +42,15 @@ def _build_uri(self, params, query_params=None): if not query_params: query_params = dict() if self._dc: - query_params['dc'] = self._dc + query_params["dc"] = self._dc if self._token: - query_params['token'] = self._token - path = '/'.join(params) + query_params["token"] = self._token + path = "/".join(params) if query_params: - return '{0}/{1}?{2}'.format(self._base_uri, path, - urlencode(query_params)) - return '{0}/{1}'.format(self._base_uri, path) + return "{0}/{1}?{2}".format(self._base_uri, path, urlencode(query_params)) + return "{0}/{1}".format(self._base_uri, path) - def _get(self, params, query_params=None, raise_on_404=False, - timeout=None): + def _get(self, params, query_params=None, raise_on_404=False, timeout=None): """Perform a GET request :param list params: List of path parts @@ -61,16 +60,15 @@ def _get(self, params, query_params=None, raise_on_404=False, :rtype: dict or list or None """ - response = self._adapter.get(self._build_uri(params, query_params), - timeout=timeout) + response = self._adapter.get(self._build_uri(params, query_params), timeout=timeout) if utils.response_ok(response, raise_on_404): return response.body return [] def _delete( - self, - params, - raise_on_404=False, + self, + params, + raise_on_404=False, ): """Perform a DELETE request @@ -103,31 +101,29 @@ def _get_stream(self, params, query_params=None): :rtype: iterator """ - for line in self._adapter.get_stream( - self._build_uri(params, query_params)): + for line in self._adapter.get_stream(self._build_uri(params, query_params)): yield line def _get_no_response_body(self, url_parts, query=None): - return utils.response_ok( - self._adapter.get(self._build_uri(url_parts, query))) + return utils.response_ok(self._adapter.get(self._build_uri(url_parts, query))) def _get_response_body(self, url_parts, query=None): response = self._adapter.get(self._build_uri(url_parts, query)) if utils.response_ok(response): return response.body + return None def _put_no_response_body(self, url_parts, query=None, payload=None): - return utils.response_ok( - self._adapter.put(self._build_uri(url_parts, query), payload)) + return utils.response_ok(self._adapter.put(self._build_uri(url_parts, query), payload)) def _put_response_body(self, url_parts, query=None, payload=None): - response = self._adapter.put(self._build_uri(url_parts, query), - data=payload) + response = self._adapter.put(self._build_uri(url_parts, query), data=payload) if utils.response_ok(response): return response.body + return None -class Response(object): +class Response: """Used to process and wrap the responses from Consul. :param int status_code: HTTP Status code @@ -135,6 +131,7 @@ class Response(object): :param dict headers: Response headers """ + status_code = None body = None headers = None @@ -149,7 +146,7 @@ def __init__(self, response): self.body = self._demarshal(response.content) self.headers = response.headers - def _demarshal(self, body): + def _demarshal(self, body): # pylint: disable=too-many-branches,too-many-return-statements """Demarshal the request payload. :param str body: The string response body @@ -158,11 +155,11 @@ def _demarshal(self, body): """ if body is None: return None - if self.status_code == 200: + if self.status_code == 200: # pylint: disable=too-many-nested-blocks try: if utils.PYTHON3 and isinstance(body, bytes): try: - body = body.decode('utf-8') + body = body.decode("utf-8") except UnicodeDecodeError: pass value = json.loads(body) @@ -172,14 +169,14 @@ def _demarshal(self, body): return None if isinstance(value, bool): return value - if 'error' not in value: + if "error" not in value: for row in value: - if 'Value' in row: + if "Value" in row: try: - row['Value'] = base64.b64decode(row['Value']) - if isinstance(row['Value'], bytes): + row["Value"] = base64.b64decode(row["Value"]) + if isinstance(row["Value"], bytes): try: - row['Value'] = row['Value'].decode('utf-8') + row["Value"] = row["Value"].decode("utf-8") except UnicodeDecodeError: pass except TypeError: diff --git a/pyms/services_discovery/consulate/api/catalog.py b/pyms/services_discovery/consulate/api/catalog.py index 055a065..2f2bfc3 100644 --- a/pyms/services_discovery/consulate/api/catalog.py +++ b/pyms/services_discovery/consulate/api/catalog.py @@ -14,13 +14,9 @@ class Catalog(base.Endpoint): """ def __init__(self, uri, adapter, dc=None, token=None): - super(Catalog, self).__init__(uri, adapter, dc, token) + super().__init__(uri, adapter, dc, token) - def register(self, node, address, - datacenter=None, - service=None, - check=None, - node_meta=None): + def register(self, node, address, datacenter=None, service=None, check=None, node_meta=None): """A a low level mechanism for directly registering or updating entries in the catalog. It is usually recommended to use the agent local endpoints, as they are simpler and perform anti-entropy. @@ -94,20 +90,19 @@ def register(self, node, address, :rtype: bool """ - payload = {'Node': node, 'Address': address} + payload = {"Node": node, "Address": address} if datacenter: - payload['Datacenter'] = datacenter + payload["Datacenter"] = datacenter if service: - payload['Service'] = service + payload["Service"] = service if check: - payload['Check'] = check + payload["Check"] = check if node_meta: - payload['NodeMeta'] = node_meta + payload["NodeMeta"] = node_meta - return self._put_response_body(['register'], None, payload) + return self._put_response_body(["register"], None, payload) - def deregister(self, node, datacenter=None, - check_id=None, service_id=None): + def deregister(self, node, datacenter=None, check_id=None, service_id=None): """Directly remove entries in the catalog. It is usually recommended to use the agent local endpoints, as they are simpler and perform anti-entropy. @@ -127,14 +122,14 @@ def deregister(self, node, datacenter=None, :rtype: bool """ - payload = {'Node': node} + payload = {"Node": node} if datacenter: - payload['Datacenter'] = datacenter + payload["Datacenter"] = datacenter if check_id: - payload['CheckID'] = check_id + payload["CheckID"] = check_id if service_id: - payload['ServiceID'] = service_id - return self._put_response_body(['deregister'], None, payload) + payload["ServiceID"] = service_id + return self._put_response_body(["deregister"], None, payload) def datacenters(self): """Return all the datacenters that are known by the Consul server. @@ -142,7 +137,7 @@ def datacenters(self): :rtype: list """ - return self._get_list(['datacenters']) + return self._get_list(["datacenters"]) def node(self, node_id): """Return the node data for the specified node @@ -151,7 +146,7 @@ def node(self, node_id): :rtype: dict """ - return self._get(['node', node_id]) + return self._get(["node", node_id]) def nodes(self, node_meta=None): """Return all of the nodes for the current datacenter. @@ -160,8 +155,8 @@ def nodes(self, node_meta=None): :rtype: list """ - query_params = {'node-meta': node_meta} if node_meta else {} - return self._get_list(['nodes'], query_params) + query_params = {"node-meta": node_meta} if node_meta else {} + return self._get_list(["nodes"], query_params) def service(self, service_id): """Return the service details for the given service @@ -170,7 +165,7 @@ def service(self, service_id): :rtype: list """ - return self._get_list(['service', service_id]) + return self._get_list(["service", service_id]) def services(self): """Return a list of all of the services for the current datacenter. @@ -178,4 +173,4 @@ def services(self): :rtype: list """ - return self._get_list(['services']) + return self._get_list(["services"]) diff --git a/pyms/services_discovery/consulate/api/coordinate.py b/pyms/services_discovery/consulate/api/coordinate.py index a2a6a81..84eff4d 100644 --- a/pyms/services_discovery/consulate/api/coordinate.py +++ b/pyms/services_discovery/consulate/api/coordinate.py @@ -2,12 +2,13 @@ Consul Coordinate Endpoint Access """ -from pyms.services_discovery.consulate.api import base from math import sqrt +from pyms.services_discovery.consulate.api import base + + class Coordinate(base.Endpoint): - """Used to query node coordinates. - """ + """Used to query node coordinates.""" def node(self, node_id): """Return coordinates for the given node. @@ -16,7 +17,7 @@ def node(self, node_id): :rtype: dict """ - return self._get(['node', node_id]) + return self._get(["node", node_id]) def nodes(self): """Return coordinates for the current datacenter. @@ -24,39 +25,38 @@ def nodes(self): :rtype: list """ - return self._get_list(['nodes']) + return self._get_list(["nodes"]) - def rtt(self, src, dst): + @staticmethod + def rtt(src, dst): """Calculated RTT between two node coordinates. - - :param dict src - :param dict dst - :rtype float - + :param src: + :param dst: + :return: """ if not isinstance(src, (dict)): - raise ValueError('coordinate object must be a dictionary') + raise ValueError("coordinate object must be a dictionary") if not isinstance(dst, (dict)): - raise ValueError('coordinate object must be a dictionary') - if 'Coord' not in src: - raise ValueError('coordinate object has no Coord key') - if 'Coord' not in dst: - raise ValueError('coordinate object has no Coord key') + raise ValueError("coordinate object must be a dictionary") + if "Coord" not in src: + raise ValueError("coordinate object has no Coord key") + if "Coord" not in dst: + raise ValueError("coordinate object has no Coord key") - src_coord = src['Coord'] - dst_coord = dst['Coord'] + src_coord = src["Coord"] + dst_coord = dst["Coord"] - if len(src_coord.get('Vec')) != len(dst_coord.get('Vec')): - raise ValueError('coordinate objects are not compatible due to different length') + if len(src_coord.get("Vec")) != len(dst_coord.get("Vec")): + raise ValueError("coordinate objects are not compatible due to different length") sumsq = 0.0 - for i in xrange(len(src_coord.get('Vec'))): - diff = src_coord.get('Vec')[i] - dst_coord.get('Vec')[i] + for i in range(len(src_coord.get("Vec"))): + diff = src_coord.get("Vec")[i] - dst_coord.get("Vec")[i] sumsq += diff * diff - rtt = sqrt(sumsq) + src_coord.get('Height') + dst_coord.get('Height') - adjusted = rtt + src_coord.get('Adjustment') + dst_coord.get('Adjustment') + rtt = sqrt(sumsq) + src_coord.get("Height") + dst_coord.get("Height") + adjusted = rtt + src_coord.get("Adjustment") + dst_coord.get("Adjustment") if adjusted > 0.0: rtt = adjusted diff --git a/pyms/services_discovery/consulate/api/event.py b/pyms/services_discovery/consulate/api/event.py index 3c215b9..415f15e 100644 --- a/pyms/services_discovery/consulate/api/event.py +++ b/pyms/services_discovery/consulate/api/event.py @@ -6,16 +6,9 @@ class Event(base.Endpoint): - """The Event endpoints are used to fire a new event and list recent events. + """The Event endpoints are used to fire a new event and list recent events.""" - """ - - def fire(self, name, - payload=None, - datacenter=None, - node=None, - service=None, - tag=None): + def fire(self, name, payload=None, datacenter=None, node=None, service=None, tag=None): """Trigger a new user Event :param str name: The name of the event @@ -29,16 +22,15 @@ def fire(self, name, """ query_args = {} if datacenter: - query_args['dc'] = datacenter + query_args["dc"] = datacenter if node: - query_args['node'] = node + query_args["node"] = node if service: - query_args['service'] = service + query_args["service"] = service if tag: - query_args['tag'] = tag - response = self._adapter.put(self._build_uri(['fire', name], - query_args), payload) - return response.body.get('ID') + query_args["tag"] = tag + response = self._adapter.put(self._build_uri(["fire", name], query_args), payload) + return response.body.get("ID") def list(self, name=None): """Returns the most recent events known by the agent. As a consequence @@ -51,5 +43,5 @@ def list(self, name=None): """ query_args = {} if name: - query_args['name'] = name - return self._get(['list'], query_args) + query_args["name"] = name + return self._get(["list"], query_args) diff --git a/pyms/services_discovery/consulate/api/health.py b/pyms/services_discovery/consulate/api/health.py index f2ca951..a7ff4b9 100644 --- a/pyms/services_discovery/consulate/api/health.py +++ b/pyms/services_discovery/consulate/api/health.py @@ -22,8 +22,8 @@ def checks(self, service_id, node_meta=None): :rtype: list """ - query_params = {'node-meta': node_meta} if node_meta else {} - return self._get_list(['checks', service_id], query_params) + query_params = {"node-meta": node_meta} if node_meta else {} + return self._get_list(["checks", service_id], query_params) def node(self, node_id): """Return the health info for a given node. @@ -32,7 +32,7 @@ def node(self, node_id): :rtype: list """ - return self._get_list(['node', node_id]) + return self._get_list(["node", node_id]) def service(self, service_id, tag=None, passing=None, node_meta=None): """Returns the nodes and health info of a service @@ -45,14 +45,13 @@ def service(self, service_id, tag=None, passing=None, node_meta=None): query_params = {} if tag: - query_params['tag'] = tag + query_params["tag"] = tag if passing: - query_params['passing'] = '' + query_params["passing"] = "" if node_meta: - query_params['node-meta'] = node_meta + query_params["node-meta"] = node_meta - return self._get_list(['service', service_id], - query_params=query_params) + return self._get_list(["service", service_id], query_params=query_params) def state(self, state): """Returns the checks in a given state where state is one of @@ -62,4 +61,4 @@ def state(self, state): :rtype: list """ - return self._get_list(['state', state]) + return self._get_list(["state", state]) diff --git a/pyms/services_discovery/consulate/api/kv.py b/pyms/services_discovery/consulate/api/kv.py index b3596eb..89feb40 100644 --- a/pyms/services_discovery/consulate/api/kv.py +++ b/pyms/services_discovery/consulate/api/kv.py @@ -36,7 +36,7 @@ def __contains__(self, item): :rtype: bool """ - item = item.lstrip('/') + item = item.lstrip("/") return self._get_no_response_body([item]) def __delitem__(self, item): @@ -58,8 +58,8 @@ def __getitem__(self, item): """ value = self._get_item(item) if not value: - raise KeyError('Key not found ({0})'.format(item)) - return value.get('Value') + raise KeyError("Key not found ({0})".format(item)) + return value.get("Value") def __iter__(self): """Iterate over all the keys in the Key/Value service @@ -103,11 +103,11 @@ def acquire_lock(self, item, session, value=None, cas=None, flags=None): :return: bool """ - query_params = {'acquire': session} + query_params = {"acquire": session} if cas is not None: - query_params['cas'] = cas + query_params["cas"] = cas if flags is not None: - query_params['flags'] = flags + query_params["flags"] = flags return self._put_response_body([item], query_params, value) def delete(self, item, recurse=False): @@ -133,7 +133,7 @@ def get(self, item, default=None, raw=False): """ response = self._get_item(item, raw) if isinstance(response, dict): - return response.get('Value', default) + return response.get("Value", default) return response or default def get_record(self, item): @@ -162,53 +162,53 @@ def find(self, prefix, separator=None): :rtype: dict """ - query_params = {'recurse': None} + query_params = {"recurse": None} if separator: - query_params['keys'] = prefix - query_params['separator'] = separator - response = self._get_list([prefix.lstrip('/')], query_params) + query_params["keys"] = prefix + query_params["separator"] = separator + response = self._get_list([prefix.lstrip("/")], query_params) if separator: results = response else: results = {} for row in response: - results[row['Key']] = row['Value'] + results[row["Key"]] = row["Value"] return results def items(self): """Return a dict of all of the key/value pairs in the Key/Value service - *Example:* + *Example:* - .. code:: python + .. code:: python - >>> consul.kv.items() - {'foo': 'bar', 'bar': 'baz', 'quz': True, 'corgie': 'dog'} + >>> consul.kv.items() + {'foo': 'bar', 'bar': 'baz', 'quz': True, 'corgie': 'dog'} - :rtype: dict + :rtype: dict """ - return [{item['Key']: item['Value']} for item in self._get_all_items()] + return [{item["Key"]: item["Value"]} for item in self._get_all_items()] def iteritems(self): """Iterate over the dict of key/value pairs in the Key/Value service - *Example:* + *Example:* - .. code:: python + .. code:: python - >>> for key, value in consul.kv.iteritems(): - ... print(key, value) - ... - (u'bar', 'baz') - (u'foo', 'bar') - (u'quz', True) + >>> for key, value in consul.kv.iteritems(): + ... print(key, value) + ... + (u'bar', 'baz') + (u'foo', 'bar') + (u'quz', True) - :rtype: iterator + :rtype: iterator """ for item in self._get_all_items(): - yield item['Key'], item['Value'] + yield item["Key"], item["Value"] def keys(self): """Return a list of all of the keys in the Key/Value service @@ -223,7 +223,7 @@ def keys(self): :rtype: list """ - return sorted([row['Key'] for row in self._get_all_items()]) + return sorted([row["Key"] for row in self._get_all_items()]) def records(self, key=None): """Return a list of tuples for all of the records in the Key/Value @@ -243,11 +243,9 @@ def records(self, key=None): """ if key: - return [(item['Key'], item['Flags'], item['Value']) - for item in self._get_list([key], {'recurse': None})] - else: - return [(item['Key'], item['Flags'], item['Value']) - for item in self._get_all_items()] + return [(item["Key"], item["Flags"], item["Value"]) for item in self._get_list([key], {"recurse": None})] + + return [(item["Key"], item["Flags"], item["Value"]) for item in self._get_all_items()] def release_lock(self, item, session): """Release an existing lock from the Consul KV database. @@ -257,7 +255,7 @@ def release_lock(self, item, session): :return: bool """ - return self._put_response_body([item], {'release': session}) + return self._put_response_body([item], {"release": session}) def set(self, item, value): """Set a value in the Key/Value service, using the CAS mechanism @@ -295,7 +293,7 @@ def values(self): :rtype: list """ - return [row['Value'] for row in self._get_all_items()] + return [row["Value"] for row in self._get_all_items()] def _delete_item(self, item, recurse=False): """Remove an item from the Consul database @@ -304,7 +302,7 @@ def _delete_item(self, item, recurse=False): :param recurse: :return: """ - query_params = {'recurse': True} if recurse else {} + query_params = {"recurse": True} if recurse else {} return self._adapter.delete(self._build_uri([item], query_params)) def _get_all_items(self): @@ -314,7 +312,7 @@ def _get_all_items(self): :rtype: list """ - return self._get_list([''], {'recurse': None}) + return self._get_list([""], {"recurse": None}) def _get_item(self, item, raw=False): """Internal method to get the full item record from the Key/Value @@ -325,8 +323,8 @@ def _get_item(self, item, raw=False): :rtype: mixed """ - item = item.lstrip('/') - query_params = {'raw': True} if raw else {} + item = item.lstrip("/") + query_params = {"raw": True} if raw else {} response = self._adapter.get(self._build_uri([item], query_params)) if response.status_code == 200: return response.body @@ -347,8 +345,8 @@ def _get_modify_index(self, item, value, replace): response = self._adapter.get(self._build_uri([item])) index = 0 if response.status_code == 200: - index = response.body.get('ModifyIndex') - rvalue = response.body.get('Value') + index = response.body.get("ModifyIndex") + rvalue = response.body.get("Value") if rvalue == value: return None if not replace: @@ -366,16 +364,11 @@ def _prepare_value(value): if not utils.is_string(value) or isinstance(value, bytes): return value try: - if utils.PYTHON3: - return value.encode('utf-8') - elif isinstance(value, unicode): - return value.encode('utf-8') + return value.encode("utf-8") except UnicodeDecodeError: return value - return value - def _set_item(self, item, value, flags=None, replace=True, - query_params=None): + def _set_item(self, item, value, flags=None, replace=True, query_params=None): """Internal method for setting a key/value pair with flags in the Key/Value service @@ -387,21 +380,19 @@ def _set_item(self, item, value, flags=None, replace=True, """ value = self._prepare_value(value) - if value and item.endswith('/'): - item = item.rstrip('/') + if value and item.endswith("/"): + item = item.rstrip("/") index = self._get_modify_index(item, value, replace) if index is None: return True query_params = query_params or {} - query_params.update({'cas': index}) + query_params.update({"cas": index}) if flags is not None: - query_params['flags'] = flags - response = self._adapter.put(self._build_uri([item], query_params), - value) + query_params["flags"] = flags + response = self._adapter.put(self._build_uri([item], query_params), value) if not response.status_code == 200 or not response.body: if response.status_code == 500: - raise exceptions.ServerError( - response.body or 'Internal Consul server error') - raise KeyError( - 'Error setting "{0}" ({1})'.format(item, response.status_code)) + raise exceptions.ServerError(response.body or "Internal Consul server error") + raise KeyError('Error setting "{0}" ({1})'.format(item, response.status_code)) + return None diff --git a/pyms/services_discovery/consulate/api/lock.py b/pyms/services_discovery/consulate/api/lock.py index 3738f9b..3e621dc 100644 --- a/pyms/services_discovery/consulate/api/lock.py +++ b/pyms/services_discovery/consulate/api/lock.py @@ -31,7 +31,8 @@ class Lock(base.Endpoint): :raises: :exc:`~consulate.exception.LockError` """ - DEFAULT_PREFIX = 'consulate/locks' + + DEFAULT_PREFIX = "consulate/locks" def __init__(self, uri, adapter, session, datacenter=None, token=None): """Create a new instance of the Lock @@ -43,8 +44,8 @@ def __init__(self, uri, adapter, session, datacenter=None, token=None): :param str token: Access Token """ - super(Lock, self).__init__(uri, adapter, datacenter, token) - self._base_uri = '{0}/kv'.format(uri) + super().__init__(uri, adapter, datacenter, token) + self._base_uri = "{0}/kv".format(uri) self._session = session self._session_id = None self._item = str(uuid.uuid4()) @@ -79,23 +80,20 @@ def prefix(self, value): :param str value: The value to set the path prefix to """ - self._prefix = value or '' + self._prefix = value or "" def _acquire(self, key=None, value=None): self._session_id = self._session.create() - self._item = '/'.join([self._prefix, (key or str(uuid.uuid4()))]) - LOGGER.debug('Acquiring a lock of %s for session %s', - self._item, self._session_id) - response = self._put_response_body([self._item], - {'acquire': self._session_id}, - value) + self._item = "/".join([self._prefix, (key or str(uuid.uuid4()))]) + LOGGER.debug("Acquiring a lock of %s for session %s", self._item, self._session_id) + response = self._put_response_body([self._item], {"acquire": self._session_id}, value) if not response: self._session.destroy(self._session_id) raise exceptions.LockFailure() def _release(self): """Release the lock""" - self._put_response_body([self._item], {'release': self._session_id}) + self._put_response_body([self._item], {"release": self._session_id}) self._adapter.delete(self._build_uri([self._item])) self._session.destroy(self._session_id) self._item, self._session_id = None, None diff --git a/pyms/services_discovery/consulate/api/session.py b/pyms/services_discovery/consulate/api/session.py index 43fe850..21761b5 100644 --- a/pyms/services_discovery/consulate/api/session.py +++ b/pyms/services_discovery/consulate/api/session.py @@ -8,13 +8,7 @@ class Session(base.Endpoint): """Create, destroy, and query Consul sessions.""" - def create(self, - name=None, - behavior='release', - node=None, - delay=None, - ttl=None, - checks=None): + def create(self, name=None, behavior="release", node=None, delay=None, ttl=None, checks=None): """Initialize a new session. None of the fields are mandatory, and in fact no body needs to be PUT @@ -53,18 +47,18 @@ def create(self, :return str: session ID """ - payload = {'name': name} if name else {} + payload = {"name": name} if name else {} if node: - payload['Node'] = node + payload["Node"] = node if behavior: - payload['Behavior'] = behavior + payload["Behavior"] = behavior if delay: - payload['LockDelay'] = delay + payload["LockDelay"] = delay if ttl: - payload['TTL'] = ttl + payload["TTL"] = ttl if checks: - payload['Checks'] = checks - return self._put_response_body(['create'], None, payload).get('ID') + payload["Checks"] = checks + return self._put_response_body(["create"], None, payload).get("ID") def destroy(self, session_id): """Destroy an existing session @@ -73,7 +67,7 @@ def destroy(self, session_id): :return: bool """ - return self._put_no_response_body(['destroy', session_id]) + return self._put_no_response_body(["destroy", session_id]) def info(self, session_id): """Returns the requested session information within a given dc. @@ -83,7 +77,7 @@ def info(self, session_id): :return: dict """ - return self._get_response_body(['info', session_id]) + return self._get_response_body(["info", session_id]) def list(self): """Returns the active sessions for a given dc. @@ -91,7 +85,7 @@ def list(self): :return: list """ - return self._get_response_body(['list']) + return self._get_response_body(["list"]) def node(self, node): """Returns the active sessions for a given node and dc. @@ -101,7 +95,7 @@ def node(self, node): :return: list """ - return self._get_response_body(['node', node]) + return self._get_response_body(["node", node]) def renew(self, session_id): """Renew the given session. This is used with sessions that have a TTL, @@ -112,4 +106,4 @@ def renew(self, session_id): :return: dict """ - return self._put_response_body(['renew', session_id]) + return self._put_response_body(["renew", session_id]) diff --git a/pyms/services_discovery/consulate/api/status.py b/pyms/services_discovery/consulate/api/status.py index 336be36..2789f47 100644 --- a/pyms/services_discovery/consulate/api/status.py +++ b/pyms/services_discovery/consulate/api/status.py @@ -17,7 +17,7 @@ def leader(self): :rtype: str """ - return self._get(['leader']) + return self._get(["leader"]) def peers(self): """Get the Raft peers for the datacenter the agent is running in. @@ -25,7 +25,7 @@ def peers(self): :rtype: list """ - value = self._get(['peers']) + value = self._get(["peers"]) if not isinstance(value, list): return [value] return value diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index 96b5853..25a55f5 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -5,20 +5,23 @@ import json import sys import os -try: - import urlparse -except ImportError: - import urllib.parse as urlparse +import time +import subprocess + + +import urllib.parse as urlparse from requests import exceptions -from pyms.services_discovery from pyms.services_discovery import consulate +from pyms.services_discovery import consulate from pyms.services_discovery.consulate import adapters from pyms.services_discovery.consulate import utils -CONSUL_ENV_VAR = 'CONSUL_RPC_ADDR' -EPILOG = ('If the CONSUL_RPC_ADDR environment variable is set, it will be ' - 'parsed and used for default values when connecting.') +CONSUL_ENV_VAR = "CONSUL_RPC_ADDR" +EPILOG = ( + "If the CONSUL_RPC_ADDR environment variable is set, it will be " + "parsed and used for default values when connecting." +) def on_error(message, exit_code=2): @@ -29,83 +32,94 @@ def on_error(message, exit_code=2): :param int exit_code: The numeric exit code """ - sys.stderr.write(message + '\n') + sys.stderr.write(message + "\n") sys.exit(exit_code) def connection_error(): """Common exit routine when consulate can't connect to Consul""" - on_error('Could not connect to consul', 1) + on_error("Could not connect to consul", 1) ACL_PARSERS = [ - ('backup', 'Backup to stdout or a JSON file', [ - [['-f', '--file'], {'help': 'JSON file to write instead of stdout', - 'nargs': '?'}], - [['-p', '--pretty'], {'help': 'pretty-print JSON output', - 'action': 'store_true'}]]), - ('restore', 'Restore from stdin or a JSON file', [ - [['-f', '--file'], - {'help': 'JSON file to read instead of stdin', - 'nargs': '?'}], - [['-n', '--no-replace'], - {'help': 'Do not replace existing entries', - 'action': 'store_true'}]]) - ] + ( + "backup", + "Backup to stdout or a JSON file", + [ + [["-f", "--file"], {"help": "JSON file to write instead of stdout", "nargs": "?"}], + [["-p", "--pretty"], {"help": "pretty-print JSON output", "action": "store_true"}], + ], + ), + ( + "restore", + "Restore from stdin or a JSON file", + [ + [["-f", "--file"], {"help": "JSON file to read instead of stdin", "nargs": "?"}], + [["-n", "--no-replace"], {"help": "Do not replace existing entries", "action": "store_true"}], + ], + ), +] KV_PARSERS = [ - ('backup', 'Backup to stdout or a JSON file', [ - [['key'], {'help': 'The key to use as target to backup a ' - 'specific key or folder.', - 'nargs': '?'}], - [['-b', '--base64'], {'help': 'Base64 encode values', - 'action': 'store_true'}], - [['-f', '--file'], {'help': 'JSON file to write instead of stdout', - 'nargs': '?'}], - [['-p', '--pretty'], {'help': 'pretty-print JSON output', - 'action': 'store_true'}]]), - ('restore', 'Restore from stdin or a JSON file', [ - [['key'], {'help': 'The key as target to restore to a specific key ' - 'or folder.', - 'nargs': '?'}], - [['-p', '--prune'], {'help': 'Remove entries from consul tree that ' - 'are not in restore file.', - 'action': 'store_true'}], - [['-b', '--base64'], {'help': 'Restore from Base64 encode values', - 'action': 'store_true'}], - [['-f', '--file'], - {'help': 'JSON file to read instead of stdin', - 'nargs': '?'}], - [['-n', '--no-replace'], - {'help': 'Do not replace existing entries', - 'action': 'store_true'}]]), - ('ls', 'List all of the keys', [ - [['key'], {'help': 'The key to use as target to list contents of ' - 'specific key or folder', - 'nargs': '?'}], - [['-l', '--long'], - {'help': 'Long format', - 'action': 'store_true'}]]), - ('mkdir', 'Create a folder', [ - [['path'], - {'help': 'The path to create'}]]), - ('get', 'Get a key from the database', [ - [['key'], {'help': 'The key to get'}], - [['-r', '--recurse'], - {'help': 'Get all keys prefixed with the specified key', - 'action': 'store_true'}], - [['-t', '--trim'], - {'help': 'Number of levels of prefix to trim from returned key', - 'type': int, - 'default': 0}]]), - ('set', 'Set a key in the database', [ - [['key'], {'help': 'The key to set'}], - [['value'], {'help': 'The value of the key'}]]), - ('rm', 'Remove a key from the database', [ - [['key'], {'help': 'The key to remove'}], - [['-r', '--recurse'], - {'help': 'Delete all keys prefixed with the specified key', - 'action': 'store_true'}]])] + ( + "backup", + "Backup to stdout or a JSON file", + [ + [["key"], {"help": "The key to use as target to backup a " "specific key or folder.", "nargs": "?"}], + [["-b", "--base64"], {"help": "Base64 encode values", "action": "store_true"}], + [["-f", "--file"], {"help": "JSON file to write instead of stdout", "nargs": "?"}], + [["-p", "--pretty"], {"help": "pretty-print JSON output", "action": "store_true"}], + ], + ), + ( + "restore", + "Restore from stdin or a JSON file", + [ + [["key"], {"help": "The key as target to restore to a specific key " "or folder.", "nargs": "?"}], + [ + ["-p", "--prune"], + {"help": "Remove entries from consul tree that " "are not in restore file.", "action": "store_true"}, + ], + [["-b", "--base64"], {"help": "Restore from Base64 encode values", "action": "store_true"}], + [["-f", "--file"], {"help": "JSON file to read instead of stdin", "nargs": "?"}], + [["-n", "--no-replace"], {"help": "Do not replace existing entries", "action": "store_true"}], + ], + ), + ( + "ls", + "List all of the keys", + [ + [["key"], {"help": "The key to use as target to list contents of " "specific key or folder", "nargs": "?"}], + [["-l", "--long"], {"help": "Long format", "action": "store_true"}], + ], + ), + ("mkdir", "Create a folder", [[["path"], {"help": "The path to create"}]]), + ( + "get", + "Get a key from the database", + [ + [["key"], {"help": "The key to get"}], + [["-r", "--recurse"], {"help": "Get all keys prefixed with the specified key", "action": "store_true"}], + [ + ["-t", "--trim"], + {"help": "Number of levels of prefix to trim from returned key", "type": int, "default": 0}, + ], + ], + ), + ( + "set", + "Set a key in the database", + [[["key"], {"help": "The key to set"}], [["value"], {"help": "The value of the key"}]], + ), + ( + "rm", + "Remove a key from the database", + [ + [["key"], {"help": "The key to remove"}], + [["-r", "--recurse"], {"help": "Delete all keys prefixed with the specified key", "action": "store_true"}], + ], + ), +] def add_acl_args(parser): @@ -114,10 +128,9 @@ def add_acl_args(parser): :param argparse.Subparser parser: parser """ - kv_parser = parser.add_parser('acl', help='ACL Utilities') + kv_parser = parser.add_parser("acl", help="ACL Utilities") - subparsers = kv_parser.add_subparsers(dest='action', - title='ACL Database Utilities') + subparsers = kv_parser.add_subparsers(dest="action", title="ACL Database Utilities") for (name, help_text, arguments) in ACL_PARSERS: parser = subparsers.add_parser(name, help=help_text) @@ -131,10 +144,9 @@ def add_kv_args(parser): :param argparse.Subparser parser: parser """ - kv_parser = parser.add_parser('kv', help='Key/Value Database Utilities') + kv_parser = parser.add_parser("kv", help="Key/Value Database Utilities") - subparsers = kv_parser.add_subparsers(dest='action', - title='Key/Value Database Utilities') + subparsers = kv_parser.add_subparsers(dest="action", title="Key/Value Database Utilities") for (name, help_text, arguments) in KV_PARSERS: parser = subparsers.add_parser(name, help=help_text) @@ -149,35 +161,22 @@ def add_register_args(parser): """ # Service registration - registerp = parser.add_parser('register', - help='Register a service for this node') - registerp.add_argument('name', help='The service name') - registerp.add_argument('-a', '--address', default=None, - help='Specify an address') - registerp.add_argument('-p', '--port', default=None, type=int, - help='Specify a port') - registerp.add_argument('-s', '--service-id', default=None, - help='Specify a service ID') - registerp.add_argument('-t', '--tags', default=[], - help='Specify a comma delimited list of tags') - rsparsers = registerp.add_subparsers(dest='ctype', - title='Service Check Options') - check = rsparsers.add_parser('check', - help='Define an external script-based check') - check.add_argument('interval', default=10, type=int, - help='How often to run the check script') - check.add_argument('path', default=None, - help='Path to the script invoked by Consul') - httpcheck = rsparsers.add_parser('httpcheck', - help='Define an HTTP-based check') - httpcheck.add_argument('interval', default=10, type=int, - help='How often to run the check script') - httpcheck.add_argument('url', default=None, - help='HTTP URL to be polled by Consul') - rsparsers.add_parser('no-check', help='Do not enable service monitoring') - ttl = rsparsers.add_parser('ttl', help='Define a duration based TTL check') - ttl.add_argument('duration', type=int, default=10, - help='TTL duration for a service with missing check data') + registerp = parser.add_parser("register", help="Register a service for this node") + registerp.add_argument("name", help="The service name") + registerp.add_argument("-a", "--address", default=None, help="Specify an address") + registerp.add_argument("-p", "--port", default=None, type=int, help="Specify a port") + registerp.add_argument("-s", "--service-id", default=None, help="Specify a service ID") + registerp.add_argument("-t", "--tags", default=[], help="Specify a comma delimited list of tags") + rsparsers = registerp.add_subparsers(dest="ctype", title="Service Check Options") + check = rsparsers.add_parser("check", help="Define an external script-based check") + check.add_argument("interval", default=10, type=int, help="How often to run the check script") + check.add_argument("path", default=None, help="Path to the script invoked by Consul") + httpcheck = rsparsers.add_parser("httpcheck", help="Define an HTTP-based check") + httpcheck.add_argument("interval", default=10, type=int, help="How often to run the check script") + httpcheck.add_argument("url", default=None, help="HTTP URL to be polled by Consul") + rsparsers.add_parser("no-check", help="Do not enable service monitoring") + ttl = rsparsers.add_parser("ttl", help="Define a duration based TTL check") + ttl.add_argument("duration", type=int, default=10, help="TTL duration for a service with missing check data") def add_run_once_args(parser): @@ -186,16 +185,10 @@ def add_run_once_args(parser): :param argparse.Subparser parser: parser """ - run_oncep = parser.add_parser('run_once', - help='Run a command locked to a single ' - 'execution') - run_oncep.add_argument('lock', - help='The name of the lock which will be ' - 'held in Consul.') - run_oncep.add_argument('command_to_run', nargs=argparse.REMAINDER, - help='The command to lock') - run_oncep.add_argument('-i', '--interval', default=None, - help='Hold the lock for X seconds') + run_oncep = parser.add_parser("run_once", help="Run a command locked to a single " "execution") + run_oncep.add_argument("lock", help="The name of the lock which will be " "held in Consul.") + run_oncep.add_argument("command_to_run", nargs=argparse.REMAINDER, help="The command to lock") + run_oncep.add_argument("-i", "--interval", default=None, help="Hold the lock for X seconds") def add_deregister_args(parser): @@ -205,9 +198,8 @@ def add_deregister_args(parser): """ # Service registration - registerp = parser.add_parser('deregister', - help='Deregister a service for this node') - registerp.add_argument('service_id', help='The service registration id') + registerp = parser.add_parser("deregister", help="Deregister a service for this node") + registerp.add_argument("service_id", help="The service registration id") def add_services_args(parser): @@ -217,38 +209,30 @@ def add_services_args(parser): """ # Service registration - registerp = parser.add_parser('services', - help='List services for this node') + registerp = parser.add_parser("services", help="List services for this node") + + registerp.add_argument("-i", "--indent", type=int, default=None, help="The indent level for output") - registerp.add_argument('-i', '--indent', type=int, default=None, help='The indent level for output') def parse_cli_args(): """Create the argument parser and add the arguments""" - parser = argparse.ArgumentParser(description='CLI utilities for Consul', - epilog=EPILOG) + parser = argparse.ArgumentParser(description="CLI utilities for Consul", epilog=EPILOG) - env_var = os.environ.get(CONSUL_ENV_VAR, '') + env_var = os.environ.get(CONSUL_ENV_VAR, "") parsed_defaults = urlparse.urlparse(env_var) - parser.add_argument('--api-scheme', - default=parsed_defaults.scheme or 'http', - help='The scheme to use for connecting to Consul with') - parser.add_argument('--api-host', - default=parsed_defaults.hostname or 'localhost', - help='The consul host to connect on') - parser.add_argument('--api-port', - default=parsed_defaults.port or 8500, - help='The consul API port to connect to') - parser.add_argument('--datacenter', - dest='dc', - default=None, - help='The datacenter to specify for the connection') - parser.add_argument('--token', default=None, help='ACL token') - parser.add_argument('--version', action='version', - version=consulate.__version__, - help='Current consulate version') - - sparser = parser.add_subparsers(title='Commands', dest='command') + parser.add_argument( + "--api-scheme", default=parsed_defaults.scheme or "http", help="The scheme to use for connecting to Consul with" + ) + parser.add_argument( + "--api-host", default=parsed_defaults.hostname or "localhost", help="The consul host to connect on" + ) + parser.add_argument("--api-port", default=parsed_defaults.port or 8500, help="The consul API port to connect to") + parser.add_argument("--datacenter", dest="dc", default=None, help="The datacenter to specify for the connection") + parser.add_argument("--token", default=None, help="ACL token") + parser.add_argument("--version", action="version", version=consulate.__version__, help="Current consulate version") + + sparser = parser.add_subparsers(title="Commands", dest="command") add_acl_args(sparser) add_kv_args(sparser) add_register_args(sparser) @@ -265,14 +249,13 @@ def acl_backup(consul, args): :param argparser.namespace args: The cli args """ - handle = open(args.file, 'w') if args.file else sys.stdout + handle = open(args.file, "w") if args.file else sys.stdout acls = consul.acl.list() try: if args.pretty: - handle.write(json.dumps(acls, sort_keys=True, indent=2, - separators=(',', ': ')) + '\n') + handle.write(json.dumps(acls, sort_keys=True, indent=2, separators=(",", ": ")) + "\n") else: - handle.write(json.dumps(acls, sort_keys=True) + '\n') + handle.write(json.dumps(acls, sort_keys=True) + "\n") except exceptions.ConnectionError: connection_error() @@ -284,17 +267,14 @@ def acl_restore(consul, args): :param argparser.namespace args: The cli args """ - handle = open(args.file, 'r') if args.file else sys.stdin + handle = open(args.file, "r") if args.file else sys.stdin data = json.load(handle) for row in data: - consul.acl.update(row['ID'], row['Name'], row['Type'], row['Rules']) - print('{0} ACLs written'.format(len(data))) + consul.acl.update(row["ID"], row["Name"], row["Type"], row["Rules"]) + print("{0} ACLs written".format(len(data))) -ACL_ACTIONS = { - 'backup': acl_backup, - 'restore': acl_restore -} +ACL_ACTIONS = {"backup": acl_backup, "restore": acl_restore} def kv_backup(consul, args): @@ -304,28 +284,23 @@ def kv_backup(consul, args): :param argparser.namespace args: The cli args """ - handle = open(args.file, 'w') if args.file else sys.stdout + handle = open(args.file, "w") if args.file else sys.stdout if args.key: - args.key = args.key.strip('/') - prefixlen = len(args.key.split('/')) - records = [('/'.join(k.split('/')[prefixlen:]), f, v) - for k, f, v in consul.kv.records(args.key)] + args.key = args.key.strip("/") + prefixlen = len(args.key.split("/")) + records = [("/".join(k.split("/")[prefixlen:]), f, v) for k, f, v in consul.kv.records(args.key)] else: records = consul.kv.records() if args.base64: if utils.PYTHON3: - records = [(k, f, str(base64.b64encode(utils.maybe_encode(v)), - 'ascii') if v else v) - for k, f, v in records] + records = [(k, f, str(base64.b64encode(utils.maybe_encode(v)), "ascii") if v else v) for k, f, v in records] else: - records = [(k, f, base64.b64encode(v) if v else v) - for k, f, v in records] + records = [(k, f, base64.b64encode(v) if v else v) for k, f, v in records] try: if args.pretty: - handle.write(json.dumps(records, sort_keys=True, indent=2, - separators=(',', ': ')) + '\n') + handle.write(json.dumps(records, sort_keys=True, indent=2, separators=(",", ": ")) + "\n") else: - handle.write(json.dumps(records) + '\n') + handle.write(json.dumps(records) + "\n") except exceptions.ConnectionError: connection_error() @@ -355,14 +330,14 @@ def kv_get(consul, args): for key in sorted(consul.kv.find(args.key)): displaykey = key if args.trim: - keyparts = displaykey.split('/') - if (args.trim >= len(keyparts)): + keyparts = displaykey.split("/") + if args.trim >= len(keyparts): displaykey = keyparts[-1] else: - displaykey = '/'.join(keyparts[args.trim:]) - sys.stdout.write('%s\t%s\n' % (displaykey, consul.kv.get(key))) + displaykey = "/".join(keyparts[args.trim :]) + sys.stdout.write("%s\t%s\n" % (displaykey, consul.kv.get(key))) else: - sys.stdout.write('%s\n' % consul.kv.get(args.key)) + sys.stdout.write("%s\n" % consul.kv.get(args.key)) except exceptions.ConnectionError: connection_error() @@ -376,7 +351,7 @@ def kv_ls(consul, args): """ try: if args.key: - args.key = args.key.lstrip('/') + args.key = args.key.lstrip("/") keylist = sorted(consul.kv.find(args.key)) else: keylist = consul.kv.keys() @@ -385,7 +360,7 @@ def kv_ls(consul, args): keylen = 0 if consul.kv[key]: keylen = len(consul.kv[key]) - print('{0:>14} {1}'.format(keylen, key)) + print("{0:>14} {1}".format(keylen, key)) else: print(key) except exceptions.ConnectionError: @@ -399,15 +374,15 @@ def kv_mkdir(consul, args): :param argparser.namespace args: The cli args """ - if not args.path[:-1] == '/': - args.path += '/' + if not args.path[:-1] == "/": + args.path += "/" try: consul.kv.set(args.path, None) except exceptions.ConnectionError: connection_error() -def kv_restore(consul, args): +def kv_restore(consul, args): # pylint: disable=too-many-branches,too-many-format-args """Restore the Consul KV store :param consulate.api_old.Consul consul: The Consul instance @@ -416,30 +391,28 @@ def kv_restore(consul, args): """ if args.prune: if args.key: - args.key = args.key.strip('/') + args.key = args.key.strip("/") keylist = consul.kv.find(args.key) else: - keylist = consul.kv.find('') - handle = open(args.file, 'r') if args.file else sys.stdin + keylist = consul.kv.find("") + handle = open(args.file, "r") if args.file else sys.stdin data = json.load(handle) for row in data: if isinstance(row, dict): # translate raw api export to internal representation - if row['Value'] is not None: - row['Value'] = base64.b64decode(row['Value']) - row = [row['Key'], row['Flags'], row['Value']] + if row["Value"] is not None: + row["Value"] = base64.b64decode(row["Value"]) + row = [row["Key"], row["Flags"], row["Value"]] if args.base64 and row[2] is not None: row[2] = base64.b64decode(row[2]) # Here's an awesome thing to make things work - if not utils.PYTHON3 and isinstance(row[2], unicode): - row[2] = row[2].encode('utf-8') if args.key: if row[0] == "": rowkey = args.key else: - rowkey = args.key + '/' + row[0] + rowkey = args.key + "/" + row[0] else: rowkey = row[0] if args.prune: @@ -486,14 +459,15 @@ def kv_set(consul, args): # Mapping dict to simplify the code in main() KV_ACTIONS = { - 'backup': kv_backup, - 'del': kv_delete, - 'get': kv_get, - 'ls': kv_ls, - 'mkdir': kv_mkdir, - 'restore': kv_restore, - 'rm': kv_rm, - 'set': kv_set} + "backup": kv_backup, + "del": kv_delete, + "get": kv_get, + "ls": kv_ls, + "mkdir": kv_mkdir, + "restore": kv_restore, + "rm": kv_rm, + "set": kv_set, +} def register(consul, args): @@ -503,16 +477,15 @@ def register(consul, args): :param argparser.namespace args: The cli args """ - check = args.path if args.ctype == 'check' else None - httpcheck = args.url if args.ctype == 'httpcheck' else None - interval = '%ss' % args.interval if args.ctype in ['check', - 'httpcheck'] else None - ttl = '%ss' % args.duration if args.ctype == 'ttl' else None - tags = args.tags.split(',') if args.tags else None + check = args.path if args.ctype == "check" else None + httpcheck = args.url if args.ctype == "httpcheck" else None + interval = "%ss" % args.interval if args.ctype in ["check", "httpcheck"] else None + ttl = "%ss" % args.duration if args.ctype == "ttl" else None + tags = args.tags.split(",") if args.tags else None try: - consul.agent.service.register(args.name, args.service_id, args.address, - int(args.port), tags, check, interval, - ttl, httpcheck) + consul.agent.service.register( + args.name, args.service_id, args.address, int(args.port), tags, check, interval, ttl, httpcheck + ) except exceptions.ConnectionError: connection_error() @@ -537,47 +510,39 @@ def run_once(consul, args): :param argparser.namespace args: The cli args """ - import time - import subprocess error_msg, error_code = None, None try: - consul.lock.prefix('') + consul.lock.prefix("") with consul.lock.acquire(args.lock): if args.interval: now = int(time.time()) last_run = consul.kv.get("{0}_last_run".format(args.lock)) - if str(last_run) not in ['null', 'None'] and \ - int(last_run) + int(args.interval) > now: - sys.stdout.write( - 'Last run happened fewer than {0} seconds ago. ' - 'Exiting\n'.format(args.interval)) + if str(last_run) not in ["null", "None"] and int(last_run) + int(args.interval) > now: + sys.stdout.write("Last run happened fewer than {0} seconds ago. " "Exiting\n".format(args.interval)) return consul.kv["{0}_last_run".format(args.lock)] = now # Should the subprocess return an error code, release the lock try: - print(subprocess.check_output(args.command_to_run[0].strip(), - stderr=subprocess.STDOUT, - shell=True)) + print(subprocess.check_output(args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True)) # If the subprocess fails except subprocess.CalledProcessError as err: error_code = 1 - error_msg = ('"{0}" exited with return code "{1}" and ' - 'output {2}'.format(args.command_to_run, - err.returncode, - err.output)) + error_msg = '"{0}" exited with return code "{1}" and ' "output {2}".format( + args.command_to_run, err.returncode, err.output + ) except OSError as err: error_code = 1 - error_msg = '"{0}" command does not exist'.format( - args.command_to_run, err) + error_msg = '"{0}" command does not exist'.format( # pylint: disable=too-many-format-args + args.command_to_run, err + ) except Exception as err: error_code = 1 - error_msg = '"{0}" exited with error "{1}"'.format( - args.command_to_run, err) + error_msg = '"{0}" exited with error "{1}"'.format(args.command_to_run, err) except consulate.LockFailure: - on_error('Cannot obtain the required lock. Exiting') + on_error("Cannot obtain the required lock. Exiting") except exceptions.ConnectionError: connection_error() @@ -595,43 +560,39 @@ def services(consul, args): """ svcs = consul.agent.services() - print(json.dumps(svcs, - sort_keys=True, - indent=args.indent, - separators=(',', ': ')) + '\n') + print(json.dumps(svcs, sort_keys=True, indent=args.indent, separators=(",", ": ")) + "\n") def main(): """Entrypoint for the consulate cli application""" args = parse_cli_args() - if args.api_scheme == 'http+unix': + if args.api_scheme == "http+unix": adapter = adapters.UnixSocketRequest port = None - api_host = os.environ.get('CONSUL_HTTP_ADDR').replace('unix://', '') + api_host = os.environ.get("CONSUL_HTTP_ADDR").replace("unix://", "") if args.api_host: api_host = args.api_host else: adapter = None port = args.api_port - api_host = 'localhost' + api_host = "localhost" if args.api_host: api_host = args.api_host - consul = consulate.Consul(api_host, port, args.dc, - args.token, args.api_scheme, adapter) + consul = consulate.Consul(api_host, port, args.dc, args.token, args.api_scheme, adapter) - if args.command == 'acl': + if args.command == "acl": ACL_ACTIONS[args.action](consul, args) - elif args.command == 'kv': + elif args.command == "kv": KV_ACTIONS[args.action](consul, args) - elif args.command == 'register': + elif args.command == "register": register(consul, args) - elif args.command == 'deregister': + elif args.command == "deregister": deregister(consul, args) - elif args.command == 'services': + elif args.command == "services": services(consul, args) - elif args.command == 'run_once': + elif args.command == "run_once": run_once(consul, args) diff --git a/pyms/services_discovery/consulate/client.py b/pyms/services_discovery/consulate/client.py index 2d3b926..d8c5dfb 100644 --- a/pyms/services_discovery/consulate/client.py +++ b/pyms/services_discovery/consulate/client.py @@ -5,15 +5,15 @@ import os from pyms.services_discovery.consulate import adapters, api, utils -DEFAULT_HOST = os.environ.get('CONSUL_HOST') or 'localhost' -DEFAULT_PORT = os.environ.get('CONSUL_PORT') or 8500 -DEFAULT_ADDR = os.environ.get('CONSUL_HTTP_ADDR') -DEFAULT_SCHEME = 'http' -DEFAULT_TOKEN = os.environ.get('CONSUL_HTTP_TOKEN') -API_VERSION = 'v1' +DEFAULT_HOST = os.environ.get("CONSUL_HOST") or "localhost" +DEFAULT_PORT = os.environ.get("CONSUL_PORT") or 8500 +DEFAULT_ADDR = os.environ.get("CONSUL_HTTP_ADDR") +DEFAULT_SCHEME = "http" +DEFAULT_TOKEN = os.environ.get("CONSUL_HTTP_TOKEN") +API_VERSION = "v1" -class Consul(object): +class Consul: # pylint: disable=too-many-instance-attributes """Access the Consul HTTP API via Python. The default values connect to Consul via ``localhost:8500`` via http. If @@ -39,36 +39,33 @@ class Consul(object): :param float timeout: Timeout in seconds for API requests (Default: None) """ - def __init__(self, - addr=DEFAULT_ADDR, - host=DEFAULT_HOST, - port=DEFAULT_PORT, - datacenter=None, - token=DEFAULT_TOKEN, - scheme=DEFAULT_SCHEME, - adapter=None, - verify=True, - cert=None, - timeout=None): + + def __init__( + self, + addr=DEFAULT_ADDR, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + datacenter=None, + token=DEFAULT_TOKEN, + scheme=DEFAULT_SCHEME, + adapter=None, + verify=True, + cert=None, + timeout=None, + ): # pylint: disable=too-many-arguments """Create a new instance of the Consul class""" - base_uri = self._base_uri(addr=addr, - scheme=scheme, - host=host, - port=port) - self._adapter = adapter() if adapter else adapters.Request( - timeout=timeout, verify=verify, cert=cert) + base_uri = self._base_uri(addr=addr, scheme=scheme, host=host, port=port) + self._adapter = adapter() if adapter else adapters.Request(timeout=timeout, verify=verify, cert=cert) self._acl = api.ACL(base_uri, self._adapter, datacenter, token) self._agent = api.Agent(base_uri, self._adapter, datacenter, token) self._catalog = api.Catalog(base_uri, self._adapter, datacenter, token) self._event = api.Event(base_uri, self._adapter, datacenter, token) self._health = api.Health(base_uri, self._adapter, datacenter, token) - self._coordinate = api.Coordinate(base_uri, self._adapter, datacenter, - token) + self._coordinate = api.Coordinate(base_uri, self._adapter, datacenter, token) self._kv = api.KV(base_uri, self._adapter, datacenter, token) self._session = api.Session(base_uri, self._adapter, datacenter, token) self._status = api.Status(base_uri, self._adapter, datacenter, token) - self._lock = api.Lock(base_uri, self._adapter, self._session, - datacenter, token) + self._lock = api.Lock(base_uri, self._adapter, self._session, datacenter, token) @property def acl(self): @@ -193,8 +190,6 @@ def _base_uri(scheme, host, port, addr=None): """ if addr is None: if port: - return '{0}://{1}:{2}/{3}'.format(scheme, host, port, - API_VERSION) - return '{0}://{1}/{2}'.format(scheme, utils.quote(host, ''), - API_VERSION) - return '{0}/{1}'.format(addr, API_VERSION) + return "{0}://{1}:{2}/{3}".format(scheme, host, port, API_VERSION) + return "{0}://{1}/{2}".format(scheme, utils.quote(host, ""), API_VERSION) + return "{0}/{1}".format(addr, API_VERSION) diff --git a/pyms/services_discovery/consulate/models/acl.py b/pyms/services_discovery/consulate/models/acl.py index 734ca07..3dea588 100644 --- a/pyms/services_discovery/consulate/models/acl.py +++ b/pyms/services_discovery/consulate/models/acl.py @@ -1,139 +1,147 @@ -# coding=utf-8 """Models for the ACL endpoints""" import uuid from pyms.services_discovery.consulate.models import base -def _validate_link_array(value, model): - """ Validate the policies or roles links are formatted correctly. +def _validate_link_array(value, model): # pylint: disable=unused-argument + """Validate the policies or roles links are formatted correctly. :param list value: An array of PolicyLink or RoleLink. :param rtype: bool """ - return all(['ID' in link or 'Name' in link for link in value]) + return all(["ID" in link or "Name" in link for link in value]) -def _validate_service_identities(value, model): - """ Validate service_identities is formatted correctly. +def _validate_service_identities(value, model): # pylint: disable=unused-argument + """Validate service_identities is formatted correctly. :param ServiceIdentities value: A ServiceIdentity list :param rtype: bool """ - return all( - ['ServiceName' in service_identity for service_identity in value]) + return all(["ServiceName" in service_identity for service_identity in value]) class ACLPolicy(base.Model): """Defines the model used for an ACL policy.""" - __slots__ = ['datacenters', 'description', 'id', 'name', 'rules'] + + __slots__ = ["datacenters", "description", "id", "name", "rules"] __attributes__ = { - 'datacenters': { - 'key': 'Datacenters', - 'type': list, - }, - 'description': { - 'key': 'Description', - 'type': str, - }, - 'id': { - 'key': 'ID', - 'type': uuid.UUID, - 'cast_from': str, - 'cast_to': str, - }, - 'name': { - 'key': 'Name', - 'type': str, - }, - 'rules': { - 'key': 'Rules', - 'type': str, - } + "datacenters": { + "key": "Datacenters", + "type": list, + }, + "description": { + "key": "Description", + "type": str, + }, + "id": { + "key": "ID", + "type": uuid.UUID, + "cast_from": str, + "cast_to": str, + }, + "name": { + "key": "Name", + "type": str, + }, + "rules": { + "key": "Rules", + "type": str, + }, } class ACLRole(base.Model): """Defines the model used for an ACL role.""" - __slots__ = ['description', 'name', 'policies', 'service_identities'] + + __slots__ = ["description", "name", "policies", "service_identities"] __attributes__ = { - 'description': { - 'key': 'Description', - 'type': str, + "description": { + "key": "Description", + "type": str, }, - 'name': { - 'key': 'Name', - 'type': str, - 'required': True, + "name": { + "key": "Name", + "type": str, + "required": True, }, - 'policies': { - 'key': 'Policies', - 'type': list, - 'validator': _validate_link_array, + "policies": { + "key": "Policies", + "type": list, + "validator": _validate_link_array, }, "service_identities": { - 'key': 'ServiceIdentities', - 'type': list, - 'validator': _validate_service_identities, - } + "key": "ServiceIdentities", + "type": list, + "validator": _validate_service_identities, + }, } class ACLToken(base.Model): """Defines the model used for an ACL token.""" + __slots__ = [ - 'accessor_id', 'description', 'expiration_time', 'expiration_ttl', - 'local', 'policies', 'roles', 'secret_id', 'service_identities' + "accessor_id", + "description", + "expiration_time", + "expiration_ttl", + "local", + "policies", + "roles", + "secret_id", + "service_identities", ] __attributes__ = { - 'accessor_id': { - 'key': 'AccessorID', - 'type': uuid.UUID, - 'cast_from': str, - 'cast_to': str, - }, - 'description': { - 'key': 'Description', - 'type': str, - }, - 'expiration_time': { - 'key': 'ExpirationTime', - 'type': str, - }, - 'expiration_ttl': { - 'key': 'ExpirationTTL', - 'type': str, - }, - 'local': { - 'key': 'Local', - 'type': bool, - }, - 'policies': { - 'key': 'Policies', - 'type': list, - 'validator': _validate_link_array, - }, - 'roles': { - 'key': 'Roles', - 'type': list, - 'validator': _validate_link_array, - }, - 'secret_id': { - 'key': 'SecretID', - 'type': uuid.UUID, - 'cast_from': str, - 'cast_to': str, + "accessor_id": { + "key": "AccessorID", + "type": uuid.UUID, + "cast_from": str, + "cast_to": str, + }, + "description": { + "key": "Description", + "type": str, + }, + "expiration_time": { + "key": "ExpirationTime", + "type": str, + }, + "expiration_ttl": { + "key": "ExpirationTTL", + "type": str, + }, + "local": { + "key": "Local", + "type": bool, + }, + "policies": { + "key": "Policies", + "type": list, + "validator": _validate_link_array, + }, + "roles": { + "key": "Roles", + "type": list, + "validator": _validate_link_array, + }, + "secret_id": { + "key": "SecretID", + "type": uuid.UUID, + "cast_from": str, + "cast_to": str, }, "service_identities": { - 'key': 'ServiceIdentities', - 'type': list, - 'validator': _validate_service_identities, - } + "key": "ServiceIdentities", + "type": list, + "validator": _validate_service_identities, + }, } @@ -142,27 +150,12 @@ class ACLToken(base.Model): class ACL(base.Model): """Defines the model used for an individual ACL token.""" - __slots__ = ['id', 'name', 'type', 'rules'] + + __slots__ = ["id", "name", "type", "rules"] __attributes__ = { - 'id': { - 'key': 'ID', - 'type': uuid.UUID, - 'cast_from': str, - 'cast_to': str - }, - 'name': { - 'key': 'Name', - 'type': str - }, - 'type': { - 'key': 'Type', - 'type': str, - 'enum': {'client', 'management'}, - 'required': True - }, - 'rules': { - 'key': 'Rules', - 'type': str - } + "id": {"key": "ID", "type": uuid.UUID, "cast_from": str, "cast_to": str}, + "name": {"key": "Name", "type": str}, + "type": {"key": "Type", "type": str, "enum": {"client", "management"}, "required": True}, + "rules": {"key": "Rules", "type": str}, } diff --git a/pyms/services_discovery/consulate/models/agent.py b/pyms/services_discovery/consulate/models/agent.py index d5862d2..2e53e5d 100644 --- a/pyms/services_discovery/consulate/models/agent.py +++ b/pyms/services_discovery/consulate/models/agent.py @@ -1,7 +1,6 @@ -# coding=utf-8 """Models for the Agent endpoints""" -from pyms.services_discovery.consulate.models import base from pyms.services_discovery.consulate import utils +from pyms.services_discovery.consulate.models import base def _validate_args(value, model): @@ -13,9 +12,13 @@ def _validate_args(value, model): :rtype: bool """ - return all([isinstance(v, str) for v in value]) \ - and not model.args and not model.grpc and not model.http \ + return ( + all([isinstance(v, str) for v in value]) + and not model.args + and not model.grpc + and not model.http and not model.ttl + ) def _validate_grpc(value, model): @@ -27,9 +30,7 @@ def _validate_grpc(value, model): :rtype: bool """ - return utils.validate_url(value) \ - and not model.args and not model.http \ - and not model.tcp and not model.ttl + return utils.validate_url(value) and not model.args and not model.http and not model.tcp and not model.ttl def _validate_http(value, model): @@ -41,9 +42,7 @@ def _validate_http(value, model): :rtype: bool """ - return utils.validate_url(value) \ - and not model.args and not model.grpc and not model.tcp \ - and not model.ttl + return utils.validate_url(value) and not model.args and not model.grpc and not model.tcp and not model.ttl def _validate_interval(value, model): @@ -65,8 +64,7 @@ def _validate_tcp(_value, model): :rtype: bool """ - return not model.args and not model.grpc \ - and not model.http and not model.ttl + return not model.args and not model.grpc and not model.http and not model.ttl def _validate_ttl(value, model): @@ -77,170 +75,98 @@ def _validate_ttl(value, model): :rtype: bool """ - return utils.validate_go_interval(value) and not model.args \ - and not model.grpc and not model.http \ - and not model.tcp and not model.interval + return ( + utils.validate_go_interval(value) + and not model.args + and not model.grpc + and not model.http + and not model.tcp + and not model.interval + ) class Check(base.Model): """Model for making Check API requests to Consul.""" - __slots__ = ['id', 'name', 'interval', 'notes', - 'deregister_critical_service_after', 'args', - 'docker_container_id', 'grpc', 'grpc_use_tls', - 'http', 'method', 'header', 'timeout', 'tls_skip_verify', - 'tcp', 'ttl', 'service_id', 'status'] + __slots__ = [ + "id", + "name", + "interval", + "notes", + "deregister_critical_service_after", + "args", + "docker_container_id", + "grpc", + "grpc_use_tls", + "http", + "method", + "header", + "timeout", + "tls_skip_verify", + "tcp", + "ttl", + "service_id", + "status", + ] __attributes__ = { - 'id': { - 'key': 'ID', - 'type': str - }, - 'name': { - 'key': 'Name', - 'type': str, - 'required': True - }, - 'interval': { - 'key': 'Interval', - 'type': str, - 'validator': _validate_interval - }, - 'notes': { - 'key': 'Notes', - 'type': str - }, - 'deregister_critical_service_after': { - 'key': 'DeregisterCriticalServiceAfter', - 'type': str, - 'validator': utils.validate_go_interval - }, - 'args': { - 'key': 'Args', - 'type': list, - 'validator': _validate_args - }, - 'docker_container_id': { - 'key': 'DockerContainerID', - 'type': str - }, - 'grpc': { - 'key': 'GRPC', - 'type': str, - 'validator': _validate_grpc - }, - 'grpc_use_tls': { - 'key': 'GRPCUseTLS', - 'type': bool - }, - 'http': { - 'key': 'HTTP', - 'type': str, - 'validator': _validate_http - }, - 'method': { - 'key': 'Method', - 'type': str, - 'enum': { - 'HEAD', 'GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'TRACE' - } - }, - 'header': { - 'key': 'Header', - 'type': dict, - 'validator': lambda h, _m: all( - [(isinstance(k, str) and isinstance(v, str)) - for k, v in h.items()]), - 'cast_to': lambda h: {k: [v] for k, v in h.items()} - }, - 'timeout': { - 'key': 'Timeout', - 'type': str, - 'validator': utils.validate_go_interval - }, - 'tls_skip_verify': { - 'key': 'TLSSkipVerify', - 'type': bool - }, - 'tcp': { - 'key': 'TCP', - 'type': str, - 'validator': _validate_tcp - }, - 'ttl': { - 'key': 'TTL', - 'type': str, - 'validator': _validate_ttl - }, - 'service_id': { - 'key': 'ServiceID', - 'type': str - }, - 'status': { - 'key': 'Status', - 'type': str, - 'enum': {'passing', 'warning', 'critical', 'maintenance'} - } + "id": {"key": "ID", "type": str}, + "name": {"key": "Name", "type": str, "required": True}, + "interval": {"key": "Interval", "type": str, "validator": _validate_interval}, + "notes": {"key": "Notes", "type": str}, + "deregister_critical_service_after": { + "key": "DeregisterCriticalServiceAfter", + "type": str, + "validator": utils.validate_go_interval, + }, + "args": {"key": "Args", "type": list, "validator": _validate_args}, + "docker_container_id": {"key": "DockerContainerID", "type": str}, + "grpc": {"key": "GRPC", "type": str, "validator": _validate_grpc}, + "grpc_use_tls": {"key": "GRPCUseTLS", "type": bool}, + "http": {"key": "HTTP", "type": str, "validator": _validate_http}, + "method": {"key": "Method", "type": str, "enum": {"HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"}}, + "header": { + "key": "Header", + "type": dict, + "validator": lambda h, _m: all([(isinstance(k, str) and isinstance(v, str)) for k, v in h.items()]), + "cast_to": lambda h: {k: [v] for k, v in h.items()}, + }, + "timeout": {"key": "Timeout", "type": str, "validator": utils.validate_go_interval}, + "tls_skip_verify": {"key": "TLSSkipVerify", "type": bool}, + "tcp": {"key": "TCP", "type": str, "validator": _validate_tcp}, + "ttl": {"key": "TTL", "type": str, "validator": _validate_ttl}, + "service_id": {"key": "ServiceID", "type": str}, + "status": {"key": "Status", "type": str, "enum": {"passing", "warning", "critical", "maintenance"}}, } def __init__(self, **kwargs): - super(Check, self).__init__(**kwargs) - if (self.args or self.grpc or self.http or self.tcp) \ - and not self.interval: - raise ValueError('"interval" must be specified when specifying ' - 'args, grpc, http, or tcp.') + super().__init__(**kwargs) + if (self.args or self.grpc or self.http or self.tcp) and not self.interval: + raise ValueError('"interval" must be specified when specifying ' "args, grpc, http, or tcp.") class Service(base.Model): """Model for making Check API requests to Consul.""" - __slots__ = ['id', 'name', 'tags', 'meta', 'address', 'port', 'check', - 'checks', 'enable_tag_override'] + __slots__ = ["id", "name", "tags", "meta", "address", "port", "check", "checks", "enable_tag_override"] __attributes__ = { - 'id': { - 'key': 'ID', - 'type': str - }, - 'name': { - 'key': 'Name', - 'type': str, - 'required': True - }, - 'tags': { - 'key': 'Tags', - 'type': list, - 'validator': lambda t, _m: all([isinstance(v, str) for v in t]) - }, - 'meta': { - 'key': 'Meta', - 'type': dict, - 'validator': lambda h, _m: all( - [(isinstance(k, str) and isinstance(v, str)) - for k, v in h.items()]), - }, - 'address': { - 'key': 'Address', - 'type': str - }, - 'port': { - 'key': 'Port', - 'type': int - }, - 'check': { - 'key': 'Check', - 'type': Check, - 'cast_to': dict - }, - 'checks': { - 'key': 'Checks', - 'type': list, - 'validator': lambda c, _m: all([isinstance(v, Check) for v in c]), - 'cast_to': lambda c: [dict(check) for check in c] - }, - 'enable_tag_override': { - 'Key': 'EnableTagOverride', - 'type': bool - } + "id": {"key": "ID", "type": str}, + "name": {"key": "Name", "type": str, "required": True}, + "tags": {"key": "Tags", "type": list, "validator": lambda t, _m: all([isinstance(v, str) for v in t])}, + "meta": { + "key": "Meta", + "type": dict, + "validator": lambda h, _m: all([(isinstance(k, str) and isinstance(v, str)) for k, v in h.items()]), + }, + "address": {"key": "Address", "type": str}, + "port": {"key": "Port", "type": int}, + "check": {"key": "Check", "type": Check, "cast_to": dict}, + "checks": { + "key": "Checks", + "type": list, + "validator": lambda c, _m: all([isinstance(v, Check) for v in c]), + "cast_to": lambda c: [dict(check) for check in c], + }, + "enable_tag_override": {"Key": "EnableTagOverride", "type": bool}, } - diff --git a/pyms/services_discovery/consulate/models/base.py b/pyms/services_discovery/consulate/models/base.py index 50802bf..3c0eb07 100644 --- a/pyms/services_discovery/consulate/models/base.py +++ b/pyms/services_discovery/consulate/models/base.py @@ -37,10 +37,11 @@ class MyModel(Model): """The attributes that define the data elements of the model""" def __init__(self, **kwargs): - super(Model, self).__init__() - [setattr(self, name, value) for name, value in kwargs.items()] - [self._set_default(name) for name in self.__attributes__.keys() - if name not in kwargs.keys()] + super().__init__() + [setattr(self, name, value) for name, value in kwargs.items()] # pylint: disable=expression-not-assigned + [ # pylint: disable=expression-not-assigned + self._set_default(name) for name in self.__attributes__ if name not in kwargs + ] def __iter__(self): """Iterate through the model's key, value pairs. @@ -48,7 +49,7 @@ def __iter__(self): :rtype: iterator """ - for name in self.__attributes__.keys(): + for name in self.__attributes__: value = self._maybe_cast_value(name) if value is not None: yield self._maybe_return_key(name), value @@ -67,7 +68,7 @@ def __setattr__(self, name, value): if name not in self.__attributes__: raise AttributeError('Invalid attribute "{}"'.format(name)) value = self._validate_value(name, value) - super(Model, self).__setattr__(name, value) + super().__setattr__(name, value) def __getattribute__(self, name): """Return the attribute from the model if it is set, otherwise @@ -78,10 +79,10 @@ def __getattribute__(self, name): """ try: - return super(Model, self).__getattribute__(name) + return super().__getattribute__(name) except AttributeError: if name in self.__attributes__: - return self.__attributes__[name].get('default', None) + return self.__attributes__[name].get("default", None) raise def _maybe_cast_value(self, name): @@ -93,8 +94,8 @@ def _maybe_cast_value(self, name): """ value = getattr(self, name) - if value is not None and self.__attributes__[name].get('cast_to'): - return self.__attributes__[name]['cast_to'](value) + if value is not None and self.__attributes__[name].get("cast_to"): + return self.__attributes__[name]["cast_to"](value) return value def _maybe_return_key(self, name): @@ -106,8 +107,8 @@ def _maybe_return_key(self, name): :rtype: mixed """ - if self.__attributes__[name].get('key'): - return self.__attributes__[name]['key'] + if self.__attributes__[name].get("key"): + return self.__attributes__[name]["key"] return name def _required_attr(self, name): @@ -117,7 +118,7 @@ def _required_attr(self, name): :rtype: bool """ - return self.__attributes__[name].get('required', False) + return self.__attributes__[name].get("required", False) def _set_default(self, name): """Set the default value for the attribute name. @@ -125,7 +126,7 @@ def _set_default(self, name): :param str name: The attribute name """ - setattr(self, name, self.__attributes__[name].get('default', None)) + setattr(self, name, self.__attributes__[name].get("default", None)) def _validate_value(self, name, value): """Ensures the the value validates based upon the type or a validation @@ -141,27 +142,24 @@ def _validate_value(self, name, value): if value is None: if self._required_attr(name): raise ValueError('Attribute "{}" is required'.format(name)) - return + return None - if not isinstance(value, self.__attributes__[name].get('type')): - cast_from = self.__attributes__[name].get('cast_from') + if not isinstance(value, self.__attributes__[name].get("type")): + cast_from = self.__attributes__[name].get("cast_from") if cast_from and isinstance(value, cast_from): - value = self.__attributes__[name]['type'](value) + value = self.__attributes__[name]["type"](value) else: raise TypeError( 'Attribute "{}" must be of type {} not {}'.format( - name, self.__attributes__[name]['type'].__name__, - value.__class__.__name__)) + name, self.__attributes__[name]["type"].__name__, value.__class__.__name__ + ) + ) - if self.__attributes__[name].get('enum') \ - and value not in self.__attributes__[name]['enum']: - raise ValueError( - 'Attribute "{}" value {!r} not valid'.format(name, value)) + if self.__attributes__[name].get("enum") and value not in self.__attributes__[name]["enum"]: + raise ValueError('Attribute "{}" value {!r} not valid'.format(name, value)) - validator = self.__attributes__[name].get('validator') + validator = self.__attributes__[name].get("validator") if callable(validator): if not validator(value, self): - raise ValueError( - 'Attribute "{}" value {!r} did not validate'.format( - name, value)) + raise ValueError('Attribute "{}" value {!r} did not validate'.format(name, value)) return value diff --git a/pyms/services_discovery/consulate/utils.py b/pyms/services_discovery/consulate/utils.py index 36a6224..c4908ae 100644 --- a/pyms/services_discovery/consulate/utils.py +++ b/pyms/services_discovery/consulate/utils.py @@ -4,22 +4,11 @@ """ import re -import sys -try: # pylint: disable=import-error - from urllib.parse import quote -except ImportError: - from urllib import quote - -try: # pylint: disable=import-error - from urllib import parse as _urlparse -except ImportError: - import urlparse as _urlparse - +from urllib import parse as _urlparse from pyms.services_discovery.consulate import exceptions -DURATION_PATTERN = re.compile(r'^(?:(?:-|)(?:\d+|\d+\.\d+)(?:µs|ms|s|m|h))+$') -PYTHON3 = True if sys.version_info > (3, 0, 0) else False +DURATION_PATTERN = re.compile(r"^(?:(?:-|)(?:\d+|\d+\.\d+)(?:µs|ms|s|m|h))+$") def is_string(value): @@ -31,8 +20,6 @@ def is_string(value): """ checks = [isinstance(value, t) for t in [bytes, str]] - if not PYTHON3: - checks.append(isinstance(value, unicode)) return any(checks) @@ -44,7 +31,7 @@ def maybe_encode(value): """ try: - return value.encode('utf-8') + return value.encode("utf-8") except AttributeError: return value @@ -56,9 +43,7 @@ def _response_error(response): :rtype: str """ - return (response.body.decode('utf-8') - if hasattr(response, 'body') and response.body - else str(response.status_code)) + return response.body.decode("utf-8") if hasattr(response, "body") and response.body else str(response.status_code) def response_ok(response, raise_on_404=False): @@ -71,8 +56,9 @@ def response_ok(response, raise_on_404=False): :raises: consulate.exceptions.ConsulateException """ + result = False if response.status_code == 200: - return True + result = True elif response.status_code == 400: raise exceptions.ClientError(_response_error(response)) elif response.status_code == 401: @@ -83,7 +69,7 @@ def response_ok(response, raise_on_404=False): raise exceptions.NotFound(_response_error(response)) elif response.status_code == 500: raise exceptions.ServerError(_response_error(response)) - return False + return result def validate_go_interval(value, _model=None): diff --git a/pyms/utils/__init__.py b/pyms/utils/__init__.py index 1583c00..2d21216 100644 --- a/pyms/utils/__init__.py +++ b/pyms/utils/__init__.py @@ -1,3 +1,3 @@ from .utils import import_from, import_package, check_package_exists -__all__ = ['import_from', 'import_package', 'check_package_exists'] +__all__ = ["import_from", "import_package", "check_package_exists"] diff --git a/pyms/utils/files.py b/pyms/utils/files.py index a9e2244..7a0bebb 100644 --- a/pyms/utils/files.py +++ b/pyms/utils/files.py @@ -49,7 +49,7 @@ def _get_conf_from_file(self, path, fn=None): if fn: files_cached[path] = fn(path) else: - file_to_read = open(path, 'rb') + file_to_read = open(path, "rb") content = file_to_read.read() # The key will be type bytes file_to_read.close() files_cached[path] = content diff --git a/pyms/utils/utils.py b/pyms/utils/utils.py index 9fee53b..0e68148 100644 --- a/pyms/utils/utils.py +++ b/pyms/utils/utils.py @@ -31,7 +31,8 @@ def check_package_exists(package_name: Text) -> Union[Exception, bool]: spec = importlib.util.find_spec(package_name) if spec is None: raise PackageNotExists( - "{package} is not installed. try with pip install -U {package}".format(package=package_name)) + "{package} is not installed. try with pip install -U {package}".format(package=package_name) + ) return True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e6c68b5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.black] +line-length = 120 +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist +)/ +''' \ No newline at end of file diff --git a/setup.py b/setup.py index 9cbcbbd..d6cd4d2 100644 --- a/setup.py +++ b/setup.py @@ -4,74 +4,81 @@ from setuptools import setup, find_packages -author = __import__('pyms').__author__ -author_email = __import__('pyms').__email__ -version = __import__('pyms').__version__ +author = __import__("pyms").__author__ +author_email = __import__("pyms").__email__ +version = __import__("pyms").__version__ -if os.path.exists('README.md'): - long_description = codecs.open('README.md', 'r', 'utf-8').read() +if os.path.exists("README.md"): + long_description = codecs.open("README.md", "r", "utf-8").read() else: - long_description = '' + long_description = "" install_min_requires = [ - 'flask>=1.1.2', - 'python-json-logger>=2.0.0', - 'pyyaml>=5.3.1', - 'anyconfig>=0.9.11', - 'cryptography>=3.2', + "flask>=1.1.2", + "python-json-logger>=2.0.0", + "pyyaml>=5.3.1", + "anyconfig>=0.9.11", + "cryptography>=3.2", ] install_crypt_requires = [ - 'cryptography>=3.1.1', + "cryptography>=3.1.1", ] install_aws_requires = [ - 'boto3>=1.15.6', + "boto3>=1.15.6", ] install_request_requires = [ - 'requests>=2.24.0', + "requests>=2.24.0", ] install_swagger_requires = [ - 'connexion[swagger-ui]>=2.7.0', - 'swagger-ui-bundle>=0.0.6', - 'semver>=2.10.1', - 'prance>=0.18.2', + "connexion[swagger-ui]>=2.7.0", + "swagger-ui-bundle>=0.0.6", + "semver>=2.10.1", + "prance>=0.18.2", ] install_traces_requires = [ - 'jaeger-client>=4.3.0', - 'flask-opentracing>=1.1.0', - 'opentracing>=2.1', - 'opentracing-instrumentation>=3.2.1', - 'tornado>=4.3,<6.0' + "jaeger-client>=4.3.0", + "flask-opentracing>=1.1.0", + "opentracing>=2.1", + "opentracing-instrumentation>=3.2.1", + "tornado>=4.3,<6.0", ] install_metrics_requires = [ - 'prometheus_client>=0.8.0', + "prometheus_client>=0.8.0", ] install_tests_requires = [ - 'requests-mock>=1.8.0', - 'coverage>=5.3', - 'pytest>=6.1.0', - 'pytest-cov>=2.10.1', - 'pylint>=2.6.0', - 'flake8>=3.8.2', - 'tox>=3.20.0', - 'bandit>=1.6.2', - 'mkdocs>=1.1.2', - 'mkdocs-material>=6.0.0', - 'lightstep>=4.4.8', - 'safety==1.9.0', - 'mypy>=0.782' - 'httmock>=1.4.0' + "requests-mock>=1.8.0", + "coverage>=5.3", + "pytest>=6.1.0", + "pytest-cov>=2.10.1", + "pylint>=2.6.0", + "flake8>=3.8.2", + "tox>=3.20.0", + "bandit>=1.6.2", + "mkdocs>=1.1.2", + "mkdocs-material>=6.0.0", + "lightstep>=4.4.8", + "safety==1.9.0", + "pre-commit>=2.8.1", + "black>=20.8b1", + "mypy>=0.782", + "httmock>=1.4.0", ] -install_all_requires = (install_request_requires + install_swagger_requires + - install_traces_requires + install_metrics_requires + install_crypt_requires + - install_aws_requires) +install_all_requires = ( + install_request_requires + + install_swagger_requires + + install_traces_requires + + install_metrics_requires + + install_crypt_requires + + install_aws_requires +) setup( name="py-ms", @@ -80,10 +87,10 @@ author_email=author_email, description="Library of utils to create REST Python Microservices", long_description=long_description, - long_description_content_type='text/markdown', + long_description_content_type="text/markdown", classifiers=[ - 'Development Status :: 4 - Beta', - 'Framework :: Flask', + "Development Status :: 4 - Beta", + "Framework :: Flask", "Intended Audience :: Developers", "Natural Language :: English", "Programming Language :: Python :: 3.6", @@ -93,29 +100,26 @@ license="License :: OSI Approved :: GNU General Public License v3 (GPLv3)", platforms=["any"], keywords="", - url='https://github.com/python-microservices/pyms/', + url="https://github.com/python-microservices/pyms/", packages=find_packages( - exclude=['*.tests', '*.tests.*', 'tests.*', 'tests', '*.examples', '*.examples.*', 'examples.*', 'examples']), + exclude=["*.tests", "*.tests.*", "tests.*", "tests", "*.examples", "*.examples.*", "examples.*", "examples"] + ), setup_requires=[ - 'pytest-runner>=5.2', + "pytest-runner>=5.2", ], install_requires=install_min_requires, extras_require={ - 'all': install_all_requires, - 'request': install_request_requires, - 'swagger': install_swagger_requires, - 'traces': install_traces_requires, - 'metrics': install_metrics_requires, - 'crypt': install_crypt_requires, - 'aws': install_aws_requires, - 'tests': install_tests_requires, + "all": install_all_requires, + "request": install_request_requires, + "swagger": install_swagger_requires, + "traces": install_traces_requires, + "metrics": install_metrics_requires, + "crypt": install_crypt_requires, + "aws": install_aws_requires, + "tests": install_tests_requires, }, tests_require=install_all_requires + install_tests_requires, include_package_data=True, - entry_points={ - 'console_scripts': [ - 'pyms = pyms.cmd.main:Command' - ] - }, + entry_points={"console_scripts": ["pyms = pyms.cmd.main:Command"]}, zip_safe=True, ) diff --git a/tests/common.py b/tests/common.py index 4d8ecbc..4249bf6 100644 --- a/tests/common.py +++ b/tests/common.py @@ -2,6 +2,7 @@ from pyms.flask.app import Microservice from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, DEFAULT_CONFIGMAP_FILENAME + class MyMicroserviceNoSingleton(Microservice): _singleton = False diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 1f5cb81..9f5f3a0 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -22,9 +22,10 @@ def test_crypt_file_error(self): cmd = Command(arguments=arguments, autorun=False) with pytest.raises(FileDoesNotExistException) as excinfo: cmd.run() - assert ("Decrypt key None not exists. You must set a correct env var PYMS_KEY_FILE or run " - "`pyms crypt create-key` command") \ - in str(excinfo.value) + assert ( + "Decrypt key None not exists. You must set a correct env var PYMS_KEY_FILE or run " + "`pyms crypt create-key` command" + ) in str(excinfo.value) def test_crypt_file_ok(self): crypt = Crypt() @@ -34,18 +35,22 @@ def test_crypt_file_ok(self): cmd.run() crypt.delete_key() - @patch('pyms.cmd.main.Command.get_input', return_value='Y') + @patch("pyms.cmd.main.Command.get_input", return_value="Y") def test_generate_file_ok(self, input): crypt = Crypt() - arguments = ["create-key", ] + arguments = [ + "create-key", + ] cmd = Command(arguments=arguments, autorun=False) cmd.run() crypt.delete_key() - @patch('pyms.cmd.main.Command.get_input', return_value='n') + @patch("pyms.cmd.main.Command.get_input", return_value="n") def test_output_key(self, input): crypt = Crypt() - arguments = ["create-key", ] + arguments = [ + "create-key", + ] cmd = Command(arguments=arguments, autorun=False) cmd.run() with pytest.raises(FileNotFoundError) as excinfo: @@ -61,23 +66,29 @@ def test_startproject_error(self): def test_get_bundled_specs(self): specs = get_bundled_specs(Path("tests/swagger_for_tests/swagger.yaml")) - self.assertEqual(specs.get('swagger'), "2.0") - self.assertEqual(specs.get('info').get('version'), "1.0.0") - self.assertEqual(specs.get('info').get('contact').get('email'), "apiteam@swagger.io") + self.assertEqual(specs.get("swagger"), "2.0") + self.assertEqual(specs.get("info").get("version"), "1.0.0") + self.assertEqual(specs.get("info").get("contact").get("email"), "apiteam@swagger.io") def test_merge_swagger_ok(self): - arguments = ["merge-swagger", "--file", "tests/swagger_for_tests/swagger.yaml", ] + arguments = [ + "merge-swagger", + "--file", + "tests/swagger_for_tests/swagger.yaml", + ] cmd = Command(arguments=arguments, autorun=False) assert cmd.run() os.remove("tests/swagger_for_tests/swagger-complete.yaml") def test_merge_swagger_error(self): - arguments = ["merge-swagger", ] + arguments = [ + "merge-swagger", + ] cmd = Command(arguments=arguments, autorun=False) with pytest.raises(ResolutionError) as excinfo: cmd.run() - @patch('pyms.cmd.main.Command.yes_no_input', return_value=True) + @patch("pyms.cmd.main.Command.yes_no_input", return_value=True) def test_create_config_all(self, input): # Remove config file if already exists for test remove_conf_file() diff --git a/tests/test_config.py b/tests/test_config.py index e0f8e4f..7e38960 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,10 +5,20 @@ from pyms.config import get_conf, ConfFile, create_conf_file from pyms.config.conf import validate_conf -from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY, LOGGER_NAME, CONFIG_BASE, \ - CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY -from pyms.exceptions import AttrDoesNotExistException, ConfigDoesNotFoundException, ServiceDoesNotExistException, \ - ConfigErrorException +from pyms.constants import ( + CONFIGMAP_FILE_ENVIRONMENT, + CONFIGMAP_FILE_ENVIRONMENT_LEGACY, + LOGGER_NAME, + CONFIG_BASE, + CRYPT_FILE_KEY_ENVIRONMENT, + CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, +) +from pyms.exceptions import ( + AttrDoesNotExistException, + ConfigDoesNotFoundException, + ServiceDoesNotExistException, + ConfigErrorException, +) logger = logging.getLogger(LOGGER_NAME) @@ -133,7 +143,7 @@ def test_default_flask(self): assert config.APP_NAME == "Python Microservice" assert config.SUBSERVICE1.test == "input" - @mock.patch('pyms.config.conf.ConfFile') + @mock.patch("pyms.config.conf.ConfFile") def test_without_params(self, mock_confile): with self.assertRaises(ServiceDoesNotExistException): get_conf() diff --git a/tests/test_crypt.py b/tests/test_crypt.py index 8c4ebe7..f480a3a 100644 --- a/tests/test_crypt.py +++ b/tests/test_crypt.py @@ -7,8 +7,13 @@ from pyms.cloud.aws.kms import Crypt as CryptAws from pyms.config import get_conf -from pyms.constants import LOGGER_NAME, CONFIGMAP_FILE_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT, CONFIG_BASE, \ - CRYPT_FILE_KEY_ENVIRONMENT_LEGACY +from pyms.constants import ( + LOGGER_NAME, + CONFIGMAP_FILE_ENVIRONMENT, + CRYPT_FILE_KEY_ENVIRONMENT, + CONFIG_BASE, + CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, +) from pyms.crypt.driver import CryptAbstract, CryptResource from pyms.crypt.fernet import Crypt as CryptFernet from pyms.exceptions import FileDoesNotExistException @@ -30,12 +35,12 @@ def decrypt(self, encrypted): class CryptTests(unittest.TestCase): - def test_ko(self): with pytest.raises(TypeError) as excinfo: MockDecrypt() assert "Can't instantiate abstract class MockDecrypt with abstract methods decrypt, encrypt" in str( - excinfo.value) + excinfo.value + ) def test_ko_enCryptFernet(self): crypt = MockDecrypt2() @@ -57,9 +62,10 @@ def test_crypt_file_error(self): crypt = CryptFernet() with pytest.raises(FileDoesNotExistException) as excinfo: crypt.read_key() - assert ("Decrypt key None not exists. You must set a correct env var PYMS_KEY_FILE or run " - "`pyms crypt create-key` command") \ - in str(excinfo.value) + assert ( + "Decrypt key None not exists. You must set a correct env var PYMS_KEY_FILE or run " + "`pyms crypt create-key` command" + ) in str(excinfo.value) def test_crypt_file_ok(self): crypt = CryptFernet() @@ -110,8 +116,8 @@ def setUp(self): def tearDown(self): del os.environ[CONFIGMAP_FILE_ENVIRONMENT] - @patch.object(CryptAws, '_init_boto') - @patch.object(CryptAws, '_aws_decrypt') + @patch.object(CryptAws, "_init_boto") + @patch.object(CryptAws, "_aws_decrypt") def test_encrypt_conf(self, mock_aws_decrypt, mock_init_boto): mock_aws_decrypt.return_value = "http://database-url" crypt = CryptResource() @@ -123,8 +129,9 @@ class FlaskWithEncryptedFernetTests(unittest.TestCase): BASE_DIR = os.path.dirname(os.path.abspath(__file__)) def setUp(self): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "config-tests-flask-encrypted-fernet.yml") + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "config-tests-flask-encrypted-fernet.yml" + ) os.environ[CRYPT_FILE_KEY_ENVIRONMENT] = os.path.join(self.BASE_DIR, "key.key") self.crypt = CryptFernet(path=self.BASE_DIR) self.crypt._loader.put_file(b"9IXx2F5d5Ob-h5xdCnFSUXhuFKLGRibvLfSbixpcfCw=", "wb") @@ -152,8 +159,9 @@ class FlaskWithEncryptedNoneTests(unittest.TestCase): BASE_DIR = os.path.dirname(os.path.abspath(__file__)) def setUp(self): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "config-tests-flask-encrypted-none.yml") + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "config-tests-flask-encrypted-none.yml" + ) ms = MyMicroserviceNoSingleton(path=__file__) ms.reload_conf() self.app = ms.create_app() diff --git a/tests/test_flask.py b/tests/test_flask.py index c9215d1..9b563bf 100644 --- a/tests/test_flask.py +++ b/tests/test_flask.py @@ -21,8 +21,9 @@ class HomeWithFlaskTests(unittest.TestCase): """ def setUp(self): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "config-tests-flask.yml") + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "config-tests-flask.yml" + ) ms = MyMicroservice() ms.reload_conf() self.app = ms.create_app() @@ -30,16 +31,16 @@ def setUp(self): self.assertEqual("Python Microservice With Flask", self.app.config["APP_NAME"]) def test_home(self): - response = self.client.get('/') + response = self.client.get("/") self.assertEqual(404, response.status_code) def test_healthcheck(self): - response = self.client.get('/healthcheck') + response = self.client.get("/healthcheck") self.assertEqual(b"OK", response.data) self.assertEqual(200, response.status_code) def test_swagger(self): - response = self.client.get('/ui/') + response = self.client.get("/ui/") self.assertEqual(404, response.status_code) @@ -49,35 +50,36 @@ class FlaskWithSwaggerTests(unittest.TestCase): """ def setUp(self): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "config-tests-flask-swagger.yml") + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "config-tests-flask-swagger.yml" + ) ms = MyMicroservice(path=__file__) self.app = ms.create_app() self.client = self.app.test_client() self.assertEqual("Python Microservice With Flask in tests", self.app.config["APP_NAME"]) def test_healthcheck(self): - response = self.client.get('/healthcheck') + response = self.client.get("/healthcheck") self.assertEqual(b"OK", response.data) self.assertEqual(200, response.status_code) def test_swagger(self): - response = self.client.get('/ui/') + response = self.client.get("/ui/") self.assertEqual(200, response.status_code) def test_exists_service(self): self.assertTrue(isinstance(self.app.ms.swagger, DriverService)) def test_reverse_proxy(self): - response = self.client.get('/my-proxy-path/ui/', headers={"X-Script-Name": "/my-proxy-path"}) + response = self.client.get("/my-proxy-path/ui/", headers={"X-Script-Name": "/my-proxy-path"}) self.assertEqual(200, response.status_code) def test_reverse_proxy_no_slash(self): - response = self.client.get('/my-proxy-path/ui/', headers={"X-Script-Name": "my-proxy-path"}) + response = self.client.get("/my-proxy-path/ui/", headers={"X-Script-Name": "my-proxy-path"}) self.assertEqual(200, response.status_code) def test_reverse_proxy_zuul(self): - response = self.client.get('/my-proxy-path-zuul/ui/', headers={"X-Forwarded-Prefix": "my-proxy-path-zuul"}) + response = self.client.get("/my-proxy-path-zuul/ui/", headers={"X-Forwarded-Prefix": "my-proxy-path-zuul"}) self.assertEqual(200, response.status_code) @@ -90,17 +92,15 @@ class ReloadTests(unittest.TestCase): file2 = "config-tests-reload2.yml" def setUp(self): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), - self.file1) + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), self.file1) ms = MyMicroservice(path=__file__) self.app = ms.create_app() self.client = self.app.test_client() self.assertEqual("reload1", self.app.config["APP_NAME"]) def test_configreload(self): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), - self.file2) - response = self.client.post('/reload-config') + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), self.file2) + response = self.client.post("/reload-config") self.assertEqual(b"OK", response.data) self.assertEqual(200, response.status_code) self.assertEqual("reload2", config()["APP_NAME"]) @@ -112,8 +112,9 @@ class MicroserviceTest(unittest.TestCase): """ def setUp(self): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "config-tests.yml") + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "config-tests.yml" + ) def test_singleton(self): ms1 = Microservice(path=__file__) @@ -141,30 +142,20 @@ def test_config_singleton(self): assert conf_one == conf_two -@pytest.mark.parametrize("payload, configfile, status_code", [ - ( - "Python Microservice", - "config-tests.yml", - 200 - ), - ( - "Python Microservice With Flask", - "config-tests-flask.yml", - 404 - ), - ( - "Python Microservice With Flask and Lightstep", - "config-tests-flask-trace-lightstep.yml", - 200 - ) -]) +@pytest.mark.parametrize( + "payload, configfile, status_code", + [ + ("Python Microservice", "config-tests.yml", 200), + ("Python Microservice With Flask", "config-tests-flask.yml", 404), + ("Python Microservice With Flask and Lightstep", "config-tests-flask-trace-lightstep.yml", 200), + ], +) def test_configfiles(payload, configfile, status_code): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), - configfile) + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(os.path.dirname(os.path.abspath(__file__)), configfile) ms = MyMicroservice(path=__file__) ms.reload_conf() app = ms.create_app() client = app.test_client() - response = client.get('/') + response = client.get("/") assert payload == app.config["APP_NAME"] assert status_code == response.status_code diff --git a/tests/test_metrics.py b/tests/test_metrics.py index c237ced..a9554db 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -63,7 +63,7 @@ def test_metrics_logger(self): def test_metrics_jaeger(self): self.client.get("/") self.client.get("/metrics") - generated_logger = b'jaeger:reporter_spans_total' + generated_logger = b"jaeger:reporter_spans_total" assert generated_logger in generate_latest() @@ -79,8 +79,9 @@ def current_test(cls): def setUpClass(cls): cls.temp_dir = TemporaryDirectory() os.environ["prometheus_multiproc_dir"] = cls.temp_dir.name - cls.patch_value_class = unittest.mock.patch.object(values, "ValueClass", - values.MultiProcessValue(cls.current_test)) + cls.patch_value_class = unittest.mock.patch.object( + values, "ValueClass", values.MultiProcessValue(cls.current_test) + ) cls.patch_value_class.start() def setUp(self): @@ -135,5 +136,5 @@ def test_metrics_logger(self): def test_metrics_jaeger(self): self.client.get("/") self.client.get("/metrics") - generated_logger = b'jaeger:reporter_spans_total' + generated_logger = b"jaeger:reporter_spans_total" assert generated_logger in generate_latest(self.app.ms.metrics.registry) diff --git a/tests/test_requests.py b/tests/test_requests.py index 2cc11a3..028b305 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -12,8 +12,7 @@ class RequestServiceNoDataTests(unittest.TestCase): - """Test common rest operations wrapper. - """ + """Test common rest operations wrapper.""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -28,8 +27,12 @@ def setUp(self): def test_get(self, mock_request): url = "http://www.my-site.com/users" full_url = url - text = json.dumps([{'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com.com'}, - {'id': 2, 'name': 'Jon', 'email': 'jon@my-site.com.com'}]) + text = json.dumps( + [ + {"id": 1, "name": "Peter", "email": "peter@my-site.com.com"}, + {"id": 2, "name": "Jon", "email": "jon@my-site.com.com"}, + ] + ) with self.app.app_context(): mock_request.get(full_url, text=text) @@ -40,8 +43,7 @@ def test_get(self, mock_request): class RequestServiceTests(unittest.TestCase): - """Test common rest operations wrapper. - """ + """Test common rest operations wrapper.""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -56,8 +58,14 @@ def setUp(self): def test_get(self, mock_request): url = "http://www.my-site.com/users" full_url = url - text = json.dumps({'data': [{'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com.com'}, - {'id': 2, 'name': 'Jon', 'email': 'jon@my-site.com.com'}]}) + text = json.dumps( + { + "data": [ + {"id": 1, "name": "Peter", "email": "peter@my-site.com.com"}, + {"id": 2, "name": "Jon", "email": "jon@my-site.com.com"}, + ] + } + ) with self.app.app_context(): mock_request.get(full_url, text=text) @@ -69,7 +77,7 @@ def test_get(self, mock_request): @requests_mock.Mocker() def test_get_for_object_without_json(self, mock_request): url = "http://www.my-site.com/users/{user-id}/posts" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123/posts" expected = {} @@ -82,10 +90,16 @@ def test_get_for_object_without_json(self, mock_request): @requests_mock.Mocker() def test_get_for_object_without_valid_json_data(self, mock_request): url = "http://www.my-site.com/users/{user-id}/posts" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123/posts" - text = json.dumps({'another_data': [{'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com.com'}, - {'id': 2, 'name': 'Jon', 'email': 'jon@my-site.com.com'}]}) + text = json.dumps( + { + "another_data": [ + {"id": 1, "name": "Peter", "email": "peter@my-site.com.com"}, + {"id": 2, "name": "Jon", "email": "jon@my-site.com.com"}, + ] + } + ) expected = {} with self.app.app_context(): @@ -97,12 +111,20 @@ def test_get_for_object_without_valid_json_data(self, mock_request): @requests_mock.Mocker() def test_get_for_object_with_valid_data(self, mock_request): url = "http://www.my-site.com/users/{user-id}/posts" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123/posts" - text = json.dumps({'data': [{'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com.com'}, - {'id': 2, 'name': 'Jon', 'email': 'jon@my-site.com.com'}]}) - expected = [{'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com.com'}, - {'id': 2, 'name': 'Jon', 'email': 'jon@my-site.com.com'}] + text = json.dumps( + { + "data": [ + {"id": 1, "name": "Peter", "email": "peter@my-site.com.com"}, + {"id": 2, "name": "Jon", "email": "jon@my-site.com.com"}, + ] + } + ) + expected = [ + {"id": 1, "name": "Peter", "email": "peter@my-site.com.com"}, + {"id": 2, "name": "Jon", "email": "jon@my-site.com.com"}, + ] with self.app.app_context(): mock_request.get(full_url, text=text) @@ -114,8 +136,8 @@ def test_get_for_object_with_valid_data(self, mock_request): def test_post(self, mock_request): url = "http://www.my-site.com/users" full_url = url - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'data': {'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com'}}) + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"data": {"id": 1, "name": "Peter", "email": "peter@my-site.com"}}) with self.app.app_context(): mock_request.post(full_url, text=text, status_code=201) @@ -128,7 +150,7 @@ def test_post(self, mock_request): def test_post_for_object_without_json(self, mock_request): url = "http://www.my-site.com/users" full_url = url - user = {'name': 'Peter', 'email': 'peter@my-site.com'} + user = {"name": "Peter", "email": "peter@my-site.com"} expected = {} with self.app.app_context(): @@ -141,8 +163,8 @@ def test_post_for_object_without_json(self, mock_request): def test_post_for_object_without_valid_json_data(self, mock_request): url = "http://www.my-site.com/users" full_url = url - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'another_data': {'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com.com'}}) + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"another_data": {"id": 1, "name": "Peter", "email": "peter@my-site.com.com"}}) expected = {} with self.app.app_context(): @@ -155,9 +177,9 @@ def test_post_for_object_without_valid_json_data(self, mock_request): def test_post_for_object_with_valid_data(self, mock_request): url = "http://www.my-site.com/users" full_url = url - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'data': {'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com.com'}}) - expected = {'id': 1, 'name': 'Peter', 'email': 'peter@my-site.com.com'} + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"data": {"id": 1, "name": "Peter", "email": "peter@my-site.com.com"}}) + expected = {"id": 1, "name": "Peter", "email": "peter@my-site.com.com"} with self.app.app_context(): mock_request.post(full_url, text=text, status_code=201) @@ -168,10 +190,10 @@ def test_post_for_object_with_valid_data(self, mock_request): @requests_mock.Mocker() def test_put(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'data': {'id': 123, 'name': 'Peter', 'email': 'peter@my-site.com'}}) + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"data": {"id": 123, "name": "Peter", "email": "peter@my-site.com"}}) with self.app.app_context(): mock_request.put(full_url, text=text, status_code=200) @@ -183,9 +205,9 @@ def test_put(self, mock_request): @requests_mock.Mocker() def test_put_for_object_without_json(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" - user = {'name': 'Peter', 'email': 'peter@my-site.com'} + user = {"name": "Peter", "email": "peter@my-site.com"} expected = {} with self.app.app_context(): @@ -197,10 +219,10 @@ def test_put_for_object_without_json(self, mock_request): @requests_mock.Mocker() def test_put_for_object_without_valid_json_data(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'another_data': {'id': 123, 'name': 'Peter', 'email': 'peter@my-site.com.com'}}) + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"another_data": {"id": 123, "name": "Peter", "email": "peter@my-site.com.com"}}) expected = {} with self.app.app_context(): @@ -212,11 +234,11 @@ def test_put_for_object_without_valid_json_data(self, mock_request): @requests_mock.Mocker() def test_put_for_object_with_valid_data(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'data': {'id': 123, 'name': 'Peter', 'email': 'peter@my-site.com.com'}}) - expected = {'id': 123, 'name': 'Peter', 'email': 'peter@my-site.com.com'} + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"data": {"id": 123, "name": "Peter", "email": "peter@my-site.com.com"}}) + expected = {"id": 123, "name": "Peter", "email": "peter@my-site.com.com"} with self.app.app_context(): mock_request.put(full_url, text=text, status_code=200) @@ -227,10 +249,10 @@ def test_put_for_object_with_valid_data(self, mock_request): @requests_mock.Mocker() def test_patch(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'data': {'id': 123, 'name': 'Peter', 'email': 'peter@my-site.com'}}) + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"data": {"id": 123, "name": "Peter", "email": "peter@my-site.com"}}) with self.app.app_context(): mock_request.patch(full_url, text=text, status_code=200) @@ -242,9 +264,9 @@ def test_patch(self, mock_request): @requests_mock.Mocker() def test_patch_for_object_without_json(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" - user = {'name': 'Peter', 'email': 'peter@my-site.com'} + user = {"name": "Peter", "email": "peter@my-site.com"} expected = {} with self.app.app_context(): @@ -256,10 +278,10 @@ def test_patch_for_object_without_json(self, mock_request): @requests_mock.Mocker() def test_patch_for_object_without_valid_json_data(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'another_data': {'id': 123, 'name': 'Peter', 'email': 'peter@my-site.com.com'}}) + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"another_data": {"id": 123, "name": "Peter", "email": "peter@my-site.com.com"}}) expected = {} with self.app.app_context(): @@ -271,11 +293,11 @@ def test_patch_for_object_without_valid_json_data(self, mock_request): @requests_mock.Mocker() def test_patch_for_object_with_valid_data(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" - user = {'name': 'Peter', 'email': 'peter@my-site.com'} - text = json.dumps({'data': {'id': 123, 'name': 'Peter', 'email': 'peter@my-site.com.com'}}) - expected = {'id': 123, 'name': 'Peter', 'email': 'peter@my-site.com.com'} + user = {"name": "Peter", "email": "peter@my-site.com"} + text = json.dumps({"data": {"id": 123, "name": "Peter", "email": "peter@my-site.com.com"}}) + expected = {"id": 123, "name": "Peter", "email": "peter@my-site.com.com"} with self.app.app_context(): mock_request.patch(full_url, text=text, status_code=200) @@ -286,7 +308,7 @@ def test_patch_for_object_with_valid_data(self, mock_request): @requests_mock.Mocker() def test_delete(self, mock_request): url = "http://www.my-site.com/users/{user-id}" - path_params = {'user-id': 123} + path_params = {"user-id": 123} full_url = "http://www.my-site.com/users/123" with self.app.app_context(): @@ -294,63 +316,54 @@ def test_delete(self, mock_request): response = self.request.delete(url, path_params) self.assertEqual(204, response.status_code) - self.assertEqual('', response.text) + self.assertEqual("", response.text) - def test_propagate_headers_empty(self, ): - input_headers = { - - } + def test_propagate_headers_empty( + self, + ): + input_headers = {} expected_headers = { - 'Content-Length': '12', - 'Content-Type': 'application/x-www-form-urlencoded', - 'Host': 'localhost' + "Content-Length": "12", + "Content-Type": "application/x-www-form-urlencoded", + "Host": "localhost", } - with self.app.test_request_context( - '/tests/', data={'format': 'short'}): + with self.app.test_request_context("/tests/", data={"format": "short"}): headers = self.request.set_propagate_headers(input_headers) self.assertEqual(expected_headers, headers) def test_propagate_headers_no_override(self): - input_headers = { - 'Host': 'my-server' - } - expected_headers = { - 'Host': 'my-server' - } - with self.app.test_request_context( - '/tests/'): + input_headers = {"Host": "my-server"} + expected_headers = {"Host": "my-server"} + with self.app.test_request_context("/tests/"): headers = self.request.set_propagate_headers(input_headers) self.assertEqual(expected_headers, headers) def test_propagate_headers_propagate(self): - input_headers = { - } + input_headers = {} expected_headers = { - 'Content-Length': '12', - 'Content-Type': 'application/x-www-form-urlencoded', - 'Host': 'localhost', - 'A': 'b', + "Content-Length": "12", + "Content-Type": "application/x-www-form-urlencoded", + "Host": "localhost", + "A": "b", } - with self.app.test_request_context( - '/tests/', data={'format': 'short'}, headers={'a': 'b'}): + with self.app.test_request_context("/tests/", data={"format": "short"}, headers={"a": "b"}): headers = self.request.set_propagate_headers(input_headers) self.assertEqual(expected_headers, headers) def test_propagate_headers_propagate_no_override(self): input_headers = { - 'Host': 'my-server', - 'Span': '1234', + "Host": "my-server", + "Span": "1234", } expected_headers = { - 'Host': 'my-server', - 'A': 'b', - 'Span': '1234', + "Host": "my-server", + "A": "b", + "Span": "1234", } - with self.app.test_request_context( - '/tests/', headers={'a': 'b', 'span': '5678'}): + with self.app.test_request_context("/tests/", headers={"a": "b", "span": "5678"}): headers = self.request.set_propagate_headers(input_headers) self.assertEqual(expected_headers, headers) @@ -358,12 +371,11 @@ def test_propagate_headers_propagate_no_override(self): def test_propagate_headers_on_get(self): url = "http://www.my-site.com/users" mock_headers = { - 'A': 'b', + "A": "b", } self.request.set_propagate_headers = unittest.mock.Mock() self.request.set_propagate_headers.return_value = mock_headers - with self.app.test_request_context( - '/tests/', data={'format': 'short'}, headers=mock_headers): + with self.app.test_request_context("/tests/", data={"format": "short"}, headers=mock_headers): self.request.get(url, propagate_headers=True) self.request.set_propagate_headers.assert_called_once_with({}) @@ -371,22 +383,21 @@ def test_propagate_headers_on_get(self): def test_propagate_headers_on_get_with_headers(self): url = "http://www.my-site.com/users" mock_headers = { - 'A': 'b', + "A": "b", } get_headers = { - 'C': 'd', + "C": "d", } self.request.set_propagate_headers = unittest.mock.Mock() self.request.set_propagate_headers.return_value = mock_headers - with self.app.test_request_context( - '/tests/', data={'format': 'short'}, headers=mock_headers): + with self.app.test_request_context("/tests/", data={"format": "short"}, headers=mock_headers): self.request.get(url, headers=get_headers, propagate_headers=True) self.request.set_propagate_headers.assert_called_once_with(get_headers) @requests_mock.Mocker() def test_retries_with_500(self, mock_request): - url = 'http://localhost:9999' + url = "http://localhost:9999" with self.app.app_context(): mock_request.get(url, text="", status_code=500) response = self.request.get(url) @@ -396,7 +407,7 @@ def test_retries_with_500(self, mock_request): @requests_mock.Mocker() def test_retries_with_200(self, mock_request): - url = 'http://localhost:9999' + url = "http://localhost:9999" with self.app.app_context(): mock_request.get(url, text="", status_code=200) response = self.request.get(url) diff --git a/tests/test_service_discovery.py b/tests/test_service_discovery.py index b67ae22..d42e4da 100644 --- a/tests/test_service_discovery.py +++ b/tests/test_service_discovery.py @@ -11,8 +11,7 @@ class ServiceDiscoveryTests(unittest.TestCase): - """Test common rest operations wrapper. - """ + """Test common rest operations wrapper.""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/test_swagger.py b/tests/test_swagger.py index 65ed4ce..b86c04c 100644 --- a/tests/test_swagger.py +++ b/tests/test_swagger.py @@ -8,8 +8,7 @@ class SwaggerTests(unittest.TestCase): - """Test common rest operations wrapper. - """ + """Test common rest operations wrapper.""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -22,17 +21,16 @@ def setUp(self): self.assertEqual("Python Microservice Swagger", self.app.config["APP_NAME"]) def test_default(self): - response = self.client.get('/test-api-path/ws-doc/') + response = self.client.get("/test-api-path/ws-doc/") self.assertEqual(200, response.status_code) def test_home(self): - response = self.client.get('/test-api-path/') + response = self.client.get("/test-api-path/") self.assertEqual(200, response.status_code) class SwaggerNoAbsPathTests(unittest.TestCase): - """Test common rest operations wrapper. - """ + """Test common rest operations wrapper.""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -45,17 +43,16 @@ def setUp(self): self.assertEqual("Python Microservice Swagger2", self.app.config["APP_NAME"]) def test_default(self): - response = self.client.get('/test-api-path2/ws-doc2/') + response = self.client.get("/test-api-path2/ws-doc2/") self.assertEqual(200, response.status_code) def test_home(self): - response = self.client.get('/test-api-path2/no-abs-path') + response = self.client.get("/test-api-path2/no-abs-path") self.assertEqual(200, response.status_code) class SwaggerOpenapi3Tests(unittest.TestCase): - """Test common rest operations wrapper. - """ + """Test common rest operations wrapper.""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -68,17 +65,16 @@ def setUp(self): self.assertEqual("Python Microservice Swagger Openapi 3", self.app.config["APP_NAME"]) def test_default(self): - response = self.client.get('/ws-doc/') + response = self.client.get("/ws-doc/") self.assertEqual(200, response.status_code) def test_home(self): - response = self.client.get('/test-url') + response = self.client.get("/test-url") self.assertEqual(200, response.status_code) class SwaggerOpenapi3NoAbsPathTests(unittest.TestCase): - """Test common rest operations wrapper. - """ + """Test common rest operations wrapper.""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -91,9 +87,9 @@ def setUp(self): self.assertEqual("Python Microservice Swagger Openapi 3 No abspath", self.app.config["APP_NAME"]) def test_default(self): - response = self.client.get('/test-api-path2/ws-doc2/') + response = self.client.get("/test-api-path2/ws-doc2/") self.assertEqual(200, response.status_code) def test_home(self): - response = self.client.get('/test-api-path2/test-url') + response = self.client.get("/test-api-path2/test-url") self.assertEqual(200, response.status_code) diff --git a/tests/test_utils.py b/tests/test_utils.py index f8404ae..ad63d9c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,11 +16,10 @@ class ConfUtils(unittest.TestCase): def test_check_package_exists_exception(self): with pytest.raises(PackageNotExists) as excinfo: check_package_exists("this-package-not-exists") - assert "this-package-not-exists is not installed. try with pip install -U this-package-not-exists" \ - in str(excinfo.value) + assert "this-package-not-exists is not installed. try with pip install -U this-package-not-exists" in str( + excinfo.value + ) def test_import_package(self): os_import = import_package("os") assert os_import == os - - From 2d3fcd762e6a9df76a78dbafc27fb2c4fd58e4d2 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 14:36:44 +0100 Subject: [PATCH 09/27] fix: removed python2 checks --- pyms/services_discovery/consulate/api/base.py | 2 +- pyms/services_discovery/consulate/cli.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pyms/services_discovery/consulate/api/base.py b/pyms/services_discovery/consulate/api/base.py index 6499b34..03b0f3f 100644 --- a/pyms/services_discovery/consulate/api/base.py +++ b/pyms/services_discovery/consulate/api/base.py @@ -157,7 +157,7 @@ def _demarshal(self, body): # pylint: disable=too-many-branches,too-many-return return None if self.status_code == 200: # pylint: disable=too-many-nested-blocks try: - if utils.PYTHON3 and isinstance(body, bytes): + if isinstance(body, bytes): try: body = body.decode("utf-8") except UnicodeDecodeError: diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index 25a55f5..8fe8449 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -292,10 +292,7 @@ def kv_backup(consul, args): else: records = consul.kv.records() if args.base64: - if utils.PYTHON3: - records = [(k, f, str(base64.b64encode(utils.maybe_encode(v)), "ascii") if v else v) for k, f, v in records] - else: - records = [(k, f, base64.b64encode(v) if v else v) for k, f, v in records] + records = [(k, f, str(base64.b64encode(utils.maybe_encode(v)), "ascii") if v else v) for k, f, v in records] try: if args.pretty: handle.write(json.dumps(records, sort_keys=True, indent=2, separators=(",", ": ")) + "\n") From 2b3850511b1e033479a1f2f63fe29ab6a6f281d0 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 14:41:20 +0100 Subject: [PATCH 10/27] fix: merge conflict, removed --- Pipfile.lock | 1271 -------------------------------------------------- 1 file changed, 1271 deletions(-) delete mode 100644 Pipfile.lock diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 6b29300..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,1271 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "30f7bae4efa04f995974cbeab5512cb644c3726a04ab10ba9cf0e4ff01c67cb5" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "anyconfig": { - "hashes": [ - "sha256:8888130cde5461cb39379afdd1d09b1b1342356210f0a6743a4b60f9973226f8", - "sha256:e8594bd9898954538ed1370bd0a5dbded9e207548f933bbcd4dac5f48deb6a38" - ], - "version": "==0.9.11" - }, - "attrs": { - "hashes": [ - "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", - "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" - ], - "version": "==20.3.0" - }, - "boto3": { - "hashes": [ - "sha256:23d2575b7bd01c4e7153f283c1d1c12d329dabf78a6279d4192f2e752bb67b1a", - "sha256:cb3f4c2f2576153b845e5b4f325d54a04f4ca00c156f2063965432bfa5d714dd" - ], - "version": "==1.16.13" - }, - "botocore": { - "hashes": [ - "sha256:1b1b4cf5efd552ecc7f1ce0fc674d5fba56857db5ffe6394ee76edd1a568d7a6", - "sha256:b3b4b8fa33620f015c52e426a92e7db21b5e667ed4785c5fbc484ebfdb2b5153" - ], - "version": "==1.19.13" - }, - "certifi": { - "hashes": [ - "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", - "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" - ], - "version": "==2020.11.8" - }, - "cffi": { - "hashes": [ - "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", - "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", - "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", - "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", - "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", - "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", - "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", - "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", - "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", - "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", - "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", - "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", - "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", - "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", - "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", - "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", - "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", - "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", - "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", - "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", - "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", - "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", - "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", - "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", - "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", - "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", - "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", - "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", - "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", - "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", - "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", - "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", - "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", - "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", - "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", - "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" - ], - "version": "==1.14.3" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" - ], - "version": "==7.1.2" - }, - "clickclick": { - "hashes": [ - "sha256:4efb13e62353e34c5eef7ed6582c4920b418d7dedc86d819e22ee089ba01802c", - "sha256:c8f33e6d9ec83f68416dd2136a7950125bd256ec39ccc9a85c6e280a16be2bb5" - ], - "version": "==20.10.2" - }, - "connexion": { - "extras": [ - "swagger-ui" - ], - "hashes": [ - "sha256:1ccfac57d4bb7adf4295ba6f5e48f5a1f66057df6a0713417766c9b5235182ee", - "sha256:5439e9659a89c4380d93a07acfbf3380d70be4130574de8881e5f0dfec7ad0e2" - ], - "version": "==2.7.0" - }, - "contextlib2": { - "hashes": [ - "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e", - "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b" - ], - "version": "==0.6.0.post1" - }, - "cryptography": { - "hashes": [ - "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538", - "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f", - "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77", - "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b", - "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33", - "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e", - "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb", - "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e", - "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7", - "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297", - "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d", - "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7", - "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b", - "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7", - "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4", - "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8", - "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b", - "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851", - "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13", - "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b", - "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3", - "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df" - ], - "version": "==3.2.1" - }, - "flask": { - "hashes": [ - "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060", - "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557" - ], - "version": "==1.1.2" - }, - "flask-opentracing": { - "hashes": [ - "sha256:a9a39d367fbe7e9ed9c77b90ac48159c1a3e82982a5abf84d3f4d710d24580ac" - ], - "version": "==1.1.0" - }, - "future": { - "hashes": [ - "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" - ], - "version": "==0.18.2" - }, - "idna": { - "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" - ], - "version": "==2.10" - }, - "importlib-metadata": { - "hashes": [ - "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da", - "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3" - ], - "markers": "python_version < '3.8'", - "version": "==2.0.0" - }, - "inflection": { - "hashes": [ - "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", - "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2" - ], - "version": "==0.5.1" - }, - "itsdangerous": { - "hashes": [ - "sha256:1f08ac48fe58cb99a1f58add0c90924b4267398bbd1640268d26a29f6a03eaa4", - "sha256:26ba8fb99157bbd5a1016f9d9dc5ed5ff1325f6f6f2c10b18107199470676b4d" - ], - "version": "==2.0.0a1" - }, - "jaeger-client": { - "hashes": [ - "sha256:7f72c341a21e3dcbc8498d1ed17c93e0eec71c5becc5c9333c4293793d4e6433" - ], - "version": "==4.3.0" - }, - "jinja2": { - "hashes": [ - "sha256:c10142f819c2d22bdcd17548c46fa9b77cf4fda45097854c689666bf425e7484", - "sha256:c922560ac46888d47384de1dbdc3daaa2ea993af4b26a436dec31fa2c19ec668" - ], - "version": "==3.0.0a1" - }, - "jmespath": { - "hashes": [ - "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", - "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" - ], - "version": "==0.10.0" - }, - "jsonschema": { - "hashes": [ - "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", - "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" - ], - "version": "==3.2.0" - }, - "markupsafe": { - "hashes": [ - "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", - "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", - "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", - "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", - "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", - "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", - "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", - "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", - "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", - "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", - "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", - "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", - "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", - "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", - "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", - "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", - "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", - "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", - "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", - "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", - "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", - "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" - ], - "version": "==2.0.0a1" - }, - "openapi-spec-validator": { - "hashes": [ - "sha256:6dd75e50c94f1bb454d0e374a56418e7e06a07affb2c7f1df88564c5d728dac3", - "sha256:79381a69b33423ee400ae1624a461dae7725e450e2e306e32f2dd8d16a4d85cb", - "sha256:ec1b01a00e20955a527358886991ae34b4b791b253027ee9f7df5f84b59d91c7" - ], - "version": "==0.2.9" - }, - "opentracing": { - "hashes": [ - "sha256:33b10634917a7496a7e3a18b18b7c055053d0a8da5101e2a95d454783cac9765" - ], - "version": "==2.3.0" - }, - "opentracing-instrumentation": { - "hashes": [ - "sha256:ae9b48a5b6e47189887fff9785230b13141659bb3bf3e2700e4c7470ae5d27b4", - "sha256:d94801aebab95b7e8f3eeb80ee0e5e886ad4f1761bcadff9b5d1d3d04560c04a" - ], - "version": "==3.3.1" - }, - "prance": { - "hashes": [ - "sha256:86ec64378036471efdd9681648d8da6b39e5143ea2e6981b7863a33fbc75d739", - "sha256:b87504dffb40c1e29aca0ca01f5f3dbb1941f29fd32c3a5e0fbd90ae37f749a3" - ], - "version": "==0.19.0" - }, - "prometheus-client": { - "hashes": [ - "sha256:983c7ac4b47478720db338f1491ef67a100b474e3bc7dafcbaefb7d0b8f9b01c", - "sha256:c6e6b706833a6bd1fd51711299edee907857be10ece535126a158f911ee80915" - ], - "version": "==0.8.0" - }, - "py-ms": { - "editable": true, - "extras": [ - "all" - ], - "path": "." - }, - "pycparser": { - "hashes": [ - "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", - "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" - ], - "version": "==2.20" - }, - "pyrsistent": { - "hashes": [ - "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" - ], - "version": "==0.17.3" - }, - "python-dateutil": { - "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" - ], - "version": "==2.8.1" - }, - "python-json-logger": { - "hashes": [ - "sha256:f26eea7898db40609563bed0a7ca11af12e2a79858632706d835a0f961b7d398" - ], - "version": "==2.0.1" - }, - "pyyaml": { - "hashes": [ - "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", - "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", - "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", - "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", - "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", - "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", - "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", - "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", - "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", - "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", - "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" - ], - "version": "==5.3.1" - }, - "requests": { - "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" - ], - "version": "==2.24.0" - }, - "s3transfer": { - "hashes": [ - "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13", - "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db" - ], - "version": "==0.3.3" - }, - "semver": { - "hashes": [ - "sha256:ced8b23dceb22134307c1b8abfa523da14198793d9787ac838e70e29e77458d4", - "sha256:fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f" - ], - "version": "==2.13.0" - }, - "six": { - "hashes": [ - "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", - "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" - ], - "version": "==1.15.0" - }, - "swagger-ui-bundle": { - "hashes": [ - "sha256:f5255f786cde67a2638111f4a7d04355836743198a83c4ecbe815d9fc384b0c8", - "sha256:f5691167f2e9f73ecbe8229a89454ae5ea958f90bb0d4583ed7adaae598c4122" - ], - "version": "==0.0.8" - }, - "threadloop": { - "hashes": [ - "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599", - "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944" - ], - "version": "==1.0.2" - }, - "thrift": { - "hashes": [ - "sha256:9af1c86bf73433afc6010ed376a6c6aca2b54099cc0d61895f640870a9ae7d89" - ], - "version": "==0.13.0" - }, - "tornado": { - "hashes": [ - "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", - "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", - "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", - "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", - "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", - "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", - "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" - ], - "version": "==5.1.1" - }, - "urllib3": { - "hashes": [ - "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", - "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" - ], - "markers": "python_version != '3.4'", - "version": "==1.25.11" - }, - "werkzeug": { - "hashes": [ - "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", - "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" - ], - "version": "==1.0.1" - }, - "wrapt": { - "hashes": [ - "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" - ], - "version": "==1.12.1" - }, - "zipp": { - "hashes": [ - "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108", - "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb" - ], - "version": "==3.4.0" - } - }, - "develop": { - "anyconfig": { - "hashes": [ - "sha256:8888130cde5461cb39379afdd1d09b1b1342356210f0a6743a4b60f9973226f8", - "sha256:e8594bd9898954538ed1370bd0a5dbded9e207548f933bbcd4dac5f48deb6a38" - ], - "version": "==0.9.11" - }, - "appdirs": { - "hashes": [ - "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", - "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" - ], - "version": "==1.4.4" - }, - "astroid": { - "hashes": [ - "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703", - "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386" - ], - "version": "==2.4.2" - }, - "attrs": { - "hashes": [ - "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", - "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" - ], - "version": "==20.3.0" - }, - "bandit": { - "hashes": [ - "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952", - "sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065" - ], - "version": "==1.6.2" - }, - "basictracer": { - "hashes": [ - "sha256:22a3b00c9a3d7b1e630078d284350a57416883686abc1ff109629750b4b58d65" - ], - "version": "==3.1.0" - }, - "black": { - "hashes": [ - "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" - ], - "version": "==20.8b1" - }, - "certifi": { - "hashes": [ - "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", - "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" - ], - "version": "==2020.11.8" - }, - "cffi": { - "hashes": [ - "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", - "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", - "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", - "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", - "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", - "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", - "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", - "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", - "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", - "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", - "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", - "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", - "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", - "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", - "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", - "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", - "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", - "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", - "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", - "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", - "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", - "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", - "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", - "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", - "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", - "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", - "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", - "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", - "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", - "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", - "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", - "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", - "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", - "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", - "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", - "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" - ], - "version": "==1.14.3" - }, - "cfgv": { - "hashes": [ - "sha256:32e43d604bbe7896fe7c248a9c2276447dbef840feb28fe20494f62af110211d", - "sha256:cf22deb93d4bcf92f345a5c3cd39d3d41d6340adc60c78bbbd6588c384fda6a1" - ], - "version": "==3.2.0" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" - ], - "version": "==7.1.2" - }, - "coverage": { - "hashes": [ - "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516", - "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259", - "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9", - "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097", - "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0", - "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f", - "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7", - "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c", - "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5", - "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7", - "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729", - "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978", - "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9", - "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f", - "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9", - "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822", - "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418", - "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82", - "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f", - "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d", - "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221", - "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4", - "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21", - "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709", - "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54", - "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d", - "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270", - "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24", - "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751", - "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a", - "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237", - "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7", - "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636", - "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8" - ], - "version": "==5.3" - }, - "cryptography": { - "hashes": [ - "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538", - "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f", - "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77", - "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b", - "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33", - "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e", - "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb", - "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e", - "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7", - "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297", - "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d", - "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7", - "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b", - "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7", - "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4", - "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8", - "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b", - "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851", - "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13", - "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b", - "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3", - "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df" - ], - "version": "==3.2.1" - }, - "distlib": { - "hashes": [ - "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", - "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1" - ], - "version": "==0.3.1" - }, - "dparse": { - "hashes": [ - "sha256:a1b5f169102e1c894f9a7d5ccf6f9402a836a5d24be80a986c7ce9eaed78f367", - "sha256:e953a25e44ebb60a5c6efc2add4420c177f1d8404509da88da9729202f306994" - ], - "version": "==0.5.1" - }, - "filelock": { - "hashes": [ - "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", - "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836" - ], - "version": "==3.0.12" - }, - "flake8": { - "hashes": [ - "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839", - "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b" - ], - "version": "==3.8.4" - }, - "flask": { - "hashes": [ - "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060", - "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557" - ], - "version": "==1.1.2" - }, - "future": { - "hashes": [ - "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" - ], - "version": "==0.18.2" - }, - "gitdb": { - "hashes": [ - "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac", - "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9" - ], - "version": "==4.0.5" - }, - "gitpython": { - "hashes": [ - "sha256:6eea89b655917b500437e9668e4a12eabdcf00229a0df1762aabd692ef9b746b", - "sha256:befa4d101f91bad1b632df4308ec64555db684c360bd7d2130b4807d49ce86b8" - ], - "version": "==3.1.11" - }, - "googleapis-common-protos": { - "hashes": [ - "sha256:560716c807117394da12cecb0a54da5a451b5cf9866f1d37e9a5e2329a665351", - "sha256:c8961760f5aad9a711d37b675be103e0cc4e9a39327e0d6d857872f698403e24" - ], - "version": "==1.52.0" - }, - "httmock": { - "hashes": [ - "sha256:13e6c63f135a928e15d386af789a2890efb03e0e280f29bdc9961f3f0dc34cb9", - "sha256:44eaf4bb59cc64cd6f5d8bf8700b46aa3097cc5651b9bc85c527dfbc71792f41" - ], - "version": "==1.4.0" - }, - "identify": { - "hashes": [ - "sha256:5dd84ac64a9a115b8e0b27d1756b244b882ad264c3c423f42af8235a6e71ca12", - "sha256:c9504ba6a043ee2db0a9d69e43246bc138034895f6338d5aed1b41e4a73b1513" - ], - "version": "==1.5.9" - }, - "idna": { - "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" - ], - "version": "==2.10" - }, - "importlib-metadata": { - "hashes": [ - "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da", - "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3" - ], - "markers": "python_version < '3.8'", - "version": "==2.0.0" - }, - "iniconfig": { - "hashes": [ - "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", - "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" - ], - "version": "==1.1.1" - }, - "isort": { - "hashes": [ - "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7", - "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58" - ], - "version": "==5.6.4" - }, - "itsdangerous": { - "hashes": [ - "sha256:1f08ac48fe58cb99a1f58add0c90924b4267398bbd1640268d26a29f6a03eaa4", - "sha256:26ba8fb99157bbd5a1016f9d9dc5ed5ff1325f6f6f2c10b18107199470676b4d" - ], - "version": "==2.0.0a1" - }, - "jinja2": { - "hashes": [ - "sha256:c10142f819c2d22bdcd17548c46fa9b77cf4fda45097854c689666bf425e7484", - "sha256:c922560ac46888d47384de1dbdc3daaa2ea993af4b26a436dec31fa2c19ec668" - ], - "version": "==3.0.0a1" - }, - "joblib": { - "hashes": [ - "sha256:698c311779f347cf6b7e6b8a39bb682277b8ee4aba8cf9507bc0cf4cd4737b72", - "sha256:9e284edd6be6b71883a63c9b7f124738a3c16195513ad940eae7e3438de885d5" - ], - "version": "==0.17.0" - }, - "jsonpickle": { - "hashes": [ - "sha256:8919c166bac0574e3d74425c7559434062002d9dfc0ac2afa6dc746ba4a19439", - "sha256:e8d4b7cd0bd6826001a74377df1079a76ad8bae0f909282de2554164c837c8ba" - ], - "version": "==1.4.1" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", - "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", - "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", - "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", - "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", - "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", - "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", - "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", - "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", - "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", - "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", - "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", - "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", - "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", - "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", - "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", - "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", - "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", - "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", - "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", - "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" - ], - "version": "==1.4.3" - }, - "lightstep": { - "hashes": [ - "sha256:4dfc5cc5c8baa47d0cf2c41e11b97511ffe218b296697f9538b0444a6b54f857", - "sha256:6bce8085d117eef6d9ac5f7e0dae901fcabd2cc6a5780decc00b61e45e7af116" - ], - "version": "==4.4.8" - }, - "livereload": { - "hashes": [ - "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869" - ], - "version": "==2.6.3" - }, - "lunr": { - "extras": [ - "languages" - ], - "hashes": [ - "sha256:aab3f489c4d4fab4c1294a257a30fec397db56f0a50273218ccc3efdbf01d6ca", - "sha256:c4fb063b98eff775dd638b3df380008ae85e6cb1d1a24d1cd81a10ef6391c26e" - ], - "version": "==0.5.8" - }, - "markdown": { - "hashes": [ - "sha256:5d9f2b5ca24bc4c7a390d22323ca4bad200368612b5aaa7796babf971d2b2f18", - "sha256:c109c15b7dc20a9ac454c9e6025927d44460b85bd039da028d85e2b6d0bcc328" - ], - "version": "==3.3.3" - }, - "markupsafe": { - "hashes": [ - "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", - "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", - "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", - "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", - "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", - "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", - "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", - "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", - "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", - "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", - "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", - "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", - "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", - "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", - "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", - "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", - "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", - "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", - "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", - "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", - "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", - "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" - ], - "version": "==2.0.0a1" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "mkdocs": { - "hashes": [ - "sha256:096f52ff52c02c7e90332d2e53da862fde5c062086e1b5356a6e392d5d60f5e9", - "sha256:f0b61e5402b99d7789efa032c7a74c90a20220a9c81749da06dbfbcbd52ffb39" - ], - "version": "==1.1.2" - }, - "mkdocs-material": { - "hashes": [ - "sha256:c0591b806e85302314bf308d61745e2cbb6b3caa3cec3d5fcb8d70a81076e7cb", - "sha256:e15b016bea35300956b66d553043331cbe1793ca53946180424b8bca21bbb781" - ], - "version": "==6.1.4" - }, - "mkdocs-material-extensions": { - "hashes": [ - "sha256:6947fb7f5e4291e3c61405bad3539d81e0b3cd62ae0d66ced018128af509c68f", - "sha256:d90c807a88348aa6d1805657ec5c0b2d8d609c110e62b9dce4daf7fa981fa338" - ], - "version": "==1.0.1" - }, - "mypy": { - "hashes": [ - "sha256:0a0d102247c16ce93c97066443d11e2d36e6cc2a32d8ccc1f705268970479324", - "sha256:0d34d6b122597d48a36d6c59e35341f410d4abfa771d96d04ae2c468dd201abc", - "sha256:2170492030f6faa537647d29945786d297e4862765f0b4ac5930ff62e300d802", - "sha256:2842d4fbd1b12ab422346376aad03ff5d0805b706102e475e962370f874a5122", - "sha256:2b21ba45ad9ef2e2eb88ce4aeadd0112d0f5026418324176fd494a6824b74975", - "sha256:72060bf64f290fb629bd4a67c707a66fd88ca26e413a91384b18db3876e57ed7", - "sha256:af4e9ff1834e565f1baa74ccf7ae2564ae38c8df2a85b057af1dbbc958eb6666", - "sha256:bd03b3cf666bff8d710d633d1c56ab7facbdc204d567715cb3b9f85c6e94f669", - "sha256:c614194e01c85bb2e551c421397e49afb2872c88b5830e3554f0519f9fb1c178", - "sha256:cf4e7bf7f1214826cf7333627cb2547c0db7e3078723227820d0a2490f117a01", - "sha256:da56dedcd7cd502ccd3c5dddc656cb36113dd793ad466e894574125945653cea", - "sha256:e86bdace26c5fe9cf8cb735e7cedfe7850ad92b327ac5d797c656717d2ca66de", - "sha256:e97e9c13d67fbe524be17e4d8025d51a7dca38f90de2e462243ab8ed8a9178d1", - "sha256:eea260feb1830a627fb526d22fbb426b750d9f5a47b624e8d5e7e004359b219c" - ], - "version": "==0.790" - }, - "mypy-extensions": { - "hashes": [ - "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", - "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" - ], - "version": "==0.4.3" - }, - "nltk": { - "hashes": [ - "sha256:845365449cd8c5f9731f7cb9f8bd6fd0767553b9d53af9eb1b3abf7700936b35" - ], - "version": "==3.5" - }, - "nodeenv": { - "hashes": [ - "sha256:5304d424c529c997bc888453aeaa6362d242b6b4631e90f3d4bf1b290f1c84a9", - "sha256:ab45090ae383b716c4ef89e690c41ff8c2b257b85b309f01f3654df3d084bd7c" - ], - "version": "==1.5.0" - }, - "opentracing": { - "hashes": [ - "sha256:33b10634917a7496a7e3a18b18b7c055053d0a8da5101e2a95d454783cac9765" - ], - "version": "==2.3.0" - }, - "packaging": { - "hashes": [ - "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", - "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" - ], - "version": "==20.4" - }, - "pathspec": { - "hashes": [ - "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", - "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" - ], - "version": "==0.8.1" - }, - "pbr": { - "hashes": [ - "sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9", - "sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00" - ], - "version": "==5.5.1" - }, - "pluggy": { - "hashes": [ - "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", - "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" - ], - "version": "==0.13.1" - }, - "pre-commit": { - "hashes": [ - "sha256:22e6aa3bd571debb01eb7d34483f11c01b65237be4eebbf30c3d4fb65762d315", - "sha256:905ebc9b534b991baec87e934431f2d0606ba27f2b90f7f652985f5a5b8b6ae6" - ], - "version": "==2.8.2" - }, - "protobuf": { - "hashes": [ - "sha256:0e55cf11e3cdc7af9e280539144c0896ff046144dc17ddf6344e33f7bd01c53b", - "sha256:2ceef235ec88363f0853679363e215aee79f2998931b83e5d1bf08289e3c6a6f", - "sha256:36583c483ddd3c60f25a9a1e1660baa1e033e9470a49e019d0705d6fefaea52b", - "sha256:3bb580ee33b844087e419a9302a255a956d695147a64d59e96a3e0abd78fa67d", - "sha256:4136552036dbff1e5841cd240d1be973348752cfdaa91c9088d8a32ae43f06c0", - "sha256:428930d8f9607723ab6482f07bc7a651e95f86c7f5db6347a0b1f24319cd1e3b", - "sha256:471b0cd067e1ea2c6c5cc82fc7c04990a0497914668ca0bdc7db7925e25a8045", - "sha256:4ed0b7df03cd668dbd4bf1a432e58fc9201ae4f15a9a6f837bce4c7496c431da", - "sha256:76f4a7f5c418167496b68a7310efe93a33066aa83f5fac3a9189bc5ace6ff905", - "sha256:991301d26c33cc25c8a81fd7d25ad5a31cc6eb166b96afcbce93675545e03532", - "sha256:ae99d4b1c15439586d7d3bfc3ec7d3e933419fdcff3e483ed5a14e653b45d39c", - "sha256:afef26dd04202c8f4efe8292bd9a683166e32b6004abb0302d6a92db1a994bf1", - "sha256:bb056806dee32e6f8ad47935ca2be34bed124675c0132a469e70aa6847ce2223", - "sha256:bb0b3df5c29a8dc51e6435505d42907ac3651e2b0be40c30e9f40d08aca6d47c", - "sha256:bd644bf3865a72d5a244e7193f784cc6cd51bb453d447ee73a5e05d60dc22717", - "sha256:d6bb25e26cbee3eebf460012e3406ad5d77ced6824abbea919d8c7a0466e82f1", - "sha256:ed7143575f71fed2599e1548a0378cfd6dc958920221666e3a33785af3f0d7e2", - "sha256:ff82b53153edda066698bae674d673566029e856fa126dcb95a432aed4161af2" - ], - "version": "==3.14.0rc1" - }, - "py": { - "hashes": [ - "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", - "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" - ], - "version": "==1.9.0" - }, - "py-ms": { - "editable": true, - "extras": [ - "all" - ], - "path": "." - }, - "pycodestyle": { - "hashes": [ - "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", - "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" - ], - "version": "==2.6.0" - }, - "pycparser": { - "hashes": [ - "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", - "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" - ], - "version": "==2.20" - }, - "pyflakes": { - "hashes": [ - "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", - "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" - ], - "version": "==2.2.0" - }, - "pygments": { - "hashes": [ - "sha256:381985fcc551eb9d37c52088a32914e00517e57f4a21609f48141ba08e193fa0", - "sha256:88a0bbcd659fcb9573703957c6b9cff9fab7295e6e76db54c9d00ae42df32773" - ], - "version": "==2.7.2" - }, - "pylint": { - "hashes": [ - "sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210", - "sha256:bfe68f020f8a0fece830a22dd4d5dddb4ecc6137db04face4c3420a46a52239f" - ], - "version": "==2.6.0" - }, - "pymdown-extensions": { - "hashes": [ - "sha256:9ba704052d4bdc04a7cd63f7db4ef6add73bafcef22c0cf6b2e3386cf4ece51e", - "sha256:a3689c04f4cbddacd9d569425c571ae07e2673cc4df63a26cdbf1abc15229137" - ], - "version": "==8.0.1" - }, - "pyparsing": { - "hashes": [ - "sha256:13140e8d0e1edd806eb50f18535d77f2143b40771d4aaef6b4950dd93d48a7db", - "sha256:38891c1032d0c759f0fa5ed3a8f249fd992b083fa2303ead58ee48a51b269e02" - ], - "version": "==3.0.0b1" - }, - "pytest": { - "hashes": [ - "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe", - "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e" - ], - "version": "==6.1.2" - }, - "pytest-cov": { - "hashes": [ - "sha256:45ec2d5182f89a81fc3eb29e3d1ed3113b9e9a873bcddb2a71faaab066110191", - "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e" - ], - "version": "==2.10.1" - }, - "python-json-logger": { - "hashes": [ - "sha256:f26eea7898db40609563bed0a7ca11af12e2a79858632706d835a0f961b7d398" - ], - "version": "==2.0.1" - }, - "pyyaml": { - "hashes": [ - "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", - "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", - "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", - "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", - "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", - "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", - "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", - "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", - "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", - "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", - "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" - ], - "version": "==5.3.1" - }, - "regex": { - "hashes": [ - "sha256:03855ee22980c3e4863dc84c42d6d2901133362db5daf4c36b710dd895d78f0a", - "sha256:06b52815d4ad38d6524666e0d50fe9173533c9cc145a5779b89733284e6f688f", - "sha256:11116d424734fe356d8777f89d625f0df783251ada95d6261b4c36ad27a394bb", - "sha256:119e0355dbdd4cf593b17f2fc5dbd4aec2b8899d0057e4957ba92f941f704bf5", - "sha256:127a9e0c0d91af572fbb9e56d00a504dbd4c65e574ddda3d45b55722462210de", - "sha256:1ec66700a10e3c75f1f92cbde36cca0d3aaee4c73dfa26699495a3a30b09093c", - "sha256:227a8d2e5282c2b8346e7f68aa759e0331a0b4a890b55a5cfbb28bd0261b84c0", - "sha256:2564def9ce0710d510b1fc7e5178ce2d20f75571f788b5197b3c8134c366f50c", - "sha256:297116e79074ec2a2f885d22db00ce6e88b15f75162c5e8b38f66ea734e73c64", - "sha256:2dc522e25e57e88b4980d2bdd334825dbf6fa55f28a922fc3bfa60cc09e5ef53", - "sha256:3a5f08039eee9ea195a89e180c5762bfb55258bfb9abb61a20d3abee3b37fd12", - "sha256:3dfca201fa6b326239e1bccb00b915e058707028809b8ecc0cf6819ad233a740", - "sha256:49461446b783945597c4076aea3f49aee4b4ce922bd241e4fcf62a3e7c61794c", - "sha256:4afa350f162551cf402bfa3cd8302165c8e03e689c897d185f16a167328cc6dd", - "sha256:4b5a9bcb56cc146c3932c648603b24514447eafa6ce9295234767bf92f69b504", - "sha256:52e83a5f28acd621ba8e71c2b816f6541af7144b69cc5859d17da76c436a5427", - "sha256:625116aca6c4b57c56ea3d70369cacc4d62fead4930f8329d242e4fe7a58ce4b", - "sha256:654c1635f2313d0843028487db2191530bca45af61ca85d0b16555c399625b0e", - "sha256:8092a5a06ad9a7a247f2a76ace121183dc4e1a84c259cf9c2ce3bbb69fac3582", - "sha256:832339223b9ce56b7b15168e691ae654d345ac1635eeb367ade9ecfe0e66bee0", - "sha256:8ca9dca965bd86ea3631b975d63b0693566d3cc347e55786d5514988b6f5b84c", - "sha256:96f99219dddb33e235a37283306834700b63170d7bb2a1ee17e41c6d589c8eb9", - "sha256:9b6305295b6591e45f069d3553c54d50cc47629eb5c218aac99e0f7fafbf90a1", - "sha256:a62162be05edf64f819925ea88d09d18b09bebf20971b363ce0c24e8b4aa14c0", - "sha256:aacc8623ffe7999a97935eeabbd24b1ae701d08ea8f874a6ff050e93c3e658cf", - "sha256:b45bab9f224de276b7bc916f6306b86283f6aa8afe7ed4133423efb42015a898", - "sha256:b88fa3b8a3469f22b4f13d045d9bd3eda797aa4e406fde0a2644bc92bbdd4bdd", - "sha256:b8a686a6c98872007aa41fdbb2e86dc03b287d951ff4a7f1da77fb7f14113e4d", - "sha256:bd904c0dec29bbd0769887a816657491721d5f545c29e30fd9d7a1a275dc80ab", - "sha256:bf4f896c42c63d1f22039ad57de2644c72587756c0cfb3cc3b7530cfe228277f", - "sha256:c13d311a4c4a8d671f5860317eb5f09591fbe8259676b86a85769423b544451e", - "sha256:c2c6c56ee97485a127555c9595c069201b5161de9d05495fbe2132b5ac104786", - "sha256:c32c91a0f1ac779cbd73e62430de3d3502bbc45ffe5bb6c376015acfa848144b", - "sha256:c3466a84fce42c2016113101018a9981804097bacbab029c2d5b4fcb224b89de", - "sha256:c454ad88e56e80e44f824ef8366bb7e4c3def12999151fd5c0ea76a18fe9aa3e", - "sha256:c8a2b7ccff330ae4c460aff36626f911f918555660cc28163417cb84ffb25789", - "sha256:cb905f3d2e290a8b8f1579d3984f2cfa7c3a29cc7cba608540ceeed18513f520", - "sha256:cfcf28ed4ce9ced47b9b9670a4f0d3d3c0e4d4779ad4dadb1ad468b097f808aa", - "sha256:dd3e6547ecf842a29cf25123fbf8d2461c53c8d37aa20d87ecee130c89b7079b", - "sha256:de7fd57765398d141949946c84f3590a68cf5887dac3fc52388df0639b01eda4", - "sha256:ea37320877d56a7f0a1e6a625d892cf963aa7f570013499f5b8d5ab8402b5625", - "sha256:f1fce1e4929157b2afeb4bb7069204d4370bab9f4fc03ca1fbec8bd601f8c87d", - "sha256:f43109822df2d3faac7aad79613f5f02e4eab0fc8ad7932d2e70e2a83bd49c26" - ], - "version": "==2020.10.28" - }, - "requests": { - "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" - ], - "version": "==2.24.0" - }, - "requests-mock": { - "hashes": [ - "sha256:11215c6f4df72702aa357f205cf1e537cffd7392b3e787b58239bde5fb3db53b", - "sha256:e68f46844e4cee9d447150343c9ae875f99fa8037c6dcf5f15bf1fe9ab43d226" - ], - "version": "==1.8.0" - }, - "safety": { - "hashes": [ - "sha256:23bf20690d4400edc795836b0c983c2b4cbbb922233108ff925b7dd7750f00c9", - "sha256:86c1c4a031fe35bd624fce143fbe642a0234d29f7cbf7a9aa269f244a955b087" - ], - "version": "==1.9.0" - }, - "six": { - "hashes": [ - "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", - "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" - ], - "version": "==1.15.0" - }, - "smmap": { - "hashes": [ - "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4", - "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24" - ], - "version": "==3.0.4" - }, - "stevedore": { - "hashes": [ - "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62", - "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0" - ], - "version": "==3.2.2" - }, - "thrift": { - "hashes": [ - "sha256:9af1c86bf73433afc6010ed376a6c6aca2b54099cc0d61895f640870a9ae7d89" - ], - "version": "==0.13.0" - }, - "toml": { - "hashes": [ - "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", - "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" - ], - "version": "==0.10.2" - }, - "tornado": { - "hashes": [ - "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", - "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", - "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", - "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", - "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", - "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", - "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" - ], - "version": "==5.1.1" - }, - "tox": { - "hashes": [ - "sha256:42ce19ce5dc2f6d6b1fdc5666c476e1f1e2897359b47e0aa3a5b774f335d57c2", - "sha256:4321052bfe28f9d85082341ca8e233e3ea901fdd14dab8a5d3fbd810269fbaf6" - ], - "version": "==3.20.1" - }, - "tqdm": { - "hashes": [ - "sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad", - "sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432" - ], - "version": "==4.51.0" - }, - "typed-ast": { - "hashes": [ - "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", - "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", - "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d", - "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", - "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", - "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", - "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c", - "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", - "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", - "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", - "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", - "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", - "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", - "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d", - "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", - "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", - "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c", - "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", - "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395", - "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", - "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", - "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", - "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", - "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", - "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072", - "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298", - "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91", - "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", - "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f", - "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" - ], - "markers": "implementation_name == 'cpython' and python_version < '3.8'", - "version": "==1.4.1" - }, - "typing-extensions": { - "hashes": [ - "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", - "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", - "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" - ], - "version": "==3.7.4.3" - }, - "urllib3": { - "hashes": [ - "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", - "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" - ], - "markers": "python_version != '3.4'", - "version": "==1.25.11" - }, - "virtualenv": { - "hashes": [ - "sha256:b0011228208944ce71052987437d3843e05690b2f23d1c7da4263fde104c97a2", - "sha256:b8d6110f493af256a40d65e29846c69340a947669eec8ce784fcf3dd3af28380" - ], - "version": "==20.1.0" - }, - "werkzeug": { - "hashes": [ - "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", - "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" - ], - "version": "==1.0.1" - }, - "wrapt": { - "hashes": [ - "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" - ], - "version": "==1.12.1" - }, - "zipp": { - "hashes": [ - "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108", - "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb" - ], - "version": "==3.4.0" - } - } -} From 5f12cd2b2f1078c44f621e40264dc39faef8df61 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 14:43:58 +0100 Subject: [PATCH 11/27] feat: re-added pipfile lock --- Pipfile.lock | 1215 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1215 insertions(+) create mode 100644 Pipfile.lock diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..1f8a17e --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,1215 @@ +{ + "_meta": { + "hash": { + "sha256": "30f7bae4efa04f995974cbeab5512cb644c3726a04ab10ba9cf0e4ff01c67cb5" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "anyconfig": { + "hashes": [ + "sha256:8888130cde5461cb39379afdd1d09b1b1342356210f0a6743a4b60f9973226f8", + "sha256:e8594bd9898954538ed1370bd0a5dbded9e207548f933bbcd4dac5f48deb6a38" + ], + "version": "==0.9.11" + }, + "attrs": { + "hashes": [ + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + ], + "version": "==20.3.0" + }, + "boto3": { + "hashes": [ + "sha256:23d2575b7bd01c4e7153f283c1d1c12d329dabf78a6279d4192f2e752bb67b1a", + "sha256:cb3f4c2f2576153b845e5b4f325d54a04f4ca00c156f2063965432bfa5d714dd" + ], + "version": "==1.16.13" + }, + "botocore": { + "hashes": [ + "sha256:1b1b4cf5efd552ecc7f1ce0fc674d5fba56857db5ffe6394ee76edd1a568d7a6", + "sha256:b3b4b8fa33620f015c52e426a92e7db21b5e667ed4785c5fbc484ebfdb2b5153" + ], + "version": "==1.19.13" + }, + "certifi": { + "hashes": [ + "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", + "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" + ], + "version": "==2020.11.8" + }, + "cffi": { + "hashes": [ + "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", + "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", + "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", + "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", + "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", + "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", + "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", + "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", + "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", + "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", + "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", + "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", + "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", + "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", + "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", + "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", + "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", + "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", + "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", + "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", + "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", + "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", + "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", + "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", + "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", + "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", + "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", + "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", + "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", + "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", + "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", + "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", + "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", + "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", + "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", + "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" + ], + "version": "==1.14.3" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "version": "==7.1.2" + }, + "clickclick": { + "hashes": [ + "sha256:4efb13e62353e34c5eef7ed6582c4920b418d7dedc86d819e22ee089ba01802c", + "sha256:c8f33e6d9ec83f68416dd2136a7950125bd256ec39ccc9a85c6e280a16be2bb5" + ], + "version": "==20.10.2" + }, + "connexion": { + "extras": [ + "swagger-ui" + ], + "hashes": [ + "sha256:1ccfac57d4bb7adf4295ba6f5e48f5a1f66057df6a0713417766c9b5235182ee", + "sha256:5439e9659a89c4380d93a07acfbf3380d70be4130574de8881e5f0dfec7ad0e2" + ], + "version": "==2.7.0" + }, + "contextlib2": { + "hashes": [ + "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e", + "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b" + ], + "version": "==0.6.0.post1" + }, + "cryptography": { + "hashes": [ + "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538", + "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f", + "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77", + "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b", + "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33", + "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e", + "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb", + "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e", + "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7", + "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297", + "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d", + "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7", + "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b", + "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7", + "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4", + "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8", + "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b", + "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851", + "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13", + "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b", + "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3", + "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df" + ], + "version": "==3.2.1" + }, + "flask": { + "hashes": [ + "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060", + "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557" + ], + "version": "==1.1.2" + }, + "flask-opentracing": { + "hashes": [ + "sha256:a9a39d367fbe7e9ed9c77b90ac48159c1a3e82982a5abf84d3f4d710d24580ac" + ], + "version": "==1.1.0" + }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "version": "==0.18.2" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "version": "==2.10" + }, + "importlib-metadata": { + "hashes": [ + "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da", + "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3" + ], + "markers": "python_version < '3.8'", + "version": "==2.0.0" + }, + "inflection": { + "hashes": [ + "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", + "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2" + ], + "version": "==0.5.1" + }, + "itsdangerous": { + "hashes": [ + "sha256:1f08ac48fe58cb99a1f58add0c90924b4267398bbd1640268d26a29f6a03eaa4", + "sha256:26ba8fb99157bbd5a1016f9d9dc5ed5ff1325f6f6f2c10b18107199470676b4d" + ], + "version": "==2.0.0a1" + }, + "jaeger-client": { + "hashes": [ + "sha256:7f72c341a21e3dcbc8498d1ed17c93e0eec71c5becc5c9333c4293793d4e6433" + ], + "version": "==4.3.0" + }, + "jinja2": { + "hashes": [ + "sha256:c10142f819c2d22bdcd17548c46fa9b77cf4fda45097854c689666bf425e7484", + "sha256:c922560ac46888d47384de1dbdc3daaa2ea993af4b26a436dec31fa2c19ec668" + ], + "version": "==3.0.0a1" + }, + "jmespath": { + "hashes": [ + "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", + "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" + ], + "version": "==0.10.0" + }, + "jsonschema": { + "hashes": [ + "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", + "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" + ], + "version": "==3.2.0" + }, + "markupsafe": { + "hashes": [ + "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", + "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", + "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", + "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", + "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", + "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", + "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", + "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", + "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", + "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", + "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", + "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", + "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", + "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", + "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", + "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", + "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", + "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", + "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", + "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", + "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", + "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" + ], + "version": "==2.0.0a1" + }, + "openapi-spec-validator": { + "hashes": [ + "sha256:6dd75e50c94f1bb454d0e374a56418e7e06a07affb2c7f1df88564c5d728dac3", + "sha256:79381a69b33423ee400ae1624a461dae7725e450e2e306e32f2dd8d16a4d85cb", + "sha256:ec1b01a00e20955a527358886991ae34b4b791b253027ee9f7df5f84b59d91c7" + ], + "version": "==0.2.9" + }, + "opentracing": { + "hashes": [ + "sha256:33b10634917a7496a7e3a18b18b7c055053d0a8da5101e2a95d454783cac9765" + ], + "version": "==2.3.0" + }, + "opentracing-instrumentation": { + "hashes": [ + "sha256:ae9b48a5b6e47189887fff9785230b13141659bb3bf3e2700e4c7470ae5d27b4", + "sha256:d94801aebab95b7e8f3eeb80ee0e5e886ad4f1761bcadff9b5d1d3d04560c04a" + ], + "version": "==3.3.1" + }, + "prance": { + "hashes": [ + "sha256:86ec64378036471efdd9681648d8da6b39e5143ea2e6981b7863a33fbc75d739", + "sha256:b87504dffb40c1e29aca0ca01f5f3dbb1941f29fd32c3a5e0fbd90ae37f749a3" + ], + "version": "==0.19.0" + }, + "prometheus-client": { + "hashes": [ + "sha256:983c7ac4b47478720db338f1491ef67a100b474e3bc7dafcbaefb7d0b8f9b01c", + "sha256:c6e6b706833a6bd1fd51711299edee907857be10ece535126a158f911ee80915" + ], + "version": "==0.8.0" + }, + "py-ms": { + "editable": true, + "extras": [ + "all" + ], + "path": "." + }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "version": "==2.20" + }, + "pyrsistent": { + "hashes": [ + "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" + ], + "version": "==0.17.3" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "version": "==2.8.1" + }, + "python-json-logger": { + "hashes": [ + "sha256:f26eea7898db40609563bed0a7ca11af12e2a79858632706d835a0f961b7d398" + ], + "version": "==2.0.1" + }, + "pyyaml": { + "hashes": [ + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" + ], + "version": "==5.3.1" + }, + "requests": { + "hashes": [ + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + ], + "version": "==2.24.0" + }, + "s3transfer": { + "hashes": [ + "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13", + "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db" + ], + "version": "==0.3.3" + }, + "semver": { + "hashes": [ + "sha256:ced8b23dceb22134307c1b8abfa523da14198793d9787ac838e70e29e77458d4", + "sha256:fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f" + ], + "version": "==2.13.0" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "version": "==1.15.0" + }, + "swagger-ui-bundle": { + "hashes": [ + "sha256:f5255f786cde67a2638111f4a7d04355836743198a83c4ecbe815d9fc384b0c8", + "sha256:f5691167f2e9f73ecbe8229a89454ae5ea958f90bb0d4583ed7adaae598c4122" + ], + "version": "==0.0.8" + }, + "threadloop": { + "hashes": [ + "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599", + "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944" + ], + "version": "==1.0.2" + }, + "thrift": { + "hashes": [ + "sha256:9af1c86bf73433afc6010ed376a6c6aca2b54099cc0d61895f640870a9ae7d89" + ], + "version": "==0.13.0" + }, + "tornado": { + "hashes": [ + "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", + "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", + "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", + "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", + "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", + "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", + "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" + ], + "version": "==5.1.1" + }, + "urllib3": { + "hashes": [ + "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", + "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" + ], + "markers": "python_version != '3.4'", + "version": "==1.25.11" + }, + "werkzeug": { + "hashes": [ + "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", + "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" + ], + "version": "==1.0.1" + }, + "wrapt": { + "hashes": [ + "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" + ], + "version": "==1.12.1" + }, + "zipp": { + "hashes": [ + "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108", + "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb" + ], + "version": "==3.4.0" + } + }, + "develop": { + "anyconfig": { + "hashes": [ + "sha256:8888130cde5461cb39379afdd1d09b1b1342356210f0a6743a4b60f9973226f8", + "sha256:e8594bd9898954538ed1370bd0a5dbded9e207548f933bbcd4dac5f48deb6a38" + ], + "version": "==0.9.11" + }, + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "astroid": { + "hashes": [ + "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703", + "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386" + ], + "version": "==2.4.2" + }, + "attrs": { + "hashes": [ + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + ], + "version": "==20.3.0" + }, + "bandit": { + "hashes": [ + "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952", + "sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065" + ], + "version": "==1.6.2" + }, + "basictracer": { + "hashes": [ + "sha256:22a3b00c9a3d7b1e630078d284350a57416883686abc1ff109629750b4b58d65" + ], + "version": "==3.1.0" + }, + "black": { + "hashes": [ + "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" + ], + "version": "==20.8b1" + }, + "certifi": { + "hashes": [ + "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", + "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" + ], + "version": "==2020.11.8" + }, + "cffi": { + "hashes": [ + "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", + "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", + "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", + "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", + "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", + "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", + "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", + "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", + "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", + "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", + "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", + "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", + "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", + "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", + "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", + "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", + "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", + "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", + "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", + "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", + "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", + "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", + "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", + "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", + "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", + "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", + "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", + "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", + "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", + "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", + "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", + "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", + "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", + "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", + "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", + "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" + ], + "version": "==1.14.3" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "version": "==7.1.2" + }, + "coverage": { + "hashes": [ + "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516", + "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259", + "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9", + "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097", + "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0", + "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f", + "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7", + "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c", + "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5", + "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7", + "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729", + "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978", + "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9", + "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f", + "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9", + "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822", + "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418", + "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82", + "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f", + "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d", + "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221", + "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4", + "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21", + "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709", + "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54", + "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d", + "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270", + "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24", + "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751", + "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a", + "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237", + "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7", + "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636", + "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8" + ], + "version": "==5.3" + }, + "cryptography": { + "hashes": [ + "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538", + "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f", + "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77", + "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b", + "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33", + "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e", + "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb", + "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e", + "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7", + "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297", + "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d", + "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7", + "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b", + "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7", + "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4", + "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8", + "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b", + "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851", + "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13", + "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b", + "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3", + "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df" + ], + "version": "==3.2.1" + }, + "distlib": { + "hashes": [ + "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", + "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1" + ], + "version": "==0.3.1" + }, + "dparse": { + "hashes": [ + "sha256:a1b5f169102e1c894f9a7d5ccf6f9402a836a5d24be80a986c7ce9eaed78f367", + "sha256:e953a25e44ebb60a5c6efc2add4420c177f1d8404509da88da9729202f306994" + ], + "version": "==0.5.1" + }, + "filelock": { + "hashes": [ + "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", + "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836" + ], + "version": "==3.0.12" + }, + "flake8": { + "hashes": [ + "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839", + "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b" + ], + "version": "==3.8.4" + }, + "flask": { + "hashes": [ + "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060", + "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557" + ], + "version": "==1.1.2" + }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "version": "==0.18.2" + }, + "gitdb": { + "hashes": [ + "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac", + "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9" + ], + "version": "==4.0.5" + }, + "gitpython": { + "hashes": [ + "sha256:6eea89b655917b500437e9668e4a12eabdcf00229a0df1762aabd692ef9b746b", + "sha256:befa4d101f91bad1b632df4308ec64555db684c360bd7d2130b4807d49ce86b8" + ], + "version": "==3.1.11" + }, + "googleapis-common-protos": { + "hashes": [ + "sha256:560716c807117394da12cecb0a54da5a451b5cf9866f1d37e9a5e2329a665351", + "sha256:c8961760f5aad9a711d37b675be103e0cc4e9a39327e0d6d857872f698403e24" + ], + "version": "==1.52.0" + }, + "httmock": { + "hashes": [ + "sha256:13e6c63f135a928e15d386af789a2890efb03e0e280f29bdc9961f3f0dc34cb9", + "sha256:44eaf4bb59cc64cd6f5d8bf8700b46aa3097cc5651b9bc85c527dfbc71792f41" + ], + "version": "==1.4.0" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "version": "==2.10" + }, + "importlib-metadata": { + "hashes": [ + "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da", + "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3" + ], + "markers": "python_version < '3.8'", + "version": "==2.0.0" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, + "isort": { + "hashes": [ + "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7", + "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58" + ], + "version": "==5.6.4" + }, + "itsdangerous": { + "hashes": [ + "sha256:1f08ac48fe58cb99a1f58add0c90924b4267398bbd1640268d26a29f6a03eaa4", + "sha256:26ba8fb99157bbd5a1016f9d9dc5ed5ff1325f6f6f2c10b18107199470676b4d" + ], + "version": "==2.0.0a1" + }, + "jinja2": { + "hashes": [ + "sha256:c10142f819c2d22bdcd17548c46fa9b77cf4fda45097854c689666bf425e7484", + "sha256:c922560ac46888d47384de1dbdc3daaa2ea993af4b26a436dec31fa2c19ec668" + ], + "version": "==3.0.0a1" + }, + "joblib": { + "hashes": [ + "sha256:698c311779f347cf6b7e6b8a39bb682277b8ee4aba8cf9507bc0cf4cd4737b72", + "sha256:9e284edd6be6b71883a63c9b7f124738a3c16195513ad940eae7e3438de885d5" + ], + "version": "==0.17.0" + }, + "jsonpickle": { + "hashes": [ + "sha256:8919c166bac0574e3d74425c7559434062002d9dfc0ac2afa6dc746ba4a19439", + "sha256:e8d4b7cd0bd6826001a74377df1079a76ad8bae0f909282de2554164c837c8ba" + ], + "version": "==1.4.1" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" + }, + "lightstep": { + "hashes": [ + "sha256:4dfc5cc5c8baa47d0cf2c41e11b97511ffe218b296697f9538b0444a6b54f857", + "sha256:6bce8085d117eef6d9ac5f7e0dae901fcabd2cc6a5780decc00b61e45e7af116" + ], + "version": "==4.4.8" + }, + "livereload": { + "hashes": [ + "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869" + ], + "version": "==2.6.3" + }, + "lunr": { + "extras": [ + "languages" + ], + "hashes": [ + "sha256:aab3f489c4d4fab4c1294a257a30fec397db56f0a50273218ccc3efdbf01d6ca", + "sha256:c4fb063b98eff775dd638b3df380008ae85e6cb1d1a24d1cd81a10ef6391c26e" + ], + "version": "==0.5.8" + }, + "markdown": { + "hashes": [ + "sha256:5d9f2b5ca24bc4c7a390d22323ca4bad200368612b5aaa7796babf971d2b2f18", + "sha256:c109c15b7dc20a9ac454c9e6025927d44460b85bd039da028d85e2b6d0bcc328" + ], + "version": "==3.3.3" + }, + "markupsafe": { + "hashes": [ + "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", + "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", + "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", + "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", + "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", + "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", + "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", + "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", + "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", + "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", + "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", + "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", + "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", + "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", + "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", + "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", + "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", + "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", + "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", + "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", + "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", + "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" + ], + "version": "==2.0.0a1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "mkdocs": { + "hashes": [ + "sha256:096f52ff52c02c7e90332d2e53da862fde5c062086e1b5356a6e392d5d60f5e9", + "sha256:f0b61e5402b99d7789efa032c7a74c90a20220a9c81749da06dbfbcbd52ffb39" + ], + "version": "==1.1.2" + }, + "mypy": { + "hashes": [ + "sha256:0a0d102247c16ce93c97066443d11e2d36e6cc2a32d8ccc1f705268970479324", + "sha256:0d34d6b122597d48a36d6c59e35341f410d4abfa771d96d04ae2c468dd201abc", + "sha256:2170492030f6faa537647d29945786d297e4862765f0b4ac5930ff62e300d802", + "sha256:2842d4fbd1b12ab422346376aad03ff5d0805b706102e475e962370f874a5122", + "sha256:2b21ba45ad9ef2e2eb88ce4aeadd0112d0f5026418324176fd494a6824b74975", + "sha256:72060bf64f290fb629bd4a67c707a66fd88ca26e413a91384b18db3876e57ed7", + "sha256:af4e9ff1834e565f1baa74ccf7ae2564ae38c8df2a85b057af1dbbc958eb6666", + "sha256:bd03b3cf666bff8d710d633d1c56ab7facbdc204d567715cb3b9f85c6e94f669", + "sha256:c614194e01c85bb2e551c421397e49afb2872c88b5830e3554f0519f9fb1c178", + "sha256:cf4e7bf7f1214826cf7333627cb2547c0db7e3078723227820d0a2490f117a01", + "sha256:da56dedcd7cd502ccd3c5dddc656cb36113dd793ad466e894574125945653cea", + "sha256:e86bdace26c5fe9cf8cb735e7cedfe7850ad92b327ac5d797c656717d2ca66de", + "sha256:e97e9c13d67fbe524be17e4d8025d51a7dca38f90de2e462243ab8ed8a9178d1", + "sha256:eea260feb1830a627fb526d22fbb426b750d9f5a47b624e8d5e7e004359b219c" + ], + "version": "==0.790" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "nltk": { + "hashes": [ + "sha256:845365449cd8c5f9731f7cb9f8bd6fd0767553b9d53af9eb1b3abf7700936b35" + ], + "version": "==3.5" + }, + "opentracing": { + "hashes": [ + "sha256:33b10634917a7496a7e3a18b18b7c055053d0a8da5101e2a95d454783cac9765" + ], + "version": "==2.3.0" + }, + "packaging": { + "hashes": [ + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + ], + "version": "==20.4" + }, + "pathspec": { + "hashes": [ + "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", + "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d" + ], + "version": "==0.8.1" + }, + "pbr": { + "hashes": [ + "sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9", + "sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00" + ], + "version": "==5.5.1" + }, + "pluggy": { + "hashes": [ + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + ], + "version": "==0.13.1" + }, + "protobuf": { + "hashes": [ + "sha256:0e55cf11e3cdc7af9e280539144c0896ff046144dc17ddf6344e33f7bd01c53b", + "sha256:2ceef235ec88363f0853679363e215aee79f2998931b83e5d1bf08289e3c6a6f", + "sha256:36583c483ddd3c60f25a9a1e1660baa1e033e9470a49e019d0705d6fefaea52b", + "sha256:3bb580ee33b844087e419a9302a255a956d695147a64d59e96a3e0abd78fa67d", + "sha256:4136552036dbff1e5841cd240d1be973348752cfdaa91c9088d8a32ae43f06c0", + "sha256:428930d8f9607723ab6482f07bc7a651e95f86c7f5db6347a0b1f24319cd1e3b", + "sha256:471b0cd067e1ea2c6c5cc82fc7c04990a0497914668ca0bdc7db7925e25a8045", + "sha256:4ed0b7df03cd668dbd4bf1a432e58fc9201ae4f15a9a6f837bce4c7496c431da", + "sha256:76f4a7f5c418167496b68a7310efe93a33066aa83f5fac3a9189bc5ace6ff905", + "sha256:991301d26c33cc25c8a81fd7d25ad5a31cc6eb166b96afcbce93675545e03532", + "sha256:ae99d4b1c15439586d7d3bfc3ec7d3e933419fdcff3e483ed5a14e653b45d39c", + "sha256:afef26dd04202c8f4efe8292bd9a683166e32b6004abb0302d6a92db1a994bf1", + "sha256:bb056806dee32e6f8ad47935ca2be34bed124675c0132a469e70aa6847ce2223", + "sha256:bb0b3df5c29a8dc51e6435505d42907ac3651e2b0be40c30e9f40d08aca6d47c", + "sha256:bd644bf3865a72d5a244e7193f784cc6cd51bb453d447ee73a5e05d60dc22717", + "sha256:d6bb25e26cbee3eebf460012e3406ad5d77ced6824abbea919d8c7a0466e82f1", + "sha256:ed7143575f71fed2599e1548a0378cfd6dc958920221666e3a33785af3f0d7e2", + "sha256:ff82b53153edda066698bae674d673566029e856fa126dcb95a432aed4161af2" + ], + "version": "==3.14.0rc1" + }, + "py": { + "hashes": [ + "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", + "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" + ], + "version": "==1.9.0" + }, + "py-ms": { + "editable": true, + "extras": [ + "all" + ], + "path": "." + }, + "pycodestyle": { + "hashes": [ + "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", + "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" + ], + "version": "==2.6.0" + }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "version": "==2.20" + }, + "pyflakes": { + "hashes": [ + "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", + "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" + ], + "version": "==2.2.0" + }, + "pylint": { + "hashes": [ + "sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210", + "sha256:bfe68f020f8a0fece830a22dd4d5dddb4ecc6137db04face4c3420a46a52239f" + ], + "version": "==2.6.0" + }, + "pyparsing": { + "hashes": [ + "sha256:13140e8d0e1edd806eb50f18535d77f2143b40771d4aaef6b4950dd93d48a7db", + "sha256:38891c1032d0c759f0fa5ed3a8f249fd992b083fa2303ead58ee48a51b269e02" + ], + "version": "==3.0.0b1" + }, + "pytest": { + "hashes": [ + "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe", + "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e" + ], + "version": "==6.1.2" + }, + "pytest-cov": { + "hashes": [ + "sha256:45ec2d5182f89a81fc3eb29e3d1ed3113b9e9a873bcddb2a71faaab066110191", + "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e" + ], + "version": "==2.10.1" + }, + "python-json-logger": { + "hashes": [ + "sha256:f26eea7898db40609563bed0a7ca11af12e2a79858632706d835a0f961b7d398" + ], + "version": "==2.0.1" + }, + "pyyaml": { + "hashes": [ + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" + ], + "version": "==5.3.1" + }, + "regex": { + "hashes": [ + "sha256:03855ee22980c3e4863dc84c42d6d2901133362db5daf4c36b710dd895d78f0a", + "sha256:06b52815d4ad38d6524666e0d50fe9173533c9cc145a5779b89733284e6f688f", + "sha256:11116d424734fe356d8777f89d625f0df783251ada95d6261b4c36ad27a394bb", + "sha256:119e0355dbdd4cf593b17f2fc5dbd4aec2b8899d0057e4957ba92f941f704bf5", + "sha256:127a9e0c0d91af572fbb9e56d00a504dbd4c65e574ddda3d45b55722462210de", + "sha256:1ec66700a10e3c75f1f92cbde36cca0d3aaee4c73dfa26699495a3a30b09093c", + "sha256:227a8d2e5282c2b8346e7f68aa759e0331a0b4a890b55a5cfbb28bd0261b84c0", + "sha256:2564def9ce0710d510b1fc7e5178ce2d20f75571f788b5197b3c8134c366f50c", + "sha256:297116e79074ec2a2f885d22db00ce6e88b15f75162c5e8b38f66ea734e73c64", + "sha256:2dc522e25e57e88b4980d2bdd334825dbf6fa55f28a922fc3bfa60cc09e5ef53", + "sha256:3a5f08039eee9ea195a89e180c5762bfb55258bfb9abb61a20d3abee3b37fd12", + "sha256:3dfca201fa6b326239e1bccb00b915e058707028809b8ecc0cf6819ad233a740", + "sha256:49461446b783945597c4076aea3f49aee4b4ce922bd241e4fcf62a3e7c61794c", + "sha256:4afa350f162551cf402bfa3cd8302165c8e03e689c897d185f16a167328cc6dd", + "sha256:4b5a9bcb56cc146c3932c648603b24514447eafa6ce9295234767bf92f69b504", + "sha256:52e83a5f28acd621ba8e71c2b816f6541af7144b69cc5859d17da76c436a5427", + "sha256:625116aca6c4b57c56ea3d70369cacc4d62fead4930f8329d242e4fe7a58ce4b", + "sha256:654c1635f2313d0843028487db2191530bca45af61ca85d0b16555c399625b0e", + "sha256:8092a5a06ad9a7a247f2a76ace121183dc4e1a84c259cf9c2ce3bbb69fac3582", + "sha256:832339223b9ce56b7b15168e691ae654d345ac1635eeb367ade9ecfe0e66bee0", + "sha256:8ca9dca965bd86ea3631b975d63b0693566d3cc347e55786d5514988b6f5b84c", + "sha256:96f99219dddb33e235a37283306834700b63170d7bb2a1ee17e41c6d589c8eb9", + "sha256:9b6305295b6591e45f069d3553c54d50cc47629eb5c218aac99e0f7fafbf90a1", + "sha256:a62162be05edf64f819925ea88d09d18b09bebf20971b363ce0c24e8b4aa14c0", + "sha256:aacc8623ffe7999a97935eeabbd24b1ae701d08ea8f874a6ff050e93c3e658cf", + "sha256:b45bab9f224de276b7bc916f6306b86283f6aa8afe7ed4133423efb42015a898", + "sha256:b88fa3b8a3469f22b4f13d045d9bd3eda797aa4e406fde0a2644bc92bbdd4bdd", + "sha256:b8a686a6c98872007aa41fdbb2e86dc03b287d951ff4a7f1da77fb7f14113e4d", + "sha256:bd904c0dec29bbd0769887a816657491721d5f545c29e30fd9d7a1a275dc80ab", + "sha256:bf4f896c42c63d1f22039ad57de2644c72587756c0cfb3cc3b7530cfe228277f", + "sha256:c13d311a4c4a8d671f5860317eb5f09591fbe8259676b86a85769423b544451e", + "sha256:c2c6c56ee97485a127555c9595c069201b5161de9d05495fbe2132b5ac104786", + "sha256:c32c91a0f1ac779cbd73e62430de3d3502bbc45ffe5bb6c376015acfa848144b", + "sha256:c3466a84fce42c2016113101018a9981804097bacbab029c2d5b4fcb224b89de", + "sha256:c454ad88e56e80e44f824ef8366bb7e4c3def12999151fd5c0ea76a18fe9aa3e", + "sha256:c8a2b7ccff330ae4c460aff36626f911f918555660cc28163417cb84ffb25789", + "sha256:cb905f3d2e290a8b8f1579d3984f2cfa7c3a29cc7cba608540ceeed18513f520", + "sha256:cfcf28ed4ce9ced47b9b9670a4f0d3d3c0e4d4779ad4dadb1ad468b097f808aa", + "sha256:dd3e6547ecf842a29cf25123fbf8d2461c53c8d37aa20d87ecee130c89b7079b", + "sha256:de7fd57765398d141949946c84f3590a68cf5887dac3fc52388df0639b01eda4", + "sha256:ea37320877d56a7f0a1e6a625d892cf963aa7f570013499f5b8d5ab8402b5625", + "sha256:f1fce1e4929157b2afeb4bb7069204d4370bab9f4fc03ca1fbec8bd601f8c87d", + "sha256:f43109822df2d3faac7aad79613f5f02e4eab0fc8ad7932d2e70e2a83bd49c26" + ], + "version": "==2020.10.28" + }, + "requests": { + "hashes": [ + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + ], + "version": "==2.24.0" + }, + "requests-mock": { + "hashes": [ + "sha256:11215c6f4df72702aa357f205cf1e537cffd7392b3e787b58239bde5fb3db53b", + "sha256:e68f46844e4cee9d447150343c9ae875f99fa8037c6dcf5f15bf1fe9ab43d226" + ], + "version": "==1.8.0" + }, + "safety": { + "hashes": [ + "sha256:23bf20690d4400edc795836b0c983c2b4cbbb922233108ff925b7dd7750f00c9", + "sha256:86c1c4a031fe35bd624fce143fbe642a0234d29f7cbf7a9aa269f244a955b087" + ], + "version": "==1.9.0" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "version": "==1.15.0" + }, + "smmap": { + "hashes": [ + "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4", + "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24" + ], + "version": "==3.0.4" + }, + "stevedore": { + "hashes": [ + "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62", + "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0" + ], + "version": "==3.2.2" + }, + "thrift": { + "hashes": [ + "sha256:9af1c86bf73433afc6010ed376a6c6aca2b54099cc0d61895f640870a9ae7d89" + ], + "version": "==0.13.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "version": "==0.10.2" + }, + "tornado": { + "hashes": [ + "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", + "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", + "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", + "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", + "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", + "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", + "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" + ], + "version": "==5.1.1" + }, + "tox": { + "hashes": [ + "sha256:42ce19ce5dc2f6d6b1fdc5666c476e1f1e2897359b47e0aa3a5b774f335d57c2", + "sha256:4321052bfe28f9d85082341ca8e233e3ea901fdd14dab8a5d3fbd810269fbaf6" + ], + "version": "==3.20.1" + }, + "tqdm": { + "hashes": [ + "sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad", + "sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432" + ], + "version": "==4.51.0" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072", + "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298", + "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "markers": "implementation_name == 'cpython' and python_version < '3.8'", + "version": "==1.4.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "version": "==3.7.4.3" + }, + "urllib3": { + "hashes": [ + "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", + "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" + ], + "markers": "python_version != '3.4'", + "version": "==1.25.11" + }, + "virtualenv": { + "hashes": [ + "sha256:b0011228208944ce71052987437d3843e05690b2f23d1c7da4263fde104c97a2", + "sha256:b8d6110f493af256a40d65e29846c69340a947669eec8ce784fcf3dd3af28380" + ], + "version": "==20.1.0" + }, + "werkzeug": { + "hashes": [ + "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", + "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" + ], + "version": "==1.0.1" + }, + "wrapt": { + "hashes": [ + "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" + ], + "version": "==1.12.1" + }, + "zipp": { + "hashes": [ + "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108", + "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb" + ], + "version": "==3.4.0" + } + } +} From 7ca0d4041c35ed6f5bb5bab70be76ad4ec9cf157 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 14:57:13 +0100 Subject: [PATCH 12/27] fix: flake8 errors --- pyms/flask/app/utils.py | 2 +- .../consulate/api/__init__.py | 2 +- pyms/services_discovery/consulate/api/base.py | 2 +- pyms/services_discovery/consulate/cli.py | 8 +++---- .../consulate/models/agent.py | 19 ++++----------- setup.py | 24 +++++++++---------- 6 files changed, 22 insertions(+), 35 deletions(-) diff --git a/pyms/flask/app/utils.py b/pyms/flask/app/utils.py index 1231ee8..8ae044e 100644 --- a/pyms/flask/app/utils.py +++ b/pyms/flask/app/utils.py @@ -58,7 +58,7 @@ def __call__(self, environ, start_response): environ["SCRIPT_NAME"] = script_name path_info = environ["PATH_INFO"] if path_info.startswith(script_name): - environ["PATH_INFO"] = path_info[len(script_name) :] + environ["PATH_INFO"] = path_info[len(script_name):] # noqa: E203 scheme = environ.get("HTTP_X_SCHEME", "") if scheme: diff --git a/pyms/services_discovery/consulate/api/__init__.py b/pyms/services_discovery/consulate/api/__init__.py index 068abc6..29d060b 100644 --- a/pyms/services_discovery/consulate/api/__init__.py +++ b/pyms/services_discovery/consulate/api/__init__.py @@ -14,4 +14,4 @@ from pyms.services_discovery.consulate.api.status import Status from pyms.services_discovery.consulate.api.base import Response -__all__ = ["ACL", "Agent", "Catalog", "Event", "Health", "KV", "Lock", "Session", "Status", "Response"] +__all__ = ["ACL", "Agent", "Catalog", "Event", "Health", "KV", "Lock", "Session", "Status", "Response", "Coordinate"] diff --git a/pyms/services_discovery/consulate/api/base.py b/pyms/services_discovery/consulate/api/base.py index 03b0f3f..e159d59 100644 --- a/pyms/services_discovery/consulate/api/base.py +++ b/pyms/services_discovery/consulate/api/base.py @@ -146,7 +146,7 @@ def __init__(self, response): self.body = self._demarshal(response.content) self.headers = response.headers - def _demarshal(self, body): # pylint: disable=too-many-branches,too-many-return-statements + def _demarshal(self, body): # pylint: disable=too-many-branches,too-many-return-statements; # noqa: C901 """Demarshal the request payload. :param str body: The string response body diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index 8fe8449..c912bc7 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -331,7 +331,7 @@ def kv_get(consul, args): if args.trim >= len(keyparts): displaykey = keyparts[-1] else: - displaykey = "/".join(keyparts[args.trim :]) + displaykey = "/".join(keyparts[args.trim:]) # noqa: E203 sys.stdout.write("%s\t%s\n" % (displaykey, consul.kv.get(key))) else: sys.stdout.write("%s\n" % consul.kv.get(args.key)) @@ -379,7 +379,7 @@ def kv_mkdir(consul, args): connection_error() -def kv_restore(consul, args): # pylint: disable=too-many-branches,too-many-format-args +def kv_restore(consul, args): # pylint: disable=too-many-branches,too-many-format-args; # noqa: C901 """Restore the Consul KV store :param consulate.api_old.Consul consul: The Consul instance @@ -531,9 +531,7 @@ def run_once(consul, args): ) except OSError as err: error_code = 1 - error_msg = '"{0}" command does not exist'.format( # pylint: disable=too-many-format-args - args.command_to_run, err - ) + error_msg = '"{0}" command does not exist "{1}"'.format(args.command_to_run, err) except Exception as err: error_code = 1 error_msg = '"{0}" exited with error "{1}"'.format(args.command_to_run, err) diff --git a/pyms/services_discovery/consulate/models/agent.py b/pyms/services_discovery/consulate/models/agent.py index 2e53e5d..8e3759b 100644 --- a/pyms/services_discovery/consulate/models/agent.py +++ b/pyms/services_discovery/consulate/models/agent.py @@ -12,13 +12,8 @@ def _validate_args(value, model): :rtype: bool """ - return ( - all([isinstance(v, str) for v in value]) - and not model.args - and not model.grpc - and not model.http - and not model.ttl - ) + is_instance = all([isinstance(v, str) for v in value]) + return is_instance and not model.args and not model.grpc and not model.http and not model.ttl def _validate_grpc(value, model): @@ -75,14 +70,8 @@ def _validate_ttl(value, model): :rtype: bool """ - return ( - utils.validate_go_interval(value) - and not model.args - and not model.grpc - and not model.http - and not model.tcp - and not model.interval - ) + is_valid = utils.validate_go_interval(value) + return is_valid and not model.args and not model.grpc and not model.http and not model.tcp and not model.interval class Check(base.Model): diff --git a/setup.py b/setup.py index b55ea6d..7a5e308 100644 --- a/setup.py +++ b/setup.py @@ -53,18 +53,18 @@ ] install_tests_requires = [ - 'requests-mock>=1.8.0', - 'coverage>=5.3', - 'pytest>=6.1.0', - 'pytest-cov>=2.10.1', - 'pylint>=2.6.0', - 'flake8>=3.8.2', - 'tox>=3.20.0', - 'bandit>=1.6.2', - 'mkdocs>=1.1.2', - 'lightstep>=4.4.8', - 'safety==1.9.0', - 'mypy>=0.782' + "requests-mock>=1.8.0", + "coverage>=5.3", + "pytest>=6.1.0", + "pytest-cov>=2.10.1", + "pylint>=2.6.0", + "flake8>=3.8.2", + "tox>=3.20.0", + "bandit>=1.6.2", + "mkdocs>=1.1.2", + "lightstep>=4.4.8", + "safety==1.9.0", + "mypy>=0.782", "pre-commit>=2.8.1", "black>=20.8b1", "httmock>=1.4.0", From 0d9bd0999382a348b4e260e53c1217182a5f813a Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 15:04:56 +0100 Subject: [PATCH 13/27] fix: bandit errors --- pyms/cmd/main.py | 5 ++--- pyms/services_discovery/consulate/cli.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pyms/cmd/main.py b/pyms/cmd/main.py index 041cde6..1ac6220 100755 --- a/pyms/cmd/main.py +++ b/pyms/cmd/main.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- from __future__ import unicode_literals, print_function import argparse @@ -143,9 +142,9 @@ def run(self): return True def yes_no_input(self, msg=""): # pragma: no cover - answer = input( + answer = input( # nosec utils.colored_text(f'{msg}{"?" if not msg.endswith("?") else ""} [Y/n] :', utils.Colors.BLUE, True) - ) # nosec + ) try: return strtobool(answer) except ValueError: diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index c912bc7..e8916c8 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -1,12 +1,11 @@ """Consulate CLI commands""" -# pragma: no cover import argparse import base64 import json import sys import os import time -import subprocess +import subprocess # nosec import urllib.parse as urlparse @@ -522,7 +521,7 @@ def run_once(consul, args): # Should the subprocess return an error code, release the lock try: - print(subprocess.check_output(args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True)) + print(subprocess.check_output(args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True)) # nosec # If the subprocess fails except subprocess.CalledProcessError as err: error_code = 1 From 2fd7b6362d7060e5ca335e0dba66324c112d5b53 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 18:05:54 +0100 Subject: [PATCH 14/27] feat: moved tests from consulate to pyms --- MANIFEST.in | 1 + Pipfile | 6 +- Pipfile.lock | 157 +++++++- docker/docker-compose.yml | 9 + docker/testing/consul.json | 16 + pyms/services_discovery/consulate/client.py | 5 +- setup.py | 1 + tests/conftest.py | 46 +++ .../services_discovery_consulate/__init__.py | 1 + tests/services_discovery_consulate/base.py | 66 ++++ .../services_discovery_consulate/consul.json | 16 + .../services_discovery_consulate/test_acl.py | 313 +++++++++++++++ .../test_agent.py | 305 +++++++++++++++ .../services_discovery_consulate/test_api.py | 363 ++++++++++++++++++ .../test_base_model.py | 139 +++++++ .../test_catalog.py | 11 + .../test_coordinate.py | 6 + .../test_event.py | 18 + tests/services_discovery_consulate/test_kv.py | 278 ++++++++++++++ .../services_discovery_consulate/test_lock.py | 10 + .../test_session.py | 43 +++ .../test_utils.py | 77 ++++ 22 files changed, 1881 insertions(+), 6 deletions(-) create mode 100644 docker/docker-compose.yml create mode 100644 docker/testing/consul.json create mode 100644 tests/conftest.py create mode 100644 tests/services_discovery_consulate/__init__.py create mode 100644 tests/services_discovery_consulate/base.py create mode 100644 tests/services_discovery_consulate/consul.json create mode 100644 tests/services_discovery_consulate/test_acl.py create mode 100644 tests/services_discovery_consulate/test_agent.py create mode 100644 tests/services_discovery_consulate/test_api.py create mode 100644 tests/services_discovery_consulate/test_base_model.py create mode 100644 tests/services_discovery_consulate/test_catalog.py create mode 100644 tests/services_discovery_consulate/test_coordinate.py create mode 100644 tests/services_discovery_consulate/test_event.py create mode 100644 tests/services_discovery_consulate/test_kv.py create mode 100644 tests/services_discovery_consulate/test_lock.py create mode 100644 tests/services_discovery_consulate/test_session.py create mode 100644 tests/services_discovery_consulate/test_utils.py diff --git a/MANIFEST.in b/MANIFEST.in index 2a5c7fe..e51798a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,5 +4,6 @@ include requirements-tests.txt recursive-include pyms * recursive-exclude tests * recursive-exclude examples * +recursive-exclude docker * prune tests prune examples \ No newline at end of file diff --git a/Pipfile b/Pipfile index b55173f..6673b6f 100644 --- a/Pipfile +++ b/Pipfile @@ -4,10 +4,10 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] -py-ms = {editable = true,extras = ["tests"], path = "."} +py-ms = {editable = true,extras = ["tests"],path = "."} [packages] -py-ms = {editable = true,extras = ["all"], path = "."} +py-ms = {editable = true,extras = ["all"],path = "."} [pipenv] -allow_prereleases = true \ No newline at end of file +allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock index 1f8a17e..2eb116c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "30f7bae4efa04f995974cbeab5512cb644c3726a04ab10ba9cf0e4ff01c67cb5" + "sha256": "499af62c8d853c966d75ee7c899c41eee7ebebba5c23c74aee2a61b231cf8e29" }, "pipfile-spec": 6, "requires": {}, @@ -472,12 +472,31 @@ ], "version": "==3.1.0" }, + "bcrypt": { + "hashes": [ + "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29", + "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7", + "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34", + "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55", + "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6", + "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1", + "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d" + ], + "version": "==3.2.0" + }, "black": { "hashes": [ "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" ], "version": "==20.8b1" }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, "certifi": { "hashes": [ "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", @@ -526,6 +545,13 @@ ], "version": "==1.14.3" }, + "cfgv": { + "hashes": [ + "sha256:32e43d604bbe7896fe7c248a9c2276447dbef840feb28fe20494f62af110211d", + "sha256:cf22deb93d4bcf92f345a5c3cd39d3d41d6340adc60c78bbbd6588c384fda6a1" + ], + "version": "==3.2.0" + }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", @@ -613,6 +639,42 @@ ], "version": "==0.3.1" }, + "distro": { + "hashes": [ + "sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92", + "sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799" + ], + "version": "==1.5.0" + }, + "docker": { + "extras": [ + "ssh" + ], + "hashes": [ + "sha256:13966471e8bc23b36bfb3a6fb4ab75043a5ef1dac86516274777576bed3b9828", + "sha256:bad94b8dd001a8a4af19ce4becc17f41b09f228173ffe6a4e0355389eef142f2" + ], + "version": "==4.3.1" + }, + "docker-compose": { + "hashes": [ + "sha256:5a5690f24c27d4b43dcbe6b3fae91ba680713208e99ee863352b3bae37bcaa83", + "sha256:84ca2edad226435e3a378ea24ca2ca4e1a77cc7c8de057e2812124c6dcb55147" + ], + "version": "==1.27.4" + }, + "dockerpty": { + "hashes": [ + "sha256:69a9d69d573a0daa31bcd1c0774eeed5c15c295fe719c61aca550ed1393156ce" + ], + "version": "==0.4.1" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, "dparse": { "hashes": [ "sha256:a1b5f169102e1c894f9a7d5ccf6f9402a836a5d24be80a986c7ce9eaed78f367", @@ -675,6 +737,13 @@ ], "version": "==1.4.0" }, + "identify": { + "hashes": [ + "sha256:5dd84ac64a9a115b8e0b27d1756b244b882ad264c3c423f42af8235a6e71ca12", + "sha256:c9504ba6a043ee2db0a9d69e43246bc138034895f6338d5aed1b41e4a73b1513" + ], + "version": "==1.5.9" + }, "idna": { "hashes": [ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", @@ -732,6 +801,13 @@ ], "version": "==1.4.1" }, + "jsonschema": { + "hashes": [ + "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", + "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" + ], + "version": "==3.2.0" + }, "lazy-object-proxy": { "hashes": [ "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", @@ -861,6 +937,13 @@ ], "version": "==3.5" }, + "nodeenv": { + "hashes": [ + "sha256:5304d424c529c997bc888453aeaa6362d242b6b4631e90f3d4bf1b290f1c84a9", + "sha256:ab45090ae383b716c4ef89e690c41ff8c2b257b85b309f01f3654df3d084bd7c" + ], + "version": "==1.5.0" + }, "opentracing": { "hashes": [ "sha256:33b10634917a7496a7e3a18b18b7c055053d0a8da5101e2a95d454783cac9765" @@ -874,6 +957,13 @@ ], "version": "==20.4" }, + "paramiko": { + "hashes": [ + "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898", + "sha256:7f36f4ba2c0d81d219f4595e35f70d56cc94f9ac40a6acdf51d6ca210ce65035" + ], + "version": "==2.7.2" + }, "pathspec": { "hashes": [ "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", @@ -895,6 +985,13 @@ ], "version": "==0.13.1" }, + "pre-commit": { + "hashes": [ + "sha256:22e6aa3bd571debb01eb7d34483f11c01b65237be4eebbf30c3d4fb65762d315", + "sha256:905ebc9b534b991baec87e934431f2d0606ba27f2b90f7f652985f5a5b8b6ae6" + ], + "version": "==2.8.2" + }, "protobuf": { "hashes": [ "sha256:0e55cf11e3cdc7af9e280539144c0896ff046144dc17ddf6344e33f7bd01c53b", @@ -960,6 +1057,29 @@ ], "version": "==2.6.0" }, + "pynacl": { + "hashes": [ + "sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4", + "sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4", + "sha256:2fe0fc5a2480361dcaf4e6e7cea00e078fcda07ba45f811b167e3f99e8cff574", + "sha256:30f9b96db44e09b3304f9ea95079b1b7316b2b4f3744fe3aaecccd95d547063d", + "sha256:4e10569f8cbed81cb7526ae137049759d2a8d57726d52c1a000a3ce366779634", + "sha256:511d269ee845037b95c9781aa702f90ccc36036f95d0f31373a6a79bd8242e25", + "sha256:537a7ccbea22905a0ab36ea58577b39d1fa9b1884869d173b5cf111f006f689f", + "sha256:54e9a2c849c742006516ad56a88f5c74bf2ce92c9f67435187c3c5953b346505", + "sha256:757250ddb3bff1eecd7e41e65f7f833a8405fede0194319f87899690624f2122", + "sha256:7757ae33dae81c300487591c68790dfb5145c7d03324000433d9a2c141f82af7", + "sha256:7c6092102219f59ff29788860ccb021e80fffd953920c4a8653889c029b2d420", + "sha256:8122ba5f2a2169ca5da936b2e5a511740ffb73979381b4229d9188f6dcb22f1f", + "sha256:9c4a7ea4fb81536c1b1f5cc44d54a296f96ae78c1ebd2311bd0b60be45a48d96", + "sha256:c914f78da4953b33d4685e3cdc7ce63401247a21425c16a39760e282075ac4a6", + "sha256:cd401ccbc2a249a47a3a1724c2918fcd04be1f7b54eb2a5a71ff915db0ac51c6", + "sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514", + "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff", + "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80" + ], + "version": "==1.4.0" + }, "pyparsing": { "hashes": [ "sha256:13140e8d0e1edd806eb50f18535d77f2143b40771d4aaef6b4950dd93d48a7db", @@ -967,6 +1087,12 @@ ], "version": "==3.0.0b1" }, + "pyrsistent": { + "hashes": [ + "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" + ], + "version": "==0.17.3" + }, "pytest": { "hashes": [ "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe", @@ -981,6 +1107,21 @@ ], "version": "==2.10.1" }, + "pytest-docker": { + "hashes": [ + "sha256:7a0544dc00c83fd7808907a55835b1b7c07e3ea2d22ee558b5a233247e462735", + "sha256:9faa9c87e6e0920612005c8d187c58b3d336f3af49a878f358578de125963858" + ], + "index": "pypi", + "version": "==0.10.1" + }, + "python-dotenv": { + "hashes": [ + "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e", + "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0" + ], + "version": "==0.15.0" + }, "python-json-logger": { "hashes": [ "sha256:f26eea7898db40609563bed0a7ca11af12e2a79858632706d835a0f961b7d398" @@ -1093,6 +1234,13 @@ ], "version": "==3.2.2" }, + "texttable": { + "hashes": [ + "sha256:ce0faf21aa77d806bbff22b107cc22cce68dc9438f97a2df32c93e9afa4ce436", + "sha256:f802f2ef8459058736264210f716c757cbf85007a30886d8541aa8c3404f1dda" + ], + "version": "==1.6.3" + }, "thrift": { "hashes": [ "sha256:9af1c86bf73433afc6010ed376a6c6aca2b54099cc0d61895f640870a9ae7d89" @@ -1191,6 +1339,13 @@ ], "version": "==20.1.0" }, + "websocket-client": { + "hashes": [ + "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549", + "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010" + ], + "version": "==0.57.0" + }, "werkzeug": { "hashes": [ "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..d4438c5 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,9 @@ +version: '3' + +services: + consul: + image: consul:1.6.0 + ports: + - 8500:8500 + volumes: + - './testing:/consul/config' \ No newline at end of file diff --git a/docker/testing/consul.json b/docker/testing/consul.json new file mode 100644 index 0000000..3368abc --- /dev/null +++ b/docker/testing/consul.json @@ -0,0 +1,16 @@ +{ + "acl": { + "enabled": true, + "enable_key_list_policy": true, + "tokens": { + "master": "9ae5fe1a-6b38-47e5-a0e7-f06b8b2fa645" + } + }, + "bootstrap_expect": 1, + "data_dir": "/tmp/consul", + "datacenter": "test", + "server": true, + "bind_addr": "{{ GetPrivateIP }}", + "client_addr": "0.0.0.0", + "enable_script_checks": true +} diff --git a/pyms/services_discovery/consulate/client.py b/pyms/services_discovery/consulate/client.py index d8c5dfb..1eb3e3a 100644 --- a/pyms/services_discovery/consulate/client.py +++ b/pyms/services_discovery/consulate/client.py @@ -3,7 +3,8 @@ """ import os -from pyms.services_discovery.consulate import adapters, api, utils +from pyms.services_discovery.consulate import adapters, api +from urllib.parse import quote DEFAULT_HOST = os.environ.get("CONSUL_HOST") or "localhost" DEFAULT_PORT = os.environ.get("CONSUL_PORT") or 8500 @@ -191,5 +192,5 @@ def _base_uri(scheme, host, port, addr=None): if addr is None: if port: return "{0}://{1}:{2}/{3}".format(scheme, host, port, API_VERSION) - return "{0}://{1}/{2}".format(scheme, utils.quote(host, ""), API_VERSION) + return "{0}://{1}/{2}".format(scheme, quote(host, ""), API_VERSION) return "{0}/{1}".format(addr, API_VERSION) diff --git a/setup.py b/setup.py index 7a5e308..ed8e13e 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ "coverage>=5.3", "pytest>=6.1.0", "pytest-cov>=2.10.1", + "pytest-docker>=0.10.1", "pylint>=2.6.0", "flake8>=3.8.2", "tox>=3.20.0", diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..da20981 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,46 @@ +import os + +import pytest +import requests +from requests.exceptions import ConnectionError + +os.environ['ASYNC_TEST_TIMEOUT'] = os.environ.get('ASYNC_TEST_TIMEOUT', '15') + + +def conf_environment(): + if not os.environ.get("PYMS_CONFIGMAP_FILE", False): + os.environ["PYMS_CONFIGMAP_FILE"] = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config-tests.yml") + + +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + """Get docker compose file""" + return os.path.join(str(pytestconfig.rootdir), "docker", "docker-compose.yml") + + +def is_responsive(url): + try: + response = requests.get(url) + if response.status_code == 200: + return True + except ConnectionError: + return False + + +@pytest.fixture(scope="session") +def http_service(docker_ip, docker_services): + """Ensure that HTTP service is up and responsive.""" + # `port_for` takes a container port and returns the corresponding host port + port = docker_services.port_for("consul", 80) + url = "http://{}:{}".format(docker_ip, port) + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_responsive(url) + ) + return url + + +@pytest.fixture(scope="session") +def config_env(docker_ip, docker_services): + """Set config for docker""" + os.environ["CONSUL_HOST"] = docker_ip + os.environ["CONSUL_PORT"] = "8500" diff --git a/tests/services_discovery_consulate/__init__.py b/tests/services_discovery_consulate/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/services_discovery_consulate/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/services_discovery_consulate/base.py b/tests/services_discovery_consulate/base.py new file mode 100644 index 0000000..6a65670 --- /dev/null +++ b/tests/services_discovery_consulate/base.py @@ -0,0 +1,66 @@ +import functools +import json +import os +import unittest +import uuid + +import httmock +import pytest + +from pyms.services_discovery import consulate +from pyms.services_discovery.consulate import exceptions + +CONSUL_TESTING_JSON = 'tests/services_discovery_consulate/consul.json' + +with open(CONSUL_TESTING_JSON, 'r') as handle: + CONSUL_CONFIG = json.load(handle) + + +def generate_key(func): + @functools.wraps(func) + def _decorator(self, *args, **kwargs): + key = str(uuid.uuid4())[0:8] + self.used_keys.append(key) + func(self, key) + + return _decorator + + +@httmock.all_requests +def raise_oserror(_url_unused, _request): + raise OSError + + +@pytest.mark.usefixtures("config_env") +class TestCase(unittest.TestCase): + def setUp(self): + self.consul = consulate.Consul( + host=os.environ['CONSUL_HOST'], + port=os.environ['CONSUL_PORT'], + token=CONSUL_CONFIG['acl']['tokens']['master']) + self.forbidden_consul = consulate.Consul( + host=os.environ['CONSUL_HOST'], + port=os.environ['CONSUL_PORT'], + token=str(uuid.uuid4())) + self.used_keys = list() + + def tearDown(self): + for key in self.consul.kv.keys(): + self.consul.kv.delete(key) + + checks = self.consul.agent.checks() + for name in checks: + self.consul.agent.check.deregister(checks[name]['CheckID']) + + services = self.consul.agent.services() + for name in services: + self.consul.agent.service.deregister(services[name]['ID']) + + for acl in self.consul.acl.list_tokens(): + if acl['AccessorID'] == CONSUL_CONFIG['acl']['tokens']['master']: + continue + try: + uuid.UUID(acl['AccessorID']) + self.consul.acl.delete_token(acl['AccessorID']) + except (ValueError, exceptions.ConsulateException): + pass diff --git a/tests/services_discovery_consulate/consul.json b/tests/services_discovery_consulate/consul.json new file mode 100644 index 0000000..3368abc --- /dev/null +++ b/tests/services_discovery_consulate/consul.json @@ -0,0 +1,16 @@ +{ + "acl": { + "enabled": true, + "enable_key_list_policy": true, + "tokens": { + "master": "9ae5fe1a-6b38-47e5-a0e7-f06b8b2fa645" + } + }, + "bootstrap_expect": 1, + "data_dir": "/tmp/consul", + "datacenter": "test", + "server": true, + "bind_addr": "{{ GetPrivateIP }}", + "client_addr": "0.0.0.0", + "enable_script_checks": true +} diff --git a/tests/services_discovery_consulate/test_acl.py b/tests/services_discovery_consulate/test_acl.py new file mode 100644 index 0000000..3ec3936 --- /dev/null +++ b/tests/services_discovery_consulate/test_acl.py @@ -0,0 +1,313 @@ +""" +Tests for Consulate.acl + +""" +import json +import uuid +import random + +import httmock + +from pyms.services_discovery import consulate +from pyms.services_discovery.consulate import exceptions + +from . import base + +ACL_OLD_RULES = """key "" { + policy = "read" +} +key "foo/" { + policy = "write" +} +""" + +ACL_NEW_RULES = """key_prefix "" { + policy = "read" +} +key "foo/" { + policy = "write" +} +""" + +ACL_NEW_UPDATE_RULES = """key_prefix "" { + policy = "deny" +} +key "foo/" { + policy = "read" +} +""" + +POLICYLINKS_SAMPLE = [ + dict(Name="policylink_sample"), +] + +POLICYLINKS_UPDATE_SAMPLE = [ + dict(Name="policylink_sample"), + dict(Name="policylink_update_sample") +] + +SERVICE_IDENTITIES_SAMPLE = [dict(ServiceName="db", Datacenters=list("dc1"))] + + +class TestCase(base.TestCase): + @staticmethod + def uuidv4(): + return str(uuid.uuid4()) + + @staticmethod + def random(): + return str(random.randint(0, 999999)) + + def _generate_policies(self): + sample = self.consul.acl.create_policy(self.random()) + sample_update = self.consul.acl.create_policy(self.random()) + return [dict(ID=sample["ID"]), dict(ID=sample_update["ID"])] + + def _generate_roles(self): + role = self.consul.acl.create_role(self.random()) + return [dict(ID=role["ID"])] + + def test_bootstrap_request_exception(self): + @httmock.all_requests + def response_content(_url_unused, _request): + raise OSError + + with httmock.HTTMock(response_content): + with self.assertRaises(exceptions.RequestError): + self.consul.acl.bootstrap() + + def test_bootstrap_success(self): + expectation = self.uuidv4() + + @httmock.all_requests + def response_content(_url_unused, request): + return httmock.response(200, json.dumps({'ID': expectation}), {}, + None, 0, request) + + with httmock.HTTMock(response_content): + result = self.consul.acl.bootstrap() + + self.assertEqual(result, expectation) + + def test_bootstrap_raises(self): + with self.assertRaises(consulate.Forbidden): + self.consul.acl.bootstrap() + + def test_clone_bad_acl_id(self): + with self.assertRaises(consulate.Forbidden): + self.consul.acl.clone(self.uuidv4()) + + def test_clone_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.acl.clone(self.uuidv4()) + + def test_create_and_destroy(self): + acl_id = self.consul.acl.create(self.uuidv4()) + self.assertTrue(self.consul.acl.destroy(acl_id)) + + def test_create_with_rules(self): + acl_id = self.consul.acl.create(self.uuidv4(), rules=ACL_OLD_RULES) + value = self.consul.acl.info(acl_id) + self.assertEqual(value['Rules'], ACL_OLD_RULES) + + def test_create_and_info(self): + acl_id = self.consul.acl.create(self.uuidv4()) + self.assertIsNotNone(acl_id) + data = self.consul.acl.info(acl_id) + self.assertIsNotNone(data) + self.assertEqual(acl_id, data.get('ID')) + + def test_create_and_list(self): + acl_id = self.consul.acl.create(self.uuidv4()) + data = self.consul.acl.list() + self.assertIn(acl_id, [r.get('ID') for r in data]) + + def test_create_and_clone(self): + acl_id = self.consul.acl.create(self.uuidv4()) + clone_id = self.consul.acl.clone(acl_id) + data = self.consul.acl.list() + self.assertIn(clone_id, [r.get('ID') for r in data]) + + def test_create_and_update(self): + acl_id = str(self.consul.acl.create(self.uuidv4())) + self.consul.acl.update(acl_id, 'Foo') + data = self.consul.acl.list() + self.assertIn('Foo', [r.get('Name') for r in data]) + self.assertIn(acl_id, [r.get('ID') for r in data]) + + def test_create_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.acl.create(self.uuidv4()) + + def test_destroy_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.acl.destroy(self.uuidv4()) + + def test_info_acl_id_not_found(self): + with self.assertRaises(consulate.NotFound): + self.forbidden_consul.acl.info(self.uuidv4()) + + def test_list_request_exception(self): + with httmock.HTTMock(base.raise_oserror): + with self.assertRaises(exceptions.RequestError): + self.consul.acl.list() + + def test_replication(self): + result = self.forbidden_consul.acl.replication() + self.assertFalse(result['Enabled']) + self.assertFalse(result['Running']) + + def test_update_not_found_adds_new_key(self): + acl_id = self.consul.acl.update(self.uuidv4(), 'Foo2') + data = self.consul.acl.list() + self.assertIn('Foo2', [r.get('Name') for r in data]) + self.assertIn(acl_id, [r.get('ID') for r in data]) + + def test_update_with_rules(self): + acl_id = self.consul.acl.update(self.uuidv4(), + name='test', + rules=ACL_OLD_RULES) + value = self.consul.acl.info(acl_id) + self.assertEqual(value['Rules'], ACL_OLD_RULES) + + def test_update_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.acl.update(self.uuidv4(), name='test') + + # NOTE: Everything above here is deprecated post consul-1.4.0 + + def test_create_policy(self): + name = self.random() + result = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) + self.assertEqual(result['Rules'], ACL_NEW_RULES) + + def test_create_and_read_policy(self): + name = self.random() + value = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) + result = self.consul.acl.read_policy(value["ID"]) + self.assertEqual(result['Rules'], ACL_NEW_RULES) + + def test_create_and_update_policy(self): + name = self.random() + value = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) + result = self.consul.acl.update_policy(value["ID"], + str(value["Name"]), + rules=ACL_NEW_UPDATE_RULES) + self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) + + def test_create_and_delete_policy(self): + name = self.random() + value = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) + result = self.consul.acl.delete_policy(value["ID"]) + self.assertTrue(result) + + def test_list_policy_exception(self): + with httmock.HTTMock(base.raise_oserror): + with self.assertRaises(exceptions.RequestError): + self.consul.acl.list_policies() + + def test_create_role(self): + name = self.random() + result = self.consul.acl.create_role( + name=name, + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + self.assertEqual(result['Name'], name) + + def test_create_and_read_role(self): + name = self.random() + value = self.consul.acl.create_role( + name=name, + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + result = self.consul.acl.read_role(value["ID"]) + self.assertEqual(result['ID'], value['ID']) + + def test_create_and_update_role(self): + name = self.random() + value = self.consul.acl.create_role( + name=name, + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + result = self.consul.acl.update_role( + value["ID"], + str(value["Name"]), + policies=self._generate_policies()) + self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) + + def test_create_and_delete_role(self): + name = self.random() + value = self.consul.acl.create_role( + name=name, + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + result = self.consul.acl.delete_role(value["ID"]) + self.assertTrue(result) + + def test_list_roles_exception(self): + with httmock.HTTMock(base.raise_oserror): + with self.assertRaises(exceptions.RequestError): + self.consul.acl.list_roles() + + def test_create_token(self): + secret_id = self.uuidv4() + accessor_id = self.uuidv4() + result = self.consul.acl.create_token( + accessor_id=accessor_id, + secret_id=secret_id, + roles=self._generate_roles(), + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + self.assertEqual(result['AccessorID'], accessor_id) + self.assertEqual(result['SecretID'], secret_id) + + def test_create_and_read_token(self): + secret_id = self.uuidv4() + accessor_id = self.uuidv4() + value = self.consul.acl.create_token( + accessor_id=accessor_id, + secret_id=secret_id, + roles=self._generate_roles(), + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + result = self.consul.acl.read_token(value["AccessorID"]) + self.assertEqual(result['AccessorID'], accessor_id) + + def test_create_and_update_token(self): + secret_id = self.uuidv4() + accessor_id = self.uuidv4() + value = self.consul.acl.create_token( + accessor_id=accessor_id, + secret_id=secret_id, + roles=self._generate_roles(), + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + result = self.consul.acl.update_token( + str(value["AccessorID"]), policies=self._generate_policies()) + self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) + + def test_create_and_clone_token(self): + secret_id = self.uuidv4() + accessor_id = self.uuidv4() + clone_description = "clone token of " + accessor_id + value = self.consul.acl.create_token( + accessor_id=accessor_id, + secret_id=secret_id, + roles=self._generate_roles(), + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + result = self.consul.acl.clone_token(value["AccessorID"], + description=clone_description) + self.assertEqual(result["Description"], clone_description) + + def test_create_and_delete_token(self): + secret_id = self.uuidv4() + accessor_id = self.uuidv4() + value = self.consul.acl.create_token( + accessor_id=accessor_id, + secret_id=secret_id, + roles=self._generate_roles(), + policies=self._generate_policies(), + service_identities=SERVICE_IDENTITIES_SAMPLE) + result = self.consul.acl.delete_token(value["AccessorID"]) + self.assertTrue(result) diff --git a/tests/services_discovery_consulate/test_agent.py b/tests/services_discovery_consulate/test_agent.py new file mode 100644 index 0000000..e50aeca --- /dev/null +++ b/tests/services_discovery_consulate/test_agent.py @@ -0,0 +1,305 @@ +""" +Tests for Consulate.agent + +""" +import uuid + +import httmock + +from pyms.services_discovery import consulate +from pyms.services_discovery.consulate import utils +from pyms.services_discovery.consulate.models import agent + +from . import base + + +class TestCase(base.TestCase): + def test_checks(self): + result = self.consul.agent.checks() + self.assertDictEqual(result, {}) + + def test_force_leave(self): + self.assertTrue(self.consul.agent.force_leave(str(uuid.uuid4()))) + + def test_force_leave_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.force_leave(str(uuid.uuid4())) + + # def test_join(self): + # self.assertTrue(self.consul.agent.join('127.0.0.1')) + + def test_join_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.join('255.255.255.255') + + def test_maintenance(self): + self.consul.agent.maintenance(True, 'testing') + self.consul.agent.maintenance(False) + + def test_maintenance_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.maintenance(True) + + def test_members(self): + result = self.consul.agent.members() + self.assertEqual(len(result), 1) + + def test_members_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.members() + + def test_metrics(self): + result = self.consul.agent.metrics() + self.assertIn('Timestamp', result) + self.assertIn('Gauges', result) + + def test_metrics_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.metrics() + + def test_monitor(self): + for offset, line in enumerate(self.consul.agent.monitor()): + self.assertTrue(utils.is_string(line)) + self.consul.agent.metrics() + if offset > 1: + break + + def test_monitor_request_exception(self): + with httmock.HTTMock(base.raise_oserror): + with self.assertRaises(consulate.RequestError): + for _line in self.consul.agent.monitor(): + break + + def test_monitor_forbidden(self): + with self.assertRaises(consulate.Forbidden): + for line in self.forbidden_consul.agent.monitor(): + self.assertIsInstance(line, str) + break + + def test_reload(self): + self.assertIsNone(self.consul.agent.reload()) + + def test_reload_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.reload() + + def test_self(self): + result = self.consul.agent.self() + self.assertIn('Config', result) + self.assertIn('Coord', result) + self.assertIn('Member', result) + + def test_self_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.self() + + def test_service_registration(self): + self.consul.agent.service.register( + 'test-service', address='10.0.0.1', port=5672, tags=['foo', 'bar'], meta={'foo' : 'bar' }) + self.assertIn('test-service', self.consul.agent.services()) + self.consul.agent.service.deregister('test-service') + + def test_service_maintenance(self): + self.consul.agent.service.register( + 'test-service', address='10.0.0.1', port=5672, tags=['foo', 'bar'], meta={'foo' : 'bar' } ) + self.assertIn('test-service', self.consul.agent.services()) + reason = 'Down for Acceptance' + self.consul.agent.service.maintenance('test-service', reason=reason) + node_in_maintenance = self.consul.catalog.nodes()[0]['Node'] + health_check = self.consul.health.node(node_in_maintenance) + self.assertEqual(len(health_check), 2) + self.assertIn(reason, [check['Notes'] for check in health_check]) + self.consul.agent.service.maintenance('test-service', enable=False) + health_check = self.consul.health.node(node_in_maintenance) + self.assertEqual(len(health_check), 1) + self.assertNotEqual(reason, health_check[0]['Notes']) + self.consul.agent.service.deregister('test-service') + + def test_token(self): + self.assertTrue( + self.consul.agent.token('acl_replication_token', 'foo')) + + def test_token_invalid(self): + with self.assertRaises(ValueError): + self.consul.agent.token('acl_replication_tokens', 'foo') + + def test_token_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.token('acl_replication_token', 'foo') + + +class CheckTestCase(base.TestCase): + + def test_register(self): + self.assertTrue(self.consul.agent.check.register( + str(uuid.uuid4()), http='http://localhost', interval='30s')) + + def test_register_args_and_no_interval(self): + with self.assertRaises(ValueError): + self.consul.agent.check.register( + str(uuid.uuid4()), args=['/bin/true']) + + def test_register_args_and_ttl(self): + with self.assertRaises(ValueError): + self.consul.agent.check.register( + str(uuid.uuid4()), args=['/bin/true'], ttl='30s') + + def test_register_http_and_no_interval(self): + with self.assertRaises(ValueError): + self.consul.agent.check.register( + str(uuid.uuid4()), http='http://localhost') + + def test_register_args_and_http(self): + with self.assertRaises(ValueError): + self.consul.agent.check.register( + str(uuid.uuid4()), args=['/bin/true'], http='http://localhost') + + def test_register_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.check.register( + str(uuid.uuid4()), args=['/bin/true'], interval='30s') + + +class TTLCheckTestCase(base.TestCase): + + def setUp(self): + super(TTLCheckTestCase, self).setUp() + name = str(uuid.uuid4()) + self.assertTrue(self.consul.agent.check.register(name, ttl='30s')) + checks = self.consul.agent.checks() + self.check_id = checks[name]['CheckID'] + + def test_pass(self): + self.assertTrue(self.consul.agent.check.ttl_pass(self.check_id)) + + def test_pass_with_note(self): + self.assertTrue( + self.consul.agent.check.ttl_pass(self.check_id, 'PASS')) + + def test_warn(self): + self.assertTrue(self.consul.agent.check.ttl_warn(self.check_id)) + + def test_warn_with_note(self): + self.assertTrue( + self.consul.agent.check.ttl_warn(self.check_id, 'WARN')) + + def test_fail(self): + self.assertTrue(self.consul.agent.check.ttl_fail(self.check_id)) + + def test_fail_with_note(self): + self.assertTrue( + self.consul.agent.check.ttl_fail(self.check_id, 'FAIL')) + + +class ServiceTestCase(base.TestCase): + + def test_register(self): + self.assertTrue( + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80, + check=agent.Check( + name='test', args=['/bin/true'], interval='30s'), + tags=[str(uuid.uuid4())])) + + def test_register_grpc(self): + self.assertTrue( + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80, + check=agent.Check( + name='test', grpc='https://grpc/status', interval='30s'))) + + def test_register_http(self): + self.assertTrue( + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80, + check=agent.Check( + name='test', http='http://localhost', interval='30s'))) + + def test_register_tcp(self): + self.assertTrue( + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80, + check=agent.Check( + name='test', tcp='localhost:80', interval='30s'))) + + def test_register_ttl(self): + self.assertTrue( + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80, + check=agent.Check(name='test', ttl='30s'))) + + def test_register_multiple_checks(self): + self.assertTrue( + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80, + checks=[ + agent.Check( + name='test1', http='http://netloc', + header={'User-Agent': 'unittest.TestCase'}, + interval='30s'), + agent.Check(name='test2', ttl='30s') + ])) + + def test_register_forbidden(self): + with self.assertRaises(consulate.Forbidden): + self.forbidden_consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80) + + def test_register_invalid_check(self): + with self.assertRaises(TypeError): + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + check=str(uuid.uuid4())) + + def test_register_invalid_checks(self): + with self.assertRaises(ValueError): + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + checks=[str(uuid.uuid4())]) + + def test_register_invalid_port(self): + with self.assertRaises(TypeError): + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port='80') + + def test_register_invalid_tags(self): + with self.assertRaises(TypeError): + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + tags=str(uuid.uuid4())) + + def test_register_invalid_interval(self): + with self.assertRaises(TypeError): + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80, + check=agent.Check( + name='test', http='http://localhost', interval=30)) + + def test_register_invalid_ttl(self): + with self.assertRaises(TypeError): + self.consul.agent.service.register( + str(uuid.uuid4()), + address='127.0.0.1', + port=80, + check=agent.Check(name='test', ttl=30)) diff --git a/tests/services_discovery_consulate/test_api.py b/tests/services_discovery_consulate/test_api.py new file mode 100644 index 0000000..369d159 --- /dev/null +++ b/tests/services_discovery_consulate/test_api.py @@ -0,0 +1,363 @@ +import json +import unittest +import uuid +from urllib import parse + +import httmock +import mock + +from pyms.services_discovery import consulate +from pyms.services_discovery.consulate import adapters +from pyms.services_discovery.consulate.api import base +from tests.services_discovery_consulate.base import CONSUL_TESTING_JSON + +with open(CONSUL_TESTING_JSON, 'r') as handle: + CONSUL_CONFIG = json.load(handle) + +SCHEME = 'http' +VERSION = 'v1' + + +class ConsulTests(unittest.TestCase): + @mock.patch('pyms.services_discovery.consulate.adapters.Request') + @mock.patch('pyms.services_discovery.consulate.api.Agent') + @mock.patch('pyms.services_discovery.consulate.api.Catalog') + @mock.patch('pyms.services_discovery.consulate.api.KV') + @mock.patch('pyms.services_discovery.consulate.api.Health') + @mock.patch('pyms.services_discovery.consulate.api.Coordinate') + @mock.patch('pyms.services_discovery.consulate.api.ACL') + @mock.patch('pyms.services_discovery.consulate.api.Event') + @mock.patch('pyms.services_discovery.consulate.api.Session') + @mock.patch('pyms.services_discovery.consulate.api.Status') + def setUp(self, status, session, event, acl, health, coordinate, kv, catalog, agent, + adapter): + self.host = '127.0.0.1' + self.port = 8500 + self.dc = CONSUL_CONFIG['datacenter'] + self.token = CONSUL_CONFIG['acl']['tokens']['master'] + + self.acl = acl + self.adapter = adapter + self.agent = agent + self.catalog = catalog + self.event = event + self.kv = kv + self.health = health + self.coordinate = coordinate + self.session = session + self.status = status + + self.base_uri = '{0}://{1}:{2}/v1'.format(SCHEME, self.host, self.port) + self.consul = consulate.Consul(self.host, self.port, self.dc, + self.token) + + def test_base_uri(self): + self.assertEquals( + self.consul._base_uri(SCHEME, self.host, self.port), self.base_uri) + + def test_unix_socket_base_uri(self): + expectation = 'http+unix://%2Fvar%2Flib%2Fconsul%2Fconsul.sock/v1' + self.assertEquals( + self.consul._base_uri('http+unix', '/var/lib/consul/consul.sock', + None), expectation) + + def test_acl_initialization(self): + self.assertTrue( + self.acl.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_adapter_initialization(self): + self.assertTrue(self.adapter.called_once_with()) + + def test_agent_initialization(self): + self.assertTrue( + self.agent.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_catalog_initialization(self): + self.assertTrue( + self.catalog.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_events_initialization(self): + self.assertTrue( + self.event.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_kv_initialization(self): + self.assertTrue( + self.kv.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_health_initialization(self): + self.assertTrue( + self.health.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_coordinate_initialization(self): + self.assertTrue( + self.coordinate.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_session_initialization(self): + self.assertTrue( + self.session.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_status_initialization(self): + self.assertTrue( + self.status.called_once_with(self.base_uri, self.adapter, self.dc, + self.token)) + + def test_acl_property(self): + self.assertEqual(self.consul.acl, self.consul._acl) + + def test_agent_property(self): + self.assertEqual(self.consul.agent, self.consul._agent) + + def test_catalog_property(self): + self.assertEqual(self.consul.catalog, self.consul._catalog) + + def test_event_property(self): + self.assertEqual(self.consul.event, self.consul._event) + + def test_health_property(self): + self.assertEqual(self.consul.health, self.consul._health) + + def test_coordinate_property(self): + self.assertEqual(self.consul.coordinate, self.consul._coordinate) + + def test_kv_property(self): + self.assertEqual(self.consul.kv, self.consul._kv) + + def test_status_property(self): + self.assertEqual(self.consul.status, self.consul._status) + + +class EndpointBuildURITests(unittest.TestCase): + def setUp(self): + self.adapter = adapters.Request() + self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.endpoint = base.Endpoint(self.base_uri, self.adapter) + + def test_adapter_assignment(self): + self.assertEqual(self.endpoint._adapter, self.adapter) + + def test_base_uri_assignment(self): + self.assertEqual(self.endpoint._base_uri, '{0}/endpoint'.format( + self.base_uri)) + + def test_dc_assignment(self): + self.assertIsNone(self.endpoint._dc) + + def test_token_assignment(self): + self.assertIsNone(self.endpoint._token) + + def test_build_uri_with_no_params(self): + result = self.endpoint._build_uri(['foo', 'bar']) + parsed = parse.urlparse(result) + query_params = parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, SCHEME) + self.assertEqual(parsed.netloc, 'localhost:8500') + self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertDictEqual(query_params, {}) + + def test_build_uri_with_params(self): + result = self.endpoint._build_uri(['foo', 'bar'], {'baz': 'qux'}) + parsed = parse.urlparse(result) + query_params = parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, SCHEME) + self.assertEqual(parsed.netloc, 'localhost:8500') + self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertDictEqual(query_params, {'baz': ['qux']}) + + +class EndpointBuildURIWithDCTests(unittest.TestCase): + def setUp(self): + self.adapter = adapters.Request() + self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.dc = str(uuid.uuid4()) + self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc) + + def test_dc_assignment(self): + self.assertEqual(self.endpoint._dc, self.dc) + + def test_token_assignment(self): + self.assertIsNone(self.endpoint._token) + + def test_build_uri_with_no_params(self): + result = self.endpoint._build_uri(['foo', 'bar']) + parsed = parse.urlparse(result) + query_params = parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, SCHEME) + self.assertEqual(parsed.netloc, 'localhost:8500') + self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertDictEqual(query_params, {'dc': [self.dc]}) + + def test_build_uri_with_params(self): + result = self.endpoint._build_uri(['foo', 'bar'], {'baz': 'qux'}) + parsed = parse.urlparse(result) + query_params = parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, SCHEME) + self.assertEqual(parsed.netloc, 'localhost:8500') + self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertDictEqual(query_params, {'dc': [self.dc], 'baz': ['qux']}) + + +class EndpointBuildURIWithTokenTests(unittest.TestCase): + def setUp(self): + self.adapter = adapters.Request() + self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.token = str(uuid.uuid4()) + self.endpoint = base.Endpoint( + self.base_uri, self.adapter, token=self.token) + + def test_dc_assignment(self): + self.assertIsNone(self.endpoint._dc) + + def test_token_assignment(self): + self.assertEqual(self.endpoint._token, self.token) + + def test_build_uri_with_no_params(self): + result = self.endpoint._build_uri(['foo', 'bar']) + parsed = parse.urlparse(result) + query_params = parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, SCHEME) + self.assertEqual(parsed.netloc, 'localhost:8500') + self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertDictEqual(query_params, {'token': [self.token]}) + + def test_build_uri_with_params(self): + result = self.endpoint._build_uri(['foo', 'bar'], {'baz': 'qux'}) + parsed = parse.urlparse(result) + query_params = parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, SCHEME) + self.assertEqual(parsed.netloc, 'localhost:8500') + self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertDictEqual(query_params, { + 'token': [self.token], + 'baz': ['qux'] + }) + + +class EndpointBuildURIWithDCAndTokenTests(unittest.TestCase): + def setUp(self): + self.adapter = adapters.Request() + self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.dc = str(uuid.uuid4()) + self.token = str(uuid.uuid4()) + self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, + self.token) + + def test_dc_assignment(self): + self.assertEqual(self.endpoint._dc, self.dc) + + def test_token_assignment(self): + self.assertEqual(self.endpoint._token, self.token) + + def test_build_uri_with_no_params(self): + result = self.endpoint._build_uri(['foo', 'bar']) + parsed = parse.urlparse(result) + query_params = parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, SCHEME) + self.assertEqual(parsed.netloc, 'localhost:8500') + self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertDictEqual(query_params, { + 'dc': [self.dc], + 'token': [self.token] + }) + + def test_build_uri_with_params(self): + result = self.endpoint._build_uri(['foo', 'bar'], {'baz': 'qux'}) + parsed = parse.urlparse(result) + query_params = parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, SCHEME) + self.assertEqual(parsed.netloc, 'localhost:8500') + self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertDictEqual(query_params, { + 'dc': [self.dc], + 'token': [self.token], + 'baz': ['qux'] + }) + + +class EndpointGetTests(unittest.TestCase): + def setUp(self): + self.adapter = adapters.Request() + self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.dc = str(uuid.uuid4()) + self.token = str(uuid.uuid4()) + self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, + self.token) + + def test_get_200_returns_response_body(self): + @httmock.all_requests + def response_content(_url_unused, request): + headers = { + 'X-Consul-Index': 4, + 'X-Consul-Knownleader': 'true', + 'X-Consul-Lastcontact': 0, + 'Date': 'Fri, 19 Dec 2014 20:44:28 GMT', + 'Content-Length': 13, + 'Content-Type': 'application/json' + } + content = b'{"consul": []}' + return httmock.response(200, content, headers, None, 0, request) + + with httmock.HTTMock(response_content): + values = self.endpoint._get([str(uuid.uuid4())]) + self.assertEqual(values, {'consul': []}) + + def test_get_404_returns_empty_list(self): + @httmock.all_requests + def response_content(_url_unused, request): + headers = { + 'content-length': 0, + 'content-type': 'text/plain; charset=utf-8' + } + return httmock.response(404, None, headers, None, 0, request) + + with httmock.HTTMock(response_content): + values = self.endpoint._get([str(uuid.uuid4())]) + self.assertEqual(values, []) + + +class EndpointGetListTests(unittest.TestCase): + def setUp(self): + self.adapter = adapters.Request() + self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.dc = str(uuid.uuid4()) + self.token = str(uuid.uuid4()) + self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, + self.token) + + def test_get_list_200_returns_response_body(self): + @httmock.all_requests + def response_content(_url_unused, request): + headers = { + 'X-Consul-Index': 4, + 'X-Consul-Knownleader': 'true', + 'X-Consul-Lastcontact': 0, + 'Date': 'Fri, 19 Dec 2014 20:44:28 GMT', + 'Content-Length': 13, + 'Content-Type': 'application/json' + } + content = b'{"consul": []}' + return httmock.response(200, content, headers, None, 0, request) + + with httmock.HTTMock(response_content): + values = self.endpoint._get_list([str(uuid.uuid4())]) + self.assertEqual(values, [{'consul': []}]) + + def test_get_list_404_returns_empty_list(self): + @httmock.all_requests + def response_content(_url_unused, request): + headers = { + 'content-length': 0, + 'content-type': 'text/plain; charset=utf-8' + } + return httmock.response(404, None, headers, None, 0, request) + + with httmock.HTTMock(response_content): + values = self.endpoint._get_list([str(uuid.uuid4())]) + self.assertEqual(values, []) diff --git a/tests/services_discovery_consulate/test_base_model.py b/tests/services_discovery_consulate/test_base_model.py new file mode 100644 index 0000000..82ec806 --- /dev/null +++ b/tests/services_discovery_consulate/test_base_model.py @@ -0,0 +1,139 @@ +# coding=utf-8 +"""Tests for the Base Model""" +import unittest +import uuid + +from pyms.services_discovery.consulate.models import base + + +class TestModel(base.Model): + """Model to perform tests against""" + __slots__ = ['id', 'serial', 'name', 'value'] + __attributes__ = { + 'id': { + 'key': 'ID', + 'type': uuid.UUID, + 'cast_from': str, + 'cast_to': str, + }, + 'serial': { + 'key': 'Serial', + 'type': int, + 'default': 0, + 'required': True, + 'validator': lambda v, _m: v >= 0, + }, + 'name': { + 'key': 'Name', + 'type': str, + 'required': True + }, + 'value': { + 'type': str + }, + 'type': { + 'key': 'Type', + 'type': str, + 'enum': {'client', 'server'} + } + } + + +class TestCase(unittest.TestCase): + + def test_happy_case_with_defaults(self): + kwargs = { + 'id': uuid.uuid4(), + 'name': str(uuid.uuid4()) + } + model = TestModel(**kwargs) + for key, value in kwargs.items(): + self.assertEqual(getattr(model, key), value) + self.assertEqual(model.serial, 0) + + def test_happy_case_with_all_values(self): + kwargs = { + 'id': uuid.uuid4(), + 'serial': 1, + 'name': str(uuid.uuid4()), + 'value': str(uuid.uuid4()) + } + model = TestModel(**kwargs) + for key, value in kwargs.items(): + self.assertEqual(getattr(model, key), value) + + def test_cast_from_str(self): + expectation = uuid.uuid4() + kwargs = { + 'id': str(expectation), + 'name': str(uuid.uuid4()) + } + model = TestModel(**kwargs) + self.assertEqual(model.id, expectation) + + def test_validator_failure(self): + kwargs = { + 'id': uuid.uuid4(), + 'name': str(uuid.uuid4()), + 'serial': -1 + } + with self.assertRaises(ValueError): + TestModel(**kwargs) + + def test_type_failure(self): + kwargs = { + 'id': True, + 'name': str(uuid.uuid4()) + } + with self.assertRaises(TypeError): + TestModel(**kwargs) + + def test_missing_requirement(self): + with self.assertRaises(ValueError): + TestModel() + + def test_invalid_attribute(self): + kwargs = {'name': str(uuid.uuid4()), 'foo': 'bar'} + with self.assertRaises(AttributeError): + TestModel(**kwargs) + + def test_invalid_attribute_assignment(self): + kwargs = {'name': str(uuid.uuid4())} + model = TestModel(**kwargs) + with self.assertRaises(AttributeError): + model.foo = 'bar' + + def test_invalid_enum_assignment(self): + kwargs = {'name': str(uuid.uuid4()), 'type': 'invalid'} + with self.assertRaises(ValueError): + TestModel(**kwargs) + + def test_cast_to_dict(self): + kwargs = { + 'id': uuid.uuid4(), + 'serial': 1, + 'name': str(uuid.uuid4()), + 'value': str(uuid.uuid4()), + 'type': 'client' + } + expectation = { + 'ID': str(kwargs['id']), + 'Serial': kwargs['serial'], + 'Name': kwargs['name'], + 'value': kwargs['value'], + 'Type': kwargs['type'] + } + model = TestModel(**kwargs) + self.assertDictEqual(dict(model), expectation) + + def test_cast_to_dict_only_requirements(self): + kwargs = { + 'serial': 1, + 'name': str(uuid.uuid4()) + } + expectation = { + 'Serial': kwargs['serial'], + 'Name': kwargs['name'] + } + model = TestModel(**kwargs) + self.assertDictEqual(dict(model), expectation) diff --git a/tests/services_discovery_consulate/test_catalog.py b/tests/services_discovery_consulate/test_catalog.py new file mode 100644 index 0000000..cb89649 --- /dev/null +++ b/tests/services_discovery_consulate/test_catalog.py @@ -0,0 +1,11 @@ +from . import base + + +class TestCatalog(base.TestCase): + def test_catalog_registration(self): + self.consul.catalog.register('test-service', address='10.0.0.1') + self.assertIn('test-service', + [n['Node'] for n in self.consul.catalog.nodes()]) + self.consul.catalog.deregister('test-service') + self.assertNotIn('test-service', + [n['Node'] for n in self.consul.catalog.nodes()]) diff --git a/tests/services_discovery_consulate/test_coordinate.py b/tests/services_discovery_consulate/test_coordinate.py new file mode 100644 index 0000000..78cb463 --- /dev/null +++ b/tests/services_discovery_consulate/test_coordinate.py @@ -0,0 +1,6 @@ +from . import base + +class TestCoordinate(base.TestCase): + def test_coordinate(self): + coordinates = self.consul.coordinate.nodes() + self.assertIsInstance(coordinates, list) diff --git a/tests/services_discovery_consulate/test_event.py b/tests/services_discovery_consulate/test_event.py new file mode 100644 index 0000000..b083a83 --- /dev/null +++ b/tests/services_discovery_consulate/test_event.py @@ -0,0 +1,18 @@ +import uuid + +from . import base + + +class TestEvent(base.TestCase): + def test_fire(self): + event_name = 'test-event-%s' % str(uuid.uuid4())[0:8] + response = self.consul.event.fire(event_name) + events = self.consul.event.list(event_name) + if isinstance(events, dict): + self.assertEqual(event_name, events.get('Name')) + self.assertEqual(response, events.get('ID')) + elif isinstance(events, dict): + self.assertIn(event_name, [e.get('Name') for e in events]) + self.assertIn(response, [e.get('ID') for e in events]) + else: + assert False, 'Unexpected return type' diff --git a/tests/services_discovery_consulate/test_kv.py b/tests/services_discovery_consulate/test_kv.py new file mode 100644 index 0000000..2fc11be --- /dev/null +++ b/tests/services_discovery_consulate/test_kv.py @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +import json +import unittest +import uuid + +import httmock + +from pyms.services_discovery.consulate import adapters, api +from . import base + +SCHEME = 'http' +VERSION = 'v1' + +ALL_DATA = (b'[{"CreateIndex":643,"ModifyIndex":643,"LockIndex":0,"Key":"bar",' + b'"Flags":0,"Value":"YmF6"},{"CreateIndex":669,"ModifyIndex":669,"' + b'LockIndex":0,"Key":"baz","Flags":0,"Value":"cXV4"},{"CreateIndex' + b'":666,"ModifyIndex":666,"LockIndex":0,"Key":"corgie","Flags":128' + b',"Value":"ZG9n"},{"CreateIndex":642,"ModifyIndex":642,"LockIndex' + b'":0,"Key":"foo","Flags":0,"Value":"YmFy"},{"CreateIndex":644,"Mo' + b'difyIndex":644,"LockIndex":0,"Key":"quz","Flags":0,"Value":"dHJ1' + b'ZQ=="}]') + +ALL_ITEMS = [{ + 'CreateIndex': 643, + 'Flags': 0, + 'Key': 'bar', + 'LockIndex': 0, + 'ModifyIndex': 643, + 'Value': 'baz' +}, { + 'CreateIndex': 669, + 'Flags': 0, + 'Key': 'baz', + 'LockIndex': 0, + 'ModifyIndex': 669, + 'Value': 'qux' +}, { + 'CreateIndex': 666, + 'Flags': 128, + 'Key': 'corgie', + 'LockIndex': 0, + 'ModifyIndex': 666, + 'Value': 'dog' +}, { + 'CreateIndex': 642, + 'Flags': 0, + 'Key': 'foo', + 'LockIndex': 0, + 'ModifyIndex': 642, + 'Value': 'bar' +}, { + 'CreateIndex': 644, + 'Flags': 0, + 'Key': 'quz', + 'LockIndex': 0, + 'ModifyIndex': 644, + 'Value': 'true' +}] + + +@httmock.all_requests +def kv_all_records_content(_url_unused, request): + return httmock.response( + 200, ALL_DATA, { + 'X-Consul-Index': 4, + 'X-Consul-Knownleader': 'true', + 'X-Consul-Lastcontact': 0, + 'Date': 'Fri, 19 Dec 2014 20:44:28 GMT', + 'Content-Length': len(ALL_DATA), + 'Content-Type': 'application/json' + }, None, 0, request) + + +class KVTests(unittest.TestCase): + def setUp(self): + self.adapter = adapters.Request() + self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.dc = str(uuid.uuid4()) + self.token = str(uuid.uuid4()) + self.kv = api.KV(self.base_uri, self.adapter, self.dc, self.token) + + def test_contains_evaluates_true(self): + @httmock.all_requests + def response_content(_url_unused, request): + return httmock.response(200, None, {}, None, 0, request) + + with httmock.HTTMock(response_content): + self.assertIn('foo', self.kv) + + def test_contains_evaluates_false(self): + @httmock.all_requests + def response_content(_url_unused, request): + return httmock.response(404, None, {}, None, 0, request) + + with httmock.HTTMock(response_content): + self.assertNotIn('foo', self.kv) + + def test_get_all_items(self): + with httmock.HTTMock(kv_all_records_content): + for index, row in enumerate(self.kv._get_all_items()): + self.assertDictEqual(row, ALL_ITEMS[index]) + + def test_items(self): + with httmock.HTTMock(kv_all_records_content): + for index, row in enumerate(self.kv.items()): + value = {ALL_ITEMS[index]['Key']: ALL_ITEMS[index]['Value']} + self.assertDictEqual(row, value) + + def test_iter(self): + with httmock.HTTMock(kv_all_records_content): + for index, row in enumerate(self.kv): + self.assertEqual(row, ALL_ITEMS[index]['Key']) + + def test_iteritems(self): + with httmock.HTTMock(kv_all_records_content): + for index, row in enumerate(self.kv.iteritems()): + value = (ALL_ITEMS[index]['Key'], ALL_ITEMS[index]['Value']) + self.assertEqual(row, value) + + def test_keys(self): + expectation = [item['Key'] for item in ALL_ITEMS] + with httmock.HTTMock(kv_all_records_content): + self.assertEqual(self.kv.keys(), expectation) + + def test_len(self): + with httmock.HTTMock(kv_all_records_content): + self.assertEqual(len(self.kv), len(ALL_ITEMS)) + + def test_values(self): + with httmock.HTTMock(kv_all_records_content): + for index, row in enumerate(self.kv.values()): + self.assertEqual(row, ALL_ITEMS[index]['Value']) + + +class TestKVGetWithNoKey(base.TestCase): + @base.generate_key + def test_get_is_none(self, key): + self.assertIsNone(self.consul.kv.get(key)) + + @base.generate_key + def test_get_item_raises_key_error(self, key): + self.assertRaises(KeyError, self.consul.kv.__getitem__, key) + + +class TestKVSet(base.TestCase): + @base.generate_key + def test_set_item_del_item(self, key): + self.consul.kv[key] = 'foo' + del self.consul.kv[key] + self.assertNotIn(key, self.consul.kv) + + @base.generate_key + def test_set_item_get_item_bool_value(self, key): + self.consul.kv[key] = True + self.assertTrue(self.consul.kv[key]) + + @base.generate_key + def test_set_path_with_value(self, key): + path = 'path/{0}/'.format(key) + self.consul.kv.set(path, 'bar') + self.assertEqual('bar', self.consul.kv[path[:-1]]) + + @base.generate_key + def test_set_item_get_item_int_value(self, key): + self.consul.kv[key] = 128 + self.assertEqual(self.consul.kv[key], '128') + + @base.generate_key + def test_set_item_get_item_str_value(self, key): + self.consul.kv[key] = b'foo' + self.assertEqual(self.consul.kv[key], 'foo') + + @base.generate_key + def test_set_item_get_item_str_value_raw(self, key): + self.consul.kv[key] = 'foo' + self.assertEqual(self.consul.kv.get(key, raw=True), 'foo') + + @base.generate_key + def test_set_get_bool_value(self, key): + self.consul.kv.set(key, True) + self.assertTrue(self.consul.kv.get(key)) + + @base.generate_key + def test_set_get_item_value(self, key): + self.consul.kv.set(key, 128) + self.assertEqual(self.consul.kv.get(key), '128') + + @base.generate_key + def test_set_item_get_item_str_value(self, key): + self.consul.kv.set(key, 'foo') + self.assertEqual(self.consul.kv.get(key), 'foo') + + @base.generate_key + def test_set_item_get_record(self, key): + self.consul.kv.set_record(key, 12, 'record') + record = self.consul.kv.get_record(key) + self.assertEqual('record', record['Value']) + self.assertEqual(12, record['Flags']) + self.assertIsInstance(record, dict) + + @base.generate_key + def test_get_record_fail(self, key): + self.assertEqual(self.consul.kv.get_record(key), None) + + @base.generate_key + def test_set_record_no_replace_get_item_str_value(self, key): + self.consul.kv.set(key, 'foo') + self.consul.kv.set_record(key, 0, 'foo', False) + self.assertEqual(self.consul.kv.get(key), 'foo') + + @base.generate_key + def test_set_record_same_value_get_item_str_value(self, key): + self.consul.kv.set(key, 'foo') + self.consul.kv.set_record(key, 0, 'foo', True) + self.assertEqual(self.consul.kv.get(key), 'foo') + + @base.generate_key + def test_set_item_get_item_dict_value(self, key): + value = {'foo': 'bar'} + expectation = json.dumps(value) + self.consul.kv.set(key, value) + self.assertEqual(self.consul.kv.get(key), expectation) + + @base.generate_key + def test_set_item_get_item_unicode_value(self, key): + self.consul.kv.set(key, 'I like to ✈') + response = self.consul.kv.get(key) + self.assertEqual(response, 'I like to ✈') + + @base.generate_key + def test_set_item_in_records(self, key): + self.consul.kv.set(key, 'zomg') + expectation = (key, 0, 'zomg') + self.assertIn(expectation, self.consul.kv.records()) + + @base.generate_key + def test_set_binary_value(self, key): + value = uuid.uuid4().bytes + self.consul.kv.set(key, value) + expectation = (key, 0, value) + self.assertIn(expectation, self.consul.kv.records()) + + +class TestKVLocking(base.TestCase): + @base.generate_key + def test_acquire_and_release_lock(self, key): + lock_key = str(uuid.uuid4())[0:8] + session_id = self.consul.session.create( + key, behavior='delete', ttl='60s') + self.assertTrue(self.consul.kv.acquire_lock(lock_key, session_id)) + self.assertTrue(self.consul.kv.release_lock(lock_key, session_id)) + self.consul.session.destroy(session_id) + + @base.generate_key + def test_acquire_and_release_lock(self, key): + lock_key = str(uuid.uuid4())[0:8] + sid = self.consul.session.create(key, behavior='delete', ttl='60s') + sid2 = self.consul.session.create( + key + '2', behavior='delete', ttl='60s') + self.assertTrue(self.consul.kv.acquire_lock(lock_key, sid)) + self.assertFalse(self.consul.kv.acquire_lock(lock_key, sid2)) + self.assertTrue(self.consul.kv.release_lock(lock_key, sid)) + self.consul.session.destroy(sid) + self.consul.session.destroy(sid2) + + @base.generate_key + def test_acquire_and_release_lock_with_value(self, key): + lock_key = str(uuid.uuid4())[0:8] + lock_value = str(uuid.uuid4()) + sid = self.consul.session.create(key, behavior='delete', ttl='60s') + sid2 = self.consul.session.create( + key + '2', behavior='delete', ttl='60s') + self.assertTrue(self.consul.kv.acquire_lock(lock_key, sid, lock_value)) + self.assertEqual(self.consul.kv.get(lock_key), lock_value) + self.assertFalse(self.consul.kv.acquire_lock(lock_key, sid2)) + self.assertTrue(self.consul.kv.release_lock(lock_key, sid)) + self.consul.session.destroy(sid) + self.consul.session.destroy(sid2) diff --git a/tests/services_discovery_consulate/test_lock.py b/tests/services_discovery_consulate/test_lock.py new file mode 100644 index 0000000..0cf3a6f --- /dev/null +++ b/tests/services_discovery_consulate/test_lock.py @@ -0,0 +1,10 @@ +import uuid + +from . import base + + +class TestLock(base.TestCase): + def test_lock_as_context_manager(self): + value = str(uuid.uuid4()) + with self.consul.lock.acquire(value=value): + self.assertEqual(self.consul.kv.get(self.consul.lock.key), value) diff --git a/tests/services_discovery_consulate/test_session.py b/tests/services_discovery_consulate/test_session.py new file mode 100644 index 0000000..4edbad9 --- /dev/null +++ b/tests/services_discovery_consulate/test_session.py @@ -0,0 +1,43 @@ +import uuid + +from . import base + + +class TestSession(base.TestCase): + def setUp(self): + super(TestSession, self).setUp() + self.sessions = list() + + def tearDown(self): + for session in self.sessions: + self.consul.session.destroy(session) + + def test_session_create(self): + name = str(uuid.uuid4())[0:8] + session_id = self.consul.session.create( + name, behavior='delete', ttl='60s') + self.sessions.append(session_id) + self.assertIsNotNone(session_id) + + def test_session_destroy(self): + name = str(uuid.uuid4())[0:8] + session_id = self.consul.session.create( + name, behavior='delete', ttl='60s') + self.consul.session.destroy(session_id) + self.assertNotIn(session_id, + [s.get('ID') for s in self.consul.session.list()]) + + def test_session_info(self): + name = str(uuid.uuid4())[0:8] + session_id = self.consul.session.create( + name, behavior='delete', ttl='60s') + result = self.consul.session.info(session_id) + self.assertEqual(session_id, result.get('ID')) + self.consul.session.destroy(session_id) + + def test_session_renew(self): + name = str(uuid.uuid4())[0:8] + session_id = self.consul.session.create( + name, behavior='delete', ttl='60s') + self.sessions.append(session_id) + self.assertTrue(self.consul.session.renew(session_id)) diff --git a/tests/services_discovery_consulate/test_utils.py b/tests/services_discovery_consulate/test_utils.py new file mode 100644 index 0000000..4432ed7 --- /dev/null +++ b/tests/services_discovery_consulate/test_utils.py @@ -0,0 +1,77 @@ +# coding=utf-8 +import unittest + +from pyms.services_discovery.consulate import exceptions, utils + + +class MaybeEncodeTestCase(unittest.TestCase): + def str_test(self): + self.assertEqual(utils.maybe_encode('foo'), b'foo') + + def byte_test(self): + self.assertEqual(utils.maybe_encode(b'bar'), b'bar') + + +class Response(object): + def __init__(self, status_code=200, body=b'content'): + self.status_code = status_code + self.body = body + + +class ResponseOkTestCase(unittest.TestCase): + + def test_200(self): + self.assertTrue(utils.response_ok(Response(200, b'ok'))) + + def test_400(self): + with self.assertRaises(exceptions.ClientError): + utils.response_ok(Response(400, b'Bad request')) + + def test_401(self): + with self.assertRaises(exceptions.ACLDisabled): + utils.response_ok(Response(401, b'What ACL?')) + + def test_403(self): + with self.assertRaises(exceptions.Forbidden): + utils.response_ok(Response(403, b'No')) + + def test_404_not_raising(self): + self.assertFalse(utils.response_ok(Response(404, b'not found'))) + + def test_404_raising(self): + with self.assertRaises(exceptions.NotFound): + utils.response_ok(Response(404, b'Not Found'), True) + + def test_500(self): + with self.assertRaises(exceptions.ServerError): + utils.response_ok(Response(500, b'Opps')) + + + + +class ValidateGoDurationTestCase(unittest.TestCase): + + def test_valid_values(self): + for value in {'5µs', '300ms', '-1.5h', '2h45m', '5m', '30s'}: + print('Testing {}'.format(value)) + self.assertTrue(utils.validate_go_interval(value)) + + def test_invalid_values(self): + for value in {'100', '1 year', '5M', '30S'}: + print('Testing {}'.format(value)) + self.assertFalse(utils.validate_go_interval(value)) + + +class ValidateURLTestCase(unittest.TestCase): + + def test_valid_values(self): + for value in {'https://foo', 'http://localhost/bar'}: + print('Testing {}'.format(value)) + self.assertTrue(utils.validate_url(value)) + + def test_invalid_values(self): + for value in {'localhost', 'a'}: + print('Testing {}'.format(value)) + self.assertFalse(utils.validate_url(value)) + + From c96aed18e3203c2e53f7cba3741d8fcb5e4d33ac Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 18:21:27 +0100 Subject: [PATCH 15/27] chore: increment coverage tests --- tests/conftest.py | 28 ------------------- .../test_event.py | 11 ++------ tests/services_discovery_consulate/test_kv.py | 2 +- .../test_utils.py | 4 +-- 4 files changed, 6 insertions(+), 39 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index da20981..19454b5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,44 +1,16 @@ import os import pytest -import requests -from requests.exceptions import ConnectionError os.environ['ASYNC_TEST_TIMEOUT'] = os.environ.get('ASYNC_TEST_TIMEOUT', '15') -def conf_environment(): - if not os.environ.get("PYMS_CONFIGMAP_FILE", False): - os.environ["PYMS_CONFIGMAP_FILE"] = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config-tests.yml") - - @pytest.fixture(scope="session") def docker_compose_file(pytestconfig): """Get docker compose file""" return os.path.join(str(pytestconfig.rootdir), "docker", "docker-compose.yml") -def is_responsive(url): - try: - response = requests.get(url) - if response.status_code == 200: - return True - except ConnectionError: - return False - - -@pytest.fixture(scope="session") -def http_service(docker_ip, docker_services): - """Ensure that HTTP service is up and responsive.""" - # `port_for` takes a container port and returns the corresponding host port - port = docker_services.port_for("consul", 80) - url = "http://{}:{}".format(docker_ip, port) - docker_services.wait_until_responsive( - timeout=30.0, pause=0.1, check=lambda: is_responsive(url) - ) - return url - - @pytest.fixture(scope="session") def config_env(docker_ip, docker_services): """Set config for docker""" diff --git a/tests/services_discovery_consulate/test_event.py b/tests/services_discovery_consulate/test_event.py index b083a83..042e380 100644 --- a/tests/services_discovery_consulate/test_event.py +++ b/tests/services_discovery_consulate/test_event.py @@ -8,11 +8,6 @@ def test_fire(self): event_name = 'test-event-%s' % str(uuid.uuid4())[0:8] response = self.consul.event.fire(event_name) events = self.consul.event.list(event_name) - if isinstance(events, dict): - self.assertEqual(event_name, events.get('Name')) - self.assertEqual(response, events.get('ID')) - elif isinstance(events, dict): - self.assertIn(event_name, [e.get('Name') for e in events]) - self.assertIn(response, [e.get('ID') for e in events]) - else: - assert False, 'Unexpected return type' + self.assertEqual(event_name, events.get('Name')) + self.assertEqual(response, events.get('ID')) + diff --git a/tests/services_discovery_consulate/test_kv.py b/tests/services_discovery_consulate/test_kv.py index 2fc11be..dfbd68b 100644 --- a/tests/services_discovery_consulate/test_kv.py +++ b/tests/services_discovery_consulate/test_kv.py @@ -166,7 +166,7 @@ def test_set_item_get_item_int_value(self, key): self.assertEqual(self.consul.kv[key], '128') @base.generate_key - def test_set_item_get_item_str_value(self, key): + def test_set_item_get_item_str_value_key(self, key): self.consul.kv[key] = b'foo' self.assertEqual(self.consul.kv[key], 'foo') diff --git a/tests/services_discovery_consulate/test_utils.py b/tests/services_discovery_consulate/test_utils.py index 4432ed7..a48ccac 100644 --- a/tests/services_discovery_consulate/test_utils.py +++ b/tests/services_discovery_consulate/test_utils.py @@ -5,10 +5,10 @@ class MaybeEncodeTestCase(unittest.TestCase): - def str_test(self): + def test_str_test(self): self.assertEqual(utils.maybe_encode('foo'), b'foo') - def byte_test(self): + def test_byte_test(self): self.assertEqual(utils.maybe_encode(b'bar'), b'bar') From 738dec3ef4aad9e1f65bb8ee224f3a4af081042d Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 18:27:13 +0100 Subject: [PATCH 16/27] fix: removed mock --- pyms/flask/app/utils.py | 2 +- pyms/services_discovery/consulate/cli.py | 6 +- tests/conftest.py | 2 +- tests/services_discovery_consulate/base.py | 26 +- .../services_discovery_consulate/test_acl.py | 104 ++++---- .../test_agent.py | 168 ++++++------- .../services_discovery_consulate/test_api.py | 224 +++++++----------- .../test_base_model.py | 109 +++------ .../test_catalog.py | 10 +- .../test_coordinate.py | 1 + .../test_event.py | 7 +- tests/services_discovery_consulate/test_kv.py | 172 ++++++-------- .../test_session.py | 17 +- .../test_utils.py | 43 ++-- 14 files changed, 370 insertions(+), 521 deletions(-) diff --git a/pyms/flask/app/utils.py b/pyms/flask/app/utils.py index 8ae044e..8f406f8 100644 --- a/pyms/flask/app/utils.py +++ b/pyms/flask/app/utils.py @@ -58,7 +58,7 @@ def __call__(self, environ, start_response): environ["SCRIPT_NAME"] = script_name path_info = environ["PATH_INFO"] if path_info.startswith(script_name): - environ["PATH_INFO"] = path_info[len(script_name):] # noqa: E203 + environ["PATH_INFO"] = path_info[len(script_name) :] # noqa: E203 scheme = environ.get("HTTP_X_SCHEME", "") if scheme: diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index e8916c8..fa6dcf7 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -330,7 +330,7 @@ def kv_get(consul, args): if args.trim >= len(keyparts): displaykey = keyparts[-1] else: - displaykey = "/".join(keyparts[args.trim:]) # noqa: E203 + displaykey = "/".join(keyparts[args.trim :]) # noqa: E203 sys.stdout.write("%s\t%s\n" % (displaykey, consul.kv.get(key))) else: sys.stdout.write("%s\n" % consul.kv.get(args.key)) @@ -521,7 +521,9 @@ def run_once(consul, args): # Should the subprocess return an error code, release the lock try: - print(subprocess.check_output(args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True)) # nosec + print( + subprocess.check_output(args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True) + ) # nosec # If the subprocess fails except subprocess.CalledProcessError as err: error_code = 1 diff --git a/tests/conftest.py b/tests/conftest.py index 19454b5..313fdbe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ import pytest -os.environ['ASYNC_TEST_TIMEOUT'] = os.environ.get('ASYNC_TEST_TIMEOUT', '15') +os.environ["ASYNC_TEST_TIMEOUT"] = os.environ.get("ASYNC_TEST_TIMEOUT", "15") @pytest.fixture(scope="session") diff --git a/tests/services_discovery_consulate/base.py b/tests/services_discovery_consulate/base.py index 6a65670..feb1acc 100644 --- a/tests/services_discovery_consulate/base.py +++ b/tests/services_discovery_consulate/base.py @@ -10,9 +10,9 @@ from pyms.services_discovery import consulate from pyms.services_discovery.consulate import exceptions -CONSUL_TESTING_JSON = 'tests/services_discovery_consulate/consul.json' +CONSUL_TESTING_JSON = "tests/services_discovery_consulate/consul.json" -with open(CONSUL_TESTING_JSON, 'r') as handle: +with open(CONSUL_TESTING_JSON, "r") as handle: CONSUL_CONFIG = json.load(handle) @@ -35,13 +35,13 @@ def raise_oserror(_url_unused, _request): class TestCase(unittest.TestCase): def setUp(self): self.consul = consulate.Consul( - host=os.environ['CONSUL_HOST'], - port=os.environ['CONSUL_PORT'], - token=CONSUL_CONFIG['acl']['tokens']['master']) + host=os.environ["CONSUL_HOST"], + port=os.environ["CONSUL_PORT"], + token=CONSUL_CONFIG["acl"]["tokens"]["master"], + ) self.forbidden_consul = consulate.Consul( - host=os.environ['CONSUL_HOST'], - port=os.environ['CONSUL_PORT'], - token=str(uuid.uuid4())) + host=os.environ["CONSUL_HOST"], port=os.environ["CONSUL_PORT"], token=str(uuid.uuid4()) + ) self.used_keys = list() def tearDown(self): @@ -50,17 +50,17 @@ def tearDown(self): checks = self.consul.agent.checks() for name in checks: - self.consul.agent.check.deregister(checks[name]['CheckID']) + self.consul.agent.check.deregister(checks[name]["CheckID"]) services = self.consul.agent.services() for name in services: - self.consul.agent.service.deregister(services[name]['ID']) + self.consul.agent.service.deregister(services[name]["ID"]) for acl in self.consul.acl.list_tokens(): - if acl['AccessorID'] == CONSUL_CONFIG['acl']['tokens']['master']: + if acl["AccessorID"] == CONSUL_CONFIG["acl"]["tokens"]["master"]: continue try: - uuid.UUID(acl['AccessorID']) - self.consul.acl.delete_token(acl['AccessorID']) + uuid.UUID(acl["AccessorID"]) + self.consul.acl.delete_token(acl["AccessorID"]) except (ValueError, exceptions.ConsulateException): pass diff --git a/tests/services_discovery_consulate/test_acl.py b/tests/services_discovery_consulate/test_acl.py index 3ec3936..b349021 100644 --- a/tests/services_discovery_consulate/test_acl.py +++ b/tests/services_discovery_consulate/test_acl.py @@ -41,10 +41,7 @@ dict(Name="policylink_sample"), ] -POLICYLINKS_UPDATE_SAMPLE = [ - dict(Name="policylink_sample"), - dict(Name="policylink_update_sample") -] +POLICYLINKS_UPDATE_SAMPLE = [dict(Name="policylink_sample"), dict(Name="policylink_update_sample")] SERVICE_IDENTITIES_SAMPLE = [dict(ServiceName="db", Datacenters=list("dc1"))] @@ -81,8 +78,7 @@ def test_bootstrap_success(self): @httmock.all_requests def response_content(_url_unused, request): - return httmock.response(200, json.dumps({'ID': expectation}), {}, - None, 0, request) + return httmock.response(200, json.dumps({"ID": expectation}), {}, None, 0, request) with httmock.HTTMock(response_content): result = self.consul.acl.bootstrap() @@ -108,32 +104,32 @@ def test_create_and_destroy(self): def test_create_with_rules(self): acl_id = self.consul.acl.create(self.uuidv4(), rules=ACL_OLD_RULES) value = self.consul.acl.info(acl_id) - self.assertEqual(value['Rules'], ACL_OLD_RULES) + self.assertEqual(value["Rules"], ACL_OLD_RULES) def test_create_and_info(self): acl_id = self.consul.acl.create(self.uuidv4()) self.assertIsNotNone(acl_id) data = self.consul.acl.info(acl_id) self.assertIsNotNone(data) - self.assertEqual(acl_id, data.get('ID')) + self.assertEqual(acl_id, data.get("ID")) def test_create_and_list(self): acl_id = self.consul.acl.create(self.uuidv4()) data = self.consul.acl.list() - self.assertIn(acl_id, [r.get('ID') for r in data]) + self.assertIn(acl_id, [r.get("ID") for r in data]) def test_create_and_clone(self): acl_id = self.consul.acl.create(self.uuidv4()) clone_id = self.consul.acl.clone(acl_id) data = self.consul.acl.list() - self.assertIn(clone_id, [r.get('ID') for r in data]) + self.assertIn(clone_id, [r.get("ID") for r in data]) def test_create_and_update(self): acl_id = str(self.consul.acl.create(self.uuidv4())) - self.consul.acl.update(acl_id, 'Foo') + self.consul.acl.update(acl_id, "Foo") data = self.consul.acl.list() - self.assertIn('Foo', [r.get('Name') for r in data]) - self.assertIn(acl_id, [r.get('ID') for r in data]) + self.assertIn("Foo", [r.get("Name") for r in data]) + self.assertIn(acl_id, [r.get("ID") for r in data]) def test_create_forbidden(self): with self.assertRaises(consulate.Forbidden): @@ -154,45 +150,41 @@ def test_list_request_exception(self): def test_replication(self): result = self.forbidden_consul.acl.replication() - self.assertFalse(result['Enabled']) - self.assertFalse(result['Running']) + self.assertFalse(result["Enabled"]) + self.assertFalse(result["Running"]) def test_update_not_found_adds_new_key(self): - acl_id = self.consul.acl.update(self.uuidv4(), 'Foo2') + acl_id = self.consul.acl.update(self.uuidv4(), "Foo2") data = self.consul.acl.list() - self.assertIn('Foo2', [r.get('Name') for r in data]) - self.assertIn(acl_id, [r.get('ID') for r in data]) + self.assertIn("Foo2", [r.get("Name") for r in data]) + self.assertIn(acl_id, [r.get("ID") for r in data]) def test_update_with_rules(self): - acl_id = self.consul.acl.update(self.uuidv4(), - name='test', - rules=ACL_OLD_RULES) + acl_id = self.consul.acl.update(self.uuidv4(), name="test", rules=ACL_OLD_RULES) value = self.consul.acl.info(acl_id) - self.assertEqual(value['Rules'], ACL_OLD_RULES) + self.assertEqual(value["Rules"], ACL_OLD_RULES) def test_update_forbidden(self): with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.acl.update(self.uuidv4(), name='test') + self.forbidden_consul.acl.update(self.uuidv4(), name="test") # NOTE: Everything above here is deprecated post consul-1.4.0 def test_create_policy(self): name = self.random() result = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) - self.assertEqual(result['Rules'], ACL_NEW_RULES) + self.assertEqual(result["Rules"], ACL_NEW_RULES) def test_create_and_read_policy(self): name = self.random() value = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) result = self.consul.acl.read_policy(value["ID"]) - self.assertEqual(result['Rules'], ACL_NEW_RULES) + self.assertEqual(result["Rules"], ACL_NEW_RULES) def test_create_and_update_policy(self): name = self.random() value = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) - result = self.consul.acl.update_policy(value["ID"], - str(value["Name"]), - rules=ACL_NEW_UPDATE_RULES) + result = self.consul.acl.update_policy(value["ID"], str(value["Name"]), rules=ACL_NEW_UPDATE_RULES) self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) def test_create_and_delete_policy(self): @@ -209,38 +201,31 @@ def test_list_policy_exception(self): def test_create_role(self): name = self.random() result = self.consul.acl.create_role( - name=name, - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) - self.assertEqual(result['Name'], name) + name=name, policies=self._generate_policies(), service_identities=SERVICE_IDENTITIES_SAMPLE + ) + self.assertEqual(result["Name"], name) def test_create_and_read_role(self): name = self.random() value = self.consul.acl.create_role( - name=name, - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) + name=name, policies=self._generate_policies(), service_identities=SERVICE_IDENTITIES_SAMPLE + ) result = self.consul.acl.read_role(value["ID"]) - self.assertEqual(result['ID'], value['ID']) + self.assertEqual(result["ID"], value["ID"]) def test_create_and_update_role(self): name = self.random() value = self.consul.acl.create_role( - name=name, - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) - result = self.consul.acl.update_role( - value["ID"], - str(value["Name"]), - policies=self._generate_policies()) + name=name, policies=self._generate_policies(), service_identities=SERVICE_IDENTITIES_SAMPLE + ) + result = self.consul.acl.update_role(value["ID"], str(value["Name"]), policies=self._generate_policies()) self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) def test_create_and_delete_role(self): name = self.random() value = self.consul.acl.create_role( - name=name, - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) + name=name, policies=self._generate_policies(), service_identities=SERVICE_IDENTITIES_SAMPLE + ) result = self.consul.acl.delete_role(value["ID"]) self.assertTrue(result) @@ -257,9 +242,10 @@ def test_create_token(self): secret_id=secret_id, roles=self._generate_roles(), policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) - self.assertEqual(result['AccessorID'], accessor_id) - self.assertEqual(result['SecretID'], secret_id) + service_identities=SERVICE_IDENTITIES_SAMPLE, + ) + self.assertEqual(result["AccessorID"], accessor_id) + self.assertEqual(result["SecretID"], secret_id) def test_create_and_read_token(self): secret_id = self.uuidv4() @@ -269,9 +255,10 @@ def test_create_and_read_token(self): secret_id=secret_id, roles=self._generate_roles(), policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) + service_identities=SERVICE_IDENTITIES_SAMPLE, + ) result = self.consul.acl.read_token(value["AccessorID"]) - self.assertEqual(result['AccessorID'], accessor_id) + self.assertEqual(result["AccessorID"], accessor_id) def test_create_and_update_token(self): secret_id = self.uuidv4() @@ -281,9 +268,9 @@ def test_create_and_update_token(self): secret_id=secret_id, roles=self._generate_roles(), policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) - result = self.consul.acl.update_token( - str(value["AccessorID"]), policies=self._generate_policies()) + service_identities=SERVICE_IDENTITIES_SAMPLE, + ) + result = self.consul.acl.update_token(str(value["AccessorID"]), policies=self._generate_policies()) self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) def test_create_and_clone_token(self): @@ -295,9 +282,9 @@ def test_create_and_clone_token(self): secret_id=secret_id, roles=self._generate_roles(), policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) - result = self.consul.acl.clone_token(value["AccessorID"], - description=clone_description) + service_identities=SERVICE_IDENTITIES_SAMPLE, + ) + result = self.consul.acl.clone_token(value["AccessorID"], description=clone_description) self.assertEqual(result["Description"], clone_description) def test_create_and_delete_token(self): @@ -308,6 +295,7 @@ def test_create_and_delete_token(self): secret_id=secret_id, roles=self._generate_roles(), policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE) + service_identities=SERVICE_IDENTITIES_SAMPLE, + ) result = self.consul.acl.delete_token(value["AccessorID"]) self.assertTrue(result) diff --git a/tests/services_discovery_consulate/test_agent.py b/tests/services_discovery_consulate/test_agent.py index e50aeca..59b97d7 100644 --- a/tests/services_discovery_consulate/test_agent.py +++ b/tests/services_discovery_consulate/test_agent.py @@ -30,10 +30,10 @@ def test_force_leave_forbidden(self): def test_join_forbidden(self): with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.join('255.255.255.255') + self.forbidden_consul.agent.join("255.255.255.255") def test_maintenance(self): - self.consul.agent.maintenance(True, 'testing') + self.consul.agent.maintenance(True, "testing") self.consul.agent.maintenance(False) def test_maintenance_forbidden(self): @@ -50,8 +50,8 @@ def test_members_forbidden(self): def test_metrics(self): result = self.consul.agent.metrics() - self.assertIn('Timestamp', result) - self.assertIn('Gauges', result) + self.assertIn("Timestamp", result) + self.assertIn("Gauges", result) def test_metrics_forbidden(self): with self.assertRaises(consulate.Forbidden): @@ -85,9 +85,9 @@ def test_reload_forbidden(self): def test_self(self): result = self.consul.agent.self() - self.assertIn('Config', result) - self.assertIn('Coord', result) - self.assertIn('Member', result) + self.assertIn("Config", result) + self.assertIn("Coord", result) + self.assertIn("Member", result) def test_self_forbidden(self): with self.assertRaises(consulate.Forbidden): @@ -95,211 +95,187 @@ def test_self_forbidden(self): def test_service_registration(self): self.consul.agent.service.register( - 'test-service', address='10.0.0.1', port=5672, tags=['foo', 'bar'], meta={'foo' : 'bar' }) - self.assertIn('test-service', self.consul.agent.services()) - self.consul.agent.service.deregister('test-service') + "test-service", address="10.0.0.1", port=5672, tags=["foo", "bar"], meta={"foo": "bar"} + ) + self.assertIn("test-service", self.consul.agent.services()) + self.consul.agent.service.deregister("test-service") def test_service_maintenance(self): self.consul.agent.service.register( - 'test-service', address='10.0.0.1', port=5672, tags=['foo', 'bar'], meta={'foo' : 'bar' } ) - self.assertIn('test-service', self.consul.agent.services()) - reason = 'Down for Acceptance' - self.consul.agent.service.maintenance('test-service', reason=reason) - node_in_maintenance = self.consul.catalog.nodes()[0]['Node'] + "test-service", address="10.0.0.1", port=5672, tags=["foo", "bar"], meta={"foo": "bar"} + ) + self.assertIn("test-service", self.consul.agent.services()) + reason = "Down for Acceptance" + self.consul.agent.service.maintenance("test-service", reason=reason) + node_in_maintenance = self.consul.catalog.nodes()[0]["Node"] health_check = self.consul.health.node(node_in_maintenance) self.assertEqual(len(health_check), 2) - self.assertIn(reason, [check['Notes'] for check in health_check]) - self.consul.agent.service.maintenance('test-service', enable=False) + self.assertIn(reason, [check["Notes"] for check in health_check]) + self.consul.agent.service.maintenance("test-service", enable=False) health_check = self.consul.health.node(node_in_maintenance) self.assertEqual(len(health_check), 1) - self.assertNotEqual(reason, health_check[0]['Notes']) - self.consul.agent.service.deregister('test-service') + self.assertNotEqual(reason, health_check[0]["Notes"]) + self.consul.agent.service.deregister("test-service") def test_token(self): - self.assertTrue( - self.consul.agent.token('acl_replication_token', 'foo')) + self.assertTrue(self.consul.agent.token("acl_replication_token", "foo")) def test_token_invalid(self): with self.assertRaises(ValueError): - self.consul.agent.token('acl_replication_tokens', 'foo') + self.consul.agent.token("acl_replication_tokens", "foo") def test_token_forbidden(self): with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.token('acl_replication_token', 'foo') + self.forbidden_consul.agent.token("acl_replication_token", "foo") class CheckTestCase(base.TestCase): - def test_register(self): - self.assertTrue(self.consul.agent.check.register( - str(uuid.uuid4()), http='http://localhost', interval='30s')) + self.assertTrue(self.consul.agent.check.register(str(uuid.uuid4()), http="http://localhost", interval="30s")) def test_register_args_and_no_interval(self): with self.assertRaises(ValueError): - self.consul.agent.check.register( - str(uuid.uuid4()), args=['/bin/true']) + self.consul.agent.check.register(str(uuid.uuid4()), args=["/bin/true"]) def test_register_args_and_ttl(self): with self.assertRaises(ValueError): - self.consul.agent.check.register( - str(uuid.uuid4()), args=['/bin/true'], ttl='30s') + self.consul.agent.check.register(str(uuid.uuid4()), args=["/bin/true"], ttl="30s") def test_register_http_and_no_interval(self): with self.assertRaises(ValueError): - self.consul.agent.check.register( - str(uuid.uuid4()), http='http://localhost') + self.consul.agent.check.register(str(uuid.uuid4()), http="http://localhost") def test_register_args_and_http(self): with self.assertRaises(ValueError): - self.consul.agent.check.register( - str(uuid.uuid4()), args=['/bin/true'], http='http://localhost') + self.consul.agent.check.register(str(uuid.uuid4()), args=["/bin/true"], http="http://localhost") def test_register_forbidden(self): with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.check.register( - str(uuid.uuid4()), args=['/bin/true'], interval='30s') + self.forbidden_consul.agent.check.register(str(uuid.uuid4()), args=["/bin/true"], interval="30s") class TTLCheckTestCase(base.TestCase): - def setUp(self): super(TTLCheckTestCase, self).setUp() name = str(uuid.uuid4()) - self.assertTrue(self.consul.agent.check.register(name, ttl='30s')) + self.assertTrue(self.consul.agent.check.register(name, ttl="30s")) checks = self.consul.agent.checks() - self.check_id = checks[name]['CheckID'] + self.check_id = checks[name]["CheckID"] def test_pass(self): self.assertTrue(self.consul.agent.check.ttl_pass(self.check_id)) def test_pass_with_note(self): - self.assertTrue( - self.consul.agent.check.ttl_pass(self.check_id, 'PASS')) + self.assertTrue(self.consul.agent.check.ttl_pass(self.check_id, "PASS")) def test_warn(self): self.assertTrue(self.consul.agent.check.ttl_warn(self.check_id)) def test_warn_with_note(self): - self.assertTrue( - self.consul.agent.check.ttl_warn(self.check_id, 'WARN')) + self.assertTrue(self.consul.agent.check.ttl_warn(self.check_id, "WARN")) def test_fail(self): self.assertTrue(self.consul.agent.check.ttl_fail(self.check_id)) def test_fail_with_note(self): - self.assertTrue( - self.consul.agent.check.ttl_fail(self.check_id, 'FAIL')) + self.assertTrue(self.consul.agent.check.ttl_fail(self.check_id, "FAIL")) class ServiceTestCase(base.TestCase): - def test_register(self): self.assertTrue( self.consul.agent.service.register( str(uuid.uuid4()), - address='127.0.0.1', + address="127.0.0.1", port=80, - check=agent.Check( - name='test', args=['/bin/true'], interval='30s'), - tags=[str(uuid.uuid4())])) + check=agent.Check(name="test", args=["/bin/true"], interval="30s"), + tags=[str(uuid.uuid4())], + ) + ) def test_register_grpc(self): self.assertTrue( self.consul.agent.service.register( str(uuid.uuid4()), - address='127.0.0.1', + address="127.0.0.1", port=80, - check=agent.Check( - name='test', grpc='https://grpc/status', interval='30s'))) + check=agent.Check(name="test", grpc="https://grpc/status", interval="30s"), + ) + ) def test_register_http(self): self.assertTrue( self.consul.agent.service.register( str(uuid.uuid4()), - address='127.0.0.1', + address="127.0.0.1", port=80, - check=agent.Check( - name='test', http='http://localhost', interval='30s'))) + check=agent.Check(name="test", http="http://localhost", interval="30s"), + ) + ) def test_register_tcp(self): self.assertTrue( self.consul.agent.service.register( str(uuid.uuid4()), - address='127.0.0.1', + address="127.0.0.1", port=80, - check=agent.Check( - name='test', tcp='localhost:80', interval='30s'))) + check=agent.Check(name="test", tcp="localhost:80", interval="30s"), + ) + ) def test_register_ttl(self): self.assertTrue( self.consul.agent.service.register( - str(uuid.uuid4()), - address='127.0.0.1', - port=80, - check=agent.Check(name='test', ttl='30s'))) + str(uuid.uuid4()), address="127.0.0.1", port=80, check=agent.Check(name="test", ttl="30s") + ) + ) def test_register_multiple_checks(self): self.assertTrue( self.consul.agent.service.register( str(uuid.uuid4()), - address='127.0.0.1', + address="127.0.0.1", port=80, checks=[ agent.Check( - name='test1', http='http://netloc', - header={'User-Agent': 'unittest.TestCase'}, - interval='30s'), - agent.Check(name='test2', ttl='30s') - ])) + name="test1", http="http://netloc", header={"User-Agent": "unittest.TestCase"}, interval="30s" + ), + agent.Check(name="test2", ttl="30s"), + ], + ) + ) def test_register_forbidden(self): with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.service.register( - str(uuid.uuid4()), - address='127.0.0.1', - port=80) + self.forbidden_consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", port=80) def test_register_invalid_check(self): with self.assertRaises(TypeError): - self.consul.agent.service.register( - str(uuid.uuid4()), - address='127.0.0.1', - check=str(uuid.uuid4())) + self.consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", check=str(uuid.uuid4())) def test_register_invalid_checks(self): with self.assertRaises(ValueError): - self.consul.agent.service.register( - str(uuid.uuid4()), - address='127.0.0.1', - checks=[str(uuid.uuid4())]) + self.consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", checks=[str(uuid.uuid4())]) def test_register_invalid_port(self): with self.assertRaises(TypeError): - self.consul.agent.service.register( - str(uuid.uuid4()), - address='127.0.0.1', - port='80') + self.consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", port="80") def test_register_invalid_tags(self): with self.assertRaises(TypeError): - self.consul.agent.service.register( - str(uuid.uuid4()), - address='127.0.0.1', - tags=str(uuid.uuid4())) + self.consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", tags=str(uuid.uuid4())) def test_register_invalid_interval(self): with self.assertRaises(TypeError): self.consul.agent.service.register( str(uuid.uuid4()), - address='127.0.0.1', + address="127.0.0.1", port=80, - check=agent.Check( - name='test', http='http://localhost', interval=30)) + check=agent.Check(name="test", http="http://localhost", interval=30), + ) def test_register_invalid_ttl(self): with self.assertRaises(TypeError): self.consul.agent.service.register( - str(uuid.uuid4()), - address='127.0.0.1', - port=80, - check=agent.Check(name='test', ttl=30)) + str(uuid.uuid4()), address="127.0.0.1", port=80, check=agent.Check(name="test", ttl=30) + ) diff --git a/tests/services_discovery_consulate/test_api.py b/tests/services_discovery_consulate/test_api.py index 369d159..9eb0e0a 100644 --- a/tests/services_discovery_consulate/test_api.py +++ b/tests/services_discovery_consulate/test_api.py @@ -4,37 +4,36 @@ from urllib import parse import httmock -import mock +from unittest.mock import patch from pyms.services_discovery import consulate from pyms.services_discovery.consulate import adapters from pyms.services_discovery.consulate.api import base from tests.services_discovery_consulate.base import CONSUL_TESTING_JSON -with open(CONSUL_TESTING_JSON, 'r') as handle: +with open(CONSUL_TESTING_JSON, "r") as handle: CONSUL_CONFIG = json.load(handle) -SCHEME = 'http' -VERSION = 'v1' +SCHEME = "http" +VERSION = "v1" class ConsulTests(unittest.TestCase): - @mock.patch('pyms.services_discovery.consulate.adapters.Request') - @mock.patch('pyms.services_discovery.consulate.api.Agent') - @mock.patch('pyms.services_discovery.consulate.api.Catalog') - @mock.patch('pyms.services_discovery.consulate.api.KV') - @mock.patch('pyms.services_discovery.consulate.api.Health') - @mock.patch('pyms.services_discovery.consulate.api.Coordinate') - @mock.patch('pyms.services_discovery.consulate.api.ACL') - @mock.patch('pyms.services_discovery.consulate.api.Event') - @mock.patch('pyms.services_discovery.consulate.api.Session') - @mock.patch('pyms.services_discovery.consulate.api.Status') - def setUp(self, status, session, event, acl, health, coordinate, kv, catalog, agent, - adapter): - self.host = '127.0.0.1' + @patch("pyms.services_discovery.consulate.adapters.Request") + @patch("pyms.services_discovery.consulate.api.Agent") + @patch("pyms.services_discovery.consulate.api.Catalog") + @patch("pyms.services_discovery.consulate.api.KV") + @patch("pyms.services_discovery.consulate.api.Health") + @patch("pyms.services_discovery.consulate.api.Coordinate") + @patch("pyms.services_discovery.consulate.api.ACL") + @patch("pyms.services_discovery.consulate.api.Event") + @patch("pyms.services_discovery.consulate.api.Session") + @patch("pyms.services_discovery.consulate.api.Status") + def setUp(self, status, session, event, acl, health, coordinate, kv, catalog, agent, adapter): + self.host = "127.0.0.1" self.port = 8500 - self.dc = CONSUL_CONFIG['datacenter'] - self.token = CONSUL_CONFIG['acl']['tokens']['master'] + self.dc = CONSUL_CONFIG["datacenter"] + self.token = CONSUL_CONFIG["acl"]["tokens"]["master"] self.acl = acl self.adapter = adapter @@ -47,67 +46,45 @@ def setUp(self, status, session, event, acl, health, coordinate, kv, catalog, ag self.session = session self.status = status - self.base_uri = '{0}://{1}:{2}/v1'.format(SCHEME, self.host, self.port) - self.consul = consulate.Consul(self.host, self.port, self.dc, - self.token) + self.base_uri = "{0}://{1}:{2}/v1".format(SCHEME, self.host, self.port) + self.consul = consulate.Consul(self.host, self.port, self.dc, self.token) def test_base_uri(self): - self.assertEquals( - self.consul._base_uri(SCHEME, self.host, self.port), self.base_uri) + self.assertEquals(self.consul._base_uri(SCHEME, self.host, self.port), self.base_uri) def test_unix_socket_base_uri(self): - expectation = 'http+unix://%2Fvar%2Flib%2Fconsul%2Fconsul.sock/v1' - self.assertEquals( - self.consul._base_uri('http+unix', '/var/lib/consul/consul.sock', - None), expectation) + expectation = "http+unix://%2Fvar%2Flib%2Fconsul%2Fconsul.sock/v1" + self.assertEquals(self.consul._base_uri("http+unix", "/var/lib/consul/consul.sock", None), expectation) def test_acl_initialization(self): - self.assertTrue( - self.acl.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.acl.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_adapter_initialization(self): self.assertTrue(self.adapter.called_once_with()) def test_agent_initialization(self): - self.assertTrue( - self.agent.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.agent.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_catalog_initialization(self): - self.assertTrue( - self.catalog.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.catalog.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_events_initialization(self): - self.assertTrue( - self.event.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.event.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_kv_initialization(self): - self.assertTrue( - self.kv.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.kv.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_health_initialization(self): - self.assertTrue( - self.health.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.health.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_coordinate_initialization(self): - self.assertTrue( - self.coordinate.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.coordinate.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_session_initialization(self): - self.assertTrue( - self.session.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.session.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_status_initialization(self): - self.assertTrue( - self.status.called_once_with(self.base_uri, self.adapter, self.dc, - self.token)) + self.assertTrue(self.status.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) def test_acl_property(self): self.assertEqual(self.consul.acl, self.consul._acl) @@ -137,15 +114,14 @@ def test_status_property(self): class EndpointBuildURITests(unittest.TestCase): def setUp(self): self.adapter = adapters.Request() - self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) self.endpoint = base.Endpoint(self.base_uri, self.adapter) def test_adapter_assignment(self): self.assertEqual(self.endpoint._adapter, self.adapter) def test_base_uri_assignment(self): - self.assertEqual(self.endpoint._base_uri, '{0}/endpoint'.format( - self.base_uri)) + self.assertEqual(self.endpoint._base_uri, "{0}/endpoint".format(self.base_uri)) def test_dc_assignment(self): self.assertIsNone(self.endpoint._dc) @@ -154,28 +130,28 @@ def test_token_assignment(self): self.assertIsNone(self.endpoint._token) def test_build_uri_with_no_params(self): - result = self.endpoint._build_uri(['foo', 'bar']) + result = self.endpoint._build_uri(["foo", "bar"]) parsed = parse.urlparse(result) query_params = parse.parse_qs(parsed.query) self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, 'localhost:8500') - self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) + self.assertEqual(parsed.netloc, "localhost:8500") + self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) self.assertDictEqual(query_params, {}) def test_build_uri_with_params(self): - result = self.endpoint._build_uri(['foo', 'bar'], {'baz': 'qux'}) + result = self.endpoint._build_uri(["foo", "bar"], {"baz": "qux"}) parsed = parse.urlparse(result) query_params = parse.parse_qs(parsed.query) self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, 'localhost:8500') - self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) - self.assertDictEqual(query_params, {'baz': ['qux']}) + self.assertEqual(parsed.netloc, "localhost:8500") + self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) + self.assertDictEqual(query_params, {"baz": ["qux"]}) class EndpointBuildURIWithDCTests(unittest.TestCase): def setUp(self): self.adapter = adapters.Request() - self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) self.dc = str(uuid.uuid4()) self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc) @@ -186,31 +162,30 @@ def test_token_assignment(self): self.assertIsNone(self.endpoint._token) def test_build_uri_with_no_params(self): - result = self.endpoint._build_uri(['foo', 'bar']) + result = self.endpoint._build_uri(["foo", "bar"]) parsed = parse.urlparse(result) query_params = parse.parse_qs(parsed.query) self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, 'localhost:8500') - self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) - self.assertDictEqual(query_params, {'dc': [self.dc]}) + self.assertEqual(parsed.netloc, "localhost:8500") + self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) + self.assertDictEqual(query_params, {"dc": [self.dc]}) def test_build_uri_with_params(self): - result = self.endpoint._build_uri(['foo', 'bar'], {'baz': 'qux'}) + result = self.endpoint._build_uri(["foo", "bar"], {"baz": "qux"}) parsed = parse.urlparse(result) query_params = parse.parse_qs(parsed.query) self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, 'localhost:8500') - self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) - self.assertDictEqual(query_params, {'dc': [self.dc], 'baz': ['qux']}) + self.assertEqual(parsed.netloc, "localhost:8500") + self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) + self.assertDictEqual(query_params, {"dc": [self.dc], "baz": ["qux"]}) class EndpointBuildURIWithTokenTests(unittest.TestCase): def setUp(self): self.adapter = adapters.Request() - self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) self.token = str(uuid.uuid4()) - self.endpoint = base.Endpoint( - self.base_uri, self.adapter, token=self.token) + self.endpoint = base.Endpoint(self.base_uri, self.adapter, token=self.token) def test_dc_assignment(self): self.assertIsNone(self.endpoint._dc) @@ -219,35 +194,31 @@ def test_token_assignment(self): self.assertEqual(self.endpoint._token, self.token) def test_build_uri_with_no_params(self): - result = self.endpoint._build_uri(['foo', 'bar']) + result = self.endpoint._build_uri(["foo", "bar"]) parsed = parse.urlparse(result) query_params = parse.parse_qs(parsed.query) self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, 'localhost:8500') - self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) - self.assertDictEqual(query_params, {'token': [self.token]}) + self.assertEqual(parsed.netloc, "localhost:8500") + self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) + self.assertDictEqual(query_params, {"token": [self.token]}) def test_build_uri_with_params(self): - result = self.endpoint._build_uri(['foo', 'bar'], {'baz': 'qux'}) + result = self.endpoint._build_uri(["foo", "bar"], {"baz": "qux"}) parsed = parse.urlparse(result) query_params = parse.parse_qs(parsed.query) self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, 'localhost:8500') - self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) - self.assertDictEqual(query_params, { - 'token': [self.token], - 'baz': ['qux'] - }) + self.assertEqual(parsed.netloc, "localhost:8500") + self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) + self.assertDictEqual(query_params, {"token": [self.token], "baz": ["qux"]}) class EndpointBuildURIWithDCAndTokenTests(unittest.TestCase): def setUp(self): self.adapter = adapters.Request() - self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) self.dc = str(uuid.uuid4()) self.token = str(uuid.uuid4()) - self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, - self.token) + self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, self.token) def test_dc_assignment(self): self.assertEqual(self.endpoint._dc, self.dc) @@ -256,65 +227,54 @@ def test_token_assignment(self): self.assertEqual(self.endpoint._token, self.token) def test_build_uri_with_no_params(self): - result = self.endpoint._build_uri(['foo', 'bar']) + result = self.endpoint._build_uri(["foo", "bar"]) parsed = parse.urlparse(result) query_params = parse.parse_qs(parsed.query) self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, 'localhost:8500') - self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) - self.assertDictEqual(query_params, { - 'dc': [self.dc], - 'token': [self.token] - }) + self.assertEqual(parsed.netloc, "localhost:8500") + self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) + self.assertDictEqual(query_params, {"dc": [self.dc], "token": [self.token]}) def test_build_uri_with_params(self): - result = self.endpoint._build_uri(['foo', 'bar'], {'baz': 'qux'}) + result = self.endpoint._build_uri(["foo", "bar"], {"baz": "qux"}) parsed = parse.urlparse(result) query_params = parse.parse_qs(parsed.query) self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, 'localhost:8500') - self.assertEqual(parsed.path, '/{0}/endpoint/foo/bar'.format(VERSION)) - self.assertDictEqual(query_params, { - 'dc': [self.dc], - 'token': [self.token], - 'baz': ['qux'] - }) + self.assertEqual(parsed.netloc, "localhost:8500") + self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) + self.assertDictEqual(query_params, {"dc": [self.dc], "token": [self.token], "baz": ["qux"]}) class EndpointGetTests(unittest.TestCase): def setUp(self): self.adapter = adapters.Request() - self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) self.dc = str(uuid.uuid4()) self.token = str(uuid.uuid4()) - self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, - self.token) + self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, self.token) def test_get_200_returns_response_body(self): @httmock.all_requests def response_content(_url_unused, request): headers = { - 'X-Consul-Index': 4, - 'X-Consul-Knownleader': 'true', - 'X-Consul-Lastcontact': 0, - 'Date': 'Fri, 19 Dec 2014 20:44:28 GMT', - 'Content-Length': 13, - 'Content-Type': 'application/json' + "X-Consul-Index": 4, + "X-Consul-Knownleader": "true", + "X-Consul-Lastcontact": 0, + "Date": "Fri, 19 Dec 2014 20:44:28 GMT", + "Content-Length": 13, + "Content-Type": "application/json", } content = b'{"consul": []}' return httmock.response(200, content, headers, None, 0, request) with httmock.HTTMock(response_content): values = self.endpoint._get([str(uuid.uuid4())]) - self.assertEqual(values, {'consul': []}) + self.assertEqual(values, {"consul": []}) def test_get_404_returns_empty_list(self): @httmock.all_requests def response_content(_url_unused, request): - headers = { - 'content-length': 0, - 'content-type': 'text/plain; charset=utf-8' - } + headers = {"content-length": 0, "content-type": "text/plain; charset=utf-8"} return httmock.response(404, None, headers, None, 0, request) with httmock.HTTMock(response_content): @@ -325,37 +285,33 @@ def response_content(_url_unused, request): class EndpointGetListTests(unittest.TestCase): def setUp(self): self.adapter = adapters.Request() - self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) self.dc = str(uuid.uuid4()) self.token = str(uuid.uuid4()) - self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, - self.token) + self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, self.token) def test_get_list_200_returns_response_body(self): @httmock.all_requests def response_content(_url_unused, request): headers = { - 'X-Consul-Index': 4, - 'X-Consul-Knownleader': 'true', - 'X-Consul-Lastcontact': 0, - 'Date': 'Fri, 19 Dec 2014 20:44:28 GMT', - 'Content-Length': 13, - 'Content-Type': 'application/json' + "X-Consul-Index": 4, + "X-Consul-Knownleader": "true", + "X-Consul-Lastcontact": 0, + "Date": "Fri, 19 Dec 2014 20:44:28 GMT", + "Content-Length": 13, + "Content-Type": "application/json", } content = b'{"consul": []}' return httmock.response(200, content, headers, None, 0, request) with httmock.HTTMock(response_content): values = self.endpoint._get_list([str(uuid.uuid4())]) - self.assertEqual(values, [{'consul': []}]) + self.assertEqual(values, [{"consul": []}]) def test_get_list_404_returns_empty_list(self): @httmock.all_requests def response_content(_url_unused, request): - headers = { - 'content-length': 0, - 'content-type': 'text/plain; charset=utf-8' - } + headers = {"content-length": 0, "content-type": "text/plain; charset=utf-8"} return httmock.response(404, None, headers, None, 0, request) with httmock.HTTMock(response_content): diff --git a/tests/services_discovery_consulate/test_base_model.py b/tests/services_discovery_consulate/test_base_model.py index 82ec806..6fd98e2 100644 --- a/tests/services_discovery_consulate/test_base_model.py +++ b/tests/services_discovery_consulate/test_base_model.py @@ -1,4 +1,3 @@ -# coding=utf-8 """Tests for the Base Model""" import unittest import uuid @@ -8,83 +7,55 @@ class TestModel(base.Model): """Model to perform tests against""" - __slots__ = ['id', 'serial', 'name', 'value'] + + __slots__ = ["id", "serial", "name", "value"] __attributes__ = { - 'id': { - 'key': 'ID', - 'type': uuid.UUID, - 'cast_from': str, - 'cast_to': str, - }, - 'serial': { - 'key': 'Serial', - 'type': int, - 'default': 0, - 'required': True, - 'validator': lambda v, _m: v >= 0, - }, - 'name': { - 'key': 'Name', - 'type': str, - 'required': True + "id": { + "key": "ID", + "type": uuid.UUID, + "cast_from": str, + "cast_to": str, }, - 'value': { - 'type': str + "serial": { + "key": "Serial", + "type": int, + "default": 0, + "required": True, + "validator": lambda v, _m: v >= 0, }, - 'type': { - 'key': 'Type', - 'type': str, - 'enum': {'client', 'server'} - } + "name": {"key": "Name", "type": str, "required": True}, + "value": {"type": str}, + "type": {"key": "Type", "type": str, "enum": {"client", "server"}}, } class TestCase(unittest.TestCase): - def test_happy_case_with_defaults(self): - kwargs = { - 'id': uuid.uuid4(), - 'name': str(uuid.uuid4()) - } + kwargs = {"id": uuid.uuid4(), "name": str(uuid.uuid4())} model = TestModel(**kwargs) for key, value in kwargs.items(): self.assertEqual(getattr(model, key), value) self.assertEqual(model.serial, 0) def test_happy_case_with_all_values(self): - kwargs = { - 'id': uuid.uuid4(), - 'serial': 1, - 'name': str(uuid.uuid4()), - 'value': str(uuid.uuid4()) - } + kwargs = {"id": uuid.uuid4(), "serial": 1, "name": str(uuid.uuid4()), "value": str(uuid.uuid4())} model = TestModel(**kwargs) for key, value in kwargs.items(): self.assertEqual(getattr(model, key), value) def test_cast_from_str(self): expectation = uuid.uuid4() - kwargs = { - 'id': str(expectation), - 'name': str(uuid.uuid4()) - } + kwargs = {"id": str(expectation), "name": str(uuid.uuid4())} model = TestModel(**kwargs) self.assertEqual(model.id, expectation) def test_validator_failure(self): - kwargs = { - 'id': uuid.uuid4(), - 'name': str(uuid.uuid4()), - 'serial': -1 - } + kwargs = {"id": uuid.uuid4(), "name": str(uuid.uuid4()), "serial": -1} with self.assertRaises(ValueError): TestModel(**kwargs) def test_type_failure(self): - kwargs = { - 'id': True, - 'name': str(uuid.uuid4()) - } + kwargs = {"id": True, "name": str(uuid.uuid4())} with self.assertRaises(TypeError): TestModel(**kwargs) @@ -93,47 +64,41 @@ def test_missing_requirement(self): TestModel() def test_invalid_attribute(self): - kwargs = {'name': str(uuid.uuid4()), 'foo': 'bar'} + kwargs = {"name": str(uuid.uuid4()), "foo": "bar"} with self.assertRaises(AttributeError): TestModel(**kwargs) def test_invalid_attribute_assignment(self): - kwargs = {'name': str(uuid.uuid4())} + kwargs = {"name": str(uuid.uuid4())} model = TestModel(**kwargs) with self.assertRaises(AttributeError): - model.foo = 'bar' + model.foo = "bar" def test_invalid_enum_assignment(self): - kwargs = {'name': str(uuid.uuid4()), 'type': 'invalid'} + kwargs = {"name": str(uuid.uuid4()), "type": "invalid"} with self.assertRaises(ValueError): TestModel(**kwargs) def test_cast_to_dict(self): kwargs = { - 'id': uuid.uuid4(), - 'serial': 1, - 'name': str(uuid.uuid4()), - 'value': str(uuid.uuid4()), - 'type': 'client' + "id": uuid.uuid4(), + "serial": 1, + "name": str(uuid.uuid4()), + "value": str(uuid.uuid4()), + "type": "client", } expectation = { - 'ID': str(kwargs['id']), - 'Serial': kwargs['serial'], - 'Name': kwargs['name'], - 'value': kwargs['value'], - 'Type': kwargs['type'] + "ID": str(kwargs["id"]), + "Serial": kwargs["serial"], + "Name": kwargs["name"], + "value": kwargs["value"], + "Type": kwargs["type"], } model = TestModel(**kwargs) self.assertDictEqual(dict(model), expectation) def test_cast_to_dict_only_requirements(self): - kwargs = { - 'serial': 1, - 'name': str(uuid.uuid4()) - } - expectation = { - 'Serial': kwargs['serial'], - 'Name': kwargs['name'] - } + kwargs = {"serial": 1, "name": str(uuid.uuid4())} + expectation = {"Serial": kwargs["serial"], "Name": kwargs["name"]} model = TestModel(**kwargs) self.assertDictEqual(dict(model), expectation) diff --git a/tests/services_discovery_consulate/test_catalog.py b/tests/services_discovery_consulate/test_catalog.py index cb89649..c7991e9 100644 --- a/tests/services_discovery_consulate/test_catalog.py +++ b/tests/services_discovery_consulate/test_catalog.py @@ -3,9 +3,7 @@ class TestCatalog(base.TestCase): def test_catalog_registration(self): - self.consul.catalog.register('test-service', address='10.0.0.1') - self.assertIn('test-service', - [n['Node'] for n in self.consul.catalog.nodes()]) - self.consul.catalog.deregister('test-service') - self.assertNotIn('test-service', - [n['Node'] for n in self.consul.catalog.nodes()]) + self.consul.catalog.register("test-service", address="10.0.0.1") + self.assertIn("test-service", [n["Node"] for n in self.consul.catalog.nodes()]) + self.consul.catalog.deregister("test-service") + self.assertNotIn("test-service", [n["Node"] for n in self.consul.catalog.nodes()]) diff --git a/tests/services_discovery_consulate/test_coordinate.py b/tests/services_discovery_consulate/test_coordinate.py index 78cb463..3089d86 100644 --- a/tests/services_discovery_consulate/test_coordinate.py +++ b/tests/services_discovery_consulate/test_coordinate.py @@ -1,5 +1,6 @@ from . import base + class TestCoordinate(base.TestCase): def test_coordinate(self): coordinates = self.consul.coordinate.nodes() diff --git a/tests/services_discovery_consulate/test_event.py b/tests/services_discovery_consulate/test_event.py index 042e380..8daa3ae 100644 --- a/tests/services_discovery_consulate/test_event.py +++ b/tests/services_discovery_consulate/test_event.py @@ -5,9 +5,8 @@ class TestEvent(base.TestCase): def test_fire(self): - event_name = 'test-event-%s' % str(uuid.uuid4())[0:8] + event_name = "test-event-%s" % str(uuid.uuid4())[0:8] response = self.consul.event.fire(event_name) events = self.consul.event.list(event_name) - self.assertEqual(event_name, events.get('Name')) - self.assertEqual(response, events.get('ID')) - + self.assertEqual(event_name, events.get("Name")) + self.assertEqual(response, events.get("ID")) diff --git a/tests/services_discovery_consulate/test_kv.py b/tests/services_discovery_consulate/test_kv.py index dfbd68b..055f107 100644 --- a/tests/services_discovery_consulate/test_kv.py +++ b/tests/services_discovery_consulate/test_kv.py @@ -8,73 +8,52 @@ from pyms.services_discovery.consulate import adapters, api from . import base -SCHEME = 'http' -VERSION = 'v1' - -ALL_DATA = (b'[{"CreateIndex":643,"ModifyIndex":643,"LockIndex":0,"Key":"bar",' - b'"Flags":0,"Value":"YmF6"},{"CreateIndex":669,"ModifyIndex":669,"' - b'LockIndex":0,"Key":"baz","Flags":0,"Value":"cXV4"},{"CreateIndex' - b'":666,"ModifyIndex":666,"LockIndex":0,"Key":"corgie","Flags":128' - b',"Value":"ZG9n"},{"CreateIndex":642,"ModifyIndex":642,"LockIndex' - b'":0,"Key":"foo","Flags":0,"Value":"YmFy"},{"CreateIndex":644,"Mo' - b'difyIndex":644,"LockIndex":0,"Key":"quz","Flags":0,"Value":"dHJ1' - b'ZQ=="}]') - -ALL_ITEMS = [{ - 'CreateIndex': 643, - 'Flags': 0, - 'Key': 'bar', - 'LockIndex': 0, - 'ModifyIndex': 643, - 'Value': 'baz' -}, { - 'CreateIndex': 669, - 'Flags': 0, - 'Key': 'baz', - 'LockIndex': 0, - 'ModifyIndex': 669, - 'Value': 'qux' -}, { - 'CreateIndex': 666, - 'Flags': 128, - 'Key': 'corgie', - 'LockIndex': 0, - 'ModifyIndex': 666, - 'Value': 'dog' -}, { - 'CreateIndex': 642, - 'Flags': 0, - 'Key': 'foo', - 'LockIndex': 0, - 'ModifyIndex': 642, - 'Value': 'bar' -}, { - 'CreateIndex': 644, - 'Flags': 0, - 'Key': 'quz', - 'LockIndex': 0, - 'ModifyIndex': 644, - 'Value': 'true' -}] +SCHEME = "http" +VERSION = "v1" + +ALL_DATA = ( + b'[{"CreateIndex":643,"ModifyIndex":643,"LockIndex":0,"Key":"bar",' + b'"Flags":0,"Value":"YmF6"},{"CreateIndex":669,"ModifyIndex":669,"' + b'LockIndex":0,"Key":"baz","Flags":0,"Value":"cXV4"},{"CreateIndex' + b'":666,"ModifyIndex":666,"LockIndex":0,"Key":"corgie","Flags":128' + b',"Value":"ZG9n"},{"CreateIndex":642,"ModifyIndex":642,"LockIndex' + b'":0,"Key":"foo","Flags":0,"Value":"YmFy"},{"CreateIndex":644,"Mo' + b'difyIndex":644,"LockIndex":0,"Key":"quz","Flags":0,"Value":"dHJ1' + b'ZQ=="}]' +) + +ALL_ITEMS = [ + {"CreateIndex": 643, "Flags": 0, "Key": "bar", "LockIndex": 0, "ModifyIndex": 643, "Value": "baz"}, + {"CreateIndex": 669, "Flags": 0, "Key": "baz", "LockIndex": 0, "ModifyIndex": 669, "Value": "qux"}, + {"CreateIndex": 666, "Flags": 128, "Key": "corgie", "LockIndex": 0, "ModifyIndex": 666, "Value": "dog"}, + {"CreateIndex": 642, "Flags": 0, "Key": "foo", "LockIndex": 0, "ModifyIndex": 642, "Value": "bar"}, + {"CreateIndex": 644, "Flags": 0, "Key": "quz", "LockIndex": 0, "ModifyIndex": 644, "Value": "true"}, +] @httmock.all_requests def kv_all_records_content(_url_unused, request): return httmock.response( - 200, ALL_DATA, { - 'X-Consul-Index': 4, - 'X-Consul-Knownleader': 'true', - 'X-Consul-Lastcontact': 0, - 'Date': 'Fri, 19 Dec 2014 20:44:28 GMT', - 'Content-Length': len(ALL_DATA), - 'Content-Type': 'application/json' - }, None, 0, request) + 200, + ALL_DATA, + { + "X-Consul-Index": 4, + "X-Consul-Knownleader": "true", + "X-Consul-Lastcontact": 0, + "Date": "Fri, 19 Dec 2014 20:44:28 GMT", + "Content-Length": len(ALL_DATA), + "Content-Type": "application/json", + }, + None, + 0, + request, + ) class KVTests(unittest.TestCase): def setUp(self): self.adapter = adapters.Request() - self.base_uri = '{0}://localhost:8500/{1}'.format(SCHEME, VERSION) + self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) self.dc = str(uuid.uuid4()) self.token = str(uuid.uuid4()) self.kv = api.KV(self.base_uri, self.adapter, self.dc, self.token) @@ -85,7 +64,7 @@ def response_content(_url_unused, request): return httmock.response(200, None, {}, None, 0, request) with httmock.HTTMock(response_content): - self.assertIn('foo', self.kv) + self.assertIn("foo", self.kv) def test_contains_evaluates_false(self): @httmock.all_requests @@ -93,7 +72,7 @@ def response_content(_url_unused, request): return httmock.response(404, None, {}, None, 0, request) with httmock.HTTMock(response_content): - self.assertNotIn('foo', self.kv) + self.assertNotIn("foo", self.kv) def test_get_all_items(self): with httmock.HTTMock(kv_all_records_content): @@ -103,22 +82,22 @@ def test_get_all_items(self): def test_items(self): with httmock.HTTMock(kv_all_records_content): for index, row in enumerate(self.kv.items()): - value = {ALL_ITEMS[index]['Key']: ALL_ITEMS[index]['Value']} + value = {ALL_ITEMS[index]["Key"]: ALL_ITEMS[index]["Value"]} self.assertDictEqual(row, value) def test_iter(self): with httmock.HTTMock(kv_all_records_content): for index, row in enumerate(self.kv): - self.assertEqual(row, ALL_ITEMS[index]['Key']) + self.assertEqual(row, ALL_ITEMS[index]["Key"]) def test_iteritems(self): with httmock.HTTMock(kv_all_records_content): for index, row in enumerate(self.kv.iteritems()): - value = (ALL_ITEMS[index]['Key'], ALL_ITEMS[index]['Value']) + value = (ALL_ITEMS[index]["Key"], ALL_ITEMS[index]["Value"]) self.assertEqual(row, value) def test_keys(self): - expectation = [item['Key'] for item in ALL_ITEMS] + expectation = [item["Key"] for item in ALL_ITEMS] with httmock.HTTMock(kv_all_records_content): self.assertEqual(self.kv.keys(), expectation) @@ -129,7 +108,7 @@ def test_len(self): def test_values(self): with httmock.HTTMock(kv_all_records_content): for index, row in enumerate(self.kv.values()): - self.assertEqual(row, ALL_ITEMS[index]['Value']) + self.assertEqual(row, ALL_ITEMS[index]["Value"]) class TestKVGetWithNoKey(base.TestCase): @@ -145,7 +124,7 @@ def test_get_item_raises_key_error(self, key): class TestKVSet(base.TestCase): @base.generate_key def test_set_item_del_item(self, key): - self.consul.kv[key] = 'foo' + self.consul.kv[key] = "foo" del self.consul.kv[key] self.assertNotIn(key, self.consul.kv) @@ -156,24 +135,24 @@ def test_set_item_get_item_bool_value(self, key): @base.generate_key def test_set_path_with_value(self, key): - path = 'path/{0}/'.format(key) - self.consul.kv.set(path, 'bar') - self.assertEqual('bar', self.consul.kv[path[:-1]]) + path = "path/{0}/".format(key) + self.consul.kv.set(path, "bar") + self.assertEqual("bar", self.consul.kv[path[:-1]]) @base.generate_key def test_set_item_get_item_int_value(self, key): self.consul.kv[key] = 128 - self.assertEqual(self.consul.kv[key], '128') + self.assertEqual(self.consul.kv[key], "128") @base.generate_key def test_set_item_get_item_str_value_key(self, key): - self.consul.kv[key] = b'foo' - self.assertEqual(self.consul.kv[key], 'foo') + self.consul.kv[key] = b"foo" + self.assertEqual(self.consul.kv[key], "foo") @base.generate_key def test_set_item_get_item_str_value_raw(self, key): - self.consul.kv[key] = 'foo' - self.assertEqual(self.consul.kv.get(key, raw=True), 'foo') + self.consul.kv[key] = "foo" + self.assertEqual(self.consul.kv.get(key, raw=True), "foo") @base.generate_key def test_set_get_bool_value(self, key): @@ -183,19 +162,19 @@ def test_set_get_bool_value(self, key): @base.generate_key def test_set_get_item_value(self, key): self.consul.kv.set(key, 128) - self.assertEqual(self.consul.kv.get(key), '128') + self.assertEqual(self.consul.kv.get(key), "128") @base.generate_key def test_set_item_get_item_str_value(self, key): - self.consul.kv.set(key, 'foo') - self.assertEqual(self.consul.kv.get(key), 'foo') + self.consul.kv.set(key, "foo") + self.assertEqual(self.consul.kv.get(key), "foo") @base.generate_key def test_set_item_get_record(self, key): - self.consul.kv.set_record(key, 12, 'record') + self.consul.kv.set_record(key, 12, "record") record = self.consul.kv.get_record(key) - self.assertEqual('record', record['Value']) - self.assertEqual(12, record['Flags']) + self.assertEqual("record", record["Value"]) + self.assertEqual(12, record["Flags"]) self.assertIsInstance(record, dict) @base.generate_key @@ -204,33 +183,33 @@ def test_get_record_fail(self, key): @base.generate_key def test_set_record_no_replace_get_item_str_value(self, key): - self.consul.kv.set(key, 'foo') - self.consul.kv.set_record(key, 0, 'foo', False) - self.assertEqual(self.consul.kv.get(key), 'foo') + self.consul.kv.set(key, "foo") + self.consul.kv.set_record(key, 0, "foo", False) + self.assertEqual(self.consul.kv.get(key), "foo") @base.generate_key def test_set_record_same_value_get_item_str_value(self, key): - self.consul.kv.set(key, 'foo') - self.consul.kv.set_record(key, 0, 'foo', True) - self.assertEqual(self.consul.kv.get(key), 'foo') + self.consul.kv.set(key, "foo") + self.consul.kv.set_record(key, 0, "foo", True) + self.assertEqual(self.consul.kv.get(key), "foo") @base.generate_key def test_set_item_get_item_dict_value(self, key): - value = {'foo': 'bar'} + value = {"foo": "bar"} expectation = json.dumps(value) self.consul.kv.set(key, value) self.assertEqual(self.consul.kv.get(key), expectation) @base.generate_key def test_set_item_get_item_unicode_value(self, key): - self.consul.kv.set(key, 'I like to ✈') + self.consul.kv.set(key, "I like to ✈") response = self.consul.kv.get(key) - self.assertEqual(response, 'I like to ✈') + self.assertEqual(response, "I like to ✈") @base.generate_key def test_set_item_in_records(self, key): - self.consul.kv.set(key, 'zomg') - expectation = (key, 0, 'zomg') + self.consul.kv.set(key, "zomg") + expectation = (key, 0, "zomg") self.assertIn(expectation, self.consul.kv.records()) @base.generate_key @@ -245,8 +224,7 @@ class TestKVLocking(base.TestCase): @base.generate_key def test_acquire_and_release_lock(self, key): lock_key = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create( - key, behavior='delete', ttl='60s') + session_id = self.consul.session.create(key, behavior="delete", ttl="60s") self.assertTrue(self.consul.kv.acquire_lock(lock_key, session_id)) self.assertTrue(self.consul.kv.release_lock(lock_key, session_id)) self.consul.session.destroy(session_id) @@ -254,9 +232,8 @@ def test_acquire_and_release_lock(self, key): @base.generate_key def test_acquire_and_release_lock(self, key): lock_key = str(uuid.uuid4())[0:8] - sid = self.consul.session.create(key, behavior='delete', ttl='60s') - sid2 = self.consul.session.create( - key + '2', behavior='delete', ttl='60s') + sid = self.consul.session.create(key, behavior="delete", ttl="60s") + sid2 = self.consul.session.create(key + "2", behavior="delete", ttl="60s") self.assertTrue(self.consul.kv.acquire_lock(lock_key, sid)) self.assertFalse(self.consul.kv.acquire_lock(lock_key, sid2)) self.assertTrue(self.consul.kv.release_lock(lock_key, sid)) @@ -267,9 +244,8 @@ def test_acquire_and_release_lock(self, key): def test_acquire_and_release_lock_with_value(self, key): lock_key = str(uuid.uuid4())[0:8] lock_value = str(uuid.uuid4()) - sid = self.consul.session.create(key, behavior='delete', ttl='60s') - sid2 = self.consul.session.create( - key + '2', behavior='delete', ttl='60s') + sid = self.consul.session.create(key, behavior="delete", ttl="60s") + sid2 = self.consul.session.create(key + "2", behavior="delete", ttl="60s") self.assertTrue(self.consul.kv.acquire_lock(lock_key, sid, lock_value)) self.assertEqual(self.consul.kv.get(lock_key), lock_value) self.assertFalse(self.consul.kv.acquire_lock(lock_key, sid2)) diff --git a/tests/services_discovery_consulate/test_session.py b/tests/services_discovery_consulate/test_session.py index 4edbad9..c05f0f3 100644 --- a/tests/services_discovery_consulate/test_session.py +++ b/tests/services_discovery_consulate/test_session.py @@ -14,30 +14,25 @@ def tearDown(self): def test_session_create(self): name = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create( - name, behavior='delete', ttl='60s') + session_id = self.consul.session.create(name, behavior="delete", ttl="60s") self.sessions.append(session_id) self.assertIsNotNone(session_id) def test_session_destroy(self): name = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create( - name, behavior='delete', ttl='60s') + session_id = self.consul.session.create(name, behavior="delete", ttl="60s") self.consul.session.destroy(session_id) - self.assertNotIn(session_id, - [s.get('ID') for s in self.consul.session.list()]) + self.assertNotIn(session_id, [s.get("ID") for s in self.consul.session.list()]) def test_session_info(self): name = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create( - name, behavior='delete', ttl='60s') + session_id = self.consul.session.create(name, behavior="delete", ttl="60s") result = self.consul.session.info(session_id) - self.assertEqual(session_id, result.get('ID')) + self.assertEqual(session_id, result.get("ID")) self.consul.session.destroy(session_id) def test_session_renew(self): name = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create( - name, behavior='delete', ttl='60s') + session_id = self.consul.session.create(name, behavior="delete", ttl="60s") self.sessions.append(session_id) self.assertTrue(self.consul.session.renew(session_id)) diff --git a/tests/services_discovery_consulate/test_utils.py b/tests/services_discovery_consulate/test_utils.py index a48ccac..0ac7b2b 100644 --- a/tests/services_discovery_consulate/test_utils.py +++ b/tests/services_discovery_consulate/test_utils.py @@ -6,72 +6,65 @@ class MaybeEncodeTestCase(unittest.TestCase): def test_str_test(self): - self.assertEqual(utils.maybe_encode('foo'), b'foo') + self.assertEqual(utils.maybe_encode("foo"), b"foo") def test_byte_test(self): - self.assertEqual(utils.maybe_encode(b'bar'), b'bar') + self.assertEqual(utils.maybe_encode(b"bar"), b"bar") class Response(object): - def __init__(self, status_code=200, body=b'content'): + def __init__(self, status_code=200, body=b"content"): self.status_code = status_code self.body = body class ResponseOkTestCase(unittest.TestCase): - def test_200(self): - self.assertTrue(utils.response_ok(Response(200, b'ok'))) + self.assertTrue(utils.response_ok(Response(200, b"ok"))) def test_400(self): with self.assertRaises(exceptions.ClientError): - utils.response_ok(Response(400, b'Bad request')) + utils.response_ok(Response(400, b"Bad request")) def test_401(self): with self.assertRaises(exceptions.ACLDisabled): - utils.response_ok(Response(401, b'What ACL?')) + utils.response_ok(Response(401, b"What ACL?")) def test_403(self): with self.assertRaises(exceptions.Forbidden): - utils.response_ok(Response(403, b'No')) + utils.response_ok(Response(403, b"No")) def test_404_not_raising(self): - self.assertFalse(utils.response_ok(Response(404, b'not found'))) + self.assertFalse(utils.response_ok(Response(404, b"not found"))) def test_404_raising(self): with self.assertRaises(exceptions.NotFound): - utils.response_ok(Response(404, b'Not Found'), True) + utils.response_ok(Response(404, b"Not Found"), True) def test_500(self): with self.assertRaises(exceptions.ServerError): - utils.response_ok(Response(500, b'Opps')) - - + utils.response_ok(Response(500, b"Opps")) class ValidateGoDurationTestCase(unittest.TestCase): - def test_valid_values(self): - for value in {'5µs', '300ms', '-1.5h', '2h45m', '5m', '30s'}: - print('Testing {}'.format(value)) + for value in {"5µs", "300ms", "-1.5h", "2h45m", "5m", "30s"}: + print("Testing {}".format(value)) self.assertTrue(utils.validate_go_interval(value)) def test_invalid_values(self): - for value in {'100', '1 year', '5M', '30S'}: - print('Testing {}'.format(value)) + for value in {"100", "1 year", "5M", "30S"}: + print("Testing {}".format(value)) self.assertFalse(utils.validate_go_interval(value)) class ValidateURLTestCase(unittest.TestCase): - def test_valid_values(self): - for value in {'https://foo', 'http://localhost/bar'}: - print('Testing {}'.format(value)) + for value in {"https://foo", "http://localhost/bar"}: + print("Testing {}".format(value)) self.assertTrue(utils.validate_url(value)) def test_invalid_values(self): - for value in {'localhost', 'a'}: - print('Testing {}'.format(value)) + for value in {"localhost", "a"}: + print("Testing {}".format(value)) self.assertFalse(utils.validate_url(value)) - - From bbc1cde783642e3c30f2624f7c3ede35596f3eab Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 18:44:51 +0100 Subject: [PATCH 17/27] fix: wait until docker is up --- docker/docker-compose.yml | 4 ++-- tests/conftest.py | 26 ++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d4438c5..d5621bc 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,9 +1,9 @@ -version: '3' +version: '2' services: consul: image: consul:1.6.0 ports: - - 8500:8500 + - "8500:8500" volumes: - './testing:/consul/config' \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 313fdbe..b005b95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,32 @@ import os import pytest +import requests +from requests.exceptions import ConnectionError os.environ["ASYNC_TEST_TIMEOUT"] = os.environ.get("ASYNC_TEST_TIMEOUT", "15") +def is_responsive(url): + try: + response = requests.get(url) + if response.status_code == 200: + return True + except ConnectionError: + return False + + +@pytest.fixture(scope="session") +def http_service(docker_ip, docker_services): + """Ensure that HTTP service is up and responsive.""" + url = "http://{}:{}".format(docker_ip, 8500) + + docker_services.wait_until_responsive( + timeout=60.0, pause=0.1, check=lambda: is_responsive(url) + ) + return docker_ip + + @pytest.fixture(scope="session") def docker_compose_file(pytestconfig): """Get docker compose file""" @@ -12,7 +34,7 @@ def docker_compose_file(pytestconfig): @pytest.fixture(scope="session") -def config_env(docker_ip, docker_services): +def config_env(http_service): """Set config for docker""" - os.environ["CONSUL_HOST"] = docker_ip + os.environ["CONSUL_HOST"] = http_service os.environ["CONSUL_PORT"] = "8500" From fd89113058110f9651304bc544792286fffb84a6 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 18:57:05 +0100 Subject: [PATCH 18/27] fix: assert raise depends of consul version --- tests/services_discovery_consulate/test_acl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/services_discovery_consulate/test_acl.py b/tests/services_discovery_consulate/test_acl.py index b349021..b8786f0 100644 --- a/tests/services_discovery_consulate/test_acl.py +++ b/tests/services_discovery_consulate/test_acl.py @@ -86,7 +86,7 @@ def response_content(_url_unused, request): self.assertEqual(result, expectation) def test_bootstrap_raises(self): - with self.assertRaises(consulate.Forbidden): + with self.assertRaises((consulate.Forbidden, consulate.ServerError)): self.consul.acl.bootstrap() def test_clone_bad_acl_id(self): From fa1752967c17ef57cd6119e46c3ef33325c6a948 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 19:10:41 +0100 Subject: [PATCH 19/27] fix: tests, isort and linters --- Pipfile.lock | 3 +-- .../ms1/main.py | 2 +- pyms/cmd/main.py | 4 +-- pyms/config/__init__.py | 2 +- pyms/config/conf.py | 14 ++++++----- pyms/config/confile.py | 6 ++--- pyms/crypt/fernet.py | 2 +- pyms/flask/app/__init__.py | 1 - pyms/flask/app/create_app.py | 6 ++--- pyms/flask/configreload/__init__.py | 1 - pyms/flask/configreload/configreload.py | 6 ++--- pyms/flask/healthcheck/__init__.py | 1 - pyms/flask/healthcheck/healthcheck.py | 2 +- pyms/flask/services/driver.py | 4 +-- pyms/flask/services/metrics.py | 4 +-- pyms/flask/services/swagger.py | 5 ++-- pyms/flask/services/tracer.py | 5 ++-- pyms/services_discovery/consulate/__init__.py | 8 +++--- .../consulate/api/__init__.py | 4 +-- pyms/services_discovery/consulate/api/kv.py | 2 +- pyms/services_discovery/consulate/api/lock.py | 2 +- pyms/services_discovery/consulate/cli.py | 9 +++---- pyms/services_discovery/consulate/client.py | 25 ++++++++++--------- pyms/utils/__init__.py | 2 +- pyms/utils/utils.py | 2 +- setup.cfg | 4 +++ setup.py | 3 ++- tests/common.py | 3 ++- .../services_discovery_consulate/test_acl.py | 2 +- .../services_discovery_consulate/test_api.py | 2 +- tests/services_discovery_consulate/test_kv.py | 1 + tests/test_cmd.py | 5 ++-- tests/test_config.py | 8 +++--- tests/test_crypt.py | 4 +-- tests/test_metrics.py | 5 ++-- tests/test_utils.py | 4 +-- 36 files changed, 82 insertions(+), 81 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 2eb116c..ef2674b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "499af62c8d853c966d75ee7c899c41eee7ebebba5c23c74aee2a61b231cf8e29" + "sha256": "30f7bae4efa04f995974cbeab5512cb644c3726a04ab10ba9cf0e4ff01c67cb5" }, "pipfile-spec": 6, "requires": {}, @@ -1112,7 +1112,6 @@ "sha256:7a0544dc00c83fd7808907a55835b1b7c07e3ea2d22ee558b5a233247e462735", "sha256:9faa9c87e6e0920612005c8d187c58b3d336f3af49a878f358578de125963858" ], - "index": "pypi", "version": "==0.10.1" }, "python-dotenv": { diff --git a/examples/microservice_distribued_tracing/ms1/main.py b/examples/microservice_distribued_tracing/ms1/main.py index 87c7b10..f92c06b 100644 --- a/examples/microservice_distribued_tracing/ms1/main.py +++ b/examples/microservice_distribued_tracing/ms1/main.py @@ -1,4 +1,4 @@ -from flask import jsonify, current_app, request +from flask import current_app, jsonify, request from pyms.flask.app import Microservice diff --git a/pyms/cmd/main.py b/pyms/cmd/main.py index 1ac6220..e622f0a 100755 --- a/pyms/cmd/main.py +++ b/pyms/cmd/main.py @@ -1,15 +1,15 @@ #!/usr/bin/env python -from __future__ import unicode_literals, print_function +from __future__ import print_function, unicode_literals import argparse import os import sys from distutils.util import strtobool +from pyms.config import create_conf_file from pyms.crypt.fernet import Crypt from pyms.flask.services.swagger import merge_swagger_file from pyms.utils import check_package_exists, import_from, utils -from pyms.config import create_conf_file class Command: diff --git a/pyms/config/__init__.py b/pyms/config/__init__.py index 3b02d25..c6e9777 100644 --- a/pyms/config/__init__.py +++ b/pyms/config/__init__.py @@ -1,4 +1,4 @@ +from .conf import create_conf_file, get_conf from .confile import ConfFile -from .conf import get_conf, create_conf_file __all__ = ["get_conf", "create_conf_file", "ConfFile"] diff --git a/pyms/config/conf.py b/pyms/config/conf.py index 9ec5561..9c4f48e 100644 --- a/pyms/config/conf.py +++ b/pyms/config/conf.py @@ -1,19 +1,21 @@ -import os import logging +import os from typing import Union + import yaml -from pyms.utils import utils + from pyms.config.confile import ConfFile from pyms.constants import ( - PYMS_CONFIG_WHITELIST_KEYWORDS, - CONFIGMAP_FILE_ENVIRONMENT_LEGACY, CONFIGMAP_FILE_ENVIRONMENT, + CONFIGMAP_FILE_ENVIRONMENT_LEGACY, CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, - LOGGER_NAME, DEFAULT_CONFIGMAP_FILENAME, + LOGGER_NAME, + PYMS_CONFIG_WHITELIST_KEYWORDS, ) -from pyms.exceptions import ServiceDoesNotExistException, ConfigErrorException, AttrDoesNotExistException +from pyms.exceptions import AttrDoesNotExistException, ConfigErrorException, ServiceDoesNotExistException +from pyms.utils import utils logger = logging.getLogger(LOGGER_NAME) diff --git a/pyms/config/confile.py b/pyms/config/confile.py index db2bd43..f028a36 100644 --- a/pyms/config/confile.py +++ b/pyms/config/confile.py @@ -2,15 +2,15 @@ import logging import os import re -from typing import Dict, Union, Text, Tuple, Iterable +from typing import Dict, Iterable, Text, Tuple, Union import anyconfig from pyms.constants import ( CONFIGMAP_FILE_ENVIRONMENT, - LOGGER_NAME, - DEFAULT_CONFIGMAP_FILENAME, CONFIGMAP_FILE_ENVIRONMENT_LEGACY, + DEFAULT_CONFIGMAP_FILENAME, + LOGGER_NAME, ) from pyms.exceptions import AttrDoesNotExistException, ConfigDoesNotFoundException from pyms.utils.files import LoadFile diff --git a/pyms/crypt/fernet.py b/pyms/crypt/fernet.py index 602d1a7..04732f3 100644 --- a/pyms/crypt/fernet.py +++ b/pyms/crypt/fernet.py @@ -7,7 +7,7 @@ from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC -from pyms.constants import CRYPT_FILE_KEY_ENVIRONMENT, DEFAULT_KEY_FILENAME, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY +from pyms.constants import CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, DEFAULT_KEY_FILENAME from pyms.crypt.driver import CryptAbstract from pyms.exceptions import FileDoesNotExistException from pyms.utils.files import LoadFile diff --git a/pyms/flask/app/__init__.py b/pyms/flask/app/__init__.py index c0b477a..cbe4ddd 100644 --- a/pyms/flask/app/__init__.py +++ b/pyms/flask/app/__init__.py @@ -1,5 +1,4 @@ from .create_app import Microservice from .create_config import config - __all__ = ["Microservice", "config"] diff --git a/pyms/flask/app/create_app.py b/pyms/flask/app/create_app.py index 333e2f1..2ab29a9 100644 --- a/pyms/flask/app/create_app.py +++ b/pyms/flask/app/create_app.py @@ -6,12 +6,12 @@ from pyms.config.conf import validate_conf from pyms.config.resource import ConfigResource -from pyms.constants import LOGGER_NAME, CONFIG_BASE +from pyms.constants import CONFIG_BASE, LOGGER_NAME from pyms.crypt.driver import CryptResource -from pyms.flask.app.utils import SingletonMeta, ReverseProxied +from pyms.flask.app.utils import ReverseProxied, SingletonMeta from pyms.flask.configreload import configreload_blueprint from pyms.flask.healthcheck import healthcheck_blueprint -from pyms.flask.services.driver import ServicesResource, DriverService +from pyms.flask.services.driver import DriverService, ServicesResource from pyms.logger import CustomJsonFormatter from pyms.utils import check_package_exists diff --git a/pyms/flask/configreload/__init__.py b/pyms/flask/configreload/__init__.py index 32c702d..482976e 100644 --- a/pyms/flask/configreload/__init__.py +++ b/pyms/flask/configreload/__init__.py @@ -1,4 +1,3 @@ from .configreload import configreload_blueprint - __all__ = ["configreload_blueprint"] diff --git a/pyms/flask/configreload/configreload.py b/pyms/flask/configreload/configreload.py index e60d3ed..55e44dc 100644 --- a/pyms/flask/configreload/configreload.py +++ b/pyms/flask/configreload/configreload.py @@ -1,6 +1,6 @@ -from __future__ import unicode_literals, print_function, absolute_import, division -from flask import Blueprint -from flask import current_app +from __future__ import absolute_import, division, print_function, unicode_literals + +from flask import Blueprint, current_app configreload_blueprint = Blueprint("configreload", __name__, static_url_path="/static") diff --git a/pyms/flask/healthcheck/__init__.py b/pyms/flask/healthcheck/__init__.py index 7374bb2..1fdbbf8 100644 --- a/pyms/flask/healthcheck/__init__.py +++ b/pyms/flask/healthcheck/__init__.py @@ -1,4 +1,3 @@ from .healthcheck import healthcheck_blueprint - __all__ = ["healthcheck_blueprint"] diff --git a/pyms/flask/healthcheck/healthcheck.py b/pyms/flask/healthcheck/healthcheck.py index e584272..de2bd41 100644 --- a/pyms/flask/healthcheck/healthcheck.py +++ b/pyms/flask/healthcheck/healthcheck.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals, print_function, absolute_import, division +from __future__ import absolute_import, division, print_function, unicode_literals from flask import Blueprint diff --git a/pyms/flask/services/driver.py b/pyms/flask/services/driver.py index 40c5e32..b6cb8e5 100644 --- a/pyms/flask/services/driver.py +++ b/pyms/flask/services/driver.py @@ -1,9 +1,9 @@ import logging -from typing import Text, Tuple, Iterator +from typing import Iterator, Text, Tuple from pyms.config import ConfFile from pyms.config.resource import ConfigResource -from pyms.constants import SERVICE_BASE, LOGGER_NAME +from pyms.constants import LOGGER_NAME, SERVICE_BASE from pyms.utils import import_from logger = logging.getLogger(LOGGER_NAME) diff --git a/pyms/flask/services/metrics.py b/pyms/flask/services/metrics.py index 279f3fa..0093cbf 100644 --- a/pyms/flask/services/metrics.py +++ b/pyms/flask/services/metrics.py @@ -1,8 +1,8 @@ import logging import time -from flask import Blueprint, Response, request, Flask -from prometheus_client import multiprocess, Counter, Histogram, generate_latest, CollectorRegistry, REGISTRY +from flask import Blueprint, Flask, Response, request +from prometheus_client import REGISTRY, CollectorRegistry, Counter, Histogram, generate_latest, multiprocess from pyms.flask.services.driver import DriverService diff --git a/pyms/flask/services/swagger.py b/pyms/flask/services/swagger.py index cd13510..1223ad1 100644 --- a/pyms/flask/services/swagger.py +++ b/pyms/flask/services/swagger.py @@ -1,11 +1,10 @@ import os from pathlib import Path -from typing import Dict, Any - -from flask import Flask +from typing import Any, Dict import connexion from connexion.resolver import RestyResolver +from flask import Flask try: import prance diff --git a/pyms/flask/services/tracer.py b/pyms/flask/services/tracer.py index 718f54b..24629f2 100644 --- a/pyms/flask/services/tracer.py +++ b/pyms/flask/services/tracer.py @@ -1,7 +1,6 @@ import logging from typing import Union - try: import opentracing except ModuleNotFoundError: # pragma: no cover @@ -15,12 +14,12 @@ except ModuleNotFoundError: # pragma: no cover get_current_span = None -from flask import current_app, request, has_request_context +from flask import current_app, has_request_context, request from pyms.config.conf import get_conf from pyms.constants import LOGGER_NAME from pyms.flask.services.driver import DriverService, get_service_name -from pyms.utils import check_package_exists, import_package, import_from +from pyms.utils import check_package_exists, import_from, import_package logger = logging.getLogger(LOGGER_NAME) diff --git a/pyms/services_discovery/consulate/__init__.py b/pyms/services_discovery/consulate/__init__.py index ed06f46..ec66677 100644 --- a/pyms/services_discovery/consulate/__init__.py +++ b/pyms/services_discovery/consulate/__init__.py @@ -7,14 +7,14 @@ from pyms.services_discovery.consulate.client import Consul from pyms.services_discovery.consulate.exceptions import ( - ConsulateException, - ClientError, - ServerError, ACLDisabled, + ClientError, + ConsulateException, Forbidden, - NotFound, LockFailure, + NotFound, RequestError, + ServerError, ) __version__ = "1.0.0" diff --git a/pyms/services_discovery/consulate/api/__init__.py b/pyms/services_discovery/consulate/api/__init__.py index 29d060b..ce58ff0 100644 --- a/pyms/services_discovery/consulate/api/__init__.py +++ b/pyms/services_discovery/consulate/api/__init__.py @@ -4,14 +4,14 @@ """ from pyms.services_discovery.consulate.api.acl import ACL from pyms.services_discovery.consulate.api.agent import Agent +from pyms.services_discovery.consulate.api.base import Response from pyms.services_discovery.consulate.api.catalog import Catalog +from pyms.services_discovery.consulate.api.coordinate import Coordinate from pyms.services_discovery.consulate.api.event import Event from pyms.services_discovery.consulate.api.health import Health -from pyms.services_discovery.consulate.api.coordinate import Coordinate from pyms.services_discovery.consulate.api.kv import KV from pyms.services_discovery.consulate.api.lock import Lock from pyms.services_discovery.consulate.api.session import Session from pyms.services_discovery.consulate.api.status import Status -from pyms.services_discovery.consulate.api.base import Response __all__ = ["ACL", "Agent", "Catalog", "Event", "Health", "KV", "Lock", "Session", "Status", "Response", "Coordinate"] diff --git a/pyms/services_discovery/consulate/api/kv.py b/pyms/services_discovery/consulate/api/kv.py index 89feb40..cacd864 100644 --- a/pyms/services_discovery/consulate/api/kv.py +++ b/pyms/services_discovery/consulate/api/kv.py @@ -2,8 +2,8 @@ Consul KV Endpoint Access """ +from pyms.services_discovery.consulate import exceptions, utils from pyms.services_discovery.consulate.api import base -from pyms.services_discovery.consulate import utils, exceptions class KV(base.Endpoint): diff --git a/pyms/services_discovery/consulate/api/lock.py b/pyms/services_discovery/consulate/api/lock.py index 3e621dc..fbf6e04 100644 --- a/pyms/services_discovery/consulate/api/lock.py +++ b/pyms/services_discovery/consulate/api/lock.py @@ -6,8 +6,8 @@ import logging import uuid -from pyms.services_discovery.consulate.api import base from pyms.services_discovery.consulate import exceptions +from pyms.services_discovery.consulate.api import base LOGGER = logging.getLogger(__name__) diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index fa6dcf7..f7c3e50 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -2,19 +2,16 @@ import argparse import base64 import json -import sys import os -import time import subprocess # nosec - - +import sys +import time import urllib.parse as urlparse from requests import exceptions from pyms.services_discovery import consulate -from pyms.services_discovery.consulate import adapters -from pyms.services_discovery.consulate import utils +from pyms.services_discovery.consulate import adapters, utils CONSUL_ENV_VAR = "CONSUL_RPC_ADDR" EPILOG = ( diff --git a/pyms/services_discovery/consulate/client.py b/pyms/services_discovery/consulate/client.py index 1eb3e3a..eac128a 100644 --- a/pyms/services_discovery/consulate/client.py +++ b/pyms/services_discovery/consulate/client.py @@ -3,9 +3,10 @@ """ import os -from pyms.services_discovery.consulate import adapters, api from urllib.parse import quote +from pyms.services_discovery.consulate import adapters, api + DEFAULT_HOST = os.environ.get("CONSUL_HOST") or "localhost" DEFAULT_PORT = os.environ.get("CONSUL_PORT") or 8500 DEFAULT_ADDR = os.environ.get("CONSUL_HTTP_ADDR") @@ -42,17 +43,17 @@ class Consul: # pylint: disable=too-many-instance-attributes """ def __init__( - self, - addr=DEFAULT_ADDR, - host=DEFAULT_HOST, - port=DEFAULT_PORT, - datacenter=None, - token=DEFAULT_TOKEN, - scheme=DEFAULT_SCHEME, - adapter=None, - verify=True, - cert=None, - timeout=None, + self, + addr=DEFAULT_ADDR, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + datacenter=None, + token=DEFAULT_TOKEN, + scheme=DEFAULT_SCHEME, + adapter=None, + verify=True, + cert=None, + timeout=None, ): # pylint: disable=too-many-arguments """Create a new instance of the Consul class""" base_uri = self._base_uri(addr=addr, scheme=scheme, host=host, port=port) diff --git a/pyms/utils/__init__.py b/pyms/utils/__init__.py index 2d21216..98f0535 100644 --- a/pyms/utils/__init__.py +++ b/pyms/utils/__init__.py @@ -1,3 +1,3 @@ -from .utils import import_from, import_package, check_package_exists +from .utils import check_package_exists, import_from, import_package __all__ = ["import_from", "import_package", "check_package_exists"] diff --git a/pyms/utils/utils.py b/pyms/utils/utils.py index 0e68148..93a8b49 100644 --- a/pyms/utils/utils.py +++ b/pyms/utils/utils.py @@ -1,6 +1,6 @@ import importlib import importlib.util -from typing import Union, Text +from typing import Text, Union from pyms.exceptions import PackageNotExists diff --git a/setup.cfg b/setup.cfg index 0b832bb..e45920b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,6 +4,10 @@ test = pytest [tool:pytest] addopts = --cov=pyms --cov=tests tests/ +[isort] +profile = black +line_length = 120 + [flake8] ignore = E501 exclude = .git,__pycache__,docs/source/conf.py,old,build,dist diff --git a/setup.py b/setup.py index ed8e13e..83d7c39 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import codecs import os -from setuptools import setup, find_packages +from setuptools import find_packages, setup author = __import__("pyms").__author__ author_email = __import__("pyms").__email__ @@ -68,6 +68,7 @@ "mypy>=0.782", "pre-commit>=2.8.1", "black>=20.8b1", + "isort>=5.6.4", "httmock>=1.4.0", ] diff --git a/tests/common.py b/tests/common.py index 4249bf6..cb52cf9 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,6 +1,7 @@ import os -from pyms.flask.app import Microservice + from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT, DEFAULT_CONFIGMAP_FILENAME +from pyms.flask.app import Microservice class MyMicroserviceNoSingleton(Microservice): diff --git a/tests/services_discovery_consulate/test_acl.py b/tests/services_discovery_consulate/test_acl.py index b8786f0..22ddc01 100644 --- a/tests/services_discovery_consulate/test_acl.py +++ b/tests/services_discovery_consulate/test_acl.py @@ -3,8 +3,8 @@ """ import json -import uuid import random +import uuid import httmock diff --git a/tests/services_discovery_consulate/test_api.py b/tests/services_discovery_consulate/test_api.py index 9eb0e0a..0a039e1 100644 --- a/tests/services_discovery_consulate/test_api.py +++ b/tests/services_discovery_consulate/test_api.py @@ -1,10 +1,10 @@ import json import unittest import uuid +from unittest.mock import patch from urllib import parse import httmock -from unittest.mock import patch from pyms.services_discovery import consulate from pyms.services_discovery.consulate import adapters diff --git a/tests/services_discovery_consulate/test_kv.py b/tests/services_discovery_consulate/test_kv.py index 055f107..a9750ce 100644 --- a/tests/services_discovery_consulate/test_kv.py +++ b/tests/services_discovery_consulate/test_kv.py @@ -6,6 +6,7 @@ import httmock from pyms.services_discovery.consulate import adapters, api + from . import base SCHEME = "http" diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 9f5f3a0..817bbd1 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -1,15 +1,16 @@ """Test common rest operations wrapper. """ import os -from pathlib import Path import unittest +from pathlib import Path from unittest.mock import patch import pytest from prance.util.url import ResolutionError + from pyms.cmd import Command -from pyms.exceptions import FileDoesNotExistException, PackageNotExists from pyms.crypt.fernet import Crypt +from pyms.exceptions import FileDoesNotExistException, PackageNotExists from pyms.flask.services.swagger import get_bundled_specs from tests.common import remove_conf_file diff --git a/tests/test_config.py b/tests/test_config.py index 7e38960..f7bf97a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,21 +3,21 @@ import unittest from unittest import mock -from pyms.config import get_conf, ConfFile, create_conf_file +from pyms.config import ConfFile, create_conf_file, get_conf from pyms.config.conf import validate_conf from pyms.constants import ( + CONFIG_BASE, CONFIGMAP_FILE_ENVIRONMENT, CONFIGMAP_FILE_ENVIRONMENT_LEGACY, - LOGGER_NAME, - CONFIG_BASE, CRYPT_FILE_KEY_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, + LOGGER_NAME, ) from pyms.exceptions import ( AttrDoesNotExistException, ConfigDoesNotFoundException, - ServiceDoesNotExistException, ConfigErrorException, + ServiceDoesNotExistException, ) logger = logging.getLogger(LOGGER_NAME) diff --git a/tests/test_crypt.py b/tests/test_crypt.py index f480a3a..09bb60c 100644 --- a/tests/test_crypt.py +++ b/tests/test_crypt.py @@ -8,11 +8,11 @@ from pyms.cloud.aws.kms import Crypt as CryptAws from pyms.config import get_conf from pyms.constants import ( - LOGGER_NAME, + CONFIG_BASE, CONFIGMAP_FILE_ENVIRONMENT, CRYPT_FILE_KEY_ENVIRONMENT, - CONFIG_BASE, CRYPT_FILE_KEY_ENVIRONMENT_LEGACY, + LOGGER_NAME, ) from pyms.crypt.driver import CryptAbstract, CryptResource from pyms.crypt.fernet import Crypt as CryptFernet diff --git a/tests/test_metrics.py b/tests/test_metrics.py index a9554db..8dfac14 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -4,11 +4,10 @@ from tempfile import TemporaryDirectory from opentracing import global_tracer -from prometheus_client import generate_latest -from prometheus_client import values +from prometheus_client import generate_latest, values from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT -from pyms.flask.services.metrics import LOGGER_TOTAL_MESSAGES, FLASK_REQUEST_COUNT, FLASK_REQUEST_LATENCY +from pyms.flask.services.metrics import FLASK_REQUEST_COUNT, FLASK_REQUEST_LATENCY, LOGGER_TOTAL_MESSAGES from tests.common import MyMicroserviceNoSingleton diff --git a/tests/test_utils.py b/tests/test_utils.py index ad63d9c..812a50a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,9 +5,9 @@ import pytest -from pyms.exceptions import PackageNotExists, FileDoesNotExistException -from pyms.utils import check_package_exists, import_package from pyms.crypt.fernet import Crypt +from pyms.exceptions import FileDoesNotExistException, PackageNotExists +from pyms.utils import check_package_exists, import_package class ConfUtils(unittest.TestCase): From 6b0614012d3c799a89aac95c1af0a5d7f7589e2e Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 19:16:19 +0100 Subject: [PATCH 20/27] fix: bandit --- pyms/services_discovery/consulate/cli.py | 6 ++++-- pyms/services_discovery/consulate/client.py | 22 ++++++++++----------- tests/conftest.py | 4 +--- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index f7c3e50..e0eb792 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -519,8 +519,10 @@ def run_once(consul, args): # Should the subprocess return an error code, release the lock try: print( - subprocess.check_output(args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True) - ) # nosec + subprocess.check_output( + args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True + ) # nosec + ) # If the subprocess fails except subprocess.CalledProcessError as err: error_code = 1 diff --git a/pyms/services_discovery/consulate/client.py b/pyms/services_discovery/consulate/client.py index eac128a..50dd615 100644 --- a/pyms/services_discovery/consulate/client.py +++ b/pyms/services_discovery/consulate/client.py @@ -43,17 +43,17 @@ class Consul: # pylint: disable=too-many-instance-attributes """ def __init__( - self, - addr=DEFAULT_ADDR, - host=DEFAULT_HOST, - port=DEFAULT_PORT, - datacenter=None, - token=DEFAULT_TOKEN, - scheme=DEFAULT_SCHEME, - adapter=None, - verify=True, - cert=None, - timeout=None, + self, + addr=DEFAULT_ADDR, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + datacenter=None, + token=DEFAULT_TOKEN, + scheme=DEFAULT_SCHEME, + adapter=None, + verify=True, + cert=None, + timeout=None, ): # pylint: disable=too-many-arguments """Create a new instance of the Consul class""" base_uri = self._base_uri(addr=addr, scheme=scheme, host=host, port=port) diff --git a/tests/conftest.py b/tests/conftest.py index b005b95..d5e6e31 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,9 +21,7 @@ def http_service(docker_ip, docker_services): """Ensure that HTTP service is up and responsive.""" url = "http://{}:{}".format(docker_ip, 8500) - docker_services.wait_until_responsive( - timeout=60.0, pause=0.1, check=lambda: is_responsive(url) - ) + docker_services.wait_until_responsive(timeout=60.0, pause=0.1, check=lambda: is_responsive(url)) return docker_ip From f947c45b76bca8e6b16803f4e8461b48ada8825d Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 19:54:56 +0100 Subject: [PATCH 21/27] fix: bandit --- pyms/services_discovery/consulate/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index e0eb792..59a3229 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -520,8 +520,8 @@ def run_once(consul, args): try: print( subprocess.check_output( - args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True - ) # nosec + args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True # nosec + ) ) # If the subprocess fails except subprocess.CalledProcessError as err: From d634a5b455e94f9d4559e12dc42252a65e96360d Mon Sep 17 00:00:00 2001 From: avara1986 Date: Mon, 9 Nov 2020 19:59:34 +0100 Subject: [PATCH 22/27] fix: lint... --- pyms/services_discovery/consulate/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py index 59a3229..487786b 100644 --- a/pyms/services_discovery/consulate/cli.py +++ b/pyms/services_discovery/consulate/cli.py @@ -520,7 +520,7 @@ def run_once(consul, args): try: print( subprocess.check_output( - args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True # nosec + args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True # nosec ) ) # If the subprocess fails From 00166e751418458ed59f4f4a124057458dc0217f Mon Sep 17 00:00:00 2001 From: avara1986 Date: Tue, 10 Nov 2020 16:23:14 +0100 Subject: [PATCH 23/27] feat: moved consulate to py-ms-consulate package --- .coveragerc | 3 +- Pipfile.lock | 152 +---- docker/docker-compose.yml | 9 - docker/testing/consul.json | 16 - pyms/flask/services/service_discovery.py | 6 +- pyms/services_discovery/__init__.py | 0 pyms/services_discovery/consulate/__init__.py | 35 -- pyms/services_discovery/consulate/adapters.py | 130 ---- .../consulate/api/__init__.py | 17 - pyms/services_discovery/consulate/api/acl.py | 438 ------------- .../services_discovery/consulate/api/agent.py | 409 ------------ pyms/services_discovery/consulate/api/base.py | 187 ------ .../consulate/api/catalog.py | 176 ------ .../consulate/api/coordinate.py | 63 -- .../services_discovery/consulate/api/event.py | 47 -- .../consulate/api/health.py | 64 -- pyms/services_discovery/consulate/api/kv.py | 398 ------------ pyms/services_discovery/consulate/api/lock.py | 99 --- .../consulate/api/session.py | 109 ---- .../consulate/api/status.py | 31 - pyms/services_discovery/consulate/cli.py | 593 ------------------ pyms/services_discovery/consulate/client.py | 197 ------ .../consulate/exceptions.py | 49 -- .../consulate/models/__init__.py | 2 - .../consulate/models/acl.py | 161 ----- .../consulate/models/agent.py | 161 ----- .../consulate/models/base.py | 165 ----- pyms/services_discovery/consulate/utils.py | 97 --- setup.py | 8 +- tests/conftest.py | 38 -- .../services_discovery_consulate/__init__.py | 1 - tests/services_discovery_consulate/base.py | 66 -- .../services_discovery_consulate/consul.json | 16 - .../services_discovery_consulate/test_acl.py | 301 --------- .../test_agent.py | 281 --------- .../services_discovery_consulate/test_api.py | 319 ---------- .../test_base_model.py | 104 --- .../test_catalog.py | 9 - .../test_coordinate.py | 7 - .../test_event.py | 12 - tests/services_discovery_consulate/test_kv.py | 255 -------- .../services_discovery_consulate/test_lock.py | 10 - .../test_session.py | 38 -- .../test_utils.py | 70 --- 44 files changed, 25 insertions(+), 5324 deletions(-) delete mode 100644 docker/docker-compose.yml delete mode 100644 docker/testing/consul.json delete mode 100644 pyms/services_discovery/__init__.py delete mode 100644 pyms/services_discovery/consulate/__init__.py delete mode 100644 pyms/services_discovery/consulate/adapters.py delete mode 100644 pyms/services_discovery/consulate/api/__init__.py delete mode 100644 pyms/services_discovery/consulate/api/acl.py delete mode 100644 pyms/services_discovery/consulate/api/agent.py delete mode 100644 pyms/services_discovery/consulate/api/base.py delete mode 100644 pyms/services_discovery/consulate/api/catalog.py delete mode 100644 pyms/services_discovery/consulate/api/coordinate.py delete mode 100644 pyms/services_discovery/consulate/api/event.py delete mode 100644 pyms/services_discovery/consulate/api/health.py delete mode 100644 pyms/services_discovery/consulate/api/kv.py delete mode 100644 pyms/services_discovery/consulate/api/lock.py delete mode 100644 pyms/services_discovery/consulate/api/session.py delete mode 100644 pyms/services_discovery/consulate/api/status.py delete mode 100644 pyms/services_discovery/consulate/cli.py delete mode 100644 pyms/services_discovery/consulate/client.py delete mode 100644 pyms/services_discovery/consulate/exceptions.py delete mode 100644 pyms/services_discovery/consulate/models/__init__.py delete mode 100644 pyms/services_discovery/consulate/models/acl.py delete mode 100644 pyms/services_discovery/consulate/models/agent.py delete mode 100644 pyms/services_discovery/consulate/models/base.py delete mode 100644 pyms/services_discovery/consulate/utils.py delete mode 100644 tests/services_discovery_consulate/__init__.py delete mode 100644 tests/services_discovery_consulate/base.py delete mode 100644 tests/services_discovery_consulate/consul.json delete mode 100644 tests/services_discovery_consulate/test_acl.py delete mode 100644 tests/services_discovery_consulate/test_agent.py delete mode 100644 tests/services_discovery_consulate/test_api.py delete mode 100644 tests/services_discovery_consulate/test_base_model.py delete mode 100644 tests/services_discovery_consulate/test_catalog.py delete mode 100644 tests/services_discovery_consulate/test_coordinate.py delete mode 100644 tests/services_discovery_consulate/test_event.py delete mode 100644 tests/services_discovery_consulate/test_kv.py delete mode 100644 tests/services_discovery_consulate/test_lock.py delete mode 100644 tests/services_discovery_consulate/test_session.py delete mode 100644 tests/services_discovery_consulate/test_utils.py diff --git a/.coveragerc b/.coveragerc index 2fcf985..dbda08f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,5 +5,4 @@ include= omit = *examples/* .tox/* - venv/* - pyms/services_discovery/consulate/* \ No newline at end of file + venv/* \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index ef2674b..4555cd2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -30,17 +30,17 @@ }, "boto3": { "hashes": [ - "sha256:23d2575b7bd01c4e7153f283c1d1c12d329dabf78a6279d4192f2e752bb67b1a", - "sha256:cb3f4c2f2576153b845e5b4f325d54a04f4ca00c156f2063965432bfa5d714dd" + "sha256:546586e56ddaf1a75970d65a890351e7093e579a7b686fd45b13e973127dbeec", + "sha256:bc77898ff92c8d328b2f2b74c05008179e6b32041c6b635f0199079ed828f60f" ], - "version": "==1.16.13" + "version": "==1.16.14" }, "botocore": { "hashes": [ - "sha256:1b1b4cf5efd552ecc7f1ce0fc674d5fba56857db5ffe6394ee76edd1a568d7a6", - "sha256:b3b4b8fa33620f015c52e426a92e7db21b5e667ed4785c5fbc484ebfdb2b5153" + "sha256:65bb3bfb81e7496c426d89900c91957b1e20c1af4c8c864d2527d623525193ee", + "sha256:cec2a66faab716773477962be782449f4abdc6475925fc259d66a729a39bc1b7" ], - "version": "==1.19.13" + "version": "==1.19.14" }, "certifi": { "hashes": [ @@ -299,6 +299,13 @@ ], "path": "." }, + "py-ms-consulate": { + "hashes": [ + "sha256:577e16e11ff01739755450c87f93b98bedda66811ebcde6e2292408622eb3fa9", + "sha256:99a03fbd23587403b8e84ac22ee05f0ecb0f9939525b80d3e0c63fdf4f4d1e0c" + ], + "version": "==1.0.0" + }, "pycparser": { "hashes": [ "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", @@ -472,31 +479,12 @@ ], "version": "==3.1.0" }, - "bcrypt": { - "hashes": [ - "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29", - "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7", - "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34", - "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55", - "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6", - "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1", - "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d" - ], - "version": "==3.2.0" - }, "black": { "hashes": [ "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" ], "version": "==20.8b1" }, - "cached-property": { - "hashes": [ - "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", - "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" - ], - "version": "==1.5.2" - }, "certifi": { "hashes": [ "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", @@ -639,42 +627,6 @@ ], "version": "==0.3.1" }, - "distro": { - "hashes": [ - "sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92", - "sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799" - ], - "version": "==1.5.0" - }, - "docker": { - "extras": [ - "ssh" - ], - "hashes": [ - "sha256:13966471e8bc23b36bfb3a6fb4ab75043a5ef1dac86516274777576bed3b9828", - "sha256:bad94b8dd001a8a4af19ce4becc17f41b09f228173ffe6a4e0355389eef142f2" - ], - "version": "==4.3.1" - }, - "docker-compose": { - "hashes": [ - "sha256:5a5690f24c27d4b43dcbe6b3fae91ba680713208e99ee863352b3bae37bcaa83", - "sha256:84ca2edad226435e3a378ea24ca2ca4e1a77cc7c8de057e2812124c6dcb55147" - ], - "version": "==1.27.4" - }, - "dockerpty": { - "hashes": [ - "sha256:69a9d69d573a0daa31bcd1c0774eeed5c15c295fe719c61aca550ed1393156ce" - ], - "version": "==0.4.1" - }, - "docopt": { - "hashes": [ - "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" - ], - "version": "==0.6.2" - }, "dparse": { "hashes": [ "sha256:a1b5f169102e1c894f9a7d5ccf6f9402a836a5d24be80a986c7ce9eaed78f367", @@ -730,13 +682,6 @@ ], "version": "==1.52.0" }, - "httmock": { - "hashes": [ - "sha256:13e6c63f135a928e15d386af789a2890efb03e0e280f29bdc9961f3f0dc34cb9", - "sha256:44eaf4bb59cc64cd6f5d8bf8700b46aa3097cc5651b9bc85c527dfbc71792f41" - ], - "version": "==1.4.0" - }, "identify": { "hashes": [ "sha256:5dd84ac64a9a115b8e0b27d1756b244b882ad264c3c423f42af8235a6e71ca12", @@ -801,13 +746,6 @@ ], "version": "==1.4.1" }, - "jsonschema": { - "hashes": [ - "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", - "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" - ], - "version": "==3.2.0" - }, "lazy-object-proxy": { "hashes": [ "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", @@ -957,13 +895,6 @@ ], "version": "==20.4" }, - "paramiko": { - "hashes": [ - "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898", - "sha256:7f36f4ba2c0d81d219f4595e35f70d56cc94f9ac40a6acdf51d6ca210ce65035" - ], - "version": "==2.7.2" - }, "pathspec": { "hashes": [ "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd", @@ -1057,29 +988,6 @@ ], "version": "==2.6.0" }, - "pynacl": { - "hashes": [ - "sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4", - "sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4", - "sha256:2fe0fc5a2480361dcaf4e6e7cea00e078fcda07ba45f811b167e3f99e8cff574", - "sha256:30f9b96db44e09b3304f9ea95079b1b7316b2b4f3744fe3aaecccd95d547063d", - "sha256:4e10569f8cbed81cb7526ae137049759d2a8d57726d52c1a000a3ce366779634", - "sha256:511d269ee845037b95c9781aa702f90ccc36036f95d0f31373a6a79bd8242e25", - "sha256:537a7ccbea22905a0ab36ea58577b39d1fa9b1884869d173b5cf111f006f689f", - "sha256:54e9a2c849c742006516ad56a88f5c74bf2ce92c9f67435187c3c5953b346505", - "sha256:757250ddb3bff1eecd7e41e65f7f833a8405fede0194319f87899690624f2122", - "sha256:7757ae33dae81c300487591c68790dfb5145c7d03324000433d9a2c141f82af7", - "sha256:7c6092102219f59ff29788860ccb021e80fffd953920c4a8653889c029b2d420", - "sha256:8122ba5f2a2169ca5da936b2e5a511740ffb73979381b4229d9188f6dcb22f1f", - "sha256:9c4a7ea4fb81536c1b1f5cc44d54a296f96ae78c1ebd2311bd0b60be45a48d96", - "sha256:c914f78da4953b33d4685e3cdc7ce63401247a21425c16a39760e282075ac4a6", - "sha256:cd401ccbc2a249a47a3a1724c2918fcd04be1f7b54eb2a5a71ff915db0ac51c6", - "sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514", - "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff", - "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80" - ], - "version": "==1.4.0" - }, "pyparsing": { "hashes": [ "sha256:13140e8d0e1edd806eb50f18535d77f2143b40771d4aaef6b4950dd93d48a7db", @@ -1087,12 +995,6 @@ ], "version": "==3.0.0b1" }, - "pyrsistent": { - "hashes": [ - "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" - ], - "version": "==0.17.3" - }, "pytest": { "hashes": [ "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe", @@ -1107,20 +1009,6 @@ ], "version": "==2.10.1" }, - "pytest-docker": { - "hashes": [ - "sha256:7a0544dc00c83fd7808907a55835b1b7c07e3ea2d22ee558b5a233247e462735", - "sha256:9faa9c87e6e0920612005c8d187c58b3d336f3af49a878f358578de125963858" - ], - "version": "==0.10.1" - }, - "python-dotenv": { - "hashes": [ - "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e", - "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0" - ], - "version": "==0.15.0" - }, "python-json-logger": { "hashes": [ "sha256:f26eea7898db40609563bed0a7ca11af12e2a79858632706d835a0f961b7d398" @@ -1233,13 +1121,6 @@ ], "version": "==3.2.2" }, - "texttable": { - "hashes": [ - "sha256:ce0faf21aa77d806bbff22b107cc22cce68dc9438f97a2df32c93e9afa4ce436", - "sha256:f802f2ef8459058736264210f716c757cbf85007a30886d8541aa8c3404f1dda" - ], - "version": "==1.6.3" - }, "thrift": { "hashes": [ "sha256:9af1c86bf73433afc6010ed376a6c6aca2b54099cc0d61895f640870a9ae7d89" @@ -1338,13 +1219,6 @@ ], "version": "==20.1.0" }, - "websocket-client": { - "hashes": [ - "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549", - "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010" - ], - "version": "==0.57.0" - }, "werkzeug": { "hashes": [ "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index d5621bc..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '2' - -services: - consul: - image: consul:1.6.0 - ports: - - "8500:8500" - volumes: - - './testing:/consul/config' \ No newline at end of file diff --git a/docker/testing/consul.json b/docker/testing/consul.json deleted file mode 100644 index 3368abc..0000000 --- a/docker/testing/consul.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "acl": { - "enabled": true, - "enable_key_list_policy": true, - "tokens": { - "master": "9ae5fe1a-6b38-47e5-a0e7-f06b8b2fa645" - } - }, - "bootstrap_expect": 1, - "data_dir": "/tmp/consul", - "datacenter": "test", - "server": true, - "bind_addr": "{{ GetPrivateIP }}", - "client_addr": "0.0.0.0", - "enable_script_checks": true -} diff --git a/pyms/flask/services/service_discovery.py b/pyms/flask/services/service_discovery.py index 7e65ea9..9339170 100644 --- a/pyms/flask/services/service_discovery.py +++ b/pyms/flask/services/service_discovery.py @@ -1,9 +1,13 @@ import logging import uuid +try: + import consulate +except ModuleNotFoundError: # pragma: no cover + consulate = None + from pyms.constants import LOGGER_NAME from pyms.flask.services.driver import DriverService -from pyms.services_discovery import consulate from pyms.utils import import_from logger = logging.getLogger(LOGGER_NAME) diff --git a/pyms/services_discovery/__init__.py b/pyms/services_discovery/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyms/services_discovery/consulate/__init__.py b/pyms/services_discovery/consulate/__init__.py deleted file mode 100644 index ec66677..0000000 --- a/pyms/services_discovery/consulate/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Consulate: A client library for Consul - -""" -import logging -from logging import NullHandler - -from pyms.services_discovery.consulate.client import Consul -from pyms.services_discovery.consulate.exceptions import ( - ACLDisabled, - ClientError, - ConsulateException, - Forbidden, - LockFailure, - NotFound, - RequestError, - ServerError, -) - -__version__ = "1.0.0" - -# Prevent undesired log output to the root logger -logging.getLogger("consulate").addHandler(NullHandler()) - -__all__ = [ - "Consul", - "ConsulateException", - "ClientError", - "ServerError", - "ACLDisabled", - "Forbidden", - "NotFound", - "LockFailure", - "RequestError", -] diff --git a/pyms/services_discovery/consulate/adapters.py b/pyms/services_discovery/consulate/adapters.py deleted file mode 100644 index 163ae3d..0000000 --- a/pyms/services_discovery/consulate/adapters.py +++ /dev/null @@ -1,130 +0,0 @@ -# coding=utf-8 -""" -HTTP Client Library Adapters - -""" -import json -import logging -import socket - -import requests -import requests.exceptions - -try: - import requests_unixsocket -except ImportError: # pragma: no cover - requests_unixsocket = None - -from pyms.services_discovery.consulate import api, exceptions, utils - -LOGGER = logging.getLogger(__name__) - -CONTENT_FORM = "application/x-www-form-urlencoded; charset=utf-8" -CONTENT_JSON = "application/json; charset=utf-8" - - -def prepare_data(fun): - """Decorator for transforming the data being submitted to Consul - - :param function fun: The decorated function - - """ - - def inner(*args, **kwargs): - """Inner wrapper function for the decorator - - :param list args: positional arguments - :param dict kwargs: keyword arguments - - """ - if kwargs.get("data"): - if not utils.is_string(kwargs.get("data")): - kwargs["data"] = json.dumps(kwargs["data"]) - elif len(args) == 3 and not (utils.is_string(args[2]) or args[2] is None): - args = args[0], args[1], json.dumps(args[2]) - return fun(*args, **kwargs) - - return inner - - -class Request: - """The Request adapter class""" - - def __init__(self, timeout=None, verify=True, cert=None): - """ - Create a new request adapter instance. - - :param int timeout: [optional] timeout to use while sending requests - to consul. - """ - self.session = requests.Session() - self.session.verify = verify - self.session.cert = cert - self.timeout = timeout - - def delete(self, uri): - """Perform a HTTP delete - - :param src uri: The URL to send the DELETE to - :rtype: consulate.api.Response - - """ - LOGGER.debug("DELETE %s", uri) - return api.Response(self.session.delete(uri, timeout=self.timeout)) - - def get(self, uri, timeout=None): - """Perform a HTTP get - - :param src uri: The URL to send the DELETE to - :param timeout: How long to wait on the response - :type timeout: int or float or None - :rtype: consulate.api.Response - - """ - LOGGER.debug("GET %s", uri) - try: - return api.Response(self.session.get(uri, timeout=timeout or self.timeout)) - except (requests.exceptions.RequestException, OSError, socket.error) as err: - raise exceptions.RequestError(str(err)) - - def get_stream(self, uri): - """Perform a HTTP get that returns the response as a stream. - - :param src uri: The URL to send the DELETE to - :rtype: iterator - - """ - LOGGER.debug("GET Stream from %s", uri) - try: - response = self.session.get(uri, stream=True) - except (requests.exceptions.RequestException, OSError, socket.error) as err: - raise exceptions.RequestError(str(err)) - if utils.response_ok(response): - for line in response.iter_lines(): # pragma: no cover - yield line.decode("utf-8") - - @prepare_data - def put(self, uri, data=None, timeout=None): - """Perform a HTTP put - - :param src uri: The URL to send the DELETE to - :param str data: The PUT data - :param timeout: How long to wait on the response - :type timeout: int or float or None - :rtype: consulate.api.Response - - """ - LOGGER.debug("PUT %s with %r", uri, data) - headers = {"Content-Type": CONTENT_FORM if utils.is_string(data) else CONTENT_JSON} - try: - return api.Response(self.session.put(uri, data=data, headers=headers, timeout=timeout or self.timeout)) - except (requests.exceptions.RequestException, OSError, socket.error) as err: - raise exceptions.RequestError(str(err)) - - -class UnixSocketRequest(Request): # pragma: no cover - """Use to communicate with Consul over a Unix socket""" - - def __init__(self, timeout=None): - super().__init__(timeout) - self.session = requests_unixsocket.Session() diff --git a/pyms/services_discovery/consulate/api/__init__.py b/pyms/services_discovery/consulate/api/__init__.py deleted file mode 100644 index ce58ff0..0000000 --- a/pyms/services_discovery/consulate/api/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -Consul API Endpoints - -""" -from pyms.services_discovery.consulate.api.acl import ACL -from pyms.services_discovery.consulate.api.agent import Agent -from pyms.services_discovery.consulate.api.base import Response -from pyms.services_discovery.consulate.api.catalog import Catalog -from pyms.services_discovery.consulate.api.coordinate import Coordinate -from pyms.services_discovery.consulate.api.event import Event -from pyms.services_discovery.consulate.api.health import Health -from pyms.services_discovery.consulate.api.kv import KV -from pyms.services_discovery.consulate.api.lock import Lock -from pyms.services_discovery.consulate.api.session import Session -from pyms.services_discovery.consulate.api.status import Status - -__all__ = ["ACL", "Agent", "Catalog", "Event", "Health", "KV", "Lock", "Session", "Status", "Response", "Coordinate"] diff --git a/pyms/services_discovery/consulate/api/acl.py b/pyms/services_discovery/consulate/api/acl.py deleted file mode 100644 index f383357..0000000 --- a/pyms/services_discovery/consulate/api/acl.py +++ /dev/null @@ -1,438 +0,0 @@ -""" -Consul ACL Endpoint Access - -""" -import logging - -from pyms.services_discovery.consulate import exceptions -from pyms.services_discovery.consulate.api import base -from pyms.services_discovery.consulate.models import acl as model - -# from typing import List, Dict, Union - -LOGGER = logging.getLogger(__name__) - - -# ServiceIdentity = Dict[str, Union[str, List[str]]] -# ServiceIdentities = List[ServiceIdentity] -# PolicyLink = Dict[str, str] -# PolicyLinks = List[PolicyLink] -# RoleLink = Dict[str, str] -# RoleLinks = List[RoleLink] - - -class ACL(base.Endpoint): # pylint: disable=too-many-public-methods - """The ACL endpoints are used to create, update, destroy, and query ACL - tokens. - - """ - - def list_policies(self): - """List all ACL policies available in cluster. - - :param rtype: list - - """ - return self._get(["policies"]) - - def read_policy(self, id): # pylint: disable=redefined-builtin - """Read an existing policy with the given ID. - - :param str id: The ID of the policy. - :param rtype: dict - - """ - return self._get(["policy", id]) - - def create_policy(self, name, datacenters=None, description=None, rules=None): - """Create policy with name given and rules. - - :param str name: name of the policy - :param list() datacenters: A list of datacenters to filter on policy. - :param str description: Human readable description of the policy. - :param str rules: A json serializable string for ACL rules. - :param rtype: dict - - """ - return self._put_response_body( - ["policy"], - {}, - dict(model.ACLPolicy(name=name, datacenters=datacenters, description=description, rules=rules)), - ) - - def update_policy( - self, id, name, datacenters=None, description=None, rules=None - ): # pylint: disable=redefined-builtin - """Update policy with id given. - - :param str id: A UUID for the policy to update. - :param str name: name of the policy - :param list() datacenters: A list of datacenters to filter on policy. - :param str description: Human readable description of the policy. - :param str rules: A json serializable string for ACL rules. - :param rtype: dict - - """ - return self._put_response_body( - ["policy", id], - {}, - dict(model.ACLPolicy(name=name, datacenters=datacenters, description=description, rules=rules)), - ) - - def delete_policy(self, id): # pylint: disable=redefined-builtin - """Delete an existing policy with the given ID. - - :param str id: The ID of the policy. - :param rtype: bool - - """ - return self._delete(["policy", id]) - - def list_roles(self): - """List all ACL roles available in cluster - :param rtype: list - - """ - return self._get(["roles"]) - - def read_role(self, id=None, name=None): # pylint: disable=redefined-builtin - """Read an existing role with the given ID or Name. - - :param str id: The ID of the role. - :param str name: The name of the role. - :param rtype: dict - - """ - if id is not None: # pylint: disable=no-else-return - return self._get(["role", id]) - elif name is not None: - return self._get(["role", "name", name]) - else: - raise exceptions.NotFound("Either id or name must be specified") - - def create_role(self, name, description=None, policies=None, service_identities=None): - """Create an ACL role from a list of policies or service identities. - - :param str name: The name of the ACL role. Must be unique. - :param str description: The description of the ACL role. - :param PolicyLinks policies: An array of PolicyLink. - :param ServiceIdentities service_identities: A ServiceIdentity array. - :param rtype: dict - - """ - return self._put_response_body( - ["role"], - {}, - dict( - model.ACLRole( - name=name, description=description, policies=policies, service_identities=service_identities - ) - ), - ) - - def update_role( - self, id, name, description=None, policies=None, service_identities=None - ): # pylint: disable=redefined-builtin - """Update role with id given. - - :param str id: A UUID for the policy to update. - :param str name: name of the policy - :param list() datacenters: A list of datacenters to filter on policy. - :param str description: Human readable description of the policy. - :param str rules: A json serializable string for ACL rules. - :param rtype: dict - - """ - return self._put_response_body( - ["role", id], - {}, - dict( - model.ACLRole( - name=name, description=description, policies=policies, service_identities=service_identities - ) - ), - ) - - def delete_role(self, id): # pylint: disable=redefined-builtin - """Delete an existing role with the given ID. - - :param str id: The ID of the role. - :param rtype: bool - - """ - return self._delete(["policy", id]) - - def list_tokens(self): - """List all ACL tokens available in cluster. - - :param rtype: list - - """ - return self._get(["tokens"]) - - def read_token(self, accessor_id): - """Read an existing token with the given ID. - - :param str id: The ID of the role. - :param rtype: dict - - """ - return self._get(["token", accessor_id]) - - def read_self_token(self): - """Retrieve the currently used token. - - :param rtype: dict - - """ - return self._get(["token", "self"]) - - def create_token( - self, - accessor_id=None, - description=None, - expiration_time=None, - expiration_ttl=None, - local=False, - policies=None, - roles=None, - secret_id=None, - service_identities=None, - ): # pylint: disable=too-many-arguments - """Create a token from the roles, policies, and service identities - provided. - - :param str accessor_id: A UUID for accessing the token. - :param str description: A human-readable description of the token. - :param str expiration_time: The amount of time till the token expires. - :param str expiration_ttl: Sets expiration_time to creation time + - expiration_ttl value. - :param bool local: Whether the token is only locally available in the - current datacenter or to all datacenters defined. - :param PolicyLinks policies: A PolicyLink array. - :param RoleLinks roles: A RoleLink array. - :param str secret_id: A UUID for making requests to consul. - :param ServiceIdentities service_identities: A ServiceIdentity array. - :param rtype: dict - - """ - return self._put_response_body( - ["token"], - {}, - dict( - model.ACLToken( - accessor_id=accessor_id, - description=description, - expiration_time=expiration_time, - expiration_ttl=expiration_ttl, - local=local, - policies=policies, - roles=roles, - secret_id=secret_id, - service_identities=service_identities, - ) - ), - ) - - def update_token( - self, - accessor_id, - description=None, - expiration_time=None, - expiration_ttl=None, - local=False, - policies=None, - roles=None, - secret_id=None, - service_identities=None, - ): # pylint: disable=too-many-arguments - """Create a token from the roles, policies, and service identities - provided. - - :param str accessor_id: A UUID for accessing the token. - :param str description: A human-readable description of the token. - :param str expiration_time: The amount of time till the token expires. - :param str expiration_ttl: Sets expiration_time to creation time + - expiration_ttl value. - :param bool local: Whether the token is only locally available in the - current datacenter or to all datacenters defined. - :param PolicyLinks policies: A PolicyLink array. - :param RoleLinks roles: A RoleLink array. - :param str secret_id: A UUID for making requests to consul. - :param ServiceIdentities service_identities: A ServiceIdentity array. - :param rtype: dict - - """ - return self._put_response_body( - ["token", accessor_id], - {}, - dict( - model.ACLToken( - accessor_id=accessor_id, - description=description, - expiration_time=expiration_time, - expiration_ttl=expiration_ttl, - local=local, - policies=policies, - roles=roles, - secret_id=secret_id, - service_identities=service_identities, - ) - ), - ) - - def clone_token(self, accessor_id, description=None): - """Clone a token by the accessor_id. - - :param str accessor_id: A UUID for accessing the token. - :param str description: A human-readable description of the token. - :param rtype: dict - - """ - return self._put_response_body( - ["token", accessor_id, "clone"], {}, dict(model.ACLToken(description=description)) - ) - - def delete_token(self, accessor_id): - """Delete an existing token with the given AcccessorID. - - :param str id: The AccessorID of the token. - :param rtype: bool - - """ - return self._delete(["token", accessor_id]) - - # NOTE: Everything below here is deprecated post consul-1.4.0. - - def bootstrap(self): - """This endpoint does a special one-time bootstrap of the ACL system, - making the first management token if the acl_master_token is not - specified in the Consul server configuration, and if the cluster has - not been bootstrapped previously. - - This is available in Consul 0.9.1 and later, and requires all Consul - servers to be upgraded in order to operate. - - You can detect if something has interfered with the ACL bootstrapping - by the response of this method. If you get a string response with the - ``ID``, the bootstrap was a success. If the method raises a - :exc:`~consulate.exceptions.Forbidden` exception, the cluster has - already been bootstrapped, at which point you should consider the - cluster in a potentially compromised state. - - .. versionadded: 1.0.0 - - :rtype: str - :raises: :exc:`~consulate.exceptions.Forbidden` - - """ - return self._put_response_body(["bootstrap"])["ID"] - - def create(self, name, acl_type="client", rules=None): - """The create endpoint is used to make a new token. A token has a name, - a type, and a set of ACL rules. - - The ``name`` property is opaque to Consul. To aid human operators, it - should be a meaningful indicator of the ACL's purpose. - - ``acl_type`` is either client or management. A management token is - comparable to a root user and has the ability to perform any action - including creating, modifying, and deleting ACLs. - - By contrast, a client token can only perform actions as permitted by - the rules associated. Client tokens can never manage ACLs. Given this - limitation, only a management token can be used to make requests to - the create endpoint. - - ``rules`` is a HCL string defining the rule policy. See - `Internals on `_ ACL - for more information on defining rules. - - The call to create will return the ID of the new ACL. - - :param str name: The name of the ACL to create - :param str acl_type: One of "client" or "management" - :param str rules: The rules HCL string - :rtype: str - :raises: consulate.exceptions.Forbidden - - """ - return self._put_response_body(["create"], {}, dict(model.ACL(name=name, type=acl_type, rules=rules)))["ID"] - - def clone(self, acl_id): - """Clone an existing ACL returning the new ACL ID - - :param str acl_id: The ACL id - :rtype: bool - :raises: consulate.exceptions.Forbidden - - """ - return self._put_response_body(["clone", acl_id])["ID"] - - def destroy(self, acl_id): - """Delete the specified ACL - - :param str acl_id: The ACL id - :rtype: bool - :raises: consulate.exceptions.Forbidden - - """ - response = self._adapter.put(self._build_uri(["destroy", acl_id])) - if response.status_code == 403: - raise exceptions.Forbidden(response.body) - return response.status_code == 200 - - def info(self, acl_id): - """Return a dict of information about the ACL - - :param str acl_id: The ACL id - :rtype: dict - :raises: consulate.exceptions.Forbidden - :raises: consulate.exceptions.NotFound - - """ - response = self._get(["info", acl_id], raise_on_404=True) - if not response: - raise exceptions.NotFound("ACL not found") - return response - - def list(self): - """Return a list of all ACLs - - :rtype: list([dict]) - :raises: consulate.exceptions.Forbidden - - """ - return self._get(["list"]) - - def replication(self): - """Return the status of the ACL replication process in the datacenter. - - This is intended to be used by operators, or by automation checking the - health of ACL replication. - - .. versionadded: 1.0.0 - - :rtype: dict - :raises: consulate.exceptions.Forbidden - - """ - return self._get(["replication"]) - - def update(self, acl_id, name, acl_type="client", rules=None): - """Update an existing ACL, updating its values or add a new ACL if - the ACL ID specified is not found. - - The call will return the ID of the ACL. - - :param str acl_id: The ACL id - :param str name: The name of the ACL - :param str acl_type: The ACL type - :param str rules: The ACL rules document - :rtype: str - :raises: consulate.exceptions.Forbidden - - """ - return self._put_response_body( - ["update"], {}, dict(model.ACL(id=acl_id, name=name, type=acl_type, rules=rules)) - )["ID"] diff --git a/pyms/services_discovery/consulate/api/agent.py b/pyms/services_discovery/consulate/api/agent.py deleted file mode 100644 index 9a1a8c6..0000000 --- a/pyms/services_discovery/consulate/api/agent.py +++ /dev/null @@ -1,409 +0,0 @@ -""" -Consul Agent Endpoint Access - -""" -from pyms.services_discovery.consulate.api import base -from pyms.services_discovery.consulate.models import agent as models - -_TOKENS = ["acl_token", "acl_agent_token", "acl_agent_master_token", "acl_replication_token"] - - -class Agent(base.Endpoint): - """The Consul agent is the core process of Consul. The agent maintains - membership information, registers services, runs checks, responds to - queries and more. The agent must run on every node that is part of a - Consul cluster. - - """ - - def __init__(self, uri, adapter, datacenter=None, token=None): - """Create a new instance of the Agent class - - :param str uri: Base URI - :param consul.adapters.Request adapter: Request adapter - :param str datacenter: datacenter - :param str token: Access Token - - """ - super().__init__(uri, adapter, datacenter, token) - self.check = Agent.Check(self._base_uri, adapter, datacenter, token) - self.service = Agent.Service(self._base_uri, adapter, datacenter, token) - - class Check(base.Endpoint): - """One of the primary roles of the agent is the management of system - and application level health checks. A health check is considered to be - application level if it associated with a service. A check is defined - in a configuration file, or added at runtime over the HTTP interface. - - There are two different kinds of checks: - - - Script + Interval: These checks depend on invoking an external - application which does the health check and - exits with an appropriate exit code, - potentially generating some output. A script - is paired with an invocation interval - (e.g. every 30 seconds). This is similar to - the Nagios plugin system. - - - TTL: These checks retain their last known state for a given TTL. - The state of the check must be updated periodically - over the HTTP interface. If an external system fails to - update the status within a given TTL, the check is set to - the failed state. This mechanism is used to allow an - application to directly report it's health. For example, - a web app can periodically curl the endpoint, and if the - app fails, then the TTL will expire and the health check - enters a critical state. This is conceptually similar to a - dead man's switch. - - """ - - def register( - self, - name, - check_id=None, - interval=None, - notes=None, - deregister_critical_service_after=None, - args=None, - docker_container_id=None, - grpc=None, - grpc_use_tls=None, - http=None, - http_method=None, - header=None, - timeout=None, - tls_skip_verify=None, - tcp=None, - ttl=None, - service_id=None, - status=None, - ): # pylint: disable=too-many-arguments,too-many-locals - """Add a new check to the local agent. Checks are either a script - or TTL type. The agent is responsible for managing the status of - the check and keeping the Catalog in sync. - - :param str name: - :param str check_id: - :param str interval: - :param str notes: - :param str deregister_critical_service_after: - :param str args: - :param str docker_container_id: - :param str grpc: - :param str grpc_use_tls: - :param str http: - :param str http_method: - :param str header: - :param str timeout: - :param str tls_skip_verify: - :param str tcp: - :param str ttl: - :param str service_id: - :param str status: - - :rtype: bool - :raises: ValueError - - """ - return self._put_no_response_body( - ["register"], - None, - dict( - models.Check( - name=name, - id=check_id, - interval=interval, - notes=notes, - deregister_critical_service_after=deregister_critical_service_after, - args=args, - docker_container_id=docker_container_id, - grpc=grpc, - grpc_use_tls=grpc_use_tls, - http=http, - method=http_method, - header=header, - timeout=timeout, - tls_skip_verify=tls_skip_verify, - tcp=tcp, - ttl=ttl, - service_id=service_id, - status=status, - ) - ), - ) - - def deregister(self, check_id): - """Remove a check from the local agent. The agent will take care - of deregistering the check with the Catalog. - - :param str check_id: The check id - :rtype: bool - - """ - return self._put_no_response_body(["deregister", check_id]) - - def ttl_pass(self, check_id, note=None): - """This endpoint is used with a check that is of the TTL type. - When this endpoint is accessed, the status of the check is set to - "passing", and the TTL clock is reset. - - :param str check_id: The check id - :param str note: Note to include with the check pass - :rtype: bool - - """ - return self._put_no_response_body(["pass", check_id], {"note": note} if note else None) - - def ttl_warn(self, check_id, note=None): - """This endpoint is used with a check that is of the TTL type. - When this endpoint is accessed, the status of the check is set - to "warning", and the TTL clock is reset. - - :param str check_id: The check id - :param str note: Note to include with the check warning - :rtype: bool - - """ - return self._put_no_response_body(["warn", check_id], {"note": note} if note else None) - - def ttl_fail(self, check_id, note=None): - """This endpoint is used with a check that is of the TTL type. - When this endpoint is accessed, the status of the check is set - to "critical", and the TTL clock is reset. - - :param str check_id: The check id - :param str note: Note to include with the check failure - :rtype: bool - - """ - return self._put_no_response_body(["fail", check_id], {"note": note} if note else None) - - class Service(base.Endpoint): - """One of the main goals of service discovery is to provide a catalog - of available services. To that end, the agent provides a simple - service definition format to declare the availability of a service, a - nd to potentially associate it with a health check. A health check is - considered to be application level if it associated with a service. A - service is defined in a configuration file, or added at runtime over - the HTTP interface. - - """ - - def register( - self, - name, - service_id=None, - address=None, - port=None, - tags=None, - meta=None, - check=None, - checks=None, - enable_tag_override=None, - ): # pylint: disable=too-many-arguments - """Add a new service to the local agent. - - :param str name: The name of the service - :param str service_id: The id for the service (optional) - :param str address: The service IP address - :param int port: The service port - :param list tags: A list of tags for the service - :param list meta: A list of KV pairs for the service - :param check: An optional check definition for the service - :type check: :class:`consulate.models.agent.Check` - :param checks: A list of check definitions for the service - :type checks: list([:class:`consulate.models.agent.Check`]) - :param bool enable_tag_override: Toggle the tag override feature - :rtype: bool - :raises: ValueError - - """ - return self._put_no_response_body( - ["register"], - None, - dict( - models.Service( - name=name, - id=service_id, - address=address, - port=port, - tags=tags, - meta=meta, - check=check, - checks=checks, - enable_tag_override=enable_tag_override, - ) - ), - ) - - def deregister(self, service_id): - """Deregister the service from the local agent. The agent will - take care of deregistering the service with the Catalog. If there - is an associated check, that is also deregistered. - - :param str service_id: The service id to deregister - :rtype: bool - - """ - return self._put_no_response_body(["deregister", service_id]) - - def maintenance(self, service_id, enable=True, reason=None): - """Place given service into "maintenance mode". - - :param str service_id: The id for the service - :param bool enable: Enable maintenance mode - :param str reason: Reason for putting node in maintenance - :rtype: bool - - """ - query_params = {"enable": enable} - if reason: - query_params["reason"] = reason - return self._put_no_response_body(["maintenance", service_id], query_params) - - def checks(self): - """Return the all the checks that are registered with the local agent. - These checks were either provided through configuration files, or - added dynamically using the HTTP API. It is important to note that - the checks known by the agent may be different than those reported - by the Catalog. This is usually due to changes being made while there - is no leader elected. The agent performs active anti-entropy, so in - most situations everything will be in sync within a few seconds. - - :rtype: dict - - """ - return self._get(["checks"]) - - def force_leave(self, node): - """Instructs the agent to force a node into the left state. If a node - fails unexpectedly, then it will be in a "failed" state. Once in this - state, Consul will attempt to reconnect, and additionally the services - and checks belonging to that node will not be cleaned up. Forcing a - node into the left state allows its old entries to be removed. - - """ - return self._put_no_response_body(["force-leave", node]) - - def join(self, address, wan=False): - """This endpoint is hit with a GET and is used to instruct the agent - to attempt to connect to a given address. For agents running in - server mode, setting wan=True causes the agent to attempt to join - using the WAN pool. - - :param str address: The address to join - :param bool wan: Join a WAN pool as a server - :rtype: bool - - """ - query_params = {"wan": 1} if wan else None - return self._put_no_response_body(["join", address], query_params) - - def maintenance(self, enable=True, reason=None): - """Places the agent into or removes the agent from "maintenance mode". - - .. versionadded:: 1.0.0 - - :param bool enable: Enable or disable maintenance. Default: `True` - :param str reason: The reason for the maintenance - :rtype: bool - - """ - query_params = {"enable": enable} - if reason: - query_params["reason"] = reason - return self._put_no_response_body(["maintenance"], query_params) - - def members(self): - """Returns the members the agent sees in the cluster gossip pool. - Due to the nature of gossip, this is eventually consistent and the - results may differ by agent. The strongly consistent view of nodes - is instead provided by ``Consulate.catalog.nodes``. - - :rtype: list - - """ - return self._get_list(["members"]) - - def metrics(self): - """Returns agent's metrics for the most recent finished interval - - .. versionadded:: 1.0.0 - - :rtype: dict - - """ - return self._get(["metrics"]) - - def monitor(self): - """Iterator over logs from the local agent. - - .. versionadded:: 1.0.0 - - :rtype: iterator - - """ - for line in self._get_stream(["monitor"]): - yield line - - def reload(self): - """This endpoint instructs the agent to reload its configuration. - Any errors encountered during this process are returned. - - .. versionadded:: 1.0.0 - - :rtype: list - - """ - return self._put_response_body(["reload"]) or None - - def services(self): - """return the all the services that are registered with the local - agent. These services were either provided through configuration - files, or added dynamically using the HTTP API. It is important to - note that the services known by the agent may be different than those - ]reported by the Catalog. This is usually due to changes being made - while there is no leader elected. The agent performs active - anti-entropy, so in most situations everything will be in sync - within a few seconds. - - :rtype: dict - - """ - return self._get(["services"]) - - def self(self): - """This endpoint is used to return the configuration and member - information of the local agent under the Config key. - - :rtype: dict - - """ - return self._get(["self"]) - - def token(self, name, value): - """Update the ACL tokens currently in use by the agent. It can be used - to introduce ACL tokens to the agent for the first time, or to update - tokens that were initially loaded from the agent's configuration. - Tokens are not persisted, so will need to be updated again if the agent - is restarted. - - Valid names: - - - ``acl_token`` - - ``acl_agent_token`` - - ``acl_agent_master_token`` - - ``acl_replication_token`` - - .. versionadded:: 1.0.0 - - :param str name: One of the valid token names. - :param str value: The new token value - :rtype: bool - :raises: ValueError - - """ - if name not in _TOKENS: - raise ValueError("Invalid token name: {}".format(name)) - return self._put_no_response_body(["token", name], {}, {"Token": value}) diff --git a/pyms/services_discovery/consulate/api/base.py b/pyms/services_discovery/consulate/api/base.py deleted file mode 100644 index e159d59..0000000 --- a/pyms/services_discovery/consulate/api/base.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Base Endpoint class used by all endpoint classes - -""" -import base64 -import json - -try: - from urllib.parse import urlencode # Python 3 -except ImportError: - from urllib import urlencode # Python 2 - -from pyms.services_discovery.consulate import utils - - -class Endpoint: - """Base class for API endpoints""" - - KEYWORD = "" - - def __init__(self, uri, adapter, datacenter=None, token=None): - """Create a new instance of the Endpoint class - - :param str uri: Base URI - :param consul.adapters.Request adapter: Request adapter - :param str datacenter: datacenter - :param str token: Access Token - - """ - self._adapter = adapter - self._base_uri = "{0}/{1}".format(uri, self.__class__.__name__.lower()) - self._dc = datacenter - self._token = token - - def _build_uri(self, params, query_params=None): - """Build the request URI - - :param list params: List of path parts - :param dict query_params: Build query parameters - - """ - if not query_params: - query_params = dict() - if self._dc: - query_params["dc"] = self._dc - if self._token: - query_params["token"] = self._token - path = "/".join(params) - if query_params: - return "{0}/{1}?{2}".format(self._base_uri, path, urlencode(query_params)) - return "{0}/{1}".format(self._base_uri, path) - - def _get(self, params, query_params=None, raise_on_404=False, timeout=None): - """Perform a GET request - - :param list params: List of path parts - :param dict query_params: Build query parameters - :param timeout: How long to wait on the request for - :type timeout: int or float or None - :rtype: dict or list or None - - """ - response = self._adapter.get(self._build_uri(params, query_params), timeout=timeout) - if utils.response_ok(response, raise_on_404): - return response.body - return [] - - def _delete( - self, - params, - raise_on_404=False, - ): - """Perform a DELETE request - - :param list params: List of path parts - :rtype: bool - - """ - response = self._adapter.delete(self._build_uri(params)) - if utils.response_ok(response, raise_on_404): - return response.body - return False - - def _get_list(self, params, query_params=None): - """Return a list queried from Consul - - :param list params: List of path parts - :param dict query_params: Build query parameters - - """ - result = self._get(params, query_params) - if isinstance(result, dict): - return [result] - return result - - def _get_stream(self, params, query_params=None): - """Return a list queried from Consul - - :param list params: List of path parts - :param dict query_params: Build query parameters - :rtype: iterator - - """ - for line in self._adapter.get_stream(self._build_uri(params, query_params)): - yield line - - def _get_no_response_body(self, url_parts, query=None): - return utils.response_ok(self._adapter.get(self._build_uri(url_parts, query))) - - def _get_response_body(self, url_parts, query=None): - response = self._adapter.get(self._build_uri(url_parts, query)) - if utils.response_ok(response): - return response.body - return None - - def _put_no_response_body(self, url_parts, query=None, payload=None): - return utils.response_ok(self._adapter.put(self._build_uri(url_parts, query), payload)) - - def _put_response_body(self, url_parts, query=None, payload=None): - response = self._adapter.put(self._build_uri(url_parts, query), data=payload) - if utils.response_ok(response): - return response.body - return None - - -class Response: - """Used to process and wrap the responses from Consul. - - :param int status_code: HTTP Status code - :param str body: The response body - :param dict headers: Response headers - - """ - - status_code = None - body = None - headers = None - - def __init__(self, response): - """Create a new instance of the Response class. - - :param requests.response response: The requests response - - """ - self.status_code = response.status_code - self.body = self._demarshal(response.content) - self.headers = response.headers - - def _demarshal(self, body): # pylint: disable=too-many-branches,too-many-return-statements; # noqa: C901 - """Demarshal the request payload. - - :param str body: The string response body - :rtype: dict or str - - """ - if body is None: - return None - if self.status_code == 200: # pylint: disable=too-many-nested-blocks - try: - if isinstance(body, bytes): - try: - body = body.decode("utf-8") - except UnicodeDecodeError: - pass - value = json.loads(body) - except (TypeError, ValueError): - return body - if value is None: - return None - if isinstance(value, bool): - return value - if "error" not in value: - for row in value: - if "Value" in row: - try: - row["Value"] = base64.b64decode(row["Value"]) - if isinstance(row["Value"], bytes): - try: - row["Value"] = row["Value"].decode("utf-8") - except UnicodeDecodeError: - pass - except TypeError: - pass - if isinstance(value, list) and len(value) == 1: - return value[0] - return value - return body diff --git a/pyms/services_discovery/consulate/api/catalog.py b/pyms/services_discovery/consulate/api/catalog.py deleted file mode 100644 index 2f2bfc3..0000000 --- a/pyms/services_discovery/consulate/api/catalog.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -Consul Catalog Endpoint Access - -""" -from pyms.services_discovery.consulate.api import base - - -class Catalog(base.Endpoint): - """The Consul agent is the core process of Consul. The agent maintains - membership information, registers services, runs checks, responds to - queries and more. The agent must run on every node that is part of a - Consul cluster. - - """ - - def __init__(self, uri, adapter, dc=None, token=None): - super().__init__(uri, adapter, dc, token) - - def register(self, node, address, datacenter=None, service=None, check=None, node_meta=None): - """A a low level mechanism for directly registering or updating - entries in the catalog. It is usually recommended to use the agent - local endpoints, as they are simpler and perform anti-entropy. - - The behavior of the endpoint depends on what keys are provided. The - endpoint requires Node and Address to be provided, while Datacenter - will be defaulted to match that of the agent. If only those are - provided, the endpoint will register the node with the catalog. - - If the Service key is provided, then the service will also be - registered. If ID is not provided, it will be defaulted to Service. - It is mandated that the ID be node-unique. Both Tags and Port can - be omitted. - - If the Check key is provided, then a health check will also be - registered. It is important to remember that this register API is - very low level. This manipulates the health check entry, but does - not setup a script or TTL to actually update the status. For that - behavior, an agent local check should be setup. - - The CheckID can be omitted, and will default to the Name. Like - before, the CheckID must be node-unique. The Notes is an opaque - field that is meant to hold human readable text. If a ServiceID is - provided that matches the ID of a service on that node, then the - check is treated as a service level health check, instead of a node - level health check. Lastly, the status must be one of "unknown", - "passing", "warning", or "critical". The "unknown" status is used to - indicate that the initial check has not been performed yet. - - It is important to note that Check does not have to be provided - with Service and visa-versa. They can be provided or omitted at will. - - Example service dict: - - .. code:: python - - 'Service': { - 'ID': 'redis1', - 'Service': 'redis', - 'Tags': ['master', 'v1'], - 'Port': 8000, - } - - Example check dict: - - .. code:: python - - 'Check': { - 'Node': 'foobar', - 'CheckID': 'service:redis1', - 'Name': 'Redis health check', - 'Notes': 'Script based health check', - 'Status': 'passing', - 'ServiceID': 'redis1' - } - - Example node_meta dict: - - .. code:: python - - 'NodeMeta': { - 'somekey': 'somevalue' - } - - :param str node: The node name - :param str address: The node address - :param str datacenter: The optional node datacenter - :param dict service: An optional node service - :param dict check: An optional node check - :param dict node_meta: Optional node metadata - :rtype: bool - - """ - payload = {"Node": node, "Address": address} - if datacenter: - payload["Datacenter"] = datacenter - if service: - payload["Service"] = service - if check: - payload["Check"] = check - if node_meta: - payload["NodeMeta"] = node_meta - - return self._put_response_body(["register"], None, payload) - - def deregister(self, node, datacenter=None, check_id=None, service_id=None): - """Directly remove entries in the catalog. It is usually recommended - to use the agent local endpoints, as they are simpler and perform - anti-entropy. - - The behavior of the endpoint depends on what keys are provided. The - endpoint requires ``node`` to be provided, while ``datacenter`` will - be defaulted to match that of the agent. If only ``node`` is provided, - then the node, and all associated services and checks are deleted. If - ``check_id`` is provided, only that check belonging to the node is - removed. If ``service_id`` is provided, then the service along with - it's associated health check (if any) is removed. - - :param str node: The node for the action - :param str datacenter: The optional datacenter for the node - :param str check_id: The optional check_id to remove - :param str service_id: The optional service_id to remove - :rtype: bool - - """ - payload = {"Node": node} - if datacenter: - payload["Datacenter"] = datacenter - if check_id: - payload["CheckID"] = check_id - if service_id: - payload["ServiceID"] = service_id - return self._put_response_body(["deregister"], None, payload) - - def datacenters(self): - """Return all the datacenters that are known by the Consul server. - - :rtype: list - - """ - return self._get_list(["datacenters"]) - - def node(self, node_id): - """Return the node data for the specified node - - :param str node_id: The node ID - :rtype: dict - - """ - return self._get(["node", node_id]) - - def nodes(self, node_meta=None): - """Return all of the nodes for the current datacenter. - - :param str node_meta: Desired node metadata - :rtype: list - - """ - query_params = {"node-meta": node_meta} if node_meta else {} - return self._get_list(["nodes"], query_params) - - def service(self, service_id): - """Return the service details for the given service - - :param str service_id: The service id - :rtype: list - - """ - return self._get_list(["service", service_id]) - - def services(self): - """Return a list of all of the services for the current datacenter. - - :rtype: list - - """ - return self._get_list(["services"]) diff --git a/pyms/services_discovery/consulate/api/coordinate.py b/pyms/services_discovery/consulate/api/coordinate.py deleted file mode 100644 index 84eff4d..0000000 --- a/pyms/services_discovery/consulate/api/coordinate.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Consul Coordinate Endpoint Access - -""" -from math import sqrt - -from pyms.services_discovery.consulate.api import base - - -class Coordinate(base.Endpoint): - """Used to query node coordinates.""" - - def node(self, node_id): - """Return coordinates for the given node. - - :param str node_id: The node ID - :rtype: dict - - """ - return self._get(["node", node_id]) - - def nodes(self): - """Return coordinates for the current datacenter. - - :rtype: list - - """ - return self._get_list(["nodes"]) - - @staticmethod - def rtt(src, dst): - """Calculated RTT between two node coordinates. - :param src: - :param dst: - :return: - """ - - if not isinstance(src, (dict)): - raise ValueError("coordinate object must be a dictionary") - if not isinstance(dst, (dict)): - raise ValueError("coordinate object must be a dictionary") - if "Coord" not in src: - raise ValueError("coordinate object has no Coord key") - if "Coord" not in dst: - raise ValueError("coordinate object has no Coord key") - - src_coord = src["Coord"] - dst_coord = dst["Coord"] - - if len(src_coord.get("Vec")) != len(dst_coord.get("Vec")): - raise ValueError("coordinate objects are not compatible due to different length") - - sumsq = 0.0 - for i in range(len(src_coord.get("Vec"))): - diff = src_coord.get("Vec")[i] - dst_coord.get("Vec")[i] - sumsq += diff * diff - - rtt = sqrt(sumsq) + src_coord.get("Height") + dst_coord.get("Height") - adjusted = rtt + src_coord.get("Adjustment") + dst_coord.get("Adjustment") - if adjusted > 0.0: - rtt = adjusted - - return rtt * 1000 diff --git a/pyms/services_discovery/consulate/api/event.py b/pyms/services_discovery/consulate/api/event.py deleted file mode 100644 index 415f15e..0000000 --- a/pyms/services_discovery/consulate/api/event.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Consul Event Endpoint Access - -""" -from pyms.services_discovery.consulate.api import base - - -class Event(base.Endpoint): - """The Event endpoints are used to fire a new event and list recent events.""" - - def fire(self, name, payload=None, datacenter=None, node=None, service=None, tag=None): - """Trigger a new user Event - - :param str name: The name of the event - :param str payload: The opaque event payload - :param str datacenter: Optional datacenter to fire the event in - :param str node: Optional node to fire the event for - :param str service: Optional service to fire the event for - :param str tag: Option tag to fire the event for - :return str: the new event ID - - """ - query_args = {} - if datacenter: - query_args["dc"] = datacenter - if node: - query_args["node"] = node - if service: - query_args["service"] = service - if tag: - query_args["tag"] = tag - response = self._adapter.put(self._build_uri(["fire", name], query_args), payload) - return response.body.get("ID") - - def list(self, name=None): - """Returns the most recent events known by the agent. As a consequence - of how the event command works, each agent may have a different view of - the events. Events are broadcast using the gossip protocol, so they - have no global ordering nor do they make a promise of delivery. - - :return: list - - """ - query_args = {} - if name: - query_args["name"] = name - return self._get(["list"], query_args) diff --git a/pyms/services_discovery/consulate/api/health.py b/pyms/services_discovery/consulate/api/health.py deleted file mode 100644 index a7ff4b9..0000000 --- a/pyms/services_discovery/consulate/api/health.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Consul Health Endpoint Access - -""" -from pyms.services_discovery.consulate.api import base - - -class Health(base.Endpoint): - """Used to query health related information. It is provided separately - from the Catalog, since users may prefer to not use the health checking - mechanisms as they are totally optional. Additionally, some of the query - results from the Health system are filtered, while the Catalog endpoints - provide the raw entries. - - """ - - def checks(self, service_id, node_meta=None): - """Return checks for the given service. - - :param str service_id: The service ID - :param str node_meta: Filter checks using node metadata - :rtype: list - - """ - query_params = {"node-meta": node_meta} if node_meta else {} - return self._get_list(["checks", service_id], query_params) - - def node(self, node_id): - """Return the health info for a given node. - - :param str node_id: The node ID - :rtype: list - - """ - return self._get_list(["node", node_id]) - - def service(self, service_id, tag=None, passing=None, node_meta=None): - """Returns the nodes and health info of a service - - :param str service_id: The service ID - :param str node_meta: Filter services using node metadata - :rtype: list - - """ - - query_params = {} - if tag: - query_params["tag"] = tag - if passing: - query_params["passing"] = "" - if node_meta: - query_params["node-meta"] = node_meta - - return self._get_list(["service", service_id], query_params=query_params) - - def state(self, state): - """Returns the checks in a given state where state is one of - "unknown", "passing", "warning", or "critical". - - :param str state: The state to get checks for - :rtype: list - - """ - return self._get_list(["state", state]) diff --git a/pyms/services_discovery/consulate/api/kv.py b/pyms/services_discovery/consulate/api/kv.py deleted file mode 100644 index cacd864..0000000 --- a/pyms/services_discovery/consulate/api/kv.py +++ /dev/null @@ -1,398 +0,0 @@ -""" -Consul KV Endpoint Access - -""" -from pyms.services_discovery.consulate import exceptions, utils -from pyms.services_discovery.consulate.api import base - - -class KV(base.Endpoint): - """The :py:class:`consul.api.KV` class implements a :py:class:`dict` like - interface for working with the Key/Value service. Simply use items on the - :py:class:`consulate.Session` like you would with a :py:class:`dict` to - :py:meth:`get `, - :py:meth:`set `, or - :py:meth:`delete ` values in the key/value store. - - Additionally, :py:class:`KV ` acts as an - :py:meth:`iterator `, providing methods to - iterate over :py:meth:`keys `, - :py:meth:`values `, - :py:meth:`keys and values `, etc. - - Should you need access to get or set the flag value, the - :py:meth:`get_record `, - :py:meth:`set_record `, - and :py:meth:`records ` provide a way to access - the additional fields exposed by the KV service. - - """ - - def __contains__(self, item): - """Return True if there is a value set in the Key/Value service for the - given key. - - :param str item: The key to check for - :rtype: bool - - """ - item = item.lstrip("/") - return self._get_no_response_body([item]) - - def __delitem__(self, item): - """Delete an item from the Key/Value service - - :param str item: The key name - - """ - self._delete_item(item) - - def __getitem__(self, item): - """Get a value from the Key/Value service, returning it fully - decoded if possible. - - :param str item: The item name - :rtype: mixed - :raises: KeyError - - """ - value = self._get_item(item) - if not value: - raise KeyError("Key not found ({0})".format(item)) - return value.get("Value") - - def __iter__(self): - """Iterate over all the keys in the Key/Value service - - :rtype: iterator - - """ - for key in self.keys(): - yield key - - def __len__(self): - """Return the number if items in the Key/Value service - - :return: int - - """ - return len(self._get_all_items()) - - def __setitem__(self, item, value): - """Set a value in the Key/Value service, using the CAS mechanism - to ensure that the set is atomic. If the value passed in is not a - string, an attempt will be made to JSON encode the value prior to - setting it. - - :param str item: The key to set - :param mixed value: The value to set - :raises: KeyError - - """ - self._set_item(item, value) - - def acquire_lock(self, item, session, value=None, cas=None, flags=None): - """Use Consul for locking by specifying the item/key to lock with - and a session value for removing the lock. - - :param str item: The item in the Consul KV database - :param str session: The session value for the lock - :param mixed value: An optional value to set for the lock - :param int cas: Optional Check-And-Set index value - :param int flags: User defined flags to set - :return: bool - - """ - query_params = {"acquire": session} - if cas is not None: - query_params["cas"] = cas - if flags is not None: - query_params["flags"] = flags - return self._put_response_body([item], query_params, value) - - def delete(self, item, recurse=False): - """Delete an item from the Key/Value service - - :param str item: The item key - :param bool recurse: Remove keys prefixed with the item pattern - :raises: KeyError - - """ - return self._delete_item(item, recurse) - - def get(self, item, default=None, raw=False): - """Get a value from the Key/Value service, returning it fully - decoded if possible. - - :param str item: The item key - :param mixed default: A default value to return if the get fails - :param bool raw: Return the raw value from Consul - :rtype: mixed - :raises: KeyError - - """ - response = self._get_item(item, raw) - if isinstance(response, dict): - return response.get("Value", default) - return response or default - - def get_record(self, item): - """Get the full record from the Key/Value service, returning - all fields including the flag. - - :param str item: The item key - :rtype: dict - :raises: KeyError - - """ - return self._get_item(item) - - def find(self, prefix, separator=None): - """Find all keys with the specified prefix, returning a dict of - matches. - - *Example:* - - .. code:: python - - >>> consul.kv.find('b') - {'baz': 'qux', 'bar': 'baz'} - - :param str prefix: The prefix to search with - :rtype: dict - - """ - query_params = {"recurse": None} - if separator: - query_params["keys"] = prefix - query_params["separator"] = separator - response = self._get_list([prefix.lstrip("/")], query_params) - if separator: - results = response - else: - results = {} - for row in response: - results[row["Key"]] = row["Value"] - return results - - def items(self): - """Return a dict of all of the key/value pairs in the Key/Value service - - *Example:* - - .. code:: python - - >>> consul.kv.items() - {'foo': 'bar', 'bar': 'baz', 'quz': True, 'corgie': 'dog'} - - :rtype: dict - - """ - return [{item["Key"]: item["Value"]} for item in self._get_all_items()] - - def iteritems(self): - """Iterate over the dict of key/value pairs in the Key/Value service - - *Example:* - - .. code:: python - - >>> for key, value in consul.kv.iteritems(): - ... print(key, value) - ... - (u'bar', 'baz') - (u'foo', 'bar') - (u'quz', True) - - :rtype: iterator - - """ - for item in self._get_all_items(): - yield item["Key"], item["Value"] - - def keys(self): - """Return a list of all of the keys in the Key/Value service - - *Example:* - - .. code:: python - - >>> consul.kv.keys() - [u'bar', u'foo', u'quz'] - - :rtype: list - - """ - return sorted([row["Key"] for row in self._get_all_items()]) - - def records(self, key=None): - """Return a list of tuples for all of the records in the Key/Value - service - - *Example:* - - .. code:: python - - >>> consul.kv.records() - [(u'bar', 0, 'baz'), - (u'corgie', 128, 'dog'), - (u'foo', 0, 'bar'), - (u'quz', 0, True)] - - :rtype: list of (Key, Flags, Value) - - """ - if key: - return [(item["Key"], item["Flags"], item["Value"]) for item in self._get_list([key], {"recurse": None})] - - return [(item["Key"], item["Flags"], item["Value"]) for item in self._get_all_items()] - - def release_lock(self, item, session): - """Release an existing lock from the Consul KV database. - - :param str item: The item in the Consul KV database - :param str session: The session value for the lock - :return: bool - - """ - return self._put_response_body([item], {"release": session}) - - def set(self, item, value): - """Set a value in the Key/Value service, using the CAS mechanism - to ensure that the set is atomic. If the value passed in is not a - string, an attempt will be made to JSON encode the value prior to - setting it. - - :param str item: The key to set - :param mixed value: The value to set - :raises: KeyError - - """ - return self.__setitem__(item, value) - - def set_record(self, item, flags=0, value=None, replace=True): - """Set a full record, including the item flag - - :param str item: The key to set - :param mixed value: The value to set - :param replace: If True existing value will be overwritten: - - """ - self._set_item(item, value, flags, replace) - - def values(self): - """Return a list of all of the values in the Key/Value service - - *Example:* - - .. code:: python - - >>> consul.kv.values() - [True, 'bar', 'baz'] - - :rtype: list - - """ - return [row["Value"] for row in self._get_all_items()] - - def _delete_item(self, item, recurse=False): - """Remove an item from the Consul database - - :param str item: - :param recurse: - :return: - """ - query_params = {"recurse": True} if recurse else {} - return self._adapter.delete(self._build_uri([item], query_params)) - - def _get_all_items(self): - """Internal method to return a list of all items in the Key/Value - service - - :rtype: list - - """ - return self._get_list([""], {"recurse": None}) - - def _get_item(self, item, raw=False): - """Internal method to get the full item record from the Key/Value - service - - :param str item: The item to get - :param bool raw: Return only the raw body - :rtype: mixed - - """ - item = item.lstrip("/") - query_params = {"raw": True} if raw else {} - response = self._adapter.get(self._build_uri([item], query_params)) - if response.status_code == 200: - return response.body - return None - - def _get_modify_index(self, item, value, replace): - """Get the modify index of the specified item. If replace is False - and an item is found, return ``None``. If the existing value - and the passed in value match, return ``None``. If no item exists in - the KV database, return ``0``, otherwise return the ``ModifyIndex``. - - :param str item: The item to get the index for - :param str value: The item to evaluate for equality - :param bool replace: Should the item be replaced - :rtype: int|None - - """ - response = self._adapter.get(self._build_uri([item])) - index = 0 - if response.status_code == 200: - index = response.body.get("ModifyIndex") - rvalue = response.body.get("Value") - if rvalue == value: - return None - if not replace: - return None - return index - - @staticmethod - def _prepare_value(value): - """Prepare the value passed in and ensure that it is properly encoded - - :param mixed value: The value to prepare - :rtype: bytes - - """ - if not utils.is_string(value) or isinstance(value, bytes): - return value - try: - return value.encode("utf-8") - except UnicodeDecodeError: - return value - - def _set_item(self, item, value, flags=None, replace=True, query_params=None): - """Internal method for setting a key/value pair with flags in the - Key/Value service - - :param str item: The key to set - :param mixed value: The value to set - :param int flags: User defined flags to set - :param bool replace: Overwrite existing values - :raises: KeyError - - """ - value = self._prepare_value(value) - if value and item.endswith("/"): - item = item.rstrip("/") - - index = self._get_modify_index(item, value, replace) - if index is None: - return True - query_params = query_params or {} - query_params.update({"cas": index}) - if flags is not None: - query_params["flags"] = flags - response = self._adapter.put(self._build_uri([item], query_params), value) - if not response.status_code == 200 or not response.body: - if response.status_code == 500: - raise exceptions.ServerError(response.body or "Internal Consul server error") - raise KeyError('Error setting "{0}" ({1})'.format(item, response.status_code)) - return None diff --git a/pyms/services_discovery/consulate/api/lock.py b/pyms/services_discovery/consulate/api/lock.py deleted file mode 100644 index fbf6e04..0000000 --- a/pyms/services_discovery/consulate/api/lock.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -Lock Object for easy locking - -""" -import contextlib -import logging -import uuid - -from pyms.services_discovery.consulate import exceptions -from pyms.services_discovery.consulate.api import base - -LOGGER = logging.getLogger(__name__) - - -class Lock(base.Endpoint): - """Wrapper for easy :class:`~consulate.api.kv.KV` locks. Keys are - automatically prefixed with ``consulate/locks/``. To change the prefix or - remove it invoke the :meth:~consulate.api.lock.Lock.prefix` method. - - Example: - - .. code:: python - - from pyms.services_discovery import consulate - - consul = consulate.Consul() - with consul.lock.acquire('my-key'): - print('Locked: {}'.format(consul.lock.key)) - # Do stuff - - :raises: :exc:`~consulate.exception.LockError` - - """ - - DEFAULT_PREFIX = "consulate/locks" - - def __init__(self, uri, adapter, session, datacenter=None, token=None): - """Create a new instance of the Lock - - :param str uri: Base URI - :param consul.adapters.Request adapter: Request adapter - :param consul.api.session.Session session: Session endpoint instance - :param str datacenter: datacenter - :param str token: Access Token - - """ - super().__init__(uri, adapter, datacenter, token) - self._base_uri = "{0}/kv".format(uri) - self._session = session - self._session_id = None - self._item = str(uuid.uuid4()) - self._prefix = self.DEFAULT_PREFIX - - @contextlib.contextmanager - def acquire(self, key=None, value=None): - """A context manager that allows you to acquire the lock, optionally - passing in a key and/or value. - - :param str key: The key to lock - :param str value: The value to set in the lock - :raises: :exc:`~consulate.exception.LockError` - - """ - self._acquire(key, value) - yield - self._release() - - @property - def key(self): - """Return the lock key - - :rtype: str - - """ - return self._item - - def prefix(self, value): - """Override the path prefix for the lock key - - :param str value: The value to set the path prefix to - - """ - self._prefix = value or "" - - def _acquire(self, key=None, value=None): - self._session_id = self._session.create() - self._item = "/".join([self._prefix, (key or str(uuid.uuid4()))]) - LOGGER.debug("Acquiring a lock of %s for session %s", self._item, self._session_id) - response = self._put_response_body([self._item], {"acquire": self._session_id}, value) - if not response: - self._session.destroy(self._session_id) - raise exceptions.LockFailure() - - def _release(self): - """Release the lock""" - self._put_response_body([self._item], {"release": self._session_id}) - self._adapter.delete(self._build_uri([self._item])) - self._session.destroy(self._session_id) - self._item, self._session_id = None, None diff --git a/pyms/services_discovery/consulate/api/session.py b/pyms/services_discovery/consulate/api/session.py deleted file mode 100644 index 21761b5..0000000 --- a/pyms/services_discovery/consulate/api/session.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Consul Session Endpoint Access - -""" -from pyms.services_discovery.consulate.api import base - - -class Session(base.Endpoint): - """Create, destroy, and query Consul sessions.""" - - def create(self, name=None, behavior="release", node=None, delay=None, ttl=None, checks=None): - """Initialize a new session. - - None of the fields are mandatory, and in fact no body needs to be PUT - if the defaults are to be used. - - Name can be used to provide a human-readable name for the Session. - - Behavior can be set to either ``release`` or ``delete``. This controls - the behavior when a session is invalidated. By default, this is - release, causing any locks that are held to be released. Changing this - to delete causes any locks that are held to be deleted. delete is - useful for creating ephemeral key/value entries. - - Node must refer to a node that is already registered, if specified. - By default, the agent's own node name is used. - - LockDelay (``delay``) can be specified as a duration string using a - "s" suffix for seconds. The default is 15s. - - The TTL field is a duration string, and like LockDelay it can use "s" - as a suffix for seconds. If specified, it must be between 10s and - 3600s currently. When provided, the session is invalidated if it is - not renewed before the TTL expires. See the session internals page - for more documentation of this feature. - - Checks is used to provide a list of associated health checks. It is - highly recommended that, if you override this list, you include the - default "serfHealth". - - :param str name: A human readable session name - :param str behavior: One of ``release`` or ``delete`` - :param str node: A node to create the session on - :param str delay: A lock delay for the session - :param str ttl: The time to live for the session - :param lists checks: A list of associated health checks - :return str: session ID - - """ - payload = {"name": name} if name else {} - if node: - payload["Node"] = node - if behavior: - payload["Behavior"] = behavior - if delay: - payload["LockDelay"] = delay - if ttl: - payload["TTL"] = ttl - if checks: - payload["Checks"] = checks - return self._put_response_body(["create"], None, payload).get("ID") - - def destroy(self, session_id): - """Destroy an existing session - - :param str session_id: The session to destroy - :return: bool - - """ - return self._put_no_response_body(["destroy", session_id]) - - def info(self, session_id): - """Returns the requested session information within a given dc. - By default, the dc of the agent is queried. - - :param str session_id: The session to get info about - :return: dict - - """ - return self._get_response_body(["info", session_id]) - - def list(self): - """Returns the active sessions for a given dc. - - :return: list - - """ - return self._get_response_body(["list"]) - - def node(self, node): - """Returns the active sessions for a given node and dc. - By default, the dc of the agent is queried. - - :param str node: The node to get active sessions for - :return: list - - """ - return self._get_response_body(["node", node]) - - def renew(self, session_id): - """Renew the given session. This is used with sessions that have a TTL, - and it extends the expiration by the TTL. By default, the dc - of the agent is queried. - - :param str session_id: The session to renew - :return: dict - - """ - return self._put_response_body(["renew", session_id]) diff --git a/pyms/services_discovery/consulate/api/status.py b/pyms/services_discovery/consulate/api/status.py deleted file mode 100644 index 2789f47..0000000 --- a/pyms/services_discovery/consulate/api/status.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Consul Status Endpoint Access - -""" -from pyms.services_discovery.consulate.api import base - - -class Status(base.Endpoint): - """Get information about the status of the Consul cluster. This are - generally very low level, and not really useful for clients. - - """ - - def leader(self): - """Get the Raft leader for the datacenter the agent is running in. - - :rtype: str - - """ - return self._get(["leader"]) - - def peers(self): - """Get the Raft peers for the datacenter the agent is running in. - - :rtype: list - - """ - value = self._get(["peers"]) - if not isinstance(value, list): - return [value] - return value diff --git a/pyms/services_discovery/consulate/cli.py b/pyms/services_discovery/consulate/cli.py deleted file mode 100644 index 487786b..0000000 --- a/pyms/services_discovery/consulate/cli.py +++ /dev/null @@ -1,593 +0,0 @@ -"""Consulate CLI commands""" -import argparse -import base64 -import json -import os -import subprocess # nosec -import sys -import time -import urllib.parse as urlparse - -from requests import exceptions - -from pyms.services_discovery import consulate -from pyms.services_discovery.consulate import adapters, utils - -CONSUL_ENV_VAR = "CONSUL_RPC_ADDR" -EPILOG = ( - "If the CONSUL_RPC_ADDR environment variable is set, it will be " - "parsed and used for default values when connecting." -) - - -def on_error(message, exit_code=2): - """Write out the specified message to stderr and exit the specified - exit code, defaulting to ``2``. - - :param str message: The exit message - :param int exit_code: The numeric exit code - - """ - sys.stderr.write(message + "\n") - sys.exit(exit_code) - - -def connection_error(): - """Common exit routine when consulate can't connect to Consul""" - on_error("Could not connect to consul", 1) - - -ACL_PARSERS = [ - ( - "backup", - "Backup to stdout or a JSON file", - [ - [["-f", "--file"], {"help": "JSON file to write instead of stdout", "nargs": "?"}], - [["-p", "--pretty"], {"help": "pretty-print JSON output", "action": "store_true"}], - ], - ), - ( - "restore", - "Restore from stdin or a JSON file", - [ - [["-f", "--file"], {"help": "JSON file to read instead of stdin", "nargs": "?"}], - [["-n", "--no-replace"], {"help": "Do not replace existing entries", "action": "store_true"}], - ], - ), -] - -KV_PARSERS = [ - ( - "backup", - "Backup to stdout or a JSON file", - [ - [["key"], {"help": "The key to use as target to backup a " "specific key or folder.", "nargs": "?"}], - [["-b", "--base64"], {"help": "Base64 encode values", "action": "store_true"}], - [["-f", "--file"], {"help": "JSON file to write instead of stdout", "nargs": "?"}], - [["-p", "--pretty"], {"help": "pretty-print JSON output", "action": "store_true"}], - ], - ), - ( - "restore", - "Restore from stdin or a JSON file", - [ - [["key"], {"help": "The key as target to restore to a specific key " "or folder.", "nargs": "?"}], - [ - ["-p", "--prune"], - {"help": "Remove entries from consul tree that " "are not in restore file.", "action": "store_true"}, - ], - [["-b", "--base64"], {"help": "Restore from Base64 encode values", "action": "store_true"}], - [["-f", "--file"], {"help": "JSON file to read instead of stdin", "nargs": "?"}], - [["-n", "--no-replace"], {"help": "Do not replace existing entries", "action": "store_true"}], - ], - ), - ( - "ls", - "List all of the keys", - [ - [["key"], {"help": "The key to use as target to list contents of " "specific key or folder", "nargs": "?"}], - [["-l", "--long"], {"help": "Long format", "action": "store_true"}], - ], - ), - ("mkdir", "Create a folder", [[["path"], {"help": "The path to create"}]]), - ( - "get", - "Get a key from the database", - [ - [["key"], {"help": "The key to get"}], - [["-r", "--recurse"], {"help": "Get all keys prefixed with the specified key", "action": "store_true"}], - [ - ["-t", "--trim"], - {"help": "Number of levels of prefix to trim from returned key", "type": int, "default": 0}, - ], - ], - ), - ( - "set", - "Set a key in the database", - [[["key"], {"help": "The key to set"}], [["value"], {"help": "The value of the key"}]], - ), - ( - "rm", - "Remove a key from the database", - [ - [["key"], {"help": "The key to remove"}], - [["-r", "--recurse"], {"help": "Delete all keys prefixed with the specified key", "action": "store_true"}], - ], - ), -] - - -def add_acl_args(parser): - """Add the acl command and arguments. - - :param argparse.Subparser parser: parser - - """ - kv_parser = parser.add_parser("acl", help="ACL Utilities") - - subparsers = kv_parser.add_subparsers(dest="action", title="ACL Database Utilities") - - for (name, help_text, arguments) in ACL_PARSERS: - parser = subparsers.add_parser(name, help=help_text) - for (args, kwargs) in arguments: - parser.add_argument(*args, **kwargs) - - -def add_kv_args(parser): - """Add the kv command and arguments. - - :param argparse.Subparser parser: parser - - """ - kv_parser = parser.add_parser("kv", help="Key/Value Database Utilities") - - subparsers = kv_parser.add_subparsers(dest="action", title="Key/Value Database Utilities") - - for (name, help_text, arguments) in KV_PARSERS: - parser = subparsers.add_parser(name, help=help_text) - for (args, kwargs) in arguments: - parser.add_argument(*args, **kwargs) - - -def add_register_args(parser): - """Add the register command and arguments. - - :param argparse.Subparser parser: parser - - """ - # Service registration - registerp = parser.add_parser("register", help="Register a service for this node") - registerp.add_argument("name", help="The service name") - registerp.add_argument("-a", "--address", default=None, help="Specify an address") - registerp.add_argument("-p", "--port", default=None, type=int, help="Specify a port") - registerp.add_argument("-s", "--service-id", default=None, help="Specify a service ID") - registerp.add_argument("-t", "--tags", default=[], help="Specify a comma delimited list of tags") - rsparsers = registerp.add_subparsers(dest="ctype", title="Service Check Options") - check = rsparsers.add_parser("check", help="Define an external script-based check") - check.add_argument("interval", default=10, type=int, help="How often to run the check script") - check.add_argument("path", default=None, help="Path to the script invoked by Consul") - httpcheck = rsparsers.add_parser("httpcheck", help="Define an HTTP-based check") - httpcheck.add_argument("interval", default=10, type=int, help="How often to run the check script") - httpcheck.add_argument("url", default=None, help="HTTP URL to be polled by Consul") - rsparsers.add_parser("no-check", help="Do not enable service monitoring") - ttl = rsparsers.add_parser("ttl", help="Define a duration based TTL check") - ttl.add_argument("duration", type=int, default=10, help="TTL duration for a service with missing check data") - - -def add_run_once_args(parser): - """Add the run_once command and arguments. - - :param argparse.Subparser parser: parser - - """ - run_oncep = parser.add_parser("run_once", help="Run a command locked to a single " "execution") - run_oncep.add_argument("lock", help="The name of the lock which will be " "held in Consul.") - run_oncep.add_argument("command_to_run", nargs=argparse.REMAINDER, help="The command to lock") - run_oncep.add_argument("-i", "--interval", default=None, help="Hold the lock for X seconds") - - -def add_deregister_args(parser): - """Add the deregister command and arguments. - - :param argparse.Subparser parser: parser - - """ - # Service registration - registerp = parser.add_parser("deregister", help="Deregister a service for this node") - registerp.add_argument("service_id", help="The service registration id") - - -def add_services_args(parser): - """Add the services command and arguments. - - :param argparse.Subparser parser: parser - - """ - # Service registration - registerp = parser.add_parser("services", help="List services for this node") - - registerp.add_argument("-i", "--indent", type=int, default=None, help="The indent level for output") - - -def parse_cli_args(): - """Create the argument parser and add the arguments""" - parser = argparse.ArgumentParser(description="CLI utilities for Consul", epilog=EPILOG) - - env_var = os.environ.get(CONSUL_ENV_VAR, "") - parsed_defaults = urlparse.urlparse(env_var) - - parser.add_argument( - "--api-scheme", default=parsed_defaults.scheme or "http", help="The scheme to use for connecting to Consul with" - ) - parser.add_argument( - "--api-host", default=parsed_defaults.hostname or "localhost", help="The consul host to connect on" - ) - parser.add_argument("--api-port", default=parsed_defaults.port or 8500, help="The consul API port to connect to") - parser.add_argument("--datacenter", dest="dc", default=None, help="The datacenter to specify for the connection") - parser.add_argument("--token", default=None, help="ACL token") - parser.add_argument("--version", action="version", version=consulate.__version__, help="Current consulate version") - - sparser = parser.add_subparsers(title="Commands", dest="command") - add_acl_args(sparser) - add_kv_args(sparser) - add_register_args(sparser) - add_deregister_args(sparser) - add_run_once_args(sparser) - add_services_args(sparser) - return parser.parse_args() - - -def acl_backup(consul, args): - """Dump the ACLs from Consul to JSON - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - handle = open(args.file, "w") if args.file else sys.stdout - acls = consul.acl.list() - try: - if args.pretty: - handle.write(json.dumps(acls, sort_keys=True, indent=2, separators=(",", ": ")) + "\n") - else: - handle.write(json.dumps(acls, sort_keys=True) + "\n") - except exceptions.ConnectionError: - connection_error() - - -def acl_restore(consul, args): - """Restore the Consul KV store - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - handle = open(args.file, "r") if args.file else sys.stdin - data = json.load(handle) - for row in data: - consul.acl.update(row["ID"], row["Name"], row["Type"], row["Rules"]) - print("{0} ACLs written".format(len(data))) - - -ACL_ACTIONS = {"backup": acl_backup, "restore": acl_restore} - - -def kv_backup(consul, args): - """Backup the Consul KV database - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - handle = open(args.file, "w") if args.file else sys.stdout - if args.key: - args.key = args.key.strip("/") - prefixlen = len(args.key.split("/")) - records = [("/".join(k.split("/")[prefixlen:]), f, v) for k, f, v in consul.kv.records(args.key)] - else: - records = consul.kv.records() - if args.base64: - records = [(k, f, str(base64.b64encode(utils.maybe_encode(v)), "ascii") if v else v) for k, f, v in records] - try: - if args.pretty: - handle.write(json.dumps(records, sort_keys=True, indent=2, separators=(",", ": ")) + "\n") - else: - handle.write(json.dumps(records) + "\n") - except exceptions.ConnectionError: - connection_error() - - -def kv_delete(consul, args): - """Remove a key from the Consulate database - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - try: - del consul.kv[args.key] - except exceptions.ConnectionError: - connection_error() - - -def kv_get(consul, args): - """Get the value of a key from the Consul database - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - try: - if args.recurse: - for key in sorted(consul.kv.find(args.key)): - displaykey = key - if args.trim: - keyparts = displaykey.split("/") - if args.trim >= len(keyparts): - displaykey = keyparts[-1] - else: - displaykey = "/".join(keyparts[args.trim :]) # noqa: E203 - sys.stdout.write("%s\t%s\n" % (displaykey, consul.kv.get(key))) - else: - sys.stdout.write("%s\n" % consul.kv.get(args.key)) - except exceptions.ConnectionError: - connection_error() - - -def kv_ls(consul, args): - """List out the keys from the Consul KV database - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - try: - if args.key: - args.key = args.key.lstrip("/") - keylist = sorted(consul.kv.find(args.key)) - else: - keylist = consul.kv.keys() - for key in keylist: - if args.long: - keylen = 0 - if consul.kv[key]: - keylen = len(consul.kv[key]) - print("{0:>14} {1}".format(keylen, key)) - else: - print(key) - except exceptions.ConnectionError: - connection_error() - - -def kv_mkdir(consul, args): - """Make a key based path/directory in the KV database - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - if not args.path[:-1] == "/": - args.path += "/" - try: - consul.kv.set(args.path, None) - except exceptions.ConnectionError: - connection_error() - - -def kv_restore(consul, args): # pylint: disable=too-many-branches,too-many-format-args; # noqa: C901 - """Restore the Consul KV store - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - if args.prune: - if args.key: - args.key = args.key.strip("/") - keylist = consul.kv.find(args.key) - else: - keylist = consul.kv.find("") - handle = open(args.file, "r") if args.file else sys.stdin - data = json.load(handle) - for row in data: - if isinstance(row, dict): - # translate raw api export to internal representation - if row["Value"] is not None: - row["Value"] = base64.b64decode(row["Value"]) - row = [row["Key"], row["Flags"], row["Value"]] - - if args.base64 and row[2] is not None: - row[2] = base64.b64decode(row[2]) - - # Here's an awesome thing to make things work - if args.key: - if row[0] == "": - rowkey = args.key - else: - rowkey = args.key + "/" + row[0] - else: - rowkey = row[0] - if args.prune: - if rowkey in keylist: - del keylist[rowkey] - try: - consul.kv.set_record(rowkey, row[1], row[2], not args.no_replace) - except exceptions.ConnectionError: - connection_error() - if args.prune: - for key in keylist: - print("Pruning {0}".format(key)) - try: - consul.kv.delete(key) - except exceptions.ConnectionError: - connection_error() - - -def kv_rm(consul, args): - """Remove a key from the Consulate database - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - try: - consul.kv.delete(args.key, args.recurse) - except exceptions.ConnectionError: - connection_error() - - -def kv_set(consul, args): - """Set a value of a key int the Consul database - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - try: - consul.kv[args.key] = args.value - except exceptions.ConnectionError: - connection_error() - - -# Mapping dict to simplify the code in main() -KV_ACTIONS = { - "backup": kv_backup, - "del": kv_delete, - "get": kv_get, - "ls": kv_ls, - "mkdir": kv_mkdir, - "restore": kv_restore, - "rm": kv_rm, - "set": kv_set, -} - - -def register(consul, args): - """Handle service registration. - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - check = args.path if args.ctype == "check" else None - httpcheck = args.url if args.ctype == "httpcheck" else None - interval = "%ss" % args.interval if args.ctype in ["check", "httpcheck"] else None - ttl = "%ss" % args.duration if args.ctype == "ttl" else None - tags = args.tags.split(",") if args.tags else None - try: - consul.agent.service.register( - args.name, args.service_id, args.address, int(args.port), tags, check, interval, ttl, httpcheck - ) - except exceptions.ConnectionError: - connection_error() - - -def deregister(consul, args): - """Handle service deregistration. - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - try: - consul.agent.service.deregister(args.service_id) - except exceptions.ConnectionError: - connection_error() - - -def run_once(consul, args): - """Ensure only one process can run a command at a time - - :param consulate.api_old.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - - error_msg, error_code = None, None - try: - consul.lock.prefix("") - with consul.lock.acquire(args.lock): - if args.interval: - now = int(time.time()) - last_run = consul.kv.get("{0}_last_run".format(args.lock)) - if str(last_run) not in ["null", "None"] and int(last_run) + int(args.interval) > now: - sys.stdout.write("Last run happened fewer than {0} seconds ago. " "Exiting\n".format(args.interval)) - return - consul.kv["{0}_last_run".format(args.lock)] = now - - # Should the subprocess return an error code, release the lock - try: - print( - subprocess.check_output( - args.command_to_run[0].strip(), stderr=subprocess.STDOUT, shell=True # nosec - ) - ) - # If the subprocess fails - except subprocess.CalledProcessError as err: - error_code = 1 - error_msg = '"{0}" exited with return code "{1}" and ' "output {2}".format( - args.command_to_run, err.returncode, err.output - ) - except OSError as err: - error_code = 1 - error_msg = '"{0}" command does not exist "{1}"'.format(args.command_to_run, err) - except Exception as err: - error_code = 1 - error_msg = '"{0}" exited with error "{1}"'.format(args.command_to_run, err) - - except consulate.LockFailure: - on_error("Cannot obtain the required lock. Exiting") - - except exceptions.ConnectionError: - connection_error() - - if error_msg: - on_error(error_msg, error_code) - - -def services(consul, args): - """Dump the list of services registered with Consul - - :param consulate.api.Consul consul: The Consul instance - :param argparser.namespace args: The cli args - - """ - - svcs = consul.agent.services() - print(json.dumps(svcs, sort_keys=True, indent=args.indent, separators=(",", ": ")) + "\n") - - -def main(): - """Entrypoint for the consulate cli application""" - args = parse_cli_args() - - if args.api_scheme == "http+unix": - adapter = adapters.UnixSocketRequest - port = None - - api_host = os.environ.get("CONSUL_HTTP_ADDR").replace("unix://", "") - if args.api_host: - api_host = args.api_host - else: - adapter = None - port = args.api_port - - api_host = "localhost" - if args.api_host: - api_host = args.api_host - - consul = consulate.Consul(api_host, port, args.dc, args.token, args.api_scheme, adapter) - - if args.command == "acl": - ACL_ACTIONS[args.action](consul, args) - elif args.command == "kv": - KV_ACTIONS[args.action](consul, args) - elif args.command == "register": - register(consul, args) - elif args.command == "deregister": - deregister(consul, args) - elif args.command == "services": - services(consul, args) - elif args.command == "run_once": - run_once(consul, args) diff --git a/pyms/services_discovery/consulate/client.py b/pyms/services_discovery/consulate/client.py deleted file mode 100644 index 50dd615..0000000 --- a/pyms/services_discovery/consulate/client.py +++ /dev/null @@ -1,197 +0,0 @@ -""" -Consul client object - -""" -import os -from urllib.parse import quote - -from pyms.services_discovery.consulate import adapters, api - -DEFAULT_HOST = os.environ.get("CONSUL_HOST") or "localhost" -DEFAULT_PORT = os.environ.get("CONSUL_PORT") or 8500 -DEFAULT_ADDR = os.environ.get("CONSUL_HTTP_ADDR") -DEFAULT_SCHEME = "http" -DEFAULT_TOKEN = os.environ.get("CONSUL_HTTP_TOKEN") -API_VERSION = "v1" - - -class Consul: # pylint: disable=too-many-instance-attributes - """Access the Consul HTTP API via Python. - - The default values connect to Consul via ``localhost:8500`` via http. If - you want to connect to Consul via a local UNIX socket, you'll need to - override both the ``scheme``, ``port`` and the ``adapter`` like so: - - .. code:: python - - consul = consulate.Consul('/path/to/socket', None, scheme='http+unix', - adapter=consulate.adapters.UnixSocketRequest) - services = consul.agent.services() - - :param str addr: The CONSUL_HTTP_ADDR if available (Default: None) - :param str host: The host name to connect to (Default: localhost) - :param int port: The port to connect on (Default: 8500) - :param str datacenter: Specify a specific data center - :param str token: Specify a ACL token to use - :param str scheme: Specify the scheme (Default: http) - :param class adapter: Specify to override the request adapter - (Default: :py:class:`consulate.adapters.Request`) - :param bool/str verify: Specify how to verify TLS certificates - :param tuple cert: Specify client TLS certificate and key files - :param float timeout: Timeout in seconds for API requests (Default: None) - - """ - - def __init__( - self, - addr=DEFAULT_ADDR, - host=DEFAULT_HOST, - port=DEFAULT_PORT, - datacenter=None, - token=DEFAULT_TOKEN, - scheme=DEFAULT_SCHEME, - adapter=None, - verify=True, - cert=None, - timeout=None, - ): # pylint: disable=too-many-arguments - """Create a new instance of the Consul class""" - base_uri = self._base_uri(addr=addr, scheme=scheme, host=host, port=port) - self._adapter = adapter() if adapter else adapters.Request(timeout=timeout, verify=verify, cert=cert) - self._acl = api.ACL(base_uri, self._adapter, datacenter, token) - self._agent = api.Agent(base_uri, self._adapter, datacenter, token) - self._catalog = api.Catalog(base_uri, self._adapter, datacenter, token) - self._event = api.Event(base_uri, self._adapter, datacenter, token) - self._health = api.Health(base_uri, self._adapter, datacenter, token) - self._coordinate = api.Coordinate(base_uri, self._adapter, datacenter, token) - self._kv = api.KV(base_uri, self._adapter, datacenter, token) - self._session = api.Session(base_uri, self._adapter, datacenter, token) - self._status = api.Status(base_uri, self._adapter, datacenter, token) - self._lock = api.Lock(base_uri, self._adapter, self._session, datacenter, token) - - @property - def acl(self): - """Access the Consul - `ACL `_ API - - :rtype: :py:class:`consulate.api.acl.ACL` - - """ - return self._acl - - @property - def agent(self): - """Access the Consul - `Agent `_ API - - :rtype: :py:class:`consulate.api.agent.Agent` - - """ - return self._agent - - @property - def catalog(self): - """Access the Consul - `Catalog `_ API - - :rtype: :py:class:`consulate.api.catalog.Catalog` - - """ - return self._catalog - - @property - def event(self): - """Access the Consul - `Events `_ API - - :rtype: :py:class:`consulate.api.event.Event` - - """ - return self._event - - @property - def health(self): - """Access the Consul - `Health `_ API - - :rtype: :py:class:`consulate.api.health.Health` - - """ - return self._health - - @property - def coordinate(self): - """Access the Consul - `Coordinate `_ API - - :rtype: :py:class:`consulate.api.coordinate.Coordinate` - - """ - return self._coordinate - - @property - def kv(self): - """Access the Consul - `KV `_ API - - :rtype: :py:class:`consulate.api.kv.KV` - - """ - return self._kv - - @property - def lock(self): - """Wrapper for easy :class:`~consulate.api.kv.KV` locks. - `Semaphore ` _Guide - Example: - - .. code:: python - - from pyms.services_discovery import consulate - - consul = consulate.Consul() - with consul.lock.acquire('my-key'): - print('Locked: {}'.format(consul.lock.key)) - # Do stuff - - :rtype: :class:`~consulate.api.lock.Lock` - - """ - return self._lock - - @property - def session(self): - """Access the Consul - `Session `_ API - - :rtype: :py:class:`consulate.api.session.Session` - - """ - return self._session - - @property - def status(self): - """Access the Consul - `Status `_ API - - :rtype: :py:class:`consulate.api.status.Status` - - """ - return self._status - - @staticmethod - def _base_uri(scheme, host, port, addr=None): - """Return the base URI to use for API requests. Set ``port`` to None - when creating a UNIX Socket URL. - - :param str scheme: The scheme to use (Default: http) - :param str host: The host name to connect to (Default: localhost) - :param int|None port: The port to connect on (Default: 8500) - :rtype: str - - """ - if addr is None: - if port: - return "{0}://{1}:{2}/{3}".format(scheme, host, port, API_VERSION) - return "{0}://{1}/{2}".format(scheme, quote(host, ""), API_VERSION) - return "{0}/{1}".format(addr, API_VERSION) diff --git a/pyms/services_discovery/consulate/exceptions.py b/pyms/services_discovery/consulate/exceptions.py deleted file mode 100644 index 591a3ca..0000000 --- a/pyms/services_discovery/consulate/exceptions.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Consulate Exceptions - -""" - - -class ConsulateException(Exception): - """Base Consul exception""" - - -class RequestError(ConsulateException): - """There was an error making the request to the consul server""" - - -class ClientError(ConsulateException): - """There was an error in the request that was made to consul""" - - -class ServerError(ConsulateException): - """An internal Consul server error occurred""" - - -class ACLDisabled(ConsulateException): - """Raised when ACL related calls are made while ACLs are disabled""" - - -class ACLFormatError(ConsulateException): - """Raised when PolicyLinks is missing 'ID' and 'Name' in a PolicyLink or - when ServiceIdentities is missing 'ServiceName' field in a ServiceIdentity. - - """ - - -class Forbidden(ConsulateException): - """Raised when ACLs are enabled and the token does not validate""" - - -class NotFound(ConsulateException): - """Raised when an operation is attempted with a value that can not be - found. - - """ - - -class LockFailure(ConsulateException): - """Raised by :class:`~consulate.api.lock.Lock` if the lock can not be - acquired. - - """ diff --git a/pyms/services_discovery/consulate/models/__init__.py b/pyms/services_discovery/consulate/models/__init__.py deleted file mode 100644 index 35c7e78..0000000 --- a/pyms/services_discovery/consulate/models/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding=utf-8 -"""Consulate Data Models""" diff --git a/pyms/services_discovery/consulate/models/acl.py b/pyms/services_discovery/consulate/models/acl.py deleted file mode 100644 index 3dea588..0000000 --- a/pyms/services_discovery/consulate/models/acl.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Models for the ACL endpoints""" -import uuid - -from pyms.services_discovery.consulate.models import base - - -def _validate_link_array(value, model): # pylint: disable=unused-argument - """Validate the policies or roles links are formatted correctly. - - :param list value: An array of PolicyLink or RoleLink. - :param rtype: bool - - """ - return all(["ID" in link or "Name" in link for link in value]) - - -def _validate_service_identities(value, model): # pylint: disable=unused-argument - """Validate service_identities is formatted correctly. - - :param ServiceIdentities value: A ServiceIdentity list - :param rtype: bool - - """ - return all(["ServiceName" in service_identity for service_identity in value]) - - -class ACLPolicy(base.Model): - """Defines the model used for an ACL policy.""" - - __slots__ = ["datacenters", "description", "id", "name", "rules"] - - __attributes__ = { - "datacenters": { - "key": "Datacenters", - "type": list, - }, - "description": { - "key": "Description", - "type": str, - }, - "id": { - "key": "ID", - "type": uuid.UUID, - "cast_from": str, - "cast_to": str, - }, - "name": { - "key": "Name", - "type": str, - }, - "rules": { - "key": "Rules", - "type": str, - }, - } - - -class ACLRole(base.Model): - """Defines the model used for an ACL role.""" - - __slots__ = ["description", "name", "policies", "service_identities"] - - __attributes__ = { - "description": { - "key": "Description", - "type": str, - }, - "name": { - "key": "Name", - "type": str, - "required": True, - }, - "policies": { - "key": "Policies", - "type": list, - "validator": _validate_link_array, - }, - "service_identities": { - "key": "ServiceIdentities", - "type": list, - "validator": _validate_service_identities, - }, - } - - -class ACLToken(base.Model): - """Defines the model used for an ACL token.""" - - __slots__ = [ - "accessor_id", - "description", - "expiration_time", - "expiration_ttl", - "local", - "policies", - "roles", - "secret_id", - "service_identities", - ] - - __attributes__ = { - "accessor_id": { - "key": "AccessorID", - "type": uuid.UUID, - "cast_from": str, - "cast_to": str, - }, - "description": { - "key": "Description", - "type": str, - }, - "expiration_time": { - "key": "ExpirationTime", - "type": str, - }, - "expiration_ttl": { - "key": "ExpirationTTL", - "type": str, - }, - "local": { - "key": "Local", - "type": bool, - }, - "policies": { - "key": "Policies", - "type": list, - "validator": _validate_link_array, - }, - "roles": { - "key": "Roles", - "type": list, - "validator": _validate_link_array, - }, - "secret_id": { - "key": "SecretID", - "type": uuid.UUID, - "cast_from": str, - "cast_to": str, - }, - "service_identities": { - "key": "ServiceIdentities", - "type": list, - "validator": _validate_service_identities, - }, - } - - -# NOTE: Everything below here is deprecated post consul-1.4.0. - - -class ACL(base.Model): - """Defines the model used for an individual ACL token.""" - - __slots__ = ["id", "name", "type", "rules"] - - __attributes__ = { - "id": {"key": "ID", "type": uuid.UUID, "cast_from": str, "cast_to": str}, - "name": {"key": "Name", "type": str}, - "type": {"key": "Type", "type": str, "enum": {"client", "management"}, "required": True}, - "rules": {"key": "Rules", "type": str}, - } diff --git a/pyms/services_discovery/consulate/models/agent.py b/pyms/services_discovery/consulate/models/agent.py deleted file mode 100644 index 8e3759b..0000000 --- a/pyms/services_discovery/consulate/models/agent.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Models for the Agent endpoints""" -from pyms.services_discovery.consulate import utils -from pyms.services_discovery.consulate.models import base - - -def _validate_args(value, model): - """Validate that the args values are all strings and that it does not - conflict with other attributes. - - :param list([str]) value: The args value - :param consulate.models.agent.Check model: The model instance. - :rtype: bool - - """ - is_instance = all([isinstance(v, str) for v in value]) - return is_instance and not model.args and not model.grpc and not model.http and not model.ttl - - -def _validate_grpc(value, model): - """Validate that the HTTP value is a URL and that it does not conflict - with other attributes. - - :param str value: The URL value - :param consulate.models.agent.Check model: The model instance. - :rtype: bool - - """ - return utils.validate_url(value) and not model.args and not model.http and not model.tcp and not model.ttl - - -def _validate_http(value, model): - """Validate that the HTTP value is a URL and that it does not conflict - with other attributes. - - :param str value: The URL value - :param consulate.models.agent.Check model: The model instance. - :rtype: bool - - """ - return utils.validate_url(value) and not model.args and not model.grpc and not model.tcp and not model.ttl - - -def _validate_interval(value, model): - """Validate that interval does not conflict with other attributes. - - :param str value: The interval value - :param consulate.models.agent.Check model: The model instance. - :rtype: bool - - """ - return utils.validate_go_interval(value) and not model.ttl - - -def _validate_tcp(_value, model): - """Validate that the TCP does not conflict with other attributes. - - :param str _value: The TCP value - :param consulate.models.agent.Check model: The model instance. - :rtype: bool - - """ - return not model.args and not model.grpc and not model.http and not model.ttl - - -def _validate_ttl(value, model): - """Validate that the TTL does not conflict with other attributes. - - :param str value: The TTL value - :param consulate.models.agent.Check model: The model instance. - :rtype: bool - - """ - is_valid = utils.validate_go_interval(value) - return is_valid and not model.args and not model.grpc and not model.http and not model.tcp and not model.interval - - -class Check(base.Model): - """Model for making Check API requests to Consul.""" - - __slots__ = [ - "id", - "name", - "interval", - "notes", - "deregister_critical_service_after", - "args", - "docker_container_id", - "grpc", - "grpc_use_tls", - "http", - "method", - "header", - "timeout", - "tls_skip_verify", - "tcp", - "ttl", - "service_id", - "status", - ] - - __attributes__ = { - "id": {"key": "ID", "type": str}, - "name": {"key": "Name", "type": str, "required": True}, - "interval": {"key": "Interval", "type": str, "validator": _validate_interval}, - "notes": {"key": "Notes", "type": str}, - "deregister_critical_service_after": { - "key": "DeregisterCriticalServiceAfter", - "type": str, - "validator": utils.validate_go_interval, - }, - "args": {"key": "Args", "type": list, "validator": _validate_args}, - "docker_container_id": {"key": "DockerContainerID", "type": str}, - "grpc": {"key": "GRPC", "type": str, "validator": _validate_grpc}, - "grpc_use_tls": {"key": "GRPCUseTLS", "type": bool}, - "http": {"key": "HTTP", "type": str, "validator": _validate_http}, - "method": {"key": "Method", "type": str, "enum": {"HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"}}, - "header": { - "key": "Header", - "type": dict, - "validator": lambda h, _m: all([(isinstance(k, str) and isinstance(v, str)) for k, v in h.items()]), - "cast_to": lambda h: {k: [v] for k, v in h.items()}, - }, - "timeout": {"key": "Timeout", "type": str, "validator": utils.validate_go_interval}, - "tls_skip_verify": {"key": "TLSSkipVerify", "type": bool}, - "tcp": {"key": "TCP", "type": str, "validator": _validate_tcp}, - "ttl": {"key": "TTL", "type": str, "validator": _validate_ttl}, - "service_id": {"key": "ServiceID", "type": str}, - "status": {"key": "Status", "type": str, "enum": {"passing", "warning", "critical", "maintenance"}}, - } - - def __init__(self, **kwargs): - super().__init__(**kwargs) - if (self.args or self.grpc or self.http or self.tcp) and not self.interval: - raise ValueError('"interval" must be specified when specifying ' "args, grpc, http, or tcp.") - - -class Service(base.Model): - """Model for making Check API requests to Consul.""" - - __slots__ = ["id", "name", "tags", "meta", "address", "port", "check", "checks", "enable_tag_override"] - - __attributes__ = { - "id": {"key": "ID", "type": str}, - "name": {"key": "Name", "type": str, "required": True}, - "tags": {"key": "Tags", "type": list, "validator": lambda t, _m: all([isinstance(v, str) for v in t])}, - "meta": { - "key": "Meta", - "type": dict, - "validator": lambda h, _m: all([(isinstance(k, str) and isinstance(v, str)) for k, v in h.items()]), - }, - "address": {"key": "Address", "type": str}, - "port": {"key": "Port", "type": int}, - "check": {"key": "Check", "type": Check, "cast_to": dict}, - "checks": { - "key": "Checks", - "type": list, - "validator": lambda c, _m: all([isinstance(v, Check) for v in c]), - "cast_to": lambda c: [dict(check) for check in c], - }, - "enable_tag_override": {"Key": "EnableTagOverride", "type": bool}, - } diff --git a/pyms/services_discovery/consulate/models/base.py b/pyms/services_discovery/consulate/models/base.py deleted file mode 100644 index 3c0eb07..0000000 --- a/pyms/services_discovery/consulate/models/base.py +++ /dev/null @@ -1,165 +0,0 @@ -# coding=utf-8 -""" -Base Model - -""" -import collections - - -class Model(collections.Iterable): - """A model contains an __attribute__ map that defines the name, - its type for type validation, an optional validation method, a method - used to - - .. python:: - - class MyModel(Model): - - __attributes__ = { - 'ID': { - 'type': uuid.UUID, - 'required': False, - 'default': None, - 'cast_from': str, - 'cast_to': str - }, - 'Serial': { - 'type': int - 'required': True, - 'default': 0, - 'validator': lambda v: v >= 0 end, - } - } - - """ - - __attributes__ = {} - """The attributes that define the data elements of the model""" - - def __init__(self, **kwargs): - super().__init__() - [setattr(self, name, value) for name, value in kwargs.items()] # pylint: disable=expression-not-assigned - [ # pylint: disable=expression-not-assigned - self._set_default(name) for name in self.__attributes__ if name not in kwargs - ] - - def __iter__(self): - """Iterate through the model's key, value pairs. - - :rtype: iterator - - """ - for name in self.__attributes__: - value = self._maybe_cast_value(name) - if value is not None: - yield self._maybe_return_key(name), value - - def __setattr__(self, name, value): - """Set the value for an attribute of the model, validating the - attribute name and its type if the type is defined in ``__types__``. - - :param str name: The attribute name - :param mixed value: The value to set - :raises: AttributeError - :raises: TypeError - :raises: ValueError - - """ - if name not in self.__attributes__: - raise AttributeError('Invalid attribute "{}"'.format(name)) - value = self._validate_value(name, value) - super().__setattr__(name, value) - - def __getattribute__(self, name): - """Return the attribute from the model if it is set, otherwise - returning the default if one is set. - - :param str name: The attribute name - :rtype: mixed - - """ - try: - return super().__getattribute__(name) - except AttributeError: - if name in self.__attributes__: - return self.__attributes__[name].get("default", None) - raise - - def _maybe_cast_value(self, name): - """Return the attribute value, possibly cast to a different type if - the ``cast_to`` item is set in the attribute definition. - - :param str name: The attribute name - :rtype: mixed - - """ - value = getattr(self, name) - if value is not None and self.__attributes__[name].get("cast_to"): - return self.__attributes__[name]["cast_to"](value) - return value - - def _maybe_return_key(self, name): - """Return the attribute name as specified in it's ``key`` definition, - if specified. This is to map python attribute names to their Consul - alternatives. - - :param str name: The attribute name - :rtype: mixed - - """ - if self.__attributes__[name].get("key"): - return self.__attributes__[name]["key"] - return name - - def _required_attr(self, name): - """Returns :data:`True` if the attribute is required. - - :param str name: The attribute name - :rtype: bool - - """ - return self.__attributes__[name].get("required", False) - - def _set_default(self, name): - """Set the default value for the attribute name. - - :param str name: The attribute name - - """ - setattr(self, name, self.__attributes__[name].get("default", None)) - - def _validate_value(self, name, value): - """Ensures the the value validates based upon the type or a validation - function, raising an error if it does not. - - :param str name: The attribute name - :param mixed value: The value that is being set - :rtype: mixed - :raises: TypeError - :raises: ValueError - - """ - if value is None: - if self._required_attr(name): - raise ValueError('Attribute "{}" is required'.format(name)) - return None - - if not isinstance(value, self.__attributes__[name].get("type")): - cast_from = self.__attributes__[name].get("cast_from") - if cast_from and isinstance(value, cast_from): - value = self.__attributes__[name]["type"](value) - else: - raise TypeError( - 'Attribute "{}" must be of type {} not {}'.format( - name, self.__attributes__[name]["type"].__name__, value.__class__.__name__ - ) - ) - - if self.__attributes__[name].get("enum") and value not in self.__attributes__[name]["enum"]: - raise ValueError('Attribute "{}" value {!r} not valid'.format(name, value)) - - validator = self.__attributes__[name].get("validator") - if callable(validator): - if not validator(value, self): - raise ValueError('Attribute "{}" value {!r} did not validate'.format(name, value)) - return value diff --git a/pyms/services_discovery/consulate/utils.py b/pyms/services_discovery/consulate/utils.py deleted file mode 100644 index c4908ae..0000000 --- a/pyms/services_discovery/consulate/utils.py +++ /dev/null @@ -1,97 +0,0 @@ -# coding=utf-8 -""" -Misc utility functions and constants - -""" -import re -from urllib import parse as _urlparse - -from pyms.services_discovery.consulate import exceptions - -DURATION_PATTERN = re.compile(r"^(?:(?:-|)(?:\d+|\d+\.\d+)(?:µs|ms|s|m|h))+$") - - -def is_string(value): - """Python 2 & 3 safe way to check if a value is either an instance of str - or unicode. - - :param mixed value: The value to evaluate - :rtype: bool - - """ - checks = [isinstance(value, t) for t in [bytes, str]] - return any(checks) - - -def maybe_encode(value): - """If the value passed in is a str, encode it as UTF-8 bytes for Python 3 - - :param str|bytes value: The value to maybe encode - :rtype: bytes - - """ - try: - return value.encode("utf-8") - except AttributeError: - return value - - -def _response_error(response): - """Return the decoded response error or status code if no content exists. - - :param requests.response response: The HTTP response - :rtype: str - - """ - return response.body.decode("utf-8") if hasattr(response, "body") and response.body else str(response.status_code) - - -def response_ok(response, raise_on_404=False): - """Evaluate the HTTP response and raise the appropriate exception if - required. - - :param requests.response response: The HTTP response - :param bool raise_on_404: Raise an exception on 404 error - :rtype: bool - :raises: consulate.exceptions.ConsulateException - - """ - result = False - if response.status_code == 200: - result = True - elif response.status_code == 400: - raise exceptions.ClientError(_response_error(response)) - elif response.status_code == 401: - raise exceptions.ACLDisabled(_response_error(response)) - elif response.status_code == 403: - raise exceptions.Forbidden(_response_error(response)) - elif response.status_code == 404 and raise_on_404: - raise exceptions.NotFound(_response_error(response)) - elif response.status_code == 500: - raise exceptions.ServerError(_response_error(response)) - return result - - -def validate_go_interval(value, _model=None): - """Validate the value passed in returning :data:`True` if it is a Go - Duration value. - - :param str value: The string to check - :param consulate.models.base.Model _model: Optional model passed in - :rtype: bool - - """ - return DURATION_PATTERN.match(value) is not None - - -def validate_url(value, _model=None): - """Validate that the value passed in is a URL, returning :data:`True` if - it is. - - :param str value: The string to check - :param consulate.models.base.Model _model: Optional model passed in - :rtype: bool - - """ - parsed = _urlparse.urlparse(value) - return parsed.scheme and parsed.netloc diff --git a/setup.py b/setup.py index 83d7c39..aaa4d35 100644 --- a/setup.py +++ b/setup.py @@ -52,12 +52,15 @@ "prometheus_client>=0.8.0", ] +install_service_discovery_requires = [ + "py-ms-consulate>=1.0.0", +] + install_tests_requires = [ "requests-mock>=1.8.0", "coverage>=5.3", "pytest>=6.1.0", "pytest-cov>=2.10.1", - "pytest-docker>=0.10.1", "pylint>=2.6.0", "flake8>=3.8.2", "tox>=3.20.0", @@ -69,7 +72,6 @@ "pre-commit>=2.8.1", "black>=20.8b1", "isort>=5.6.4", - "httmock>=1.4.0", ] install_all_requires = ( @@ -79,6 +81,7 @@ + install_metrics_requires + install_crypt_requires + install_aws_requires + + install_service_discovery_requires ) setup( @@ -117,6 +120,7 @@ "metrics": install_metrics_requires, "crypt": install_crypt_requires, "aws": install_aws_requires, + "consul": install_service_discovery_requires, "tests": install_tests_requires, }, tests_require=install_all_requires + install_tests_requires, diff --git a/tests/conftest.py b/tests/conftest.py index d5e6e31..e69de29 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,38 +0,0 @@ -import os - -import pytest -import requests -from requests.exceptions import ConnectionError - -os.environ["ASYNC_TEST_TIMEOUT"] = os.environ.get("ASYNC_TEST_TIMEOUT", "15") - - -def is_responsive(url): - try: - response = requests.get(url) - if response.status_code == 200: - return True - except ConnectionError: - return False - - -@pytest.fixture(scope="session") -def http_service(docker_ip, docker_services): - """Ensure that HTTP service is up and responsive.""" - url = "http://{}:{}".format(docker_ip, 8500) - - docker_services.wait_until_responsive(timeout=60.0, pause=0.1, check=lambda: is_responsive(url)) - return docker_ip - - -@pytest.fixture(scope="session") -def docker_compose_file(pytestconfig): - """Get docker compose file""" - return os.path.join(str(pytestconfig.rootdir), "docker", "docker-compose.yml") - - -@pytest.fixture(scope="session") -def config_env(http_service): - """Set config for docker""" - os.environ["CONSUL_HOST"] = http_service - os.environ["CONSUL_PORT"] = "8500" diff --git a/tests/services_discovery_consulate/__init__.py b/tests/services_discovery_consulate/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/tests/services_discovery_consulate/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/services_discovery_consulate/base.py b/tests/services_discovery_consulate/base.py deleted file mode 100644 index feb1acc..0000000 --- a/tests/services_discovery_consulate/base.py +++ /dev/null @@ -1,66 +0,0 @@ -import functools -import json -import os -import unittest -import uuid - -import httmock -import pytest - -from pyms.services_discovery import consulate -from pyms.services_discovery.consulate import exceptions - -CONSUL_TESTING_JSON = "tests/services_discovery_consulate/consul.json" - -with open(CONSUL_TESTING_JSON, "r") as handle: - CONSUL_CONFIG = json.load(handle) - - -def generate_key(func): - @functools.wraps(func) - def _decorator(self, *args, **kwargs): - key = str(uuid.uuid4())[0:8] - self.used_keys.append(key) - func(self, key) - - return _decorator - - -@httmock.all_requests -def raise_oserror(_url_unused, _request): - raise OSError - - -@pytest.mark.usefixtures("config_env") -class TestCase(unittest.TestCase): - def setUp(self): - self.consul = consulate.Consul( - host=os.environ["CONSUL_HOST"], - port=os.environ["CONSUL_PORT"], - token=CONSUL_CONFIG["acl"]["tokens"]["master"], - ) - self.forbidden_consul = consulate.Consul( - host=os.environ["CONSUL_HOST"], port=os.environ["CONSUL_PORT"], token=str(uuid.uuid4()) - ) - self.used_keys = list() - - def tearDown(self): - for key in self.consul.kv.keys(): - self.consul.kv.delete(key) - - checks = self.consul.agent.checks() - for name in checks: - self.consul.agent.check.deregister(checks[name]["CheckID"]) - - services = self.consul.agent.services() - for name in services: - self.consul.agent.service.deregister(services[name]["ID"]) - - for acl in self.consul.acl.list_tokens(): - if acl["AccessorID"] == CONSUL_CONFIG["acl"]["tokens"]["master"]: - continue - try: - uuid.UUID(acl["AccessorID"]) - self.consul.acl.delete_token(acl["AccessorID"]) - except (ValueError, exceptions.ConsulateException): - pass diff --git a/tests/services_discovery_consulate/consul.json b/tests/services_discovery_consulate/consul.json deleted file mode 100644 index 3368abc..0000000 --- a/tests/services_discovery_consulate/consul.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "acl": { - "enabled": true, - "enable_key_list_policy": true, - "tokens": { - "master": "9ae5fe1a-6b38-47e5-a0e7-f06b8b2fa645" - } - }, - "bootstrap_expect": 1, - "data_dir": "/tmp/consul", - "datacenter": "test", - "server": true, - "bind_addr": "{{ GetPrivateIP }}", - "client_addr": "0.0.0.0", - "enable_script_checks": true -} diff --git a/tests/services_discovery_consulate/test_acl.py b/tests/services_discovery_consulate/test_acl.py deleted file mode 100644 index 22ddc01..0000000 --- a/tests/services_discovery_consulate/test_acl.py +++ /dev/null @@ -1,301 +0,0 @@ -""" -Tests for Consulate.acl - -""" -import json -import random -import uuid - -import httmock - -from pyms.services_discovery import consulate -from pyms.services_discovery.consulate import exceptions - -from . import base - -ACL_OLD_RULES = """key "" { - policy = "read" -} -key "foo/" { - policy = "write" -} -""" - -ACL_NEW_RULES = """key_prefix "" { - policy = "read" -} -key "foo/" { - policy = "write" -} -""" - -ACL_NEW_UPDATE_RULES = """key_prefix "" { - policy = "deny" -} -key "foo/" { - policy = "read" -} -""" - -POLICYLINKS_SAMPLE = [ - dict(Name="policylink_sample"), -] - -POLICYLINKS_UPDATE_SAMPLE = [dict(Name="policylink_sample"), dict(Name="policylink_update_sample")] - -SERVICE_IDENTITIES_SAMPLE = [dict(ServiceName="db", Datacenters=list("dc1"))] - - -class TestCase(base.TestCase): - @staticmethod - def uuidv4(): - return str(uuid.uuid4()) - - @staticmethod - def random(): - return str(random.randint(0, 999999)) - - def _generate_policies(self): - sample = self.consul.acl.create_policy(self.random()) - sample_update = self.consul.acl.create_policy(self.random()) - return [dict(ID=sample["ID"]), dict(ID=sample_update["ID"])] - - def _generate_roles(self): - role = self.consul.acl.create_role(self.random()) - return [dict(ID=role["ID"])] - - def test_bootstrap_request_exception(self): - @httmock.all_requests - def response_content(_url_unused, _request): - raise OSError - - with httmock.HTTMock(response_content): - with self.assertRaises(exceptions.RequestError): - self.consul.acl.bootstrap() - - def test_bootstrap_success(self): - expectation = self.uuidv4() - - @httmock.all_requests - def response_content(_url_unused, request): - return httmock.response(200, json.dumps({"ID": expectation}), {}, None, 0, request) - - with httmock.HTTMock(response_content): - result = self.consul.acl.bootstrap() - - self.assertEqual(result, expectation) - - def test_bootstrap_raises(self): - with self.assertRaises((consulate.Forbidden, consulate.ServerError)): - self.consul.acl.bootstrap() - - def test_clone_bad_acl_id(self): - with self.assertRaises(consulate.Forbidden): - self.consul.acl.clone(self.uuidv4()) - - def test_clone_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.acl.clone(self.uuidv4()) - - def test_create_and_destroy(self): - acl_id = self.consul.acl.create(self.uuidv4()) - self.assertTrue(self.consul.acl.destroy(acl_id)) - - def test_create_with_rules(self): - acl_id = self.consul.acl.create(self.uuidv4(), rules=ACL_OLD_RULES) - value = self.consul.acl.info(acl_id) - self.assertEqual(value["Rules"], ACL_OLD_RULES) - - def test_create_and_info(self): - acl_id = self.consul.acl.create(self.uuidv4()) - self.assertIsNotNone(acl_id) - data = self.consul.acl.info(acl_id) - self.assertIsNotNone(data) - self.assertEqual(acl_id, data.get("ID")) - - def test_create_and_list(self): - acl_id = self.consul.acl.create(self.uuidv4()) - data = self.consul.acl.list() - self.assertIn(acl_id, [r.get("ID") for r in data]) - - def test_create_and_clone(self): - acl_id = self.consul.acl.create(self.uuidv4()) - clone_id = self.consul.acl.clone(acl_id) - data = self.consul.acl.list() - self.assertIn(clone_id, [r.get("ID") for r in data]) - - def test_create_and_update(self): - acl_id = str(self.consul.acl.create(self.uuidv4())) - self.consul.acl.update(acl_id, "Foo") - data = self.consul.acl.list() - self.assertIn("Foo", [r.get("Name") for r in data]) - self.assertIn(acl_id, [r.get("ID") for r in data]) - - def test_create_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.acl.create(self.uuidv4()) - - def test_destroy_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.acl.destroy(self.uuidv4()) - - def test_info_acl_id_not_found(self): - with self.assertRaises(consulate.NotFound): - self.forbidden_consul.acl.info(self.uuidv4()) - - def test_list_request_exception(self): - with httmock.HTTMock(base.raise_oserror): - with self.assertRaises(exceptions.RequestError): - self.consul.acl.list() - - def test_replication(self): - result = self.forbidden_consul.acl.replication() - self.assertFalse(result["Enabled"]) - self.assertFalse(result["Running"]) - - def test_update_not_found_adds_new_key(self): - acl_id = self.consul.acl.update(self.uuidv4(), "Foo2") - data = self.consul.acl.list() - self.assertIn("Foo2", [r.get("Name") for r in data]) - self.assertIn(acl_id, [r.get("ID") for r in data]) - - def test_update_with_rules(self): - acl_id = self.consul.acl.update(self.uuidv4(), name="test", rules=ACL_OLD_RULES) - value = self.consul.acl.info(acl_id) - self.assertEqual(value["Rules"], ACL_OLD_RULES) - - def test_update_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.acl.update(self.uuidv4(), name="test") - - # NOTE: Everything above here is deprecated post consul-1.4.0 - - def test_create_policy(self): - name = self.random() - result = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) - self.assertEqual(result["Rules"], ACL_NEW_RULES) - - def test_create_and_read_policy(self): - name = self.random() - value = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) - result = self.consul.acl.read_policy(value["ID"]) - self.assertEqual(result["Rules"], ACL_NEW_RULES) - - def test_create_and_update_policy(self): - name = self.random() - value = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) - result = self.consul.acl.update_policy(value["ID"], str(value["Name"]), rules=ACL_NEW_UPDATE_RULES) - self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) - - def test_create_and_delete_policy(self): - name = self.random() - value = self.consul.acl.create_policy(name=name, rules=ACL_NEW_RULES) - result = self.consul.acl.delete_policy(value["ID"]) - self.assertTrue(result) - - def test_list_policy_exception(self): - with httmock.HTTMock(base.raise_oserror): - with self.assertRaises(exceptions.RequestError): - self.consul.acl.list_policies() - - def test_create_role(self): - name = self.random() - result = self.consul.acl.create_role( - name=name, policies=self._generate_policies(), service_identities=SERVICE_IDENTITIES_SAMPLE - ) - self.assertEqual(result["Name"], name) - - def test_create_and_read_role(self): - name = self.random() - value = self.consul.acl.create_role( - name=name, policies=self._generate_policies(), service_identities=SERVICE_IDENTITIES_SAMPLE - ) - result = self.consul.acl.read_role(value["ID"]) - self.assertEqual(result["ID"], value["ID"]) - - def test_create_and_update_role(self): - name = self.random() - value = self.consul.acl.create_role( - name=name, policies=self._generate_policies(), service_identities=SERVICE_IDENTITIES_SAMPLE - ) - result = self.consul.acl.update_role(value["ID"], str(value["Name"]), policies=self._generate_policies()) - self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) - - def test_create_and_delete_role(self): - name = self.random() - value = self.consul.acl.create_role( - name=name, policies=self._generate_policies(), service_identities=SERVICE_IDENTITIES_SAMPLE - ) - result = self.consul.acl.delete_role(value["ID"]) - self.assertTrue(result) - - def test_list_roles_exception(self): - with httmock.HTTMock(base.raise_oserror): - with self.assertRaises(exceptions.RequestError): - self.consul.acl.list_roles() - - def test_create_token(self): - secret_id = self.uuidv4() - accessor_id = self.uuidv4() - result = self.consul.acl.create_token( - accessor_id=accessor_id, - secret_id=secret_id, - roles=self._generate_roles(), - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE, - ) - self.assertEqual(result["AccessorID"], accessor_id) - self.assertEqual(result["SecretID"], secret_id) - - def test_create_and_read_token(self): - secret_id = self.uuidv4() - accessor_id = self.uuidv4() - value = self.consul.acl.create_token( - accessor_id=accessor_id, - secret_id=secret_id, - roles=self._generate_roles(), - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE, - ) - result = self.consul.acl.read_token(value["AccessorID"]) - self.assertEqual(result["AccessorID"], accessor_id) - - def test_create_and_update_token(self): - secret_id = self.uuidv4() - accessor_id = self.uuidv4() - value = self.consul.acl.create_token( - accessor_id=accessor_id, - secret_id=secret_id, - roles=self._generate_roles(), - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE, - ) - result = self.consul.acl.update_token(str(value["AccessorID"]), policies=self._generate_policies()) - self.assertGreater(result["ModifyIndex"], result["CreateIndex"]) - - def test_create_and_clone_token(self): - secret_id = self.uuidv4() - accessor_id = self.uuidv4() - clone_description = "clone token of " + accessor_id - value = self.consul.acl.create_token( - accessor_id=accessor_id, - secret_id=secret_id, - roles=self._generate_roles(), - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE, - ) - result = self.consul.acl.clone_token(value["AccessorID"], description=clone_description) - self.assertEqual(result["Description"], clone_description) - - def test_create_and_delete_token(self): - secret_id = self.uuidv4() - accessor_id = self.uuidv4() - value = self.consul.acl.create_token( - accessor_id=accessor_id, - secret_id=secret_id, - roles=self._generate_roles(), - policies=self._generate_policies(), - service_identities=SERVICE_IDENTITIES_SAMPLE, - ) - result = self.consul.acl.delete_token(value["AccessorID"]) - self.assertTrue(result) diff --git a/tests/services_discovery_consulate/test_agent.py b/tests/services_discovery_consulate/test_agent.py deleted file mode 100644 index 59b97d7..0000000 --- a/tests/services_discovery_consulate/test_agent.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -Tests for Consulate.agent - -""" -import uuid - -import httmock - -from pyms.services_discovery import consulate -from pyms.services_discovery.consulate import utils -from pyms.services_discovery.consulate.models import agent - -from . import base - - -class TestCase(base.TestCase): - def test_checks(self): - result = self.consul.agent.checks() - self.assertDictEqual(result, {}) - - def test_force_leave(self): - self.assertTrue(self.consul.agent.force_leave(str(uuid.uuid4()))) - - def test_force_leave_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.force_leave(str(uuid.uuid4())) - - # def test_join(self): - # self.assertTrue(self.consul.agent.join('127.0.0.1')) - - def test_join_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.join("255.255.255.255") - - def test_maintenance(self): - self.consul.agent.maintenance(True, "testing") - self.consul.agent.maintenance(False) - - def test_maintenance_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.maintenance(True) - - def test_members(self): - result = self.consul.agent.members() - self.assertEqual(len(result), 1) - - def test_members_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.members() - - def test_metrics(self): - result = self.consul.agent.metrics() - self.assertIn("Timestamp", result) - self.assertIn("Gauges", result) - - def test_metrics_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.metrics() - - def test_monitor(self): - for offset, line in enumerate(self.consul.agent.monitor()): - self.assertTrue(utils.is_string(line)) - self.consul.agent.metrics() - if offset > 1: - break - - def test_monitor_request_exception(self): - with httmock.HTTMock(base.raise_oserror): - with self.assertRaises(consulate.RequestError): - for _line in self.consul.agent.monitor(): - break - - def test_monitor_forbidden(self): - with self.assertRaises(consulate.Forbidden): - for line in self.forbidden_consul.agent.monitor(): - self.assertIsInstance(line, str) - break - - def test_reload(self): - self.assertIsNone(self.consul.agent.reload()) - - def test_reload_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.reload() - - def test_self(self): - result = self.consul.agent.self() - self.assertIn("Config", result) - self.assertIn("Coord", result) - self.assertIn("Member", result) - - def test_self_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.self() - - def test_service_registration(self): - self.consul.agent.service.register( - "test-service", address="10.0.0.1", port=5672, tags=["foo", "bar"], meta={"foo": "bar"} - ) - self.assertIn("test-service", self.consul.agent.services()) - self.consul.agent.service.deregister("test-service") - - def test_service_maintenance(self): - self.consul.agent.service.register( - "test-service", address="10.0.0.1", port=5672, tags=["foo", "bar"], meta={"foo": "bar"} - ) - self.assertIn("test-service", self.consul.agent.services()) - reason = "Down for Acceptance" - self.consul.agent.service.maintenance("test-service", reason=reason) - node_in_maintenance = self.consul.catalog.nodes()[0]["Node"] - health_check = self.consul.health.node(node_in_maintenance) - self.assertEqual(len(health_check), 2) - self.assertIn(reason, [check["Notes"] for check in health_check]) - self.consul.agent.service.maintenance("test-service", enable=False) - health_check = self.consul.health.node(node_in_maintenance) - self.assertEqual(len(health_check), 1) - self.assertNotEqual(reason, health_check[0]["Notes"]) - self.consul.agent.service.deregister("test-service") - - def test_token(self): - self.assertTrue(self.consul.agent.token("acl_replication_token", "foo")) - - def test_token_invalid(self): - with self.assertRaises(ValueError): - self.consul.agent.token("acl_replication_tokens", "foo") - - def test_token_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.token("acl_replication_token", "foo") - - -class CheckTestCase(base.TestCase): - def test_register(self): - self.assertTrue(self.consul.agent.check.register(str(uuid.uuid4()), http="http://localhost", interval="30s")) - - def test_register_args_and_no_interval(self): - with self.assertRaises(ValueError): - self.consul.agent.check.register(str(uuid.uuid4()), args=["/bin/true"]) - - def test_register_args_and_ttl(self): - with self.assertRaises(ValueError): - self.consul.agent.check.register(str(uuid.uuid4()), args=["/bin/true"], ttl="30s") - - def test_register_http_and_no_interval(self): - with self.assertRaises(ValueError): - self.consul.agent.check.register(str(uuid.uuid4()), http="http://localhost") - - def test_register_args_and_http(self): - with self.assertRaises(ValueError): - self.consul.agent.check.register(str(uuid.uuid4()), args=["/bin/true"], http="http://localhost") - - def test_register_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.check.register(str(uuid.uuid4()), args=["/bin/true"], interval="30s") - - -class TTLCheckTestCase(base.TestCase): - def setUp(self): - super(TTLCheckTestCase, self).setUp() - name = str(uuid.uuid4()) - self.assertTrue(self.consul.agent.check.register(name, ttl="30s")) - checks = self.consul.agent.checks() - self.check_id = checks[name]["CheckID"] - - def test_pass(self): - self.assertTrue(self.consul.agent.check.ttl_pass(self.check_id)) - - def test_pass_with_note(self): - self.assertTrue(self.consul.agent.check.ttl_pass(self.check_id, "PASS")) - - def test_warn(self): - self.assertTrue(self.consul.agent.check.ttl_warn(self.check_id)) - - def test_warn_with_note(self): - self.assertTrue(self.consul.agent.check.ttl_warn(self.check_id, "WARN")) - - def test_fail(self): - self.assertTrue(self.consul.agent.check.ttl_fail(self.check_id)) - - def test_fail_with_note(self): - self.assertTrue(self.consul.agent.check.ttl_fail(self.check_id, "FAIL")) - - -class ServiceTestCase(base.TestCase): - def test_register(self): - self.assertTrue( - self.consul.agent.service.register( - str(uuid.uuid4()), - address="127.0.0.1", - port=80, - check=agent.Check(name="test", args=["/bin/true"], interval="30s"), - tags=[str(uuid.uuid4())], - ) - ) - - def test_register_grpc(self): - self.assertTrue( - self.consul.agent.service.register( - str(uuid.uuid4()), - address="127.0.0.1", - port=80, - check=agent.Check(name="test", grpc="https://grpc/status", interval="30s"), - ) - ) - - def test_register_http(self): - self.assertTrue( - self.consul.agent.service.register( - str(uuid.uuid4()), - address="127.0.0.1", - port=80, - check=agent.Check(name="test", http="http://localhost", interval="30s"), - ) - ) - - def test_register_tcp(self): - self.assertTrue( - self.consul.agent.service.register( - str(uuid.uuid4()), - address="127.0.0.1", - port=80, - check=agent.Check(name="test", tcp="localhost:80", interval="30s"), - ) - ) - - def test_register_ttl(self): - self.assertTrue( - self.consul.agent.service.register( - str(uuid.uuid4()), address="127.0.0.1", port=80, check=agent.Check(name="test", ttl="30s") - ) - ) - - def test_register_multiple_checks(self): - self.assertTrue( - self.consul.agent.service.register( - str(uuid.uuid4()), - address="127.0.0.1", - port=80, - checks=[ - agent.Check( - name="test1", http="http://netloc", header={"User-Agent": "unittest.TestCase"}, interval="30s" - ), - agent.Check(name="test2", ttl="30s"), - ], - ) - ) - - def test_register_forbidden(self): - with self.assertRaises(consulate.Forbidden): - self.forbidden_consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", port=80) - - def test_register_invalid_check(self): - with self.assertRaises(TypeError): - self.consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", check=str(uuid.uuid4())) - - def test_register_invalid_checks(self): - with self.assertRaises(ValueError): - self.consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", checks=[str(uuid.uuid4())]) - - def test_register_invalid_port(self): - with self.assertRaises(TypeError): - self.consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", port="80") - - def test_register_invalid_tags(self): - with self.assertRaises(TypeError): - self.consul.agent.service.register(str(uuid.uuid4()), address="127.0.0.1", tags=str(uuid.uuid4())) - - def test_register_invalid_interval(self): - with self.assertRaises(TypeError): - self.consul.agent.service.register( - str(uuid.uuid4()), - address="127.0.0.1", - port=80, - check=agent.Check(name="test", http="http://localhost", interval=30), - ) - - def test_register_invalid_ttl(self): - with self.assertRaises(TypeError): - self.consul.agent.service.register( - str(uuid.uuid4()), address="127.0.0.1", port=80, check=agent.Check(name="test", ttl=30) - ) diff --git a/tests/services_discovery_consulate/test_api.py b/tests/services_discovery_consulate/test_api.py deleted file mode 100644 index 0a039e1..0000000 --- a/tests/services_discovery_consulate/test_api.py +++ /dev/null @@ -1,319 +0,0 @@ -import json -import unittest -import uuid -from unittest.mock import patch -from urllib import parse - -import httmock - -from pyms.services_discovery import consulate -from pyms.services_discovery.consulate import adapters -from pyms.services_discovery.consulate.api import base -from tests.services_discovery_consulate.base import CONSUL_TESTING_JSON - -with open(CONSUL_TESTING_JSON, "r") as handle: - CONSUL_CONFIG = json.load(handle) - -SCHEME = "http" -VERSION = "v1" - - -class ConsulTests(unittest.TestCase): - @patch("pyms.services_discovery.consulate.adapters.Request") - @patch("pyms.services_discovery.consulate.api.Agent") - @patch("pyms.services_discovery.consulate.api.Catalog") - @patch("pyms.services_discovery.consulate.api.KV") - @patch("pyms.services_discovery.consulate.api.Health") - @patch("pyms.services_discovery.consulate.api.Coordinate") - @patch("pyms.services_discovery.consulate.api.ACL") - @patch("pyms.services_discovery.consulate.api.Event") - @patch("pyms.services_discovery.consulate.api.Session") - @patch("pyms.services_discovery.consulate.api.Status") - def setUp(self, status, session, event, acl, health, coordinate, kv, catalog, agent, adapter): - self.host = "127.0.0.1" - self.port = 8500 - self.dc = CONSUL_CONFIG["datacenter"] - self.token = CONSUL_CONFIG["acl"]["tokens"]["master"] - - self.acl = acl - self.adapter = adapter - self.agent = agent - self.catalog = catalog - self.event = event - self.kv = kv - self.health = health - self.coordinate = coordinate - self.session = session - self.status = status - - self.base_uri = "{0}://{1}:{2}/v1".format(SCHEME, self.host, self.port) - self.consul = consulate.Consul(self.host, self.port, self.dc, self.token) - - def test_base_uri(self): - self.assertEquals(self.consul._base_uri(SCHEME, self.host, self.port), self.base_uri) - - def test_unix_socket_base_uri(self): - expectation = "http+unix://%2Fvar%2Flib%2Fconsul%2Fconsul.sock/v1" - self.assertEquals(self.consul._base_uri("http+unix", "/var/lib/consul/consul.sock", None), expectation) - - def test_acl_initialization(self): - self.assertTrue(self.acl.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_adapter_initialization(self): - self.assertTrue(self.adapter.called_once_with()) - - def test_agent_initialization(self): - self.assertTrue(self.agent.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_catalog_initialization(self): - self.assertTrue(self.catalog.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_events_initialization(self): - self.assertTrue(self.event.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_kv_initialization(self): - self.assertTrue(self.kv.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_health_initialization(self): - self.assertTrue(self.health.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_coordinate_initialization(self): - self.assertTrue(self.coordinate.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_session_initialization(self): - self.assertTrue(self.session.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_status_initialization(self): - self.assertTrue(self.status.called_once_with(self.base_uri, self.adapter, self.dc, self.token)) - - def test_acl_property(self): - self.assertEqual(self.consul.acl, self.consul._acl) - - def test_agent_property(self): - self.assertEqual(self.consul.agent, self.consul._agent) - - def test_catalog_property(self): - self.assertEqual(self.consul.catalog, self.consul._catalog) - - def test_event_property(self): - self.assertEqual(self.consul.event, self.consul._event) - - def test_health_property(self): - self.assertEqual(self.consul.health, self.consul._health) - - def test_coordinate_property(self): - self.assertEqual(self.consul.coordinate, self.consul._coordinate) - - def test_kv_property(self): - self.assertEqual(self.consul.kv, self.consul._kv) - - def test_status_property(self): - self.assertEqual(self.consul.status, self.consul._status) - - -class EndpointBuildURITests(unittest.TestCase): - def setUp(self): - self.adapter = adapters.Request() - self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) - self.endpoint = base.Endpoint(self.base_uri, self.adapter) - - def test_adapter_assignment(self): - self.assertEqual(self.endpoint._adapter, self.adapter) - - def test_base_uri_assignment(self): - self.assertEqual(self.endpoint._base_uri, "{0}/endpoint".format(self.base_uri)) - - def test_dc_assignment(self): - self.assertIsNone(self.endpoint._dc) - - def test_token_assignment(self): - self.assertIsNone(self.endpoint._token) - - def test_build_uri_with_no_params(self): - result = self.endpoint._build_uri(["foo", "bar"]) - parsed = parse.urlparse(result) - query_params = parse.parse_qs(parsed.query) - self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, "localhost:8500") - self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) - self.assertDictEqual(query_params, {}) - - def test_build_uri_with_params(self): - result = self.endpoint._build_uri(["foo", "bar"], {"baz": "qux"}) - parsed = parse.urlparse(result) - query_params = parse.parse_qs(parsed.query) - self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, "localhost:8500") - self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) - self.assertDictEqual(query_params, {"baz": ["qux"]}) - - -class EndpointBuildURIWithDCTests(unittest.TestCase): - def setUp(self): - self.adapter = adapters.Request() - self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) - self.dc = str(uuid.uuid4()) - self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc) - - def test_dc_assignment(self): - self.assertEqual(self.endpoint._dc, self.dc) - - def test_token_assignment(self): - self.assertIsNone(self.endpoint._token) - - def test_build_uri_with_no_params(self): - result = self.endpoint._build_uri(["foo", "bar"]) - parsed = parse.urlparse(result) - query_params = parse.parse_qs(parsed.query) - self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, "localhost:8500") - self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) - self.assertDictEqual(query_params, {"dc": [self.dc]}) - - def test_build_uri_with_params(self): - result = self.endpoint._build_uri(["foo", "bar"], {"baz": "qux"}) - parsed = parse.urlparse(result) - query_params = parse.parse_qs(parsed.query) - self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, "localhost:8500") - self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) - self.assertDictEqual(query_params, {"dc": [self.dc], "baz": ["qux"]}) - - -class EndpointBuildURIWithTokenTests(unittest.TestCase): - def setUp(self): - self.adapter = adapters.Request() - self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) - self.token = str(uuid.uuid4()) - self.endpoint = base.Endpoint(self.base_uri, self.adapter, token=self.token) - - def test_dc_assignment(self): - self.assertIsNone(self.endpoint._dc) - - def test_token_assignment(self): - self.assertEqual(self.endpoint._token, self.token) - - def test_build_uri_with_no_params(self): - result = self.endpoint._build_uri(["foo", "bar"]) - parsed = parse.urlparse(result) - query_params = parse.parse_qs(parsed.query) - self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, "localhost:8500") - self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) - self.assertDictEqual(query_params, {"token": [self.token]}) - - def test_build_uri_with_params(self): - result = self.endpoint._build_uri(["foo", "bar"], {"baz": "qux"}) - parsed = parse.urlparse(result) - query_params = parse.parse_qs(parsed.query) - self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, "localhost:8500") - self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) - self.assertDictEqual(query_params, {"token": [self.token], "baz": ["qux"]}) - - -class EndpointBuildURIWithDCAndTokenTests(unittest.TestCase): - def setUp(self): - self.adapter = adapters.Request() - self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) - self.dc = str(uuid.uuid4()) - self.token = str(uuid.uuid4()) - self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, self.token) - - def test_dc_assignment(self): - self.assertEqual(self.endpoint._dc, self.dc) - - def test_token_assignment(self): - self.assertEqual(self.endpoint._token, self.token) - - def test_build_uri_with_no_params(self): - result = self.endpoint._build_uri(["foo", "bar"]) - parsed = parse.urlparse(result) - query_params = parse.parse_qs(parsed.query) - self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, "localhost:8500") - self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) - self.assertDictEqual(query_params, {"dc": [self.dc], "token": [self.token]}) - - def test_build_uri_with_params(self): - result = self.endpoint._build_uri(["foo", "bar"], {"baz": "qux"}) - parsed = parse.urlparse(result) - query_params = parse.parse_qs(parsed.query) - self.assertEqual(parsed.scheme, SCHEME) - self.assertEqual(parsed.netloc, "localhost:8500") - self.assertEqual(parsed.path, "/{0}/endpoint/foo/bar".format(VERSION)) - self.assertDictEqual(query_params, {"dc": [self.dc], "token": [self.token], "baz": ["qux"]}) - - -class EndpointGetTests(unittest.TestCase): - def setUp(self): - self.adapter = adapters.Request() - self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) - self.dc = str(uuid.uuid4()) - self.token = str(uuid.uuid4()) - self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, self.token) - - def test_get_200_returns_response_body(self): - @httmock.all_requests - def response_content(_url_unused, request): - headers = { - "X-Consul-Index": 4, - "X-Consul-Knownleader": "true", - "X-Consul-Lastcontact": 0, - "Date": "Fri, 19 Dec 2014 20:44:28 GMT", - "Content-Length": 13, - "Content-Type": "application/json", - } - content = b'{"consul": []}' - return httmock.response(200, content, headers, None, 0, request) - - with httmock.HTTMock(response_content): - values = self.endpoint._get([str(uuid.uuid4())]) - self.assertEqual(values, {"consul": []}) - - def test_get_404_returns_empty_list(self): - @httmock.all_requests - def response_content(_url_unused, request): - headers = {"content-length": 0, "content-type": "text/plain; charset=utf-8"} - return httmock.response(404, None, headers, None, 0, request) - - with httmock.HTTMock(response_content): - values = self.endpoint._get([str(uuid.uuid4())]) - self.assertEqual(values, []) - - -class EndpointGetListTests(unittest.TestCase): - def setUp(self): - self.adapter = adapters.Request() - self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) - self.dc = str(uuid.uuid4()) - self.token = str(uuid.uuid4()) - self.endpoint = base.Endpoint(self.base_uri, self.adapter, self.dc, self.token) - - def test_get_list_200_returns_response_body(self): - @httmock.all_requests - def response_content(_url_unused, request): - headers = { - "X-Consul-Index": 4, - "X-Consul-Knownleader": "true", - "X-Consul-Lastcontact": 0, - "Date": "Fri, 19 Dec 2014 20:44:28 GMT", - "Content-Length": 13, - "Content-Type": "application/json", - } - content = b'{"consul": []}' - return httmock.response(200, content, headers, None, 0, request) - - with httmock.HTTMock(response_content): - values = self.endpoint._get_list([str(uuid.uuid4())]) - self.assertEqual(values, [{"consul": []}]) - - def test_get_list_404_returns_empty_list(self): - @httmock.all_requests - def response_content(_url_unused, request): - headers = {"content-length": 0, "content-type": "text/plain; charset=utf-8"} - return httmock.response(404, None, headers, None, 0, request) - - with httmock.HTTMock(response_content): - values = self.endpoint._get_list([str(uuid.uuid4())]) - self.assertEqual(values, []) diff --git a/tests/services_discovery_consulate/test_base_model.py b/tests/services_discovery_consulate/test_base_model.py deleted file mode 100644 index 6fd98e2..0000000 --- a/tests/services_discovery_consulate/test_base_model.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Tests for the Base Model""" -import unittest -import uuid - -from pyms.services_discovery.consulate.models import base - - -class TestModel(base.Model): - """Model to perform tests against""" - - __slots__ = ["id", "serial", "name", "value"] - __attributes__ = { - "id": { - "key": "ID", - "type": uuid.UUID, - "cast_from": str, - "cast_to": str, - }, - "serial": { - "key": "Serial", - "type": int, - "default": 0, - "required": True, - "validator": lambda v, _m: v >= 0, - }, - "name": {"key": "Name", "type": str, "required": True}, - "value": {"type": str}, - "type": {"key": "Type", "type": str, "enum": {"client", "server"}}, - } - - -class TestCase(unittest.TestCase): - def test_happy_case_with_defaults(self): - kwargs = {"id": uuid.uuid4(), "name": str(uuid.uuid4())} - model = TestModel(**kwargs) - for key, value in kwargs.items(): - self.assertEqual(getattr(model, key), value) - self.assertEqual(model.serial, 0) - - def test_happy_case_with_all_values(self): - kwargs = {"id": uuid.uuid4(), "serial": 1, "name": str(uuid.uuid4()), "value": str(uuid.uuid4())} - model = TestModel(**kwargs) - for key, value in kwargs.items(): - self.assertEqual(getattr(model, key), value) - - def test_cast_from_str(self): - expectation = uuid.uuid4() - kwargs = {"id": str(expectation), "name": str(uuid.uuid4())} - model = TestModel(**kwargs) - self.assertEqual(model.id, expectation) - - def test_validator_failure(self): - kwargs = {"id": uuid.uuid4(), "name": str(uuid.uuid4()), "serial": -1} - with self.assertRaises(ValueError): - TestModel(**kwargs) - - def test_type_failure(self): - kwargs = {"id": True, "name": str(uuid.uuid4())} - with self.assertRaises(TypeError): - TestModel(**kwargs) - - def test_missing_requirement(self): - with self.assertRaises(ValueError): - TestModel() - - def test_invalid_attribute(self): - kwargs = {"name": str(uuid.uuid4()), "foo": "bar"} - with self.assertRaises(AttributeError): - TestModel(**kwargs) - - def test_invalid_attribute_assignment(self): - kwargs = {"name": str(uuid.uuid4())} - model = TestModel(**kwargs) - with self.assertRaises(AttributeError): - model.foo = "bar" - - def test_invalid_enum_assignment(self): - kwargs = {"name": str(uuid.uuid4()), "type": "invalid"} - with self.assertRaises(ValueError): - TestModel(**kwargs) - - def test_cast_to_dict(self): - kwargs = { - "id": uuid.uuid4(), - "serial": 1, - "name": str(uuid.uuid4()), - "value": str(uuid.uuid4()), - "type": "client", - } - expectation = { - "ID": str(kwargs["id"]), - "Serial": kwargs["serial"], - "Name": kwargs["name"], - "value": kwargs["value"], - "Type": kwargs["type"], - } - model = TestModel(**kwargs) - self.assertDictEqual(dict(model), expectation) - - def test_cast_to_dict_only_requirements(self): - kwargs = {"serial": 1, "name": str(uuid.uuid4())} - expectation = {"Serial": kwargs["serial"], "Name": kwargs["name"]} - model = TestModel(**kwargs) - self.assertDictEqual(dict(model), expectation) diff --git a/tests/services_discovery_consulate/test_catalog.py b/tests/services_discovery_consulate/test_catalog.py deleted file mode 100644 index c7991e9..0000000 --- a/tests/services_discovery_consulate/test_catalog.py +++ /dev/null @@ -1,9 +0,0 @@ -from . import base - - -class TestCatalog(base.TestCase): - def test_catalog_registration(self): - self.consul.catalog.register("test-service", address="10.0.0.1") - self.assertIn("test-service", [n["Node"] for n in self.consul.catalog.nodes()]) - self.consul.catalog.deregister("test-service") - self.assertNotIn("test-service", [n["Node"] for n in self.consul.catalog.nodes()]) diff --git a/tests/services_discovery_consulate/test_coordinate.py b/tests/services_discovery_consulate/test_coordinate.py deleted file mode 100644 index 3089d86..0000000 --- a/tests/services_discovery_consulate/test_coordinate.py +++ /dev/null @@ -1,7 +0,0 @@ -from . import base - - -class TestCoordinate(base.TestCase): - def test_coordinate(self): - coordinates = self.consul.coordinate.nodes() - self.assertIsInstance(coordinates, list) diff --git a/tests/services_discovery_consulate/test_event.py b/tests/services_discovery_consulate/test_event.py deleted file mode 100644 index 8daa3ae..0000000 --- a/tests/services_discovery_consulate/test_event.py +++ /dev/null @@ -1,12 +0,0 @@ -import uuid - -from . import base - - -class TestEvent(base.TestCase): - def test_fire(self): - event_name = "test-event-%s" % str(uuid.uuid4())[0:8] - response = self.consul.event.fire(event_name) - events = self.consul.event.list(event_name) - self.assertEqual(event_name, events.get("Name")) - self.assertEqual(response, events.get("ID")) diff --git a/tests/services_discovery_consulate/test_kv.py b/tests/services_discovery_consulate/test_kv.py deleted file mode 100644 index a9750ce..0000000 --- a/tests/services_discovery_consulate/test_kv.py +++ /dev/null @@ -1,255 +0,0 @@ -# -*- coding: utf-8 -*- -import json -import unittest -import uuid - -import httmock - -from pyms.services_discovery.consulate import adapters, api - -from . import base - -SCHEME = "http" -VERSION = "v1" - -ALL_DATA = ( - b'[{"CreateIndex":643,"ModifyIndex":643,"LockIndex":0,"Key":"bar",' - b'"Flags":0,"Value":"YmF6"},{"CreateIndex":669,"ModifyIndex":669,"' - b'LockIndex":0,"Key":"baz","Flags":0,"Value":"cXV4"},{"CreateIndex' - b'":666,"ModifyIndex":666,"LockIndex":0,"Key":"corgie","Flags":128' - b',"Value":"ZG9n"},{"CreateIndex":642,"ModifyIndex":642,"LockIndex' - b'":0,"Key":"foo","Flags":0,"Value":"YmFy"},{"CreateIndex":644,"Mo' - b'difyIndex":644,"LockIndex":0,"Key":"quz","Flags":0,"Value":"dHJ1' - b'ZQ=="}]' -) - -ALL_ITEMS = [ - {"CreateIndex": 643, "Flags": 0, "Key": "bar", "LockIndex": 0, "ModifyIndex": 643, "Value": "baz"}, - {"CreateIndex": 669, "Flags": 0, "Key": "baz", "LockIndex": 0, "ModifyIndex": 669, "Value": "qux"}, - {"CreateIndex": 666, "Flags": 128, "Key": "corgie", "LockIndex": 0, "ModifyIndex": 666, "Value": "dog"}, - {"CreateIndex": 642, "Flags": 0, "Key": "foo", "LockIndex": 0, "ModifyIndex": 642, "Value": "bar"}, - {"CreateIndex": 644, "Flags": 0, "Key": "quz", "LockIndex": 0, "ModifyIndex": 644, "Value": "true"}, -] - - -@httmock.all_requests -def kv_all_records_content(_url_unused, request): - return httmock.response( - 200, - ALL_DATA, - { - "X-Consul-Index": 4, - "X-Consul-Knownleader": "true", - "X-Consul-Lastcontact": 0, - "Date": "Fri, 19 Dec 2014 20:44:28 GMT", - "Content-Length": len(ALL_DATA), - "Content-Type": "application/json", - }, - None, - 0, - request, - ) - - -class KVTests(unittest.TestCase): - def setUp(self): - self.adapter = adapters.Request() - self.base_uri = "{0}://localhost:8500/{1}".format(SCHEME, VERSION) - self.dc = str(uuid.uuid4()) - self.token = str(uuid.uuid4()) - self.kv = api.KV(self.base_uri, self.adapter, self.dc, self.token) - - def test_contains_evaluates_true(self): - @httmock.all_requests - def response_content(_url_unused, request): - return httmock.response(200, None, {}, None, 0, request) - - with httmock.HTTMock(response_content): - self.assertIn("foo", self.kv) - - def test_contains_evaluates_false(self): - @httmock.all_requests - def response_content(_url_unused, request): - return httmock.response(404, None, {}, None, 0, request) - - with httmock.HTTMock(response_content): - self.assertNotIn("foo", self.kv) - - def test_get_all_items(self): - with httmock.HTTMock(kv_all_records_content): - for index, row in enumerate(self.kv._get_all_items()): - self.assertDictEqual(row, ALL_ITEMS[index]) - - def test_items(self): - with httmock.HTTMock(kv_all_records_content): - for index, row in enumerate(self.kv.items()): - value = {ALL_ITEMS[index]["Key"]: ALL_ITEMS[index]["Value"]} - self.assertDictEqual(row, value) - - def test_iter(self): - with httmock.HTTMock(kv_all_records_content): - for index, row in enumerate(self.kv): - self.assertEqual(row, ALL_ITEMS[index]["Key"]) - - def test_iteritems(self): - with httmock.HTTMock(kv_all_records_content): - for index, row in enumerate(self.kv.iteritems()): - value = (ALL_ITEMS[index]["Key"], ALL_ITEMS[index]["Value"]) - self.assertEqual(row, value) - - def test_keys(self): - expectation = [item["Key"] for item in ALL_ITEMS] - with httmock.HTTMock(kv_all_records_content): - self.assertEqual(self.kv.keys(), expectation) - - def test_len(self): - with httmock.HTTMock(kv_all_records_content): - self.assertEqual(len(self.kv), len(ALL_ITEMS)) - - def test_values(self): - with httmock.HTTMock(kv_all_records_content): - for index, row in enumerate(self.kv.values()): - self.assertEqual(row, ALL_ITEMS[index]["Value"]) - - -class TestKVGetWithNoKey(base.TestCase): - @base.generate_key - def test_get_is_none(self, key): - self.assertIsNone(self.consul.kv.get(key)) - - @base.generate_key - def test_get_item_raises_key_error(self, key): - self.assertRaises(KeyError, self.consul.kv.__getitem__, key) - - -class TestKVSet(base.TestCase): - @base.generate_key - def test_set_item_del_item(self, key): - self.consul.kv[key] = "foo" - del self.consul.kv[key] - self.assertNotIn(key, self.consul.kv) - - @base.generate_key - def test_set_item_get_item_bool_value(self, key): - self.consul.kv[key] = True - self.assertTrue(self.consul.kv[key]) - - @base.generate_key - def test_set_path_with_value(self, key): - path = "path/{0}/".format(key) - self.consul.kv.set(path, "bar") - self.assertEqual("bar", self.consul.kv[path[:-1]]) - - @base.generate_key - def test_set_item_get_item_int_value(self, key): - self.consul.kv[key] = 128 - self.assertEqual(self.consul.kv[key], "128") - - @base.generate_key - def test_set_item_get_item_str_value_key(self, key): - self.consul.kv[key] = b"foo" - self.assertEqual(self.consul.kv[key], "foo") - - @base.generate_key - def test_set_item_get_item_str_value_raw(self, key): - self.consul.kv[key] = "foo" - self.assertEqual(self.consul.kv.get(key, raw=True), "foo") - - @base.generate_key - def test_set_get_bool_value(self, key): - self.consul.kv.set(key, True) - self.assertTrue(self.consul.kv.get(key)) - - @base.generate_key - def test_set_get_item_value(self, key): - self.consul.kv.set(key, 128) - self.assertEqual(self.consul.kv.get(key), "128") - - @base.generate_key - def test_set_item_get_item_str_value(self, key): - self.consul.kv.set(key, "foo") - self.assertEqual(self.consul.kv.get(key), "foo") - - @base.generate_key - def test_set_item_get_record(self, key): - self.consul.kv.set_record(key, 12, "record") - record = self.consul.kv.get_record(key) - self.assertEqual("record", record["Value"]) - self.assertEqual(12, record["Flags"]) - self.assertIsInstance(record, dict) - - @base.generate_key - def test_get_record_fail(self, key): - self.assertEqual(self.consul.kv.get_record(key), None) - - @base.generate_key - def test_set_record_no_replace_get_item_str_value(self, key): - self.consul.kv.set(key, "foo") - self.consul.kv.set_record(key, 0, "foo", False) - self.assertEqual(self.consul.kv.get(key), "foo") - - @base.generate_key - def test_set_record_same_value_get_item_str_value(self, key): - self.consul.kv.set(key, "foo") - self.consul.kv.set_record(key, 0, "foo", True) - self.assertEqual(self.consul.kv.get(key), "foo") - - @base.generate_key - def test_set_item_get_item_dict_value(self, key): - value = {"foo": "bar"} - expectation = json.dumps(value) - self.consul.kv.set(key, value) - self.assertEqual(self.consul.kv.get(key), expectation) - - @base.generate_key - def test_set_item_get_item_unicode_value(self, key): - self.consul.kv.set(key, "I like to ✈") - response = self.consul.kv.get(key) - self.assertEqual(response, "I like to ✈") - - @base.generate_key - def test_set_item_in_records(self, key): - self.consul.kv.set(key, "zomg") - expectation = (key, 0, "zomg") - self.assertIn(expectation, self.consul.kv.records()) - - @base.generate_key - def test_set_binary_value(self, key): - value = uuid.uuid4().bytes - self.consul.kv.set(key, value) - expectation = (key, 0, value) - self.assertIn(expectation, self.consul.kv.records()) - - -class TestKVLocking(base.TestCase): - @base.generate_key - def test_acquire_and_release_lock(self, key): - lock_key = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create(key, behavior="delete", ttl="60s") - self.assertTrue(self.consul.kv.acquire_lock(lock_key, session_id)) - self.assertTrue(self.consul.kv.release_lock(lock_key, session_id)) - self.consul.session.destroy(session_id) - - @base.generate_key - def test_acquire_and_release_lock(self, key): - lock_key = str(uuid.uuid4())[0:8] - sid = self.consul.session.create(key, behavior="delete", ttl="60s") - sid2 = self.consul.session.create(key + "2", behavior="delete", ttl="60s") - self.assertTrue(self.consul.kv.acquire_lock(lock_key, sid)) - self.assertFalse(self.consul.kv.acquire_lock(lock_key, sid2)) - self.assertTrue(self.consul.kv.release_lock(lock_key, sid)) - self.consul.session.destroy(sid) - self.consul.session.destroy(sid2) - - @base.generate_key - def test_acquire_and_release_lock_with_value(self, key): - lock_key = str(uuid.uuid4())[0:8] - lock_value = str(uuid.uuid4()) - sid = self.consul.session.create(key, behavior="delete", ttl="60s") - sid2 = self.consul.session.create(key + "2", behavior="delete", ttl="60s") - self.assertTrue(self.consul.kv.acquire_lock(lock_key, sid, lock_value)) - self.assertEqual(self.consul.kv.get(lock_key), lock_value) - self.assertFalse(self.consul.kv.acquire_lock(lock_key, sid2)) - self.assertTrue(self.consul.kv.release_lock(lock_key, sid)) - self.consul.session.destroy(sid) - self.consul.session.destroy(sid2) diff --git a/tests/services_discovery_consulate/test_lock.py b/tests/services_discovery_consulate/test_lock.py deleted file mode 100644 index 0cf3a6f..0000000 --- a/tests/services_discovery_consulate/test_lock.py +++ /dev/null @@ -1,10 +0,0 @@ -import uuid - -from . import base - - -class TestLock(base.TestCase): - def test_lock_as_context_manager(self): - value = str(uuid.uuid4()) - with self.consul.lock.acquire(value=value): - self.assertEqual(self.consul.kv.get(self.consul.lock.key), value) diff --git a/tests/services_discovery_consulate/test_session.py b/tests/services_discovery_consulate/test_session.py deleted file mode 100644 index c05f0f3..0000000 --- a/tests/services_discovery_consulate/test_session.py +++ /dev/null @@ -1,38 +0,0 @@ -import uuid - -from . import base - - -class TestSession(base.TestCase): - def setUp(self): - super(TestSession, self).setUp() - self.sessions = list() - - def tearDown(self): - for session in self.sessions: - self.consul.session.destroy(session) - - def test_session_create(self): - name = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create(name, behavior="delete", ttl="60s") - self.sessions.append(session_id) - self.assertIsNotNone(session_id) - - def test_session_destroy(self): - name = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create(name, behavior="delete", ttl="60s") - self.consul.session.destroy(session_id) - self.assertNotIn(session_id, [s.get("ID") for s in self.consul.session.list()]) - - def test_session_info(self): - name = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create(name, behavior="delete", ttl="60s") - result = self.consul.session.info(session_id) - self.assertEqual(session_id, result.get("ID")) - self.consul.session.destroy(session_id) - - def test_session_renew(self): - name = str(uuid.uuid4())[0:8] - session_id = self.consul.session.create(name, behavior="delete", ttl="60s") - self.sessions.append(session_id) - self.assertTrue(self.consul.session.renew(session_id)) diff --git a/tests/services_discovery_consulate/test_utils.py b/tests/services_discovery_consulate/test_utils.py deleted file mode 100644 index 0ac7b2b..0000000 --- a/tests/services_discovery_consulate/test_utils.py +++ /dev/null @@ -1,70 +0,0 @@ -# coding=utf-8 -import unittest - -from pyms.services_discovery.consulate import exceptions, utils - - -class MaybeEncodeTestCase(unittest.TestCase): - def test_str_test(self): - self.assertEqual(utils.maybe_encode("foo"), b"foo") - - def test_byte_test(self): - self.assertEqual(utils.maybe_encode(b"bar"), b"bar") - - -class Response(object): - def __init__(self, status_code=200, body=b"content"): - self.status_code = status_code - self.body = body - - -class ResponseOkTestCase(unittest.TestCase): - def test_200(self): - self.assertTrue(utils.response_ok(Response(200, b"ok"))) - - def test_400(self): - with self.assertRaises(exceptions.ClientError): - utils.response_ok(Response(400, b"Bad request")) - - def test_401(self): - with self.assertRaises(exceptions.ACLDisabled): - utils.response_ok(Response(401, b"What ACL?")) - - def test_403(self): - with self.assertRaises(exceptions.Forbidden): - utils.response_ok(Response(403, b"No")) - - def test_404_not_raising(self): - self.assertFalse(utils.response_ok(Response(404, b"not found"))) - - def test_404_raising(self): - with self.assertRaises(exceptions.NotFound): - utils.response_ok(Response(404, b"Not Found"), True) - - def test_500(self): - with self.assertRaises(exceptions.ServerError): - utils.response_ok(Response(500, b"Opps")) - - -class ValidateGoDurationTestCase(unittest.TestCase): - def test_valid_values(self): - for value in {"5µs", "300ms", "-1.5h", "2h45m", "5m", "30s"}: - print("Testing {}".format(value)) - self.assertTrue(utils.validate_go_interval(value)) - - def test_invalid_values(self): - for value in {"100", "1 year", "5M", "30S"}: - print("Testing {}".format(value)) - self.assertFalse(utils.validate_go_interval(value)) - - -class ValidateURLTestCase(unittest.TestCase): - def test_valid_values(self): - for value in {"https://foo", "http://localhost/bar"}: - print("Testing {}".format(value)) - self.assertTrue(utils.validate_url(value)) - - def test_invalid_values(self): - for value in {"localhost", "a"}: - print("Testing {}".format(value)) - self.assertFalse(utils.validate_url(value)) From acbbf368a80f57bd891aac6258c42f06dae512d8 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Thu, 12 Nov 2020 17:34:49 +0100 Subject: [PATCH 24/27] fix: removed unused code --- pyms/flask/app/create_app.py | 2 -- pyms/flask/services/service_discovery.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/pyms/flask/app/create_app.py b/pyms/flask/app/create_app.py index 2ab29a9..272545e 100644 --- a/pyms/flask/app/create_app.py +++ b/pyms/flask/app/create_app.py @@ -59,8 +59,6 @@ def init_services(self) -> None: if service_name not in self.services or not getattr(self, service_name, False): self.services.append(service_name) setattr(self, service_name, service) - # if getattr(service, "init_action"): - # service.init_action(self) def init_services_actions(self): for service_name in self.services: diff --git a/pyms/flask/services/service_discovery.py b/pyms/flask/services/service_discovery.py index 9339170..68bb64f 100644 --- a/pyms/flask/services/service_discovery.py +++ b/pyms/flask/services/service_discovery.py @@ -14,8 +14,6 @@ CONSUL_SERVICE_DISCOVERY = "consul" -EUREKA_SERVICE_DISCOVERY = "eureka" - DEFAULT_SERVICE_DISCOVERY = CONSUL_SERVICE_DISCOVERY From 91c8e8f4923e725d0fc491e8bf9ca869f5165e4e Mon Sep 17 00:00:00 2001 From: avara1986 Date: Thu, 12 Nov 2020 17:37:38 +0100 Subject: [PATCH 25/27] docs: updated docstrings --- tests/test_service_discovery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_service_discovery.py b/tests/test_service_discovery.py index d42e4da..df18671 100644 --- a/tests/test_service_discovery.py +++ b/tests/test_service_discovery.py @@ -11,7 +11,7 @@ class ServiceDiscoveryTests(unittest.TestCase): - """Test common rest operations wrapper.""" + """Test service discovery services""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -22,6 +22,7 @@ def setUp(self): @requests_mock.Mocker() def test_init(self, mock_request): + """ test if the initializacion of service discovery call to the Service Discovery server to autoregister""" url = "http://localhost:8500/v1/agent/check/register" mock_request.put(url) From 021b261970228fcd3b7b7237e5404c94f2e4321a Mon Sep 17 00:00:00 2001 From: avara1986 Date: Thu, 12 Nov 2020 22:07:59 +0100 Subject: [PATCH 26/27] fix: updated example, added init in ServiceDiscoveryBase --- .../microservice_service_discovery/service.py | 19 +++++++++++++++---- pyms/flask/services/service_discovery.py | 11 +++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/examples/microservice_service_discovery/service.py b/examples/microservice_service_discovery/service.py index 7327021..3d9f995 100644 --- a/examples/microservice_service_discovery/service.py +++ b/examples/microservice_service_discovery/service.py @@ -1,4 +1,5 @@ import json +import uuid import requests @@ -6,15 +7,25 @@ class ServiceDiscoveryConsulBasic(ServiceDiscoveryBase): - def register_service(self, id_app, host, port, healtcheck_url, app_name): + id_app = str(uuid.uuid1()) + + def __init__(self, config): + super().__init__(config) + self.host = config.host + self.port = config.port + + def register_service(self, *args, **kwargs): + app_name = kwargs["app_name"] + healtcheck_url = kwargs["healtcheck_url"] + interval = kwargs["interval"] headers = {"Content-Type": "application/json; charset=utf-8"} data = { - "id": app_name + "-" + id_app, + "id": app_name + "-" + self.id_app, "name": app_name, - "check": {"name": "ping check", "http": healtcheck_url, "interval": "30s", "status": "passing"}, + "check": {"name": "ping check", "http": healtcheck_url, "interval": interval, "status": "passing"}, } response = requests.put( - "http://{host}:{port}/v1/agent/service/register".format(host=host, port=port), + "http://{host}:{port}/v1/agent/service/register".format(host=self.host, port=self.port), data=json.dumps(data), headers=headers, ) diff --git a/pyms/flask/services/service_discovery.py b/pyms/flask/services/service_discovery.py index 68bb64f..b546bd1 100644 --- a/pyms/flask/services/service_discovery.py +++ b/pyms/flask/services/service_discovery.py @@ -1,5 +1,4 @@ import logging -import uuid try: import consulate @@ -20,12 +19,16 @@ class ServiceDiscoveryBase: client = None + def __init__(self, config): + pass + def register_service(self, *args, **kwargs): pass class ServiceDiscoveryConsul(ServiceDiscoveryBase): def __init__(self, config): + super().__init__(config) self.client = consulate.Consul( host=config.host, port=config.port, token=config.token, scheme=config.scheme, adapter=config.adapter ) @@ -37,7 +40,6 @@ def register_service(self, *args, **kwargs): class Service(DriverService): - id_app = str(uuid.uuid1()) config_resource = "service_discovery" default_values = { "service": DEFAULT_SERVICE_DISCOVERY, @@ -45,13 +47,14 @@ class Service(DriverService): "scheme": "http", "port": 8500, "healtcheck_url": "http://127.0.0.1.nip.io:5000/healthcheck", + "interval": "10s", "autoregister": False, } def init_action(self, microservice_instance): if self.autoregister: app_name = microservice_instance.application.config["APP_NAME"] - self._client.register_service(healtcheck_url=self.healtcheck_url, app_name=app_name) + self._client.register_service(healtcheck_url=self.healtcheck_url, app_name=app_name, interval=self.interval) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -64,7 +67,7 @@ def get_client(self) -> ServiceDiscoveryBase: else: service_paths = self.service.split(".") package = ".".join(service_paths[:-1]) - client = import_from(package, service_paths[-1])() + client = import_from(package, service_paths[-1])(self) logger.debug("Init %s as service discovery", client) return client From 5ebfc242f70c802750cc359e637cc964eb291c01 Mon Sep 17 00:00:00 2001 From: avara1986 Date: Wed, 18 Nov 2020 19:29:36 +0100 Subject: [PATCH 27/27] tests: updated service discovery --- .../config-tests-service-discovery-consul.yml | 9 ++++ tests/config-tests-service-discovery.yml | 2 +- tests/test_service_discovery.py | 46 ++++++++++++++----- 3 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 tests/config-tests-service-discovery-consul.yml diff --git a/tests/config-tests-service-discovery-consul.yml b/tests/config-tests-service-discovery-consul.yml new file mode 100644 index 0000000..3d79d58 --- /dev/null +++ b/tests/config-tests-service-discovery-consul.yml @@ -0,0 +1,9 @@ +pyms: + services: + service_discovery: + service: "consul" + autoregister: true + config: + DEBUG: true + TESTING: true + APP_NAME: "Python Microservice Service Discovery" \ No newline at end of file diff --git a/tests/config-tests-service-discovery.yml b/tests/config-tests-service-discovery.yml index 3d79d58..ad6dea9 100644 --- a/tests/config-tests-service-discovery.yml +++ b/tests/config-tests-service-discovery.yml @@ -1,7 +1,7 @@ pyms: services: service_discovery: - service: "consul" + service: "tests.test_service_discovery.MockServiceDiscovery" autoregister: true config: DEBUG: true diff --git a/tests/test_service_discovery.py b/tests/test_service_discovery.py index df18671..d584c79 100644 --- a/tests/test_service_discovery.py +++ b/tests/test_service_discovery.py @@ -2,35 +2,59 @@ """ import os import unittest - -import requests_mock +from unittest.mock import patch from pyms.constants import CONFIGMAP_FILE_ENVIRONMENT from pyms.flask.app import Microservice -from pyms.flask.services.service_discovery import ServiceDiscoveryConsul +from pyms.flask.services.service_discovery import ServiceDiscoveryBase, ServiceDiscoveryConsul -class ServiceDiscoveryTests(unittest.TestCase): +class MockServiceDiscovery(ServiceDiscoveryBase): + pass + + +class ServiceDiscoveryConsulTests(unittest.TestCase): """Test service discovery services""" BASE_DIR = os.path.dirname(os.path.abspath(__file__)) def setUp(self): - os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(self.BASE_DIR, "config-tests-service-discovery.yml") + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join( + self.BASE_DIR, "config-tests-service-discovery-consul.yml" + ) ms = Microservice(path=__file__) self.ms = ms - @requests_mock.Mocker() - def test_init(self, mock_request): + @patch.object(ServiceDiscoveryConsul, "register_service", return_value=None) + def test_init(self, mock_consul): """ test if the initializacion of service discovery call to the Service Discovery server to autoregister""" - url = "http://localhost:8500/v1/agent/check/register" - - mock_request.put(url) self.ms.create_app() - self.assertTrue(True) + mock_consul.assert_called_once_with( + app_name="Python Microservice Service Discovery", + healtcheck_url="http://127.0.0.1.nip.io:5000/healthcheck", + interval="10s", + ) def test_get_client(self): client = self.ms.service_discovery.get_client() self.assertTrue(isinstance(client, ServiceDiscoveryConsul)) + + +class ServiceDiscoveryTests(unittest.TestCase): + """Test service discovery services""" + + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + def setUp(self): + os.environ[CONFIGMAP_FILE_ENVIRONMENT] = os.path.join(self.BASE_DIR, "config-tests-service-discovery.yml") + ms = Microservice(path=__file__) + ms.reload_conf() + self.ms = ms + + def test_get_client(self): + self.ms.create_app() + client = self.ms.service_discovery.get_client() + + self.assertTrue(isinstance(client, MockServiceDiscovery))