From af4ad3bf330a60fc6ee2dcab6a2b8953cd300a20 Mon Sep 17 00:00:00 2001 From: Fabio Caccamo Date: Wed, 1 Feb 2023 13:53:50 +0100 Subject: [PATCH 1/6] Add `mypy` to CI. --- .github/workflows/test-package.yml | 8 ++++++++ requirements-test.txt | 1 + setup.cfg | 5 +++++ tox.ini | 1 + 4 files changed, 15 insertions(+) diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index 443ec916..416ed6c3 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -33,6 +33,14 @@ jobs: pip install -r requirements.txt pip install -r requirements-test.txt + - name: Run pre-commit + run: | + pre-commit run -a + + - name: Run mypy + run: | + mypy --install-types --non-interactive + - name: Run tests run: | coverage run --append --source=benedict -m unittest diff --git a/requirements-test.txt b/requirements-test.txt index d8fac991..9d7f9702 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ coverage == 7.1.* orjson == 3.8.* +mypy == 0.991 pre-commit == 3.0.* python-decouple == 3.7 tox == 4.4.* diff --git a/setup.cfg b/setup.cfg index 1a3d4b89..2ab0af25 100644 --- a/setup.cfg +++ b/setup.cfg @@ -123,3 +123,8 @@ ignore = E501, W503 max-line-length = 89 max-complexity = 10 select = B,C,E,F,W,T4,B9 + +[mypy] +files = benedict +ignore_missing_imports = True +strict = True diff --git a/tox.ini b/tox.ini index ac7d89b6..d9c36c0e 100644 --- a/tox.ini +++ b/tox.ini @@ -17,5 +17,6 @@ deps = commands = pre-commit run -a + mypy --install-types --non-interactive coverage run --append -m unittest discover coverage report --show-missing --ignore-errors From f82d629868f6f6fec8e55a11a4aaaa4efa5ebc9a Mon Sep 17 00:00:00 2001 From: Fabio Caccamo Date: Wed, 1 Feb 2023 13:54:35 +0100 Subject: [PATCH 2/6] Run tests with `unittest discover`. --- .github/workflows/test-package.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index 416ed6c3..06645375 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -43,7 +43,8 @@ jobs: - name: Run tests run: | - coverage run --append --source=benedict -m unittest + coverage run --append -m unittest discover + coverage report --show-missing coverage xml -o ./coverage.xml - name: Upload coverage to Codecov From d29e97438395414f5c7388cc7a067919352c384c Mon Sep 17 00:00:00 2001 From: Fabio Caccamo Date: Wed, 1 Feb 2023 15:46:52 +0100 Subject: [PATCH 3/6] Improve tests. --- tests/__init__.py | 4 ---- tests/aws.py | 20 +++++++++++++++++++ tests/dicts/base/test_base_dict.py | 11 ----------- tests/dicts/io/test_io_dict_xls.py | 19 +++++++----------- tests/dicts/io/test_io_util.py | 16 +++++---------- tests/dicts/test_benedict.py | 25 ++---------------------- tests/github/test_issue_0020.py | 3 --- tests/github/test_issue_0198.py | 31 ++++++++++-------------------- 8 files changed, 44 insertions(+), 85 deletions(-) create mode 100644 tests/aws.py diff --git a/tests/__init__.py b/tests/__init__.py index fba0404a..e69de29b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +0,0 @@ -import unittest - -if __name__ == "__main__": - unittest.main() diff --git a/tests/aws.py b/tests/aws.py new file mode 100644 index 00000000..d0c93ee9 --- /dev/null +++ b/tests/aws.py @@ -0,0 +1,20 @@ +from decouple import config + + +def get_aws_credentials(): + aws_access_key_id = config("AWS_ACCESS_KEY_ID", default=None) + aws_secret_access_key = config("AWS_SECRET_ACCESS_KEY", default=None) + return aws_access_key_id, aws_secret_access_key + + +def get_aws_credentials_options(): + aws_access_key_id, aws_secret_access_key = get_aws_credentials() + return { + "aws_access_key_id": aws_access_key_id, + "aws_secret_access_key": aws_secret_access_key, + } + + +def has_aws_credentials(): + aws_access_key_id, aws_secret_access_key = get_aws_credentials() + return all([aws_access_key_id, aws_secret_access_key]) diff --git a/tests/dicts/base/test_base_dict.py b/tests/dicts/base/test_base_dict.py index 5369b332..79d7c9a4 100644 --- a/tests/dicts/base/test_base_dict.py +++ b/tests/dicts/base/test_base_dict.py @@ -224,17 +224,6 @@ def test__str__with_pointer(self): self.assertEqual(str(d), str(b)) self.assertEqual(b, b.dict()) - @unittest.skipIf(sys.version_info[0] > 2, "No unicode in Python 3") - def test__unicode__(self): - d = BaseDict() - d["name"] = "pythòn-bènèdìçt" - # print(unicode(d)) - - @unittest.skipIf(sys.version_info[0] > 2, "No unicode in Python > 2") - def test__unicode__with_pointer(self): - d = BaseDict({"name": "pythòn-bènèdìçt"}) - # print(unicode(d)) - def test_clear(self): d = { "a": 1, diff --git a/tests/dicts/io/test_io_dict_xls.py b/tests/dicts/io/test_io_dict_xls.py index 8955b51a..c2332154 100644 --- a/tests/dicts/io/test_io_dict_xls.py +++ b/tests/dicts/io/test_io_dict_xls.py @@ -1,8 +1,8 @@ -from decouple import config +import unittest from benedict.dicts.io import IODict - -from .test_io_dict import io_dict_test_case +from tests.aws import get_aws_credentials_options, has_aws_credentials +from tests.dicts.io.test_io_dict import io_dict_test_case class io_dict_xls_test_case(io_dict_test_case): @@ -119,16 +119,11 @@ def test_from_xls_with_valid_url_valid_content(self): self.assertTrue(isinstance(d, dict)) self.assertEqual(d, expected_dict) + @unittest.skipUnless( + has_aws_credentials(), "Skip because aws credentials are not set." + ) def test_from_xls_with_valid_s3_url_valid_content(self): - aws_access_key_id = config("AWS_ACCESS_KEY_ID", default=None) - aws_secret_access_key = config("AWS_SECRET_ACCESS_KEY", default=None) - if not all([aws_access_key_id, aws_secret_access_key]): - # don't use s3 on GH CI - return - s3_options = { - "aws_access_key_id": aws_access_key_id, - "aws_secret_access_key": aws_secret_access_key, - } + s3_options = get_aws_credentials_options() expected_dict = { "values": [ { diff --git a/tests/dicts/io/test_io_util.py b/tests/dicts/io/test_io_util.py index d8001051..e1571d01 100644 --- a/tests/dicts/io/test_io_util.py +++ b/tests/dicts/io/test_io_util.py @@ -1,8 +1,7 @@ import unittest -from decouple import config - from benedict.dicts.io import io_util +from tests.aws import get_aws_credentials_options, has_aws_credentials class io_util_test_case(unittest.TestCase): @@ -87,16 +86,11 @@ def test_write_file_to_s3(self): # io_util.write_file_to_s3("s3://test-bucket/my-file.txt", "ok", anon=True) pass + @unittest.skipUnless( + has_aws_credentials(), "Skip because aws credentials are not set." + ) def test_write_and_read_content_s3(self): - aws_access_key_id = config("AWS_ACCESS_KEY_ID", default=None) - aws_secret_access_key = config("AWS_SECRET_ACCESS_KEY", default=None) - if not all([aws_access_key_id, aws_secret_access_key]): - # skip s3 on GH CI - return - s3_options = { - "aws_access_key_id": aws_access_key_id, - "aws_secret_access_key": aws_secret_access_key, - } + s3_options = get_aws_credentials_options() filepath = "s3://python-benedict/test-io.txt" io_util.write_content_to_s3(filepath, "ok", s3_options=s3_options) content = io_util.read_content_from_s3(filepath, s3_options=s3_options) diff --git a/tests/dicts/test_benedict.py b/tests/dicts/test_benedict.py index 5ea37db1..4610fa91 100644 --- a/tests/dicts/test_benedict.py +++ b/tests/dicts/test_benedict.py @@ -370,7 +370,7 @@ def test_filter_with_parse(self): self.assertEqual(f, r) self.assertTrue(isinstance(f, benedict)) - def test_filter(self): + def test_find(self): d = { "a": 1, "b": 2, @@ -727,30 +727,9 @@ def test_from_yaml_with_keypath_separator_in_keys(self): with self.assertRaises(ValueError): # static method d = benedict.from_yaml(j) - self.assertTrue(isinstance(d, dict)) - self.assertEqual( - d, - { - "a": 1, - "b": { - "c": 3, - "d": 4, - }, - }, - ) + with self.assertRaises(ValueError): # constructor d = benedict(j, format="yaml") - self.assertTrue(isinstance(d, dict)) - self.assertEqual( - d, - { - "a": 1, - "b": { - "c": 3, - "d": 4, - }, - }, - ) r = { "192.168.0.1": { "test": "value", diff --git a/tests/github/test_issue_0020.py b/tests/github/test_issue_0020.py index 6de21669..d3d7389f 100644 --- a/tests/github/test_issue_0020.py +++ b/tests/github/test_issue_0020.py @@ -15,9 +15,6 @@ def __init__(self, val): def from_yaml(cls, loader, node): return cls(node.value) - def __repr__(self): - return f"GetAtt({self.val})" - yaml.add_constructor("!GetAtt", GetAtt.from_yaml) diff --git a/tests/github/test_issue_0198.py b/tests/github/test_issue_0198.py index 04a4a572..7f6e3c54 100644 --- a/tests/github/test_issue_0198.py +++ b/tests/github/test_issue_0198.py @@ -1,8 +1,7 @@ import unittest -from decouple import config - from benedict import benedict +from tests.aws import get_aws_credentials_options, has_aws_credentials class github_issue_0198_test_case(unittest.TestCase): @@ -14,33 +13,23 @@ class github_issue_0198_test_case(unittest.TestCase): - Run python -m unittest tests.github.test_issue_0198 """ + @unittest.skipUnless( + has_aws_credentials(), "Skip because aws credentials are not set." + ) def test_constructor_with_s3_url_and_s3_options_with_file_json(self): - aws_access_key_id = config("AWS_ACCESS_KEY_ID", default=None) - aws_secret_access_key = config("AWS_SECRET_ACCESS_KEY", default=None) - if not all([aws_access_key_id, aws_secret_access_key]): - # don't use s3 on GH CI - return - s3_options = { - "aws_access_key_id": aws_access_key_id, - "aws_secret_access_key": aws_secret_access_key, - } + s3_options = get_aws_credentials_options() d = benedict( "s3://python-benedict/valid-content.json", - s3_options=s3_options, + s3_options=get_aws_credentials_options(), ) expected_dict = {"a": 1, "b": 2, "c": 3, "x": 7, "y": 8, "z": 9} self.assertEqual(d, expected_dict) + @unittest.skipUnless( + has_aws_credentials(), "Skip because aws credentials are not set." + ) def test_constructor_with_s3_url_and_s3_options_with_file_xlsx(self): - aws_access_key_id = config("AWS_ACCESS_KEY_ID", default=None) - aws_secret_access_key = config("AWS_SECRET_ACCESS_KEY", default=None) - if not all([aws_access_key_id, aws_secret_access_key]): - # don't use s3 on GH CI - return - s3_options = { - "aws_access_key_id": aws_access_key_id, - "aws_secret_access_key": aws_secret_access_key, - } + s3_options = get_aws_credentials_options() d = benedict( "s3://python-benedict/valid-content.xlsx", s3_options=s3_options, From acb60ec1817857a01e7d9d05a4735e8349de543f Mon Sep 17 00:00:00 2001 From: Fabio Caccamo Date: Wed, 1 Feb 2023 15:48:35 +0100 Subject: [PATCH 4/6] Start adding type hints. --- benedict/core/clean.py | 4 +- benedict/core/clone.py | 3 +- benedict/core/dump.py | 4 +- benedict/core/filter.py | 6 ++- benedict/core/flatten.py | 10 ++--- benedict/core/invert.py | 8 ++-- benedict/core/items_sorted.py | 6 +-- benedict/core/keylists.py | 2 +- benedict/core/keypaths.py | 4 +- benedict/core/match.py | 2 +- benedict/core/merge.py | 8 ++-- benedict/core/move.py | 2 +- benedict/core/nest.py | 2 +- benedict/core/remove.py | 2 +- benedict/core/rename.py | 2 +- benedict/core/search.py | 13 +++++-- benedict/core/standardize.py | 4 +- benedict/core/swap.py | 2 +- benedict/core/traverse.py | 8 ++-- benedict/core/unflatten.py | 4 +- benedict/core/unique.py | 2 +- benedict/dicts/__init__.py | 9 +++-- benedict/dicts/base/base_dict.py | 6 ++- benedict/dicts/io/io_dict.py | 52 +++++++++++++------------- benedict/dicts/io/io_util.py | 37 ++++++++++-------- benedict/dicts/keylist/keylist_dict.py | 14 ++++--- benedict/dicts/keypath/keypath_dict.py | 16 ++++---- benedict/dicts/parse/parse_dict.py | 27 ++++++------- benedict/dicts/parse/parse_util.py | 5 ++- benedict/py.typed | 0 benedict/serializers/__init__.py | 2 +- benedict/serializers/abstract.py | 11 ++++-- benedict/serializers/base64.py | 21 ++++++----- benedict/serializers/csv.py | 5 ++- benedict/serializers/ini.py | 5 ++- benedict/serializers/json.py | 5 ++- benedict/serializers/pickle.py | 5 ++- benedict/serializers/plist.py | 5 ++- benedict/serializers/query_string.py | 9 +++-- benedict/serializers/toml.py | 6 ++- benedict/serializers/xls.py | 18 +++++---- benedict/serializers/xml.py | 6 ++- benedict/serializers/yaml.py | 6 ++- benedict/utils/type_util.py | 45 +++++++++++----------- 44 files changed, 233 insertions(+), 180 deletions(-) create mode 100644 benedict/py.typed diff --git a/benedict/core/clean.py b/benedict/core/clean.py index da1b6d0e..ef0e620b 100644 --- a/benedict/core/clean.py +++ b/benedict/core/clean.py @@ -1,7 +1,7 @@ from benedict.utils import type_util -def _clean_item(d, key, strings, collections): +def _clean_item(d, key, strings: bool, collections: bool): value = d.get(key, None) if not value: del_none = value is None @@ -12,7 +12,7 @@ def _clean_item(d, key, strings, collections): return False -def clean(d, strings=True, collections=True): +def clean(d, strings: bool = True, collections: bool = True): keys = list(d.keys()) for key in keys: if _clean_item(d, key, strings, collections): diff --git a/benedict/core/clone.py b/benedict/core/clone.py index ed34f84f..aa3b8c86 100644 --- a/benedict/core/clone.py +++ b/benedict/core/clone.py @@ -1,7 +1,8 @@ import copy +from typing import Any, Dict, Optional -def clone(obj, empty=False, memo=None): +def clone(obj, empty: bool = False, memo: Optional[Dict[int, Any]] = None): d = copy.deepcopy(obj, memo) if empty: d.clear() diff --git a/benedict/core/dump.py b/benedict/core/dump.py index 735ffe70..c6117ef3 100644 --- a/benedict/core/dump.py +++ b/benedict/core/dump.py @@ -1,7 +1,9 @@ +from typing import Any + from benedict.serializers import JSONSerializer -def dump(obj, **kwargs): +def dump(obj: Any, **kwargs: Any) -> str: serializer = JSONSerializer() options = {"indent": 4, "sort_keys": True} options.update(**kwargs) diff --git a/benedict/core/filter.py b/benedict/core/filter.py index d823a576..a745aacb 100644 --- a/benedict/core/filter.py +++ b/benedict/core/filter.py @@ -1,7 +1,9 @@ -from benedict.core import clone +from typing import Any, Callable +from benedict.core.clone import clone -def filter(d, predicate): + +def filter(d, predicate: Callable[[Any, Any], bool]): if not callable(predicate): raise ValueError("predicate argument must be a callable.") new_dict = clone(d, empty=True) diff --git a/benedict/core/flatten.py b/benedict/core/flatten.py index 9d119c65..a8311641 100644 --- a/benedict/core/flatten.py +++ b/benedict/core/flatten.py @@ -1,14 +1,14 @@ -from benedict.core import clone +from benedict.core.clone import clone from benedict.utils import type_util -def _flatten_key(base_key, key, separator): +def _flatten_key(base_key: str, key, separator: str) -> str: if base_key and separator: return f"{base_key}{separator}{key}" - return key + return f"{key}" -def _flatten_item(d, base_dict, base_key, separator): +def _flatten_item(d, base_dict, base_key, separator: str): new_dict = base_dict keys = list(d.keys()) for key in keys: @@ -26,6 +26,6 @@ def _flatten_item(d, base_dict, base_key, separator): return new_dict -def flatten(d, separator="_"): +def flatten(d, separator: str = "_"): new_dict = clone(d, empty=True) return _flatten_item(d, base_dict=new_dict, base_key="", separator=separator) diff --git a/benedict/core/invert.py b/benedict/core/invert.py index 6d33f4e9..88f8030c 100644 --- a/benedict/core/invert.py +++ b/benedict/core/invert.py @@ -1,20 +1,20 @@ -from benedict.core import clone +from benedict.core.clone import clone from benedict.utils import type_util -def _invert_item(d, key, value, flat): +def _invert_item(d, key, value, flat: bool) -> None: if flat: d.setdefault(value, key) else: d.setdefault(value, []).append(key) -def _invert_list(d, key, value, flat): +def _invert_list(d, key, value, flat: bool) -> None: for value_item in value: _invert_item(d, key, value_item, flat) -def invert(d, flat=False): +def invert(d, flat: bool = False): new_dict = clone(d, empty=True) for key, value in d.items(): if type_util.is_list_or_tuple(value): diff --git a/benedict/core/items_sorted.py b/benedict/core/items_sorted.py index 4237ffe2..b06fdba9 100644 --- a/benedict/core/items_sorted.py +++ b/benedict/core/items_sorted.py @@ -1,10 +1,10 @@ -def _items_sorted_by_item_at_index(d, index, reverse): +def _items_sorted_by_item_at_index(d, index, reverse: bool): return sorted(d.items(), key=lambda item: item[index], reverse=reverse) -def items_sorted_by_keys(d, reverse=False): +def items_sorted_by_keys(d, reverse: bool = False): return _items_sorted_by_item_at_index(d, 0, reverse) -def items_sorted_by_values(d, reverse=False): +def items_sorted_by_values(d, reverse: bool = False): return _items_sorted_by_item_at_index(d, 1, reverse) diff --git a/benedict/core/keylists.py b/benedict/core/keylists.py index 9d516dd8..6d928a95 100644 --- a/benedict/core/keylists.py +++ b/benedict/core/keylists.py @@ -28,5 +28,5 @@ def _get_keylist_for_value(value, parent_keys, indexes): return [] -def keylists(d, indexes=False): +def keylists(d, indexes: bool = False): return _get_keylist_for_value(d, [], indexes) diff --git a/benedict/core/keypaths.py b/benedict/core/keypaths.py index 1785f926..7b32dd99 100644 --- a/benedict/core/keypaths.py +++ b/benedict/core/keypaths.py @@ -1,8 +1,10 @@ +from typing import List + from benedict.core.keylists import keylists from benedict.utils import type_util -def keypaths(d, separator=".", indexes=False): +def keypaths(d, separator: str = ".", indexes: bool = False) -> List[str]: separator = separator or "." if not type_util.is_string(separator): raise ValueError("separator argument must be a (non-empty) string.") diff --git a/benedict/core/match.py b/benedict/core/match.py index 53643c74..b9b07ec6 100644 --- a/benedict/core/match.py +++ b/benedict/core/match.py @@ -4,7 +4,7 @@ from benedict.utils import type_util -def match(d, pattern, separator=".", indexes=True): +def match(d, pattern, separator: str = ".", indexes: bool = True): if type_util.is_regex(pattern): regex = pattern elif type_util.is_string(pattern): diff --git a/benedict/core/merge.py b/benedict/core/merge.py index 0156e675..d1f8f3b5 100644 --- a/benedict/core/merge.py +++ b/benedict/core/merge.py @@ -1,12 +1,14 @@ +from typing import Any + from benedict.utils import type_util -def _merge_dict(d, other, overwrite=True, concat=False): +def _merge_dict(d, other, overwrite: bool = True, concat: bool = False) -> None: for key, value in other.items(): _merge_item(d, key, value, overwrite=overwrite, concat=concat) -def _merge_item(d, key, value, overwrite=True, concat=False): +def _merge_item(d, key, value, overwrite: bool = True, concat: bool = False) -> None: if key in d: item = d.get(key, None) if type_util.is_dict(item) and type_util.is_dict(value): @@ -19,7 +21,7 @@ def _merge_item(d, key, value, overwrite=True, concat=False): d[key] = value -def merge(d, other, *args, **kwargs): +def merge(d, other, *args, **kwargs: Any): overwrite = kwargs.get("overwrite", True) concat = kwargs.get("concat", False) others = [other] + list(args) diff --git a/benedict/core/move.py b/benedict/core/move.py index 4bd62fdd..1232380b 100644 --- a/benedict/core/move.py +++ b/benedict/core/move.py @@ -1,4 +1,4 @@ -def move(d, key_src, key_dest, overwrite=True): +def move(d, key_src, key_dest, overwrite: bool = True) -> None: if key_dest == key_src: return if key_dest in d and not overwrite: diff --git a/benedict/core/nest.py b/benedict/core/nest.py index 1f13930c..6a544a43 100644 --- a/benedict/core/nest.py +++ b/benedict/core/nest.py @@ -1,7 +1,7 @@ from benedict.core.groupby import groupby -def _nest_items(nested_items, item, id_key, children_key): +def _nest_items(nested_items, item, id_key, children_key) -> None: children_items = nested_items.pop(item[id_key], []) item[children_key] = children_items for child_item in children_items: diff --git a/benedict/core/remove.py b/benedict/core/remove.py index d68838f2..09368aed 100644 --- a/benedict/core/remove.py +++ b/benedict/core/remove.py @@ -1,7 +1,7 @@ from benedict.utils import type_util -def remove(d, keys, *args): +def remove(d, keys, *args) -> None: if type_util.is_string(keys): keys = [keys] keys += args diff --git a/benedict/core/rename.py b/benedict/core/rename.py index a616fc67..47a3457a 100644 --- a/benedict/core/rename.py +++ b/benedict/core/rename.py @@ -1,5 +1,5 @@ from benedict.core.move import move -def rename(d, key, key_new): +def rename(d, key, key_new) -> None: move(d, key, key_new, overwrite=False) diff --git a/benedict/core/search.py b/benedict/core/search.py index 34637246..50a7a560 100644 --- a/benedict/core/search.py +++ b/benedict/core/search.py @@ -2,13 +2,13 @@ from benedict.utils import type_util -def _get_term(value, case_sensitive): +def _get_term(value, case_sensitive: bool): v_is_str = type_util.is_string(value) v = value.lower() if (v_is_str and not case_sensitive) else value return (v, v_is_str) -def _get_match(query, value, exact, case_sensitive): +def _get_match(query, value, exact: bool, case_sensitive: bool) -> bool: q, q_is_str = _get_term(query, case_sensitive) v, v_is_str = _get_term(value, case_sensitive) # TODO: add regex support @@ -17,7 +17,14 @@ def _get_match(query, value, exact, case_sensitive): return q == v -def search(d, query, in_keys=True, in_values=True, exact=False, case_sensitive=True): +def search( + d, + query, + in_keys: bool = True, + in_values: bool = True, + exact: bool = False, + case_sensitive: bool = True, +): items = [] def _search_item(item_dict, item_key, item_value): diff --git a/benedict/core/standardize.py b/benedict/core/standardize.py index d255ad66..4a1314d0 100644 --- a/benedict/core/standardize.py +++ b/benedict/core/standardize.py @@ -7,7 +7,7 @@ from benedict.utils import type_util -def _standardize_item(d, key, value): +def _standardize_item(d, key, value) -> None: if type_util.is_string(key): # https://stackoverflow.com/a/12867228/2096218 norm_key = re.sub(r"((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))", r"_\1", key) @@ -15,5 +15,5 @@ def _standardize_item(d, key, value): rename(d, key, norm_key) -def standardize(d): +def standardize(d) -> None: traverse(d, _standardize_item) diff --git a/benedict/core/swap.py b/benedict/core/swap.py index 5c6e5d79..d204d85b 100644 --- a/benedict/core/swap.py +++ b/benedict/core/swap.py @@ -1,4 +1,4 @@ -def swap(d, key1, key2): +def swap(d, key1, key2) -> None: if key1 == key2: return val1 = d[key1] diff --git a/benedict/core/traverse.py b/benedict/core/traverse.py index 8019b235..555cf073 100644 --- a/benedict/core/traverse.py +++ b/benedict/core/traverse.py @@ -1,14 +1,14 @@ from benedict.utils import type_util -def _traverse_collection(d, callback): +def _traverse_collection(d, callback) -> None: if type_util.is_dict(d): _traverse_dict(d, callback) elif type_util.is_list_or_tuple(d): _traverse_list(d, callback) -def _traverse_dict(d, callback): +def _traverse_dict(d, callback) -> None: keys = list(d.keys()) for key in keys: value = d.get(key, None) @@ -16,14 +16,14 @@ def _traverse_dict(d, callback): _traverse_collection(value, callback) -def _traverse_list(ls, callback): +def _traverse_list(ls, callback) -> None: items = list(enumerate(ls)) for index, value in items: callback(ls, index, value) _traverse_collection(value, callback) -def traverse(d, callback): +def traverse(d, callback) -> None: if not callable(callback): raise ValueError("callback argument must be a callable.") _traverse_collection(d, callback) diff --git a/benedict/core/unflatten.py b/benedict/core/unflatten.py index 23002e12..53fbf8bd 100644 --- a/benedict/core/unflatten.py +++ b/benedict/core/unflatten.py @@ -1,4 +1,4 @@ -from benedict.core import clone +from benedict.core.clone import clone from benedict.dicts.keylist import keylist_util from benedict.utils import type_util @@ -10,7 +10,7 @@ def _unflatten_item(key, value, separator): return (keys, value) -def unflatten(d, separator="_"): +def unflatten(d, separator: str = "_"): new_dict = clone(d, empty=True) keys = list(d.keys()) for key in keys: diff --git a/benedict/core/unique.py b/benedict/core/unique.py index bd22e6bc..d26ea253 100644 --- a/benedict/core/unique.py +++ b/benedict/core/unique.py @@ -1,4 +1,4 @@ -def unique(d): +def unique(d) -> None: values = [] keys = list(d.keys()) for key in keys: diff --git a/benedict/dicts/__init__.py b/benedict/dicts/__init__.py index 8fefd3c8..84bd787c 100644 --- a/benedict/dicts/__init__.py +++ b/benedict/dicts/__init__.py @@ -1,5 +1,6 @@ # fix benedict json dumps support - #57 #59 #61 from json import encoder +from typing import Any # fix benedict yaml representer - #43 from yaml import SafeDumper @@ -38,7 +39,7 @@ class benedict(KeypathDict, IODict, ParseDict): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs: Any): """ Constructs a new instance. """ @@ -188,7 +189,7 @@ def match(self, pattern, indexes=True): """ return _match(self, pattern, separator=self._keypath_separator, indexes=indexes) - def merge(self, other, *args, **kwargs): + def merge(self, other, *args, **kwargs: Any): """ Merge one or more dict objects into current instance (deepupdate). Sub-dictionaries will be merged toghether. @@ -282,7 +283,7 @@ def unique(self): # fix benedict json dumps support - #57 #59 #61 -encoder.c_make_encoder = None +encoder.c_make_encoder = None # type: ignore # fix benedict yaml representer - #43 -SafeDumper.yaml_representers[benedict] = SafeRepresenter.represent_dict +SafeDumper.yaml_representers[benedict] = SafeRepresenter.represent_dict # type: ignore diff --git a/benedict/dicts/base/base_dict.py b/benedict/dicts/base/base_dict.py index 2a85c2d0..78731134 100644 --- a/benedict/dicts/base/base_dict.py +++ b/benedict/dicts/base/base_dict.py @@ -1,3 +1,5 @@ +from typing import Any + from benedict.core import clone as _clone @@ -17,7 +19,7 @@ def _get_dict_or_value(cls, value): value[key] = key_val return value - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs: Any): if len(args) == 1 and isinstance(args[0], dict): self._dict = self._get_dict_or_value(args[0]) self._pointer = True @@ -134,7 +136,7 @@ def setdefault(self, key, default=None): return self._dict.setdefault(key, default) return super().setdefault(key, default) - def update(self, other): + def update(self, other): # type: ignore # TODO: remove this ignore other = self._get_dict_or_value(other) if self._pointer: self._dict.update(other) diff --git a/benedict/dicts/io/io_dict.py b/benedict/dicts/io/io_dict.py index 77aa8c75..f8b2c4aa 100644 --- a/benedict/dicts/io/io_dict.py +++ b/benedict/dicts/io/io_dict.py @@ -1,10 +1,12 @@ +from typing import Any + from benedict.dicts.base import BaseDict from benedict.dicts.io import io_util from benedict.utils import type_util class IODict(BaseDict): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs: Any): """ Constructs a new instance. """ @@ -19,7 +21,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @staticmethod - def _decode_init(s, **kwargs): + def _decode_init(s, **kwargs: Any): autodetected_format = io_util.autodetect_format(s) default_format = autodetected_format or "json" format = kwargs.pop("format", default_format).lower() @@ -27,7 +29,7 @@ def _decode_init(s, **kwargs): return IODict._decode(s, format, **kwargs) @staticmethod - def _decode(s, format, **kwargs): + def _decode(s, format, **kwargs: Any): data = None try: data = io_util.decode(s, format, **kwargs) @@ -43,12 +45,12 @@ def _decode(s, format, **kwargs): raise ValueError(f"Invalid data type: {type(data)}, expected dict or list.") @staticmethod - def _encode(d, format, **kwargs): + def _encode(d, format, **kwargs: Any) -> str: s = io_util.encode(d, format, **kwargs) return s @classmethod - def from_base64(cls, s, subformat="json", encoding="utf-8", **kwargs): + def from_base64(cls, s: str, subformat="json", encoding="utf-8", **kwargs: Any): """ Load and decode Base64 data from url, filepath or data-string. Data is decoded according to subformat and encoding. @@ -60,7 +62,7 @@ def from_base64(cls, s, subformat="json", encoding="utf-8", **kwargs): return cls(s, format="base64", **kwargs) @classmethod - def from_csv(cls, s, columns=None, columns_row=True, **kwargs): + def from_csv(cls, s: str, columns=None, columns_row=True, **kwargs: Any): """ Load and decode CSV data from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -72,7 +74,7 @@ def from_csv(cls, s, columns=None, columns_row=True, **kwargs): return cls(s, format="csv", **kwargs) @classmethod - def from_ini(cls, s, **kwargs): + def from_ini(cls, s: str, **kwargs: Any): """ Load and decode INI data from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -82,7 +84,7 @@ def from_ini(cls, s, **kwargs): return cls(s, format="ini", **kwargs) @classmethod - def from_json(cls, s, **kwargs): + def from_json(cls, s: str, **kwargs: Any): """ Load and decode JSON data from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -92,7 +94,7 @@ def from_json(cls, s, **kwargs): return cls(s, format="json", **kwargs) @classmethod - def from_pickle(cls, s, **kwargs): + def from_pickle(cls, s: str, **kwargs: Any): """ Load and decode a pickle encoded in Base64 format data from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -102,7 +104,7 @@ def from_pickle(cls, s, **kwargs): return cls(s, format="pickle", **kwargs) @classmethod - def from_plist(cls, s, **kwargs): + def from_plist(cls, s: str, **kwargs: Any): """ Load and decode p-list data from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -112,7 +114,7 @@ def from_plist(cls, s, **kwargs): return cls(s, format="plist", **kwargs) @classmethod - def from_query_string(cls, s, **kwargs): + def from_query_string(cls, s: str, **kwargs: Any): """ Load and decode query-string from url, filepath or data-string. Return a new dict instance. A ValueError is raised in case of failure. @@ -120,7 +122,7 @@ def from_query_string(cls, s, **kwargs): return cls(s, format="query_string", **kwargs) @classmethod - def from_toml(cls, s, **kwargs): + def from_toml(cls, s: str, **kwargs: Any): """ Load and decode TOML data from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -130,7 +132,7 @@ def from_toml(cls, s, **kwargs): return cls(s, format="toml", **kwargs) @classmethod - def from_xls(cls, s, sheet=0, columns=None, columns_row=True, **kwargs): + def from_xls(cls, s: str, sheet=0, columns=None, columns_row=True, **kwargs: Any): """ Load and decode XLS files (".xls", ".xlsx", ".xlsm") from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -144,7 +146,7 @@ def from_xls(cls, s, sheet=0, columns=None, columns_row=True, **kwargs): return cls(s, format="xls", **kwargs) @classmethod - def from_xml(cls, s, **kwargs): + def from_xml(cls, s: str, **kwargs: Any): """ Load and decode XML data from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -154,7 +156,7 @@ def from_xml(cls, s, **kwargs): return cls(s, format="xml", **kwargs) @classmethod - def from_yaml(cls, s, **kwargs): + def from_yaml(cls, s: str, **kwargs: Any): """ Load and decode YAML data from url, filepath or data-string. Decoder specific options can be passed using kwargs: @@ -163,7 +165,7 @@ def from_yaml(cls, s, **kwargs): """ return cls(s, format="yaml", **kwargs) - def to_base64(self, subformat="json", encoding="utf-8", **kwargs): + def to_base64(self, subformat="json", encoding="utf-8", **kwargs: Any): """ Encode the current dict instance in Base64 format using the given subformat and encoding. @@ -175,7 +177,7 @@ def to_base64(self, subformat="json", encoding="utf-8", **kwargs): kwargs["encoding"] = encoding return self._encode(self.dict(), "base64", **kwargs) - def to_csv(self, key="values", columns=None, columns_row=True, **kwargs): + def to_csv(self, key="values", columns=None, columns_row=True, **kwargs: Any): """ Encode a list of dicts in the current dict instance in CSV format. Encoder specific options can be passed using kwargs: @@ -187,7 +189,7 @@ def to_csv(self, key="values", columns=None, columns_row=True, **kwargs): kwargs["columns_row"] = columns_row return self._encode(self.dict()[key], "csv", **kwargs) - def to_ini(self, **kwargs): + def to_ini(self, **kwargs: Any): """ Encode the current dict instance in INI format. Encoder specific options can be passed using kwargs: @@ -197,7 +199,7 @@ def to_ini(self, **kwargs): """ return self._encode(self.dict(), "ini", **kwargs) - def to_json(self, **kwargs): + def to_json(self, **kwargs: Any): """ Encode the current dict instance in JSON format. Encoder specific options can be passed using kwargs: @@ -207,7 +209,7 @@ def to_json(self, **kwargs): """ return self._encode(self.dict(), "json", **kwargs) - def to_pickle(self, **kwargs): + def to_pickle(self, **kwargs: Any): """ Encode the current dict instance as pickle (encoded in Base64). The pickle protocol used by default is 2. @@ -218,7 +220,7 @@ def to_pickle(self, **kwargs): """ return self._encode(self.dict(), "pickle", **kwargs) - def to_plist(self, **kwargs): + def to_plist(self, **kwargs: Any): """ Encode the current dict instance as p-list. Encoder specific options can be passed using kwargs: @@ -228,7 +230,7 @@ def to_plist(self, **kwargs): """ return self._encode(self.dict(), "plist", **kwargs) - def to_query_string(self, **kwargs): + def to_query_string(self, **kwargs: Any): """ Encode the current dict instance in query-string format. Return the encoded string and optionally save it at 'filepath'. @@ -236,7 +238,7 @@ def to_query_string(self, **kwargs): """ return self._encode(self.dict(), "query_string", **kwargs) - def to_toml(self, **kwargs): + def to_toml(self, **kwargs: Any): """ Encode the current dict instance in TOML format. Encoder specific options can be passed using kwargs: @@ -246,7 +248,7 @@ def to_toml(self, **kwargs): """ return self._encode(self.dict(), "toml", **kwargs) - def to_xml(self, **kwargs): + def to_xml(self, **kwargs: Any): """ Encode the current dict instance in XML format. Encoder specific options can be passed using kwargs: @@ -280,7 +282,7 @@ def to_xls( # return self._encode(self.dict()[key], "xls", **kwargs) raise NotImplementedError - def to_yaml(self, **kwargs): + def to_yaml(self, **kwargs: Any): """ Encode the current dict instance in YAML format. Encoder specific options can be passed using kwargs: diff --git a/benedict/dicts/io/io_util.py b/benedict/dicts/io/io_util.py index 61ca645d..f8dae7f0 100644 --- a/benedict/dicts/io/io_util.py +++ b/benedict/dicts/io/io_util.py @@ -1,4 +1,5 @@ import tempfile +from typing import Any, Dict, Optional # from botocore.exceptions import ClientError from urllib.parse import urlparse @@ -16,7 +17,7 @@ def autodetect_format(s): return None -def decode(s, format, **kwargs): +def decode(s: str, format: str, **kwargs: Any) -> Any: s = str(s) serializer = get_serializer_by_format(format) if not serializer: @@ -29,7 +30,7 @@ def decode(s, format, **kwargs): return data -def encode(d, format, filepath=None, **kwargs): +def encode(d, format: str, filepath: Optional[str] = None, **kwargs: Any) -> str: serializer = get_serializer_by_format(format) if not serializer: raise ValueError(f"Invalid format: {format}.") @@ -41,7 +42,7 @@ def encode(d, format, filepath=None, **kwargs): return content -def is_binary_format(format): +def is_binary_format(format: Optional[str]) -> bool: return format in [ "xls", "xlsx", @@ -49,23 +50,23 @@ def is_binary_format(format): ] -def is_data(s): +def is_data(s: str) -> bool: return len(s.splitlines()) > 1 -def is_filepath(s): +def is_filepath(s: str) -> bool: return fsutil.is_file(s) or get_format_by_path(s) -def is_s3(s): +def is_s3(s: str) -> bool: return s.startswith("s3://") and get_format_by_path(s) -def is_url(s): +def is_url(s: str) -> bool: return any([s.startswith(protocol) for protocol in ["http://", "https://"]]) -def parse_s3_url(url): +def parse_s3_url(url: str) -> Dict[str, str]: parsed = urlparse(url, allow_fragments=False) bucket = parsed.netloc key = parsed.path.lstrip("/") @@ -79,7 +80,9 @@ def parse_s3_url(url): } -def read_content(s, format=None, options=None): +def read_content( + s: str, format: Optional[str] = None, options: Optional[Dict[str, Any]] = None +) -> str: # s -> filepath or url or data # options.setdefault("format", format) options = options or {} @@ -98,14 +101,16 @@ def read_content(s, format=None, options=None): return s -def read_content_from_file(filepath, format=None): +def read_content_from_file(filepath: str, format: Optional[str] = None) -> str: binary_format = is_binary_format(format) if binary_format: return filepath return fsutil.read_file(filepath) -def read_content_from_s3(url, s3_options, format=None): +def read_content_from_s3( + url: str, s3_options=Dict[str, Any], format: Optional[str] = None +) -> str: s3_url = parse_s3_url(url) dirpath = tempfile.gettempdir() filename = fsutil.get_filename(s3_url["key"]) @@ -117,7 +122,9 @@ def read_content_from_s3(url, s3_options, format=None): return content -def read_content_from_url(url, requests_options, format=None): +def read_content_from_url( + url: str, requests_options: Dict[str, Any], format: Optional[str] = None +) -> str: binary_format = is_binary_format(format) if binary_format: dirpath = tempfile.gettempdir() @@ -126,18 +133,18 @@ def read_content_from_url(url, requests_options, format=None): return fsutil.read_file_from_url(url, **requests_options) -def write_content(filepath, content, **options): +def write_content(filepath: str, content: str, **options: Any) -> None: if is_s3(filepath): write_content_to_s3(filepath, content, **options) else: write_content_to_file(filepath, content, **options) -def write_content_to_file(filepath, content, **options): +def write_content_to_file(filepath: str, content: str, **options: Any) -> None: fsutil.write_file(filepath, content) -def write_content_to_s3(url, content, s3_options, **options): +def write_content_to_s3(url: str, content: str, s3_options, **options: Any) -> None: s3_url = parse_s3_url(url) dirpath = tempfile.gettempdir() filename = fsutil.get_filename(s3_url["key"]) diff --git a/benedict/dicts/keylist/keylist_dict.py b/benedict/dicts/keylist/keylist_dict.py index 2f82dcea..98735b9b 100644 --- a/benedict/dicts/keylist/keylist_dict.py +++ b/benedict/dicts/keylist/keylist_dict.py @@ -1,30 +1,32 @@ +from typing import Any + from benedict.dicts.base import BaseDict from benedict.dicts.keylist import keylist_util from benedict.utils import type_util class KeylistDict(BaseDict): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs: Any): super().__init__(*args, **kwargs) - def __contains__(self, key): + def __contains__(self, key) -> bool: if type_util.is_list_or_tuple(key): return self._contains_by_keys(key) return super().__contains__(key) - def _contains_by_keys(self, keys): + def _contains_by_keys(self, keys) -> bool: parent, _, _ = keylist_util.get_item(self, keys) if type_util.is_dict_or_list_or_tuple(parent): return True return False - def __delitem__(self, key): + def __delitem__(self, key) -> None: if type_util.is_list_or_tuple(key): self._delitem_by_keys(key) return super().__delitem__(key) - def _delitem_by_keys(self, keys): + def _delitem_by_keys(self, keys) -> None: parent, key, _ = keylist_util.get_item(self, keys) if type_util.is_dict_or_list(parent): del parent[key] @@ -85,7 +87,7 @@ def _pop_by_keys(self, keys, *args): return args[0] raise KeyError(f"Invalid keys: '{keys}'") - def set(self, key, value): + def set(self, key, value) -> None: self[key] = value def setdefault(self, key, default=None): diff --git a/benedict/dicts/keypath/keypath_dict.py b/benedict/dicts/keypath/keypath_dict.py index 7425f36f..990b0762 100644 --- a/benedict/dicts/keypath/keypath_dict.py +++ b/benedict/dicts/keypath/keypath_dict.py @@ -1,3 +1,5 @@ +from typing import Any, Optional + from benedict.dicts import KeylistDict from benedict.dicts.keypath import keypath_util @@ -6,7 +8,7 @@ class KeypathDict(KeylistDict): _keypath_separator = None - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs: Any): self._keypath_separator = kwargs.pop("keypath_separator", ".") check_keys = kwargs.pop("check_keys", True) super().__init__(*args, **kwargs) @@ -14,24 +16,24 @@ def __init__(self, *args, **kwargs): keypath_util.check_keys(self, self._keypath_separator) @property - def keypath_separator(self): + def keypath_separator(self) -> Optional[str]: return self._keypath_separator @keypath_separator.setter - def keypath_separator(self, value): + def keypath_separator(self, value: str) -> None: keypath_util.check_keys(self, value) self._keypath_separator = value - def __contains__(self, key): + def __contains__(self, key) -> bool: return super().__contains__(self._parse_key(key)) - def __delitem__(self, key): + def __delitem__(self, key) -> None: super().__delitem__(self._parse_key(key)) def __getitem__(self, key): return super().__getitem__(self._parse_key(key)) - def __setitem__(self, key, value): + def __setitem__(self, key, value) -> None: keypath_util.check_keys(value, self._keypath_separator) super().__setitem__(self._parse_key(key), value) @@ -57,6 +59,6 @@ def get(self, key, default=None): def pop(self, key, *args): return super().pop(self._parse_key(key), *args) - def update(self, other): + def update(self, other) -> None: # type: ignore # TODO: remove this ignore keypath_util.check_keys(other, self._keypath_separator) super().update(other) diff --git a/benedict/dicts/parse/parse_dict.py b/benedict/dicts/parse/parse_dict.py index ef421bd9..d45607b8 100644 --- a/benedict/dicts/parse/parse_dict.py +++ b/benedict/dicts/parse/parse_dict.py @@ -1,4 +1,5 @@ from decimal import Decimal +from typing import Any from benedict.dicts.base import BaseDict from benedict.dicts.parse import parse_util @@ -6,7 +7,7 @@ class ParseDict(BaseDict): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs: Any): """ Constructs a new instance. """ @@ -52,7 +53,7 @@ def _get_values_list( values_list = self.get_list(key, [], separator) return [parser_func(value, **(parser_kwargs or {})) for value in values_list] - def get_bool(self, key, default=False): + def get_bool(self, key, default: bool = False) -> bool: """ Get value by key or keypath trying to return it as bool. Values like `1`, `true`, `yes`, `on` will be returned as `True`. @@ -76,7 +77,7 @@ def get_date(self, key, default=None, format=None, choices=None): key, default, choices, parse_util.parse_date, {"format": format} ) - def get_date_list(self, key, default=None, format=None, separator=","): + def get_date_list(self, key, default=None, format=None, separator: str = ","): """ Get value by key or keypath trying to return it as list of date values. If separator is specified and value is a string it will be splitted. @@ -95,7 +96,7 @@ def get_datetime(self, key, default=None, format=None, choices=None): key, default, choices, parse_util.parse_datetime, {"format": format} ) - def get_datetime_list(self, key, default=None, format=None, separator=","): + def get_datetime_list(self, key, default=None, format=None, separator: str = ","): """ Get value by key or keypath trying to return it as list of datetime values. If separator is specified and value is a string it will be splitted. @@ -111,7 +112,7 @@ def get_decimal(self, key, default=Decimal("0.0"), choices=None): """ return self._get_value(key, default, choices, parse_util.parse_decimal) - def get_decimal_list(self, key, default=None, separator=","): + def get_decimal_list(self, key, default=None, separator: str = ","): """ Get value by key or keypath trying to return it as list of Decimal values. If separator is specified and value is a string it will be splitted. @@ -146,7 +147,7 @@ def get_float(self, key, default=0.0, choices=None): """ return self._get_value(key, default, choices, parse_util.parse_float) - def get_float_list(self, key, default=None, separator=","): + def get_float_list(self, key, default=None, separator: str = ","): """ Get value by key or keypath trying to return it as list of float values. If separator is specified and value is a string it will be splitted. @@ -176,7 +177,7 @@ def get_list(self, key, default=None, separator=","): key, default or [], None, parse_util.parse_list, {"separator": separator} ) - def get_list_item(self, key, index=0, default=None, separator=","): + def get_list_item(self, key, index: int = 0, default=None, separator: str = ","): """ Get list by key or keypath and return value at the specified index. If separator is specified and list value is a string it will be splitted. @@ -206,21 +207,21 @@ def get_phonenumber(self, key, country_code=None, default=None): {"country_code": country_code}, ) - def get_slug(self, key, default="", choices=None): + def get_slug(self, key, default: str = "", choices=None) -> str: """ Get value by key or keypath trying to return it as slug. If choices and value is in choices return value otherwise default. """ return self._get_value(key, default, choices, parse_util.parse_slug) - def get_slug_list(self, key, default=None, separator=","): + def get_slug_list(self, key, default=None, separator: str = ","): """ Get value by key or keypath trying to return it as list of slug values. If separator is specified and value is a string it will be splitted. """ return self._get_values_list(key, default, separator, parse_util.parse_slug) - def get_str(self, key, default="", choices=None): + def get_str(self, key, default: str = "", choices=None) -> str: """ Get value by key or keypath trying to return it as string. Encoding issues will be automatically fixed. @@ -228,21 +229,21 @@ def get_str(self, key, default="", choices=None): """ return self._get_value(key, default, choices, parse_util.parse_str) - def get_str_list(self, key, default=None, separator=","): + def get_str_list(self, key, default=None, separator: str = ","): """ Get value by key or keypath trying to return it as list of str values. If separator is specified and value is a string it will be splitted. """ return self._get_values_list(key, default, separator, parse_util.parse_str) - def get_uuid(self, key, default="", choices=None): + def get_uuid(self, key, default: str = "", choices=None) -> str: """ Get value by key or keypath trying to return it as valid uuid. If choices and value is in choices return value otherwise default. """ return self._get_value(key, default, choices, parse_util.parse_uuid) - def get_uuid_list(self, key, default=None, separator=","): + def get_uuid_list(self, key, default=None, separator: str = ","): """ Get value by key or keypath trying to return it as list of valid uuid values. If separator is specified and value is a string it will be splitted. diff --git a/benedict/dicts/parse/parse_util.py b/benedict/dicts/parse/parse_util.py index 71460690..91bb66b1 100644 --- a/benedict/dicts/parse/parse_util.py +++ b/benedict/dicts/parse/parse_util.py @@ -1,6 +1,7 @@ import re from datetime import datetime from decimal import Decimal, DecimalException +from typing import Any import ftfy import phonenumbers @@ -13,7 +14,7 @@ from benedict.utils import type_util -def _parse_with(val, type_checker, parser, **kwargs): +def _parse_with(val, type_checker, parser, **kwargs: Any): if val is None: return None if callable(type_checker) and type_checker(val): @@ -44,7 +45,7 @@ def parse_date(val, format=None): return None -def _parse_datetime_with_format(val, format): +def _parse_datetime_with_format(val, format: str): try: return datetime.strptime(val, format) except Exception: diff --git a/benedict/py.typed b/benedict/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/benedict/serializers/__init__.py b/benedict/serializers/__init__.py index 48a35fed..b51da027 100644 --- a/benedict/serializers/__init__.py +++ b/benedict/serializers/__init__.py @@ -73,7 +73,7 @@ def get_format_by_path(path): return None -def get_serializer_by_format(format): +def get_serializer_by_format(format: str): format_key = (format or "").lower().strip() format_key = re.sub(r"[\s\-\_]*", "", format_key) serializer = _SERIALIZERS_BY_EXTENSION.get(format_key, None) diff --git a/benedict/serializers/abstract.py b/benedict/serializers/abstract.py index 67137b65..5fbfef07 100644 --- a/benedict/serializers/abstract.py +++ b/benedict/serializers/abstract.py @@ -1,17 +1,20 @@ +from typing import Any, List, Optional + + class AbstractSerializer: """ This class describes an abstract serializer. """ - def __init__(self, extensions=None): + def __init__(self, extensions: Optional[List[str]] = None): super().__init__() self._extensions = (extensions or []).copy() - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): raise NotImplementedError() - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: raise NotImplementedError() - def extensions(self): + def extensions(self) -> List[str]: return self._extensions.copy() diff --git a/benedict/serializers/base64.py b/benedict/serializers/base64.py index bf872e83..048ea11c 100644 --- a/benedict/serializers/base64.py +++ b/benedict/serializers/base64.py @@ -1,4 +1,5 @@ import base64 +from typing import Any from urllib.parse import unquote from benedict.serializers.abstract import AbstractSerializer @@ -18,7 +19,7 @@ def __init__(self): ], ) - def _fix_url_encoding_and_padding(self, s): + def _fix_url_encoding_and_padding(self, s: str) -> str: # fix urlencoded chars s = unquote(s) # fix padding @@ -27,17 +28,17 @@ def _fix_url_encoding_and_padding(self, s): s += "=" * (4 - m) return s - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): value = self._fix_url_encoding_and_padding(s) encoding = kwargs.pop("encoding", "utf-8") - if encoding: - value = value.encode(encoding) - value = base64.b64decode(value) - if encoding: - return value.decode(encoding) + # if encoding: + # value = value.encode(encoding) + value = base64.b64decode(value).decode(encoding) + # if encoding: + # return value.decode(encoding) return value - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: value = d encoding = kwargs.pop("encoding", "utf-8") if encoding and type_util.is_string(value): @@ -60,14 +61,14 @@ def _pop_options(self, options): serializer = get_serializer_by_format(subformat) return (serializer, encoding) - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): serializer, encoding = self._pop_options(kwargs) value = super().decode(s, encoding=encoding) if serializer: value = serializer.decode(value, **kwargs) return value - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: serializer, encoding = self._pop_options(kwargs) value = d if serializer: diff --git a/benedict/serializers/csv.py b/benedict/serializers/csv.py index 97ea5481..10a41e2b 100644 --- a/benedict/serializers/csv.py +++ b/benedict/serializers/csv.py @@ -1,5 +1,6 @@ import csv from io import StringIO +from typing import Any from benedict.serializers.abstract import AbstractSerializer from benedict.utils import type_util @@ -17,7 +18,7 @@ def __init__(self): ], ) - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): # kwargs.setdefault('delimiter', ',') if kwargs.pop("quote", False): # TODO: add tests coverage @@ -39,7 +40,7 @@ def decode(self, s, **kwargs): ln += 1 return data - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: ls = d # kwargs.setdefault('delimiter', ',') if kwargs.pop("quote", False): diff --git a/benedict/serializers/ini.py b/benedict/serializers/ini.py index 09e2f1d7..608b2617 100644 --- a/benedict/serializers/ini.py +++ b/benedict/serializers/ini.py @@ -1,6 +1,7 @@ from configparser import DEFAULTSECT as default_section from configparser import ConfigParser from io import StringIO +from typing import Any from benedict.serializers.abstract import AbstractSerializer from benedict.utils import type_util @@ -30,7 +31,7 @@ def _get_section_option_value(parser, section, option): continue return value - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): parser = ConfigParser(**kwargs) parser.read_string(s) data = {} @@ -46,7 +47,7 @@ def decode(self, s, **kwargs): ) return data - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: parser = ConfigParser(**kwargs) for key, value in d.items(): if not type_util.is_dict(value): diff --git a/benedict/serializers/json.py b/benedict/serializers/json.py index 89b9af6b..d703624f 100644 --- a/benedict/serializers/json.py +++ b/benedict/serializers/json.py @@ -1,4 +1,5 @@ import json +from typing import Any from benedict.serializers.abstract import AbstractSerializer from benedict.utils import type_util @@ -16,11 +17,11 @@ def __init__(self): ], ) - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): data = json.loads(s, **kwargs) return data - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: kwargs.setdefault("default", self._encode_default) data = json.dumps(d, **kwargs) return data diff --git a/benedict/serializers/pickle.py b/benedict/serializers/pickle.py index 20680139..76fee208 100644 --- a/benedict/serializers/pickle.py +++ b/benedict/serializers/pickle.py @@ -1,5 +1,6 @@ import base64 import pickle +from typing import Any from benedict.serializers.abstract import AbstractSerializer @@ -16,11 +17,11 @@ def __init__(self): ], ) - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): encoding = kwargs.pop("encoding", "utf-8") return pickle.loads(base64.b64decode(s.encode(encoding)), **kwargs) - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: encoding = kwargs.pop("encoding", "utf-8") kwargs.setdefault("protocol", 2) return base64.b64encode(pickle.dumps(d, **kwargs)).decode(encoding) diff --git a/benedict/serializers/plist.py b/benedict/serializers/plist.py index 10619b24..77a1291c 100644 --- a/benedict/serializers/plist.py +++ b/benedict/serializers/plist.py @@ -1,4 +1,5 @@ import plistlib +from typing import Any from benedict.serializers.abstract import AbstractSerializer @@ -16,11 +17,11 @@ def __init__(self): ], ) - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): kwargs.setdefault("fmt", plistlib.FMT_XML) encoding = kwargs.pop("encoding", "utf-8") return plistlib.loads(s.encode(encoding), **kwargs) - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: encoding = kwargs.pop("encoding", "utf-8") return plistlib.dumps(d, **kwargs).decode(encoding) diff --git a/benedict/serializers/query_string.py b/benedict/serializers/query_string.py index 1816379b..29c9959d 100644 --- a/benedict/serializers/query_string.py +++ b/benedict/serializers/query_string.py @@ -1,4 +1,5 @@ import re +from typing import Any from urllib.parse import parse_qs, urlencode from benedict.serializers.abstract import AbstractSerializer @@ -17,17 +18,17 @@ def __init__(self): ], ) - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): flat = kwargs.pop("flat", True) qs_re = r"(?:([\w\-\%\+\.\|]+\=[\w\-\%\+\.\|]*)+(?:[\&]{1})?)+" qs_pattern = re.compile(qs_re) if qs_pattern.match(s): - data = parse_qs(s) + qs_data = parse_qs(s) if flat: - data = {key: value[0] for key, value in data.items()} + data = {key: value[0] for key, value in qs_data.items()} return data raise ValueError(f"Invalid query string: {s}") - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: data = urlencode(d, **kwargs) return data diff --git a/benedict/serializers/toml.py b/benedict/serializers/toml.py index 304aadaf..172827e3 100644 --- a/benedict/serializers/toml.py +++ b/benedict/serializers/toml.py @@ -1,3 +1,5 @@ +from typing import Any + import toml try: @@ -23,13 +25,13 @@ def __init__(self): ], ) - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): if tomllib_available: data = tomllib.loads(s, **kwargs) else: data = toml.loads(s, **kwargs) return data - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: data = toml.dumps(dict(d), **kwargs) return data diff --git a/benedict/serializers/xls.py b/benedict/serializers/xls.py index e49d5673..dbffa650 100644 --- a/benedict/serializers/xls.py +++ b/benedict/serializers/xls.py @@ -1,3 +1,5 @@ +from typing import Any, List + import fsutil from openpyxl import load_workbook from slugify import slugify @@ -20,7 +22,7 @@ def __init__(self): ], ) - def _get_sheet_index_and_name_from_options(self, **kwargs): + def _get_sheet_index_and_name_from_options(self, **kwargs: Any): sheet_index_or_name = kwargs.pop("sheet", 0) sheet_index = 0 sheet_name = "" @@ -30,7 +32,7 @@ def _get_sheet_index_and_name_from_options(self, **kwargs): sheet_name = sheet_index_or_name return (sheet_index, sheet_name) - def _get_sheet_index_by_name(self, sheet_name, sheet_names): + def _get_sheet_index_by_name(self, sheet_name: str, sheet_names: List[str]) -> int: sheet_names = list([slugify(name) for name in sheet_names]) try: sheet_index = sheet_names.index(slugify(sheet_name)) @@ -38,10 +40,10 @@ def _get_sheet_index_by_name(self, sheet_name, sheet_names): except ValueError: raise Exception(f"Invalid sheet name '{sheet_name}', sheet not found.") - def _get_sheet_columns_indexes(self, columns_count): + def _get_sheet_columns_indexes(self, columns_count: int) -> List[int]: return [column_index for column_index in range(columns_count)] - def _decode_legacy(self, s, **kwargs): + def _decode_legacy(self, s: str, **kwargs: Any): filepath = s # load the worksheet @@ -69,7 +71,7 @@ def _decode_legacy(self, s, **kwargs): else: # otherwise use columns indexes as column names # for row in sheet.iter_rows(min_row=1, max_row=1): - columns = self._get_sheet_columns_indexes(sheet_columns_range) + columns = self._get_sheet_columns_indexes(len(sheet_columns_range)) # standardize column names, eg. "Date Created" -> "date_created" if columns_standardized: @@ -89,7 +91,7 @@ def _decode_legacy(self, s, **kwargs): # print(items) return items - def _decode(self, s, **kwargs): + def _decode(self, s: str, **kwargs: Any): filepath = s # load the worksheet @@ -135,12 +137,12 @@ def _decode(self, s, **kwargs): # print(items) return items - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): extension = fsutil.get_file_extension(s) if extension in ["xlsx", "xlsm"]: return self._decode(s, **kwargs) elif extension in ["xls", "xlt"]: return self._decode_legacy(s, **kwargs) - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: raise NotImplementedError diff --git a/benedict/serializers/xml.py b/benedict/serializers/xml.py index b6ce1c1a..d01d97f2 100644 --- a/benedict/serializers/xml.py +++ b/benedict/serializers/xml.py @@ -1,3 +1,5 @@ +from typing import Any + import xmltodict from benedict.serializers.abstract import AbstractSerializer @@ -15,11 +17,11 @@ def __init__(self): ], ) - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): kwargs.setdefault("dict_constructor", dict) data = xmltodict.parse(s, **kwargs) return data - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any): data = xmltodict.unparse(d, **kwargs) return data diff --git a/benedict/serializers/yaml.py b/benedict/serializers/yaml.py index cdc61eda..75b5eeff 100644 --- a/benedict/serializers/yaml.py +++ b/benedict/serializers/yaml.py @@ -1,3 +1,5 @@ +from typing import Any + import yaml from benedict.serializers.abstract import AbstractSerializer @@ -18,11 +20,11 @@ def __init__(self): ) self._json_serializer = JSONSerializer() - def decode(self, s, **kwargs): + def decode(self, s: str, **kwargs: Any): data = yaml.safe_load(s, **kwargs) return data - def encode(self, d, **kwargs): + def encode(self, d, **kwargs: Any) -> str: d = self._json_serializer.decode(self._json_serializer.encode(d)) data = yaml.dump(d, **kwargs) return data diff --git a/benedict/utils/type_util.py b/benedict/utils/type_util.py index 476d66e5..abb00eaf 100644 --- a/benedict/utils/type_util.py +++ b/benedict/utils/type_util.py @@ -2,6 +2,7 @@ import re from datetime import datetime from decimal import Decimal +from typing import Any regex = re.compile("").__class__ uuid_re = re.compile( @@ -10,86 +11,86 @@ ) -def is_bool(val): +def is_bool(val: Any) -> bool: return isinstance(val, bool) -def is_collection(val): +def is_collection(val: Any) -> bool: return isinstance(val, (dict, list, set, tuple)) -def is_datetime(val): +def is_datetime(val: Any) -> bool: return isinstance(val, datetime) -def is_decimal(val): +def is_decimal(val: Any) -> bool: return isinstance(val, Decimal) -def is_dict(val): +def is_dict(val: Any) -> bool: return isinstance(val, dict) -def is_dict_or_list(val): +def is_dict_or_list(val: Any) -> bool: return isinstance(val, (dict, list)) -def is_dict_or_list_or_tuple(val): +def is_dict_or_list_or_tuple(val: Any) -> bool: return isinstance(val, (dict, list, tuple)) -def is_float(val): +def is_float(val: Any) -> bool: return isinstance(val, float) -def is_function(val): +def is_function(val: Any) -> bool: return callable(val) -def is_integer(val): +def is_integer(val: Any) -> bool: return isinstance(val, int) -def is_json_serializable(val): +def is_json_serializable(val: Any) -> bool: json_types = (type(None), bool, dict, float, int, list, str, tuple) return isinstance(val, json_types) -def is_list(val): +def is_list(val: Any) -> bool: return isinstance(val, list) -def is_list_or_tuple(val): +def is_list_or_tuple(val: Any) -> bool: return isinstance(val, (list, tuple)) -def is_none(val): +def is_none(val: Any) -> bool: return val is None -def is_not_none(val): +def is_not_none(val: Any) -> bool: return val is not None -def is_path(val): +def is_path(val: Any) -> bool: return isinstance(val, pathlib.Path) -def is_regex(val): +def is_regex(val: Any) -> bool: return isinstance(val, regex) -def is_set(val): +def is_set(val: Any) -> bool: return isinstance(val, set) -def is_string(val): +def is_string(val: Any) -> bool: return isinstance(val, str) -def is_tuple(val): +def is_tuple(val: Any) -> bool: return isinstance(val, tuple) -def is_uuid(val): - return is_string(val) and uuid_re.match(val) +def is_uuid(val: Any) -> bool: + return is_string(val) and bool(uuid_re.match(val)) From 1550628290aa223c356a96913b2ec0dcb98fdbf4 Mon Sep 17 00:00:00 2001 From: Fabio Caccamo Date: Thu, 9 Feb 2023 10:49:04 +0100 Subject: [PATCH 5/6] Fix too many positional arguments for `download_file`. --- benedict/dicts/io/io_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benedict/dicts/io/io_util.py b/benedict/dicts/io/io_util.py index f8dae7f0..d416b8c1 100644 --- a/benedict/dicts/io/io_util.py +++ b/benedict/dicts/io/io_util.py @@ -128,7 +128,7 @@ def read_content_from_url( binary_format = is_binary_format(format) if binary_format: dirpath = tempfile.gettempdir() - filepath = fsutil.download_file(url, dirpath, **requests_options) + filepath = fsutil.download_file(url, dirpath=dirpath, **requests_options) return filepath return fsutil.read_file_from_url(url, **requests_options) From 2f65b9dba6d9d88b62d75e05dff8f26082466016 Mon Sep 17 00:00:00 2001 From: Fabio Caccamo Date: Thu, 9 Feb 2023 11:54:12 +0100 Subject: [PATCH 6/6] Add `fix-future-annotations` `pre-commit` hook. --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0acf44f1..156848ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,11 @@ repos: hooks: - id: pyupgrade + - repo: https://github.com/frostming/fix-future-annotations + rev: 0.5.0 + hooks: + - id: fix-future-annotations + - repo: https://github.com/psf/black rev: 23.1.0 hooks: