set "secret" on ListConfigEntry directly to prevent nested items from…

… showing up even if they are secret reworked formatter base class added SimpleFormatter added YamlFormatter use YamlFormatter in YamlSource when writing reference
markusressel · Oct 22, 2019 · f1a41ca · f1a41ca
1 parent b2089a2
commit f1a41ca
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 41 deletions.
diff --git a/container_app_conf/__init__.py b/container_app_conf/__init__.py
@@ -23,9 +23,9 @@
 
 from container_app_conf.const import DEFAULT_CONFIG_FILE_PATHS
 from container_app_conf.entry import ConfigEntry
-from container_app_conf.formatter import ConfigFormatter
+from container_app_conf.formatter import ConfigFormatter, SimpleFormatter
 from container_app_conf.source import DataSource
-from container_app_conf.util import find_duplicates, generate_reference_config
+from container_app_conf.util import find_duplicates, generate_reference_config, config_entries_to_dict
 
 LOGGER = logging.getLogger(__name__)
 
@@ -121,8 +121,10 @@ def print(self, formatter: ConfigFormatter = None) -> str:
         :return: printable description of the current configuration
         """
         if formatter is None:
-            formatter = ConfigFormatter()
-        output = formatter.format(self)
+            formatter = SimpleFormatter()
+
+        data = config_entries_to_dict(list(self._config_entries.values()), hide_secrets=True)
+        output = formatter.format(data)
         return output
 
     def _find_config_entries(self) -> Dict[str, ConfigEntry]:

diff --git a/container_app_conf/entry/list.py b/container_app_conf/entry/list.py
@@ -60,6 +60,7 @@ def __init__(self, item_type: Type[ConfigEntry], key_path: [str], example: any =
             example=example,
             default=default,
             required=required,
+            secret=secret
         )
 
     @property

diff --git a/container_app_conf/formatter/__init__.py b/container_app_conf/formatter/__init__.py
@@ -17,46 +17,41 @@
 #  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 #  SOFTWARE.
-from typing import List
-
-from container_app_conf import ConfigEntry
-from container_app_conf.source.env_source import EnvSource
 
 
 class ConfigFormatter:
     """
     Allows config entries to be formatted into a string
     """
 
-    def format(self, config) -> str:
+    def format(self, data: dict) -> str:
         """
-        Formats the given configuration
-        :param config: config to format
+        Formats the given entry data
+        :param data: entries to format
         :return: formatted string
         """
-        lines = list(map(self.format_entries, config._config_entries.values()))
-        output = "\n".join(lines)
-        return output
+        raise NotImplementedError()
 
-    def format_entries(self, entries: List[ConfigEntry] or ConfigEntry) -> str:
-        """
-        Formats the given entries
-        :param entries: entries to format
-        :return: formatted string
-        """
-        entries = entries if isinstance(entries, list) else [entries]
-        output = "\n".join(list(map(self.format_entry, entries)))
-        return output
 
-    def format_entry(self, entry: ConfigEntry) -> str:
+class SimpleFormatter(ConfigFormatter):
+    """
+    Prints all config entries in a human readable manner
+    """
+
+    def format(self, data: dict) -> str:
+        return "\n".join(self._format(data)).strip()
+
+    def _format(self, data: dict, prefix: str = "") -> [str]:
         """
-        Formats the given entry
-        :param entry: entry to format
-        :return: formatted string
+        Recursively formats the dictionary
+        :param data:
+        :return:
         """
-        if entry.secret:
-            value = "*****"
-        else:
-            value = entry._type_to_value(entry.value)
-
-        return "{}: {}".format(EnvSource.env_key(entry), value)
+        lines = []
+        for key, value in data.items():
+            output = prefix + key
+            if isinstance(value, dict):
+                lines.extend(self._format(value, "{}->".format(output)))
+            else:
+                lines.append(output + ": {}".format(value))
+        return lines
diff --git a/container_app_conf/formatter/yaml.py b/container_app_conf/formatter/yaml.py
@@ -0,0 +1,39 @@
+#  Copyright (c) 2019 Markus Ressel
+#  .
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#  .
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#  .
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  SOFTWARE.
+import io
+
+from ruamel.yaml import YAML
+
+from container_app_conf import ConfigFormatter
+
+
+class YamlFormatter(ConfigFormatter):
+    """
+    Formats config entries like a YAML config file
+    """
+    yaml = YAML()
+    yaml.default_style = False
+    yaml.default_flow_style = False
+
+    def format(self, data: dict) -> str:
+        output = io.StringIO()
+        self.yaml.dump(data, output)
+        output.seek(0)
+        return output.read()
diff --git a/container_app_conf/source/yaml_source.py b/container_app_conf/source/yaml_source.py
@@ -19,8 +19,9 @@
 #  SOFTWARE.
 import logging
 
-import yaml
+from ruamel.yaml import YAML
 
+from container_app_conf.formatter.yaml import YamlFormatter
 from container_app_conf.source import FilesystemSource
 
 LOGGER = logging.getLogger(__name__)
@@ -31,12 +32,19 @@ class YamlSource(FilesystemSource):
     Data source utilizing YAML files
     """
     DEFAULT_FILE_EXTENSIONS = ['yaml', 'yml']
+    formatter = YamlFormatter()
+
+    yaml = YAML()
+    yaml.default_style = False
+    yaml.default_flow_style = False
 
     def _load_file(self, file_path: str) -> dict:
         with open(file_path, 'r') as ymlfile:
-            return yaml.load(ymlfile, Loader=yaml.FullLoader)
+            return self.yaml.load(ymlfile)
 
     def _write_reference(self, reference: dict, file_path: str):
-        text = yaml.dump(reference)
+        text = self.formatter.format(reference)
         with open(file_path, "w") as file:
+            file.seek(0)
             file.write(text)
+            file.truncate()
diff --git a/container_app_conf/util.py b/container_app_conf/util.py
@@ -52,6 +52,15 @@ def generate_reference_config(config_entries: List[ConfigEntry]) -> {}:
     Generates a dictionary containing the expected config tree filled with default and example values
     :return: a dictionary containing the expected config tree
     """
+    return config_entries_to_dict(config_entries, use_examples=True)
+
+
+def config_entries_to_dict(config_entries: List[ConfigEntry], hide_secrets: bool = False,
+                           use_examples: bool = False) -> {}:
+    """
+    Converts a list of config entries to a dictionary
+    :return: a dictionary containing the expected config tree
+    """
     config_tree = {}
     for entry in config_entries:
         current_level = config_tree
@@ -60,6 +69,11 @@ def generate_reference_config(config_entries: List[ConfigEntry]) -> {}:
                 current_level[path] = {}
             current_level = current_level[path]
 
-        current_level[entry.key_path[-1]] = entry._type_to_value(entry.example)
+        if hide_secrets and entry.secret:
+            value = "_REDACTED_"
+        else:
+            value = entry._type_to_value(entry.example if use_examples else entry.value)
+
+        current_level[entry.key_path[-1]] = value
 
     return config_tree
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
 pytimeparse
-pyyaml
+ruamel.yaml
 toml
 python-dateutil
 py-range-parse>=1.0.5
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -47,7 +47,8 @@ class TestConfigBase2(ConfigBase):
 class TestConfigBase(ConfigBase):
     BOOL = BoolConfigEntry(
         key_path=["test", "bool"],
-        default=False
+        default=False,
+        secret=True
     )
     STRING = StringConfigEntry(
         key_path=["test", "string"],
@@ -92,7 +93,7 @@ class TestConfigBase(ConfigBase):
             "test",
             "values"
         ],
-        secret=True
+        secret=False
     )
 
     SECRET_REGEX = RegexConfigEntry(
@@ -101,6 +102,13 @@ class TestConfigBase(ConfigBase):
         secret=True
     )
 
+    SECRET_LIST = ListConfigEntry(
+        item_type=RegexConfigEntry,
+        key_path=["secret", "list"],
+        default=["[a-zA-Z]*"],
+        secret=True
+    )
+
 
 class TestBase(unittest.TestCase):
     under_test = TestConfigBase()

diff --git a/tests/secret_test.py b/tests/secret_test.py
@@ -33,7 +33,9 @@ def test_secret_entry(self):
 
     def test_secret_current_config(self):
         config = TestConfigBase()
-        output = config.print()
+        from container_app_conf.formatter.yaml import YamlFormatter
+        output = config.print(YamlFormatter())
+        # output = config.print()
 
         for secret_entry in list(filter(lambda x: x.secret, config._config_entries.values())):
             self.assertNotIn(str(secret_entry.value), output)