pyyaml -> ruamel.yaml, 100% cov on config.py

example/kiwi.yml

@@ -2,7 +2,7 @@
 # kiwi-scp instance configuration #
 ###################################
 
-version: '0.2.0'
+version: 0.2.0
 
 shells:
   - /bin/bash

kiwi_scp/commands/cmd_init.py

@@ -72,4 +72,4 @@ def cmd(ctx: Instance, output: Path, force: bool, show: bool):
 
     # write out the new kiwi.yml
     with open(ctx.directory.joinpath(KIWI_CONF_NAME), "w") as file:
-        file.write(Config.parse_obj(kiwi_dict).kiwi_yml)
+        Config.parse_obj(kiwi_dict).dump_kiwi_yml(file)

kiwi_scp/config.py

@@ -1,19 +1,14 @@
 import functools
-import re
+import io
 from ipaddress import IPv4Network
 from pathlib import Path
-from typing import Optional, Dict, List, Any
+from typing import Optional, Dict, List, Any, TextIO
 
-import yaml
+import ruamel.yaml
 from pydantic import BaseModel, constr, root_validator, validator
 
-from ._constants import RE_SEMVER, RE_VARNAME, HEADER_KIWI_CONF_NAME, KIWI_CONF_NAME
+from ._constants import RE_SEMVER, RE_VARNAME, KIWI_CONF_NAME
-
+from .misc import _format_kiwi_yml
-
-# indent yaml lists
-class _KiwiDumper(yaml.Dumper):
-    def increase_indent(self, flow=False, indentless=False):
-        return super().increase_indent(flow, False)  # pragma: no cover
 
 
 class _Storage(BaseModel):
@@ -75,7 +70,8 @@ class _Project(BaseModel):
             return {"directory": value[0]}
 
         else:
-            raise ValueError("Invalid Storage Format")
+            # undefined format
+            return {}
 
     @root_validator(pre=True)
     @classmethod
@@ -141,12 +137,12 @@ class Config(BaseModel):
 
     @classmethod
     @functools.lru_cache(maxsize=5)
-    def from_instance(cls, instance: Path):
+    def from_directory(cls, instance: Path):
         """parses an actual kiwi.yml from disk (cached)"""
 
         try:
             with open(instance.joinpath(KIWI_CONF_NAME)) as kc:
-                yml = yaml.safe_load(kc)
+                yml = ruamel.yaml.round_trip_load(kc)
                 return cls.parse_obj(yml)
 
         except FileNotFoundError:
@@ -184,25 +180,23 @@ class Config(BaseModel):
 
         return result
 
-    @property
+    def dump_kiwi_yml(self, stream: TextIO) -> None:
-    def kiwi_yml(self) -> str:
         """dump a kiwi.yml file"""
 
-        yml_string = yaml.dump(
+        yml = ruamel.yaml.YAML()
-            self.kiwi_dict,
+        yml.indent(offset=2)
-            Dumper=_KiwiDumper,
+        yml.dump(self.kiwi_dict, stream=stream, transform=_format_kiwi_yml)
-            default_flow_style=False,
-            sort_keys=False,
-        )
 
-        # insert newline before every main key
+    @property
-        yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE)
+    def kiwi_yml(self) -> str:
+        """get a kiwi.yml dump as a string"""
 
-        # load header comment from file
+        sio = io.StringIO()
-        with open(HEADER_KIWI_CONF_NAME, 'r') as stream:
+        self.dump_kiwi_yml(sio)
-            yml_string = stream.read() + yml_string
+        result: str = sio.getvalue()
+        sio.close()
 
-        return yml_string
+        return result
 
     @validator("shells", pre=True)
     @classmethod
@@ -341,7 +335,7 @@ class Config(BaseModel):
 
         else:
             # undefined format
-            raise ValueError("Invalid Storage Format")
+            return {}
 
     @validator("network", pre=True)
     @classmethod

kiwi_scp/misc.py

@@ -1,9 +1,12 @@
+import re
 from typing import Any, Type, List, Callable
 
 import attr
 import click
 from click.decorators import FC
 
+from ._constants import HEADER_KIWI_CONF_NAME
+
 
 @attr.s
 class _MultiDecorator:
@@ -19,8 +22,7 @@ class _MultiDecorator:
 _project_arg = click.argument(
     "project",
     required=False,
-    type=click.Path(exists=True),
+    type=str,
-    default=".",
 )
 
 _service_arg = click.argument(
@@ -52,6 +54,17 @@ def user_query(description: str, default: Any, cast_to: Type[Any] = str):
             click.echo(f"Invalid input: {e}")
 
 
+def _format_kiwi_yml(yml_string: str):
+    # insert newline before every main key
+    yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE)
+
+    # load header comment from file
+    with open(HEADER_KIWI_CONF_NAME, 'r') as stream:
+        yml_string = stream.read() + yml_string
+
+    return yml_string
+
+
 def _surround(string, bang):
     midlane = f"{bang * 3} {string} {bang * 3}"
     sidelane = bang * len(midlane)

poetry.lock

@@ -248,12 +248,27 @@ pytest = ">=4.6"
 testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"]
 
 [[package]]
-name = "pyyaml"
+name = "ruamel.yaml"
-version = "5.4.1"
+version = "0.17.16"
-description = "YAML parser and emitter for Python"
+description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order"
 category = "main"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+python-versions = ">=3"
+
+[package.dependencies]
+"ruamel.yaml.clib" = {version = ">=0.1.2", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.10\""}
+
+[package.extras]
+docs = ["ryd"]
+jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"]
+
+[[package]]
+name = "ruamel.yaml.clib"
+version = "0.2.6"
+description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml"
+category = "main"
+optional = false
+python-versions = ">=3.5"
 
 [[package]]
 name = "six"
@@ -323,7 +338,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.6.1"
-content-hash = "a5e47a9cdd079f42b70c323cf971290e0da9be66a15317b609a1f937a892fa7a"
+content-hash = "631c4b1d21036305a2e9c891a631e66106549e1873b91264f481ba29207789fd"
 
 [metadata.files]
 atomicwrites = [
@@ -457,36 +472,32 @@ pytest-cov = [
     {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"},
     {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"},
 ]
-pyyaml = [
+"ruamel.yaml" = [
-    {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"},
+    {file = "ruamel.yaml-0.17.16-py3-none-any.whl", hash = "sha256:ea21da1198c4b41b8e7a259301cc9710d3b972bf8ba52f06218478e6802dd1f1"},
-    {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"},
+    {file = "ruamel.yaml-0.17.16.tar.gz", hash = "sha256:1a771fc92d3823682b7f0893ad56cb5a5c87c48e62b5399d6f42c8759a583b33"},
-    {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"},
+]
-    {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"},
+"ruamel.yaml.clib" = [
-    {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"},
+    {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751"},
-    {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"},
+    {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527"},
-    {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"},
+    {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win32.whl", hash = "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5"},
-    {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"},
+    {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win_amd64.whl", hash = "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c"},
-    {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"},
+    {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502"},
-    {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"},
+    {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78"},
-    {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"},
+    {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win32.whl", hash = "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94"},
-    {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"},
+    {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win_amd64.whl", hash = "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468"},
-    {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"},
+    {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd"},
-    {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"},
+    {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99"},
-    {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"},
+    {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win32.whl", hash = "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb"},
-    {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"},
+    {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe"},
-    {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"},
+    {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233"},
-    {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"},
+    {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84"},
-    {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"},
+    {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win32.whl", hash = "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b"},
-    {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"},
+    {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277"},
-    {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"},
+    {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed"},
-    {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"},
+    {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0"},
-    {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"},
+    {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win32.whl", hash = "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104"},
-    {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"},
+    {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7"},
-    {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"},
+    {file = "ruamel.yaml.clib-0.2.6.tar.gz", hash = "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd"},
-    {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"},
-    {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"},
-    {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"},
-    {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"},
 ]
 six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},

pyproject.toml

@@ -9,7 +9,7 @@ python = "^3.6.1"
 attrs = "^21.2.0"
 click = "^8.0.3"
 pydantic = "^1.8.2"
-PyYAML = "^5.4.1"
+"ruamel.yaml" = "^0.17.16"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"

tests/test_config.py

@@ -1,7 +1,9 @@
+import io
 from ipaddress import IPv4Network
 from pathlib import Path
 
 import pytest
+import ruamel.yaml
 from pydantic import ValidationError
 
 from kiwi_scp.config import Config
@@ -31,6 +33,28 @@ def test_default():
     assert c.network.name == "kiwi_hub"
     assert c.network.cidr == IPv4Network("10.22.46.0/24")
 
+    kiwi_dict = {
+        "version": version,
+        "shells": ["/bin/bash"],
+        "storage": {"directory": "/var/local/kiwi"},
+        "network": {
+            "name": "kiwi_hub",
+            "cidr": "10.22.46.0/24",
+        },
+    }
+    assert c.kiwi_dict == kiwi_dict
+
+    yml = ruamel.yaml.YAML()
+    yml.indent(offset=2)
+
+    sio = io.StringIO()
+    from kiwi_scp.misc import _format_kiwi_yml
+    yml.dump(kiwi_dict, stream=sio, transform=_format_kiwi_yml)
+    yml_string = sio.getvalue()
+    sio.close()
+
+    assert c.kiwi_yml == yml_string
+
 
 #########
 # VERSION
@@ -156,30 +180,79 @@ def test_proj_empty():
 
 
 def test_proj_long():
-    c = Config(projects=[{
+    kiwi_dict = {
         "name": "project",
         "enabled": False,
         "override_storage": {"directory": "/test/directory"},
-    }])
+    }
+    c = Config(projects=[kiwi_dict])
 
     assert len(c.projects) == 1
     p = c.projects[0]
     assert p.name == "project"
     assert not p.enabled
     assert p.override_storage is not None
-    assert p.override_storage.directory == Path("/test/directory")
+
+    assert c.kiwi_dict["projects"][0] == kiwi_dict
+
+
+def test_proj_storage_str():
+    kiwi_dict = {
+        "name": "project",
+        "enabled": False,
+        "override_storage": "/test/directory",
+    }
+    c = Config(projects=[kiwi_dict])
+
+    assert len(c.projects) == 1
+    p = c.projects[0]
+    assert p.name == "project"
+    assert not p.enabled
+    assert p.override_storage is not None
+
+
+def test_proj_storage_list():
+    kiwi_dict = {
+        "name": "project",
+        "enabled": False,
+        "override_storage": ["/test/directory"],
+    }
+    c = Config(projects=[kiwi_dict])
+
+    assert len(c.projects) == 1
+    p = c.projects[0]
+    assert p.name == "project"
+    assert not p.enabled
+    assert p.override_storage is not None
+
+
+def test_proj_storage_invalid():
+    kiwi_dict = {
+        "name": "project",
+        "enabled": False,
+        "override_storage": True,
+    }
+    with pytest.raises(ValidationError) as exc_info:
+        Config(projects=[kiwi_dict])
+
+    assert len(exc_info.value.errors()) == 1
+    error = exc_info.value.errors()[0]
+    assert error["msg"] == "Invalid Storage Format"
+    assert error["type"] == "value_error"
 
 
 def test_proj_short():
-    c = Config(projects=[{
+    kiwi_dict = {
         "project": False,
-    }])
+    }
+    c = Config(projects=[kiwi_dict])
 
     assert len(c.projects) == 1
     p = c.projects[0]
     assert p.name == "project"
     assert not p.enabled
     assert p.override_storage is None
+    assert p.kiwi_dict == kiwi_dict
 
 
 def test_proj_dict():
@@ -252,12 +325,15 @@ def test_env_dict():
 
     assert c.environment == {}
 
-    c = Config(environment={"variable": "value"})
+    kiwi_dict = {"variable": "value"}
+    c = Config(environment=kiwi_dict)
 
     assert len(c.environment) == 1
     assert "variable" in c.environment
     assert c.environment["variable"] == "value"
 
+    assert c.kiwi_dict["environment"] == kiwi_dict
+
 
 def test_env_list():
     c = Config(environment=[])
@@ -348,9 +424,11 @@ def test_storage_empty():
 
 
 def test_storage_dict():
-    c = Config(storage={"directory": "/test/directory"})
+    kiwi_dict = {"directory": "/test/directory"}
+    c = Config(storage=kiwi_dict)
 
     assert c.storage.directory == Path("/test/directory")
+    assert c.storage.kiwi_dict == kiwi_dict
 
 
 def test_storage_invalid_dict():
@@ -400,10 +478,11 @@ def test_network_empty():
 
 
 def test_network_dict():
-    c = Config(network={
+    kiwi_dict = {
         "name": "test_hub",
         "cidr": "1.2.3.4/32",
-    })
+    }
+    c = Config(network=kiwi_dict)
 
     assert c == Config(network={
         "name": "TEST_HUB",
@@ -412,6 +491,7 @@ def test_network_dict():
 
     assert c.network.name == "test_hub"
     assert c.network.cidr == IPv4Network("1.2.3.4/32")
+    assert c.network.kiwi_dict == kiwi_dict
 
 
 def test_network_invalid_dict():