From 3c81021f1483251407e2be7d2f82e92af19e2f68 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 26 Oct 2021 12:19:02 +0200 Subject: [PATCH] pyyaml -> ruamel.yaml, 100% cov on config.py --- example/kiwi.yml | 2 +- kiwi_scp/commands/cmd_init.py | 2 +- kiwi_scp/config.py | 50 ++++++++---------- kiwi_scp/misc.py | 17 +++++- poetry.lock | 81 ++++++++++++++++------------- pyproject.toml | 2 +- tests/test_config.py | 98 +++++++++++++++++++++++++++++++---- 7 files changed, 175 insertions(+), 77 deletions(-) diff --git a/example/kiwi.yml b/example/kiwi.yml index e40d97d..581505a 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -2,7 +2,7 @@ # kiwi-scp instance configuration # ################################### -version: '0.2.0' +version: 0.2.0 shells: - /bin/bash diff --git a/kiwi_scp/commands/cmd_init.py b/kiwi_scp/commands/cmd_init.py index 65a3ca1..c8b68e7 100644 --- a/kiwi_scp/commands/cmd_init.py +++ b/kiwi_scp/commands/cmd_init.py @@ -72,4 +72,4 @@ def cmd(ctx: Instance, output: Path, force: bool, show: bool): # write out the new kiwi.yml with open(ctx.directory.joinpath(KIWI_CONF_NAME), "w") as file: - file.write(Config.parse_obj(kiwi_dict).kiwi_yml) + Config.parse_obj(kiwi_dict).dump_kiwi_yml(file) diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index cc7f895..e117ff6 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -1,19 +1,14 @@ import functools -import re +import io from ipaddress import IPv4Network from pathlib import Path -from typing import Optional, Dict, List, Any +from typing import Optional, Dict, List, Any, TextIO -import yaml +import ruamel.yaml from pydantic import BaseModel, constr, root_validator, validator -from ._constants import RE_SEMVER, RE_VARNAME, HEADER_KIWI_CONF_NAME, KIWI_CONF_NAME - - -# indent yaml lists -class _KiwiDumper(yaml.Dumper): - def increase_indent(self, flow=False, indentless=False): - return super().increase_indent(flow, False) # pragma: no cover +from ._constants import RE_SEMVER, RE_VARNAME, KIWI_CONF_NAME +from .misc import _format_kiwi_yml class _Storage(BaseModel): @@ -75,7 +70,8 @@ class _Project(BaseModel): return {"directory": value[0]} else: - raise ValueError("Invalid Storage Format") + # undefined format + return {} @root_validator(pre=True) @classmethod @@ -141,12 +137,12 @@ class Config(BaseModel): @classmethod @functools.lru_cache(maxsize=5) - def from_instance(cls, instance: Path): + def from_directory(cls, instance: Path): """parses an actual kiwi.yml from disk (cached)""" try: with open(instance.joinpath(KIWI_CONF_NAME)) as kc: - yml = yaml.safe_load(kc) + yml = ruamel.yaml.round_trip_load(kc) return cls.parse_obj(yml) except FileNotFoundError: @@ -184,25 +180,23 @@ class Config(BaseModel): return result - @property - def kiwi_yml(self) -> str: + def dump_kiwi_yml(self, stream: TextIO) -> None: """dump a kiwi.yml file""" - yml_string = yaml.dump( - self.kiwi_dict, - Dumper=_KiwiDumper, - default_flow_style=False, - sort_keys=False, - ) + yml = ruamel.yaml.YAML() + yml.indent(offset=2) + yml.dump(self.kiwi_dict, stream=stream, transform=_format_kiwi_yml) - # insert newline before every main key - yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) + @property + def kiwi_yml(self) -> str: + """get a kiwi.yml dump as a string""" - # load header comment from file - with open(HEADER_KIWI_CONF_NAME, 'r') as stream: - yml_string = stream.read() + yml_string + sio = io.StringIO() + self.dump_kiwi_yml(sio) + result: str = sio.getvalue() + sio.close() - return yml_string + return result @validator("shells", pre=True) @classmethod @@ -341,7 +335,7 @@ class Config(BaseModel): else: # undefined format - raise ValueError("Invalid Storage Format") + return {} @validator("network", pre=True) @classmethod diff --git a/kiwi_scp/misc.py b/kiwi_scp/misc.py index 580d5ef..845acf1 100644 --- a/kiwi_scp/misc.py +++ b/kiwi_scp/misc.py @@ -1,9 +1,12 @@ +import re from typing import Any, Type, List, Callable import attr import click from click.decorators import FC +from ._constants import HEADER_KIWI_CONF_NAME + @attr.s class _MultiDecorator: @@ -19,8 +22,7 @@ class _MultiDecorator: _project_arg = click.argument( "project", required=False, - type=click.Path(exists=True), - default=".", + type=str, ) _service_arg = click.argument( @@ -52,6 +54,17 @@ def user_query(description: str, default: Any, cast_to: Type[Any] = str): click.echo(f"Invalid input: {e}") +def _format_kiwi_yml(yml_string: str): + # insert newline before every main key + yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) + + # load header comment from file + with open(HEADER_KIWI_CONF_NAME, 'r') as stream: + yml_string = stream.read() + yml_string + + return yml_string + + def _surround(string, bang): midlane = f"{bang * 3} {string} {bang * 3}" sidelane = bang * len(midlane) diff --git a/poetry.lock b/poetry.lock index 5c3cb7e..eb51feb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -248,12 +248,27 @@ pytest = ">=4.6" testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] [[package]] -name = "pyyaml" -version = "5.4.1" -description = "YAML parser and emitter for Python" +name = "ruamel.yaml" +version = "0.17.16" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3" + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.1.2", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.10\""} + +[package.extras] +docs = ["ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel.yaml.clib" +version = "0.2.6" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +category = "main" +optional = false +python-versions = ">=3.5" [[package]] name = "six" @@ -323,7 +338,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6.1" -content-hash = "a5e47a9cdd079f42b70c323cf971290e0da9be66a15317b609a1f937a892fa7a" +content-hash = "631c4b1d21036305a2e9c891a631e66106549e1873b91264f481ba29207789fd" [metadata.files] atomicwrites = [ @@ -457,36 +472,32 @@ pytest-cov = [ {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, ] -pyyaml = [ - {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, - {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, - {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, - {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, - {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, - {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, - {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, - {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, - {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, +"ruamel.yaml" = [ + {file = "ruamel.yaml-0.17.16-py3-none-any.whl", hash = "sha256:ea21da1198c4b41b8e7a259301cc9710d3b972bf8ba52f06218478e6802dd1f1"}, + {file = "ruamel.yaml-0.17.16.tar.gz", hash = "sha256:1a771fc92d3823682b7f0893ad56cb5a5c87c48e62b5399d6f42c8759a583b33"}, +] +"ruamel.yaml.clib" = [ + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win32.whl", hash = "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win_amd64.whl", hash = "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win32.whl", hash = "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win_amd64.whl", hash = "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win32.whl", hash = "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win32.whl", hash = "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win32.whl", hash = "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7"}, + {file = "ruamel.yaml.clib-0.2.6.tar.gz", hash = "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, diff --git a/pyproject.toml b/pyproject.toml index f65fb53..b9b0370 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ python = "^3.6.1" attrs = "^21.2.0" click = "^8.0.3" pydantic = "^1.8.2" -PyYAML = "^5.4.1" +"ruamel.yaml" = "^0.17.16" [tool.poetry.dev-dependencies] pytest = "^6.2.5" diff --git a/tests/test_config.py b/tests/test_config.py index 6871cd2..e6b6e42 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,7 +1,9 @@ +import io from ipaddress import IPv4Network from pathlib import Path import pytest +import ruamel.yaml from pydantic import ValidationError from kiwi_scp.config import Config @@ -31,6 +33,28 @@ def test_default(): assert c.network.name == "kiwi_hub" assert c.network.cidr == IPv4Network("10.22.46.0/24") + kiwi_dict = { + "version": version, + "shells": ["/bin/bash"], + "storage": {"directory": "/var/local/kiwi"}, + "network": { + "name": "kiwi_hub", + "cidr": "10.22.46.0/24", + }, + } + assert c.kiwi_dict == kiwi_dict + + yml = ruamel.yaml.YAML() + yml.indent(offset=2) + + sio = io.StringIO() + from kiwi_scp.misc import _format_kiwi_yml + yml.dump(kiwi_dict, stream=sio, transform=_format_kiwi_yml) + yml_string = sio.getvalue() + sio.close() + + assert c.kiwi_yml == yml_string + ######### # VERSION @@ -156,30 +180,79 @@ def test_proj_empty(): def test_proj_long(): - c = Config(projects=[{ + kiwi_dict = { "name": "project", "enabled": False, "override_storage": {"directory": "/test/directory"}, - }]) + } + c = Config(projects=[kiwi_dict]) assert len(c.projects) == 1 p = c.projects[0] assert p.name == "project" assert not p.enabled assert p.override_storage is not None - assert p.override_storage.directory == Path("/test/directory") + + assert c.kiwi_dict["projects"][0] == kiwi_dict + + +def test_proj_storage_str(): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": "/test/directory", + } + c = Config(projects=[kiwi_dict]) + + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert not p.enabled + assert p.override_storage is not None + + +def test_proj_storage_list(): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": ["/test/directory"], + } + c = Config(projects=[kiwi_dict]) + + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert not p.enabled + assert p.override_storage is not None + + +def test_proj_storage_invalid(): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": True, + } + with pytest.raises(ValidationError) as exc_info: + Config(projects=[kiwi_dict]) + + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Storage Format" + assert error["type"] == "value_error" def test_proj_short(): - c = Config(projects=[{ + kiwi_dict = { "project": False, - }]) + } + c = Config(projects=[kiwi_dict]) assert len(c.projects) == 1 p = c.projects[0] assert p.name == "project" assert not p.enabled assert p.override_storage is None + assert p.kiwi_dict == kiwi_dict def test_proj_dict(): @@ -252,12 +325,15 @@ def test_env_dict(): assert c.environment == {} - c = Config(environment={"variable": "value"}) + kiwi_dict = {"variable": "value"} + c = Config(environment=kiwi_dict) assert len(c.environment) == 1 assert "variable" in c.environment assert c.environment["variable"] == "value" + assert c.kiwi_dict["environment"] == kiwi_dict + def test_env_list(): c = Config(environment=[]) @@ -348,9 +424,11 @@ def test_storage_empty(): def test_storage_dict(): - c = Config(storage={"directory": "/test/directory"}) + kiwi_dict = {"directory": "/test/directory"} + c = Config(storage=kiwi_dict) assert c.storage.directory == Path("/test/directory") + assert c.storage.kiwi_dict == kiwi_dict def test_storage_invalid_dict(): @@ -400,10 +478,11 @@ def test_network_empty(): def test_network_dict(): - c = Config(network={ + kiwi_dict = { "name": "test_hub", "cidr": "1.2.3.4/32", - }) + } + c = Config(network=kiwi_dict) assert c == Config(network={ "name": "TEST_HUB", @@ -412,6 +491,7 @@ def test_network_dict(): assert c.network.name == "test_hub" assert c.network.cidr == IPv4Network("1.2.3.4/32") + assert c.network.kiwi_dict == kiwi_dict def test_network_invalid_dict():