From 5ffb9cc41bd8a4f0de06bfbb257faa361c33a16c Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Mon, 11 Oct 2021 02:58:05 +0200 Subject: [PATCH 001/141] dist scripts, search upwards for kiwi instance --- .dockerignore | 10 ++ bump-version.sh => dist/bump-version.sh | 0 install.sh => dist/install.sh | 0 kiwi => dist/kiwi | 20 ++- kiwi_scp/data/etc/version_tag | 1 - poetry.lock | 225 +++++++++++++++++++++++- pyproject.toml | 3 +- 7 files changed, 249 insertions(+), 10 deletions(-) create mode 100644 .dockerignore rename bump-version.sh => dist/bump-version.sh (100%) rename install.sh => dist/install.sh (100%) rename kiwi => dist/kiwi (90%) delete mode 100644 kiwi_scp/data/etc/version_tag diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3930b8b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.git/ +.idea/ + +dist/ +example/ + +Dockerfile +.dockerignore +.gitignore +.drone.yml \ No newline at end of file diff --git a/bump-version.sh b/dist/bump-version.sh similarity index 100% rename from bump-version.sh rename to dist/bump-version.sh diff --git a/install.sh b/dist/install.sh similarity index 100% rename from install.sh rename to dist/install.sh diff --git a/kiwi b/dist/kiwi similarity index 90% rename from kiwi rename to dist/kiwi index 4038eb4..012fda1 100755 --- a/kiwi +++ b/dist/kiwi @@ -156,12 +156,20 @@ if [ "${run_kiwi_check}" = "yes" ]; then chmod 0777 "${CANARY_FILENAME}" fi -# check if pwd is a kiwi folder -if [ -f "./${KIWI_CONF_NAME}" ]; then - # determine needed kiwi-scp version - re_version_line='version\s*:\s*' - eval "$(grep -E "${re_version_line}" "./${KIWI_CONF_NAME}" | sed -r "s/${re_version_line}/KIWI_VERSION=/")" -fi +# check if pwd is a kiwi instance +path="$(pwd)" +while [ "${path}" != "" ]; do + if [ -e "${path}/${KIWI_CONF_NAME}" ]; then + # cd into kiwi instance + cd "${path}" || yes_no "Could not enter kiwi instance at '${path}'. Continue anyway?" || exit 1 + + # determine needed kiwi-scp version + re_version_line='version\s*:\s*' + eval "$(grep -E "${re_version_line}" "./${KIWI_CONF_NAME}" | sed -r "s/${re_version_line}/KIWI_VERSION=/")" + break; + fi + path="${path%/*}" +done # install if kiwi-scp not found if [ ! -x "$(kiwi_executable)" ]; then diff --git a/kiwi_scp/data/etc/version_tag b/kiwi_scp/data/etc/version_tag deleted file mode 100644 index 1180819..0000000 --- a/kiwi_scp/data/etc/version_tag +++ /dev/null @@ -1 +0,0 @@ -0.1.7 diff --git a/poetry.lock b/poetry.lock index dd53b7f..4a2f0af 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,106 @@ +[[package]] +name = "backports.entry-points-selectable" +version = "1.1.0" +description = "Compatibility shim providing selectable entry points for older implementations" +category = "dev" +optional = false +python-versions = ">=2.7" + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3.7)", "pytest-mypy", "pytest-checkdocs (>=2.4)", "pytest-enabler (>=1.0.1)"] + +[[package]] +name = "dataclasses" +version = "0.8" +description = "A backport of the dataclasses module for Python 3.6" +category = "main" +optional = false +python-versions = ">=3.6, <3.7" + +[[package]] +name = "distlib" +version = "0.3.3" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "filelock" +version = "3.3.0" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] +testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] + +[[package]] +name = "importlib-metadata" +version = "4.8.1" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +perf = ["ipython"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] + +[[package]] +name = "importlib-resources" +version = "5.2.2" +description = "Read resources from Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[[package]] +name = "platformdirs" +version = "2.4.0" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] +test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] + +[[package]] +name = "pydantic" +version = "1.8.2" +description = "Data validation and settings management using python 3.6 type hinting" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "pyyaml" version = "5.4.1" @@ -6,12 +109,113 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "typing-extensions" +version = "3.10.0.2" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "virtualenv" +version = "20.8.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +"backports.entry-points-selectable" = ">=1.0.4" +distlib = ">=0.3.1,<1" +filelock = ">=3.0.0,<4" +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +importlib-resources = {version = ">=1.0", markers = "python_version < \"3.7\""} +platformdirs = ">=2,<3" +six = ">=1.9.0,<2" + +[package.extras] +docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] + +[[package]] +name = "zipp" +version = "3.6.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] + [metadata] lock-version = "1.1" -python-versions = "^3.6" -content-hash = "36970da0e8c6151dcf68abd9008ecef35673f04db53952bfb3fd7544c0516b7f" +python-versions = "^3.6.1" +content-hash = "c449a1d6048636533037af38cb647b63b19f8027b0a12d82408dd05ed4164c06" [metadata.files] +"backports.entry-points-selectable" = [ + {file = "backports.entry_points_selectable-1.1.0-py2.py3-none-any.whl", hash = "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc"}, + {file = "backports.entry_points_selectable-1.1.0.tar.gz", hash = "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a"}, +] +dataclasses = [ + {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, + {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, +] +distlib = [ + {file = "distlib-0.3.3-py2.py3-none-any.whl", hash = "sha256:c8b54e8454e5bf6237cc84c20e8264c3e991e824ef27e8f1e81049867d861e31"}, + {file = "distlib-0.3.3.zip", hash = "sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05"}, +] +filelock = [ + {file = "filelock-3.3.0-py3-none-any.whl", hash = "sha256:bbc6a0382fe8ec4744ecdf6683a2e07f65eb10ff1aff53fc02a202565446cde0"}, + {file = "filelock-3.3.0.tar.gz", hash = "sha256:8c7eab13dc442dc249e95158bcc12dec724465919bdc9831fdbf0660f03d1785"}, +] +importlib-metadata = [ + {file = "importlib_metadata-4.8.1-py3-none-any.whl", hash = "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15"}, + {file = "importlib_metadata-4.8.1.tar.gz", hash = "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1"}, +] +importlib-resources = [ + {file = "importlib_resources-5.2.2-py3-none-any.whl", hash = "sha256:2480d8e07d1890056cb53c96e3de44fead9c62f2ba949b0f2e4c4345f4afa977"}, + {file = "importlib_resources-5.2.2.tar.gz", hash = "sha256:a65882a4d0fe5fbf702273456ba2ce74fe44892c25e42e057aca526b702a6d4b"}, +] +platformdirs = [ + {file = "platformdirs-2.4.0-py3-none-any.whl", hash = "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"}, + {file = "platformdirs-2.4.0.tar.gz", hash = "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2"}, +] +pydantic = [ + {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e"}, + {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840"}, + {file = "pydantic-1.8.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b"}, + {file = "pydantic-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1"}, + {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23"}, + {file = "pydantic-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287"}, + {file = "pydantic-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e"}, + {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820"}, + {file = "pydantic-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3"}, + {file = "pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f"}, + {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b"}, + {file = "pydantic-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3"}, + {file = "pydantic-1.8.2-py3-none-any.whl", hash = "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833"}, + {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, +] pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, @@ -43,3 +247,20 @@ pyyaml = [ {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +typing-extensions = [ + {file = "typing_extensions-3.10.0.2-py2-none-any.whl", hash = "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7"}, + {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, + {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, +] +virtualenv = [ + {file = "virtualenv-20.8.1-py2.py3-none-any.whl", hash = "sha256:10062e34c204b5e4ec5f62e6ef2473f8ba76513a9a617e873f1f8fb4a519d300"}, + {file = "virtualenv-20.8.1.tar.gz", hash = "sha256:bcc17f0b3a29670dd777d6f0755a4c04f28815395bca279cdcb213b97199a6b8"}, +] +zipp = [ + {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, + {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, +] diff --git a/pyproject.toml b/pyproject.toml index 9e5d4ca..1cb8df1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,10 +5,11 @@ description = "kiwi is the simple tool for managing container servers." authors = ["ldericher <40151420+ldericher@users.noreply.github.com>"] [tool.poetry.dependencies] -python = "^3.6" +python = "^3.6.1" PyYAML = "^5.4.1" [tool.poetry.dev-dependencies] +virtualenv = "^20.8.1" [tool.poetry.scripts] kiwi = "kiwi_scp.scripts.kiwi:main" From c0edd3bcc576cb0ec934a25407e54a012068ad27 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Mon, 11 Oct 2021 02:58:49 +0200 Subject: [PATCH 002/141] new kiwi.yml format --- .idea/runConfigurations/kiwi_next.xml | 23 +++ kiwi_scp/__init__.py | 20 --- kiwi_scp/config.py | 208 ++++++++------------------ kiwi_scp/data/etc/kiwi_default.yml | 19 +-- kiwi_scp/scripts/kiwi_next.py | 14 ++ pyproject.toml | 1 + 6 files changed, 113 insertions(+), 172 deletions(-) create mode 100644 .idea/runConfigurations/kiwi_next.xml create mode 100644 kiwi_scp/scripts/kiwi_next.py diff --git a/.idea/runConfigurations/kiwi_next.xml b/.idea/runConfigurations/kiwi_next.xml new file mode 100644 index 0000000..9b3bd56 --- /dev/null +++ b/.idea/runConfigurations/kiwi_next.xml @@ -0,0 +1,23 @@ + + + + + \ No newline at end of file diff --git a/kiwi_scp/__init__.py b/kiwi_scp/__init__.py index ba8912f..e69de29 100644 --- a/kiwi_scp/__init__.py +++ b/kiwi_scp/__init__.py @@ -1,20 +0,0 @@ -# local -from .parser import Parser -from .runner import Runner - - -def verbosity(): - # ensure singleton is instantiated: runs subcommand setup routines - _ = Runner() - return Parser().get_args().verbosity - - -def run(): - # pass down - return Runner().run() - - -__all__ = [ - 'verbosity', - 'run' -] diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index 00c0dfc..b276fa6 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -1,157 +1,79 @@ -# system -import copy -import logging -import os import re -import yaml +from typing import Optional, Dict, List -# local -from ._constants import KIWI_CONF_NAME, HEADER_KIWI_CONF_NAME, DEFAULT_KIWI_CONF_NAME, VERSION_TAG_NAME +import pydantic -class Config: +class _Storage(pydantic.BaseModel): + """a storage subsection""" + + directory: str + + +class _Project(pydantic.BaseModel): + """a project subsection""" + + name: str + enabled: bool = True + storage: Optional[_Storage] + + @pydantic.root_validator(pre=True) + @classmethod + def check_grammar(cls, values): + if isinstance(values, dict): + if "name" in values: + return values + + elif len(values) == 1: + name, enabled = list(values.items())[0] + return {"name": name, "enabled": True if enabled is None else enabled} + + elif isinstance(values, str): + return {"name": values} + + +class _Network(pydantic.BaseModel): + """a network subsection""" + + name: str + cidr: str + + +class Config(pydantic.BaseModel): """represents a kiwi.yml""" - __yml_content = {} - __keys = { - 'version': "kiwi-scp version to use in this instance", + version: str + shells: Optional[List[str]] + environment: Optional[Dict[str, Optional[str]]] - 'runtime:storage': "local directory for service data", - 'runtime:shells': "shell preference for working in service containers", - 'runtime:env': "common environment for compose yml", - - 'markers:project': "marker string for project directories", - 'markers:disabled': "marker string for disabled projects", - - 'network:name': "name for local network hub", - 'network:cidr': "CIDR block for local network hub", - } - - def __key_resolve(self, key): - """ - Resolve nested dictionaries - - If __yml_content is {'a': {'b': {'c': "val"}}} and key is 'a:b:c', - this returns a single dict {'c': "val"} and the direct key 'c' - """ - - # "a:b:c" => path = ['a', 'b'], key = 'c' - path = key.split(':') - path, key = path[:-1], path[-1] - - # resolve path - container = self.__yml_content - for step in path: - container = container[step] - - return container, key - - def __getitem__(self, key): - """array-like read access to __yml_content""" - - container, key = self.__key_resolve(key) - return container[key] - - def __setitem__(self, key, value): - """array-like write access to __yml_content""" - - container, key = self.__key_resolve(key) - container[key] = value - - def __str__(self): - """dump into textual representation""" - - # dump yml content - yml_string = yaml.dump( - self.__yml_content, - default_flow_style=False, sort_keys=False - ).strip() - - # insert newline before every main key - yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) - - # load header comment from file - with open(HEADER_KIWI_CONF_NAME, 'r') as stream: - yml_string = stream.read() + yml_string - - return yml_string - - def _update_from_file(self, filename): - """return a copy updated using a kiwi.yml file""" - - with open(filename, 'r') as stream: - try: - # create copy - result = Config() - result.__yml_content = copy.deepcopy(self.__yml_content) - - # read file - logging.debug(f"Reading '{filename}' into '{id(result.__yml_content)}'") - result.__yml_content.update(yaml.safe_load(stream)) - - return result - except yaml.YAMLError as exc: - logging.error(exc) - - def user_query(self, key): - """query user for new config value""" - - # prompt user as per argument - try: - result = input(f"Enter {self.__keys[key]} [{self[key]}] ").strip() - except EOFError: - print() - result = None - - # store result if present - if result: - self[key] = result - - def save(self): - """save current yml representation in current directory's kiwi.yml""" - - with open(KIWI_CONF_NAME, 'w') as stream: - stream.write(str(self)) - stream.write('\n') - - -class DefaultConfig(Config): - """Singleton: The default kiwi.yml file""" - - __instance = None + projects: Optional[List[_Project]] + storage: _Storage + network: _Network + @pydantic.validator("version") @classmethod - def get(cls): - if cls.__instance is None: - # create singleton - cls.__instance = cls()._update_from_file(DEFAULT_KIWI_CONF_NAME) + def check_version(cls, value: str) -> str: + if not re.match(r"^[0-9]+(\.[0-9]+(\.[0-9]+)?)?$", value): + raise ValueError - # add version data from separate file (keeps default config cleaner) - with open(VERSION_TAG_NAME, 'r') as stream: - cls.__instance['version'] = stream.read().strip() - - # return singleton - return cls.__instance - - -class LoadedConfig(Config): - """Singleton collection: kiwi.yml files by path""" - - __instances = {} + return value + @pydantic.validator("environment", pre=True) @classmethod - def get(cls, directory='.'): - if directory not in LoadedConfig.__instances: - # create singleton for new path - result = DefaultConfig.get() + def unify_env(cls, value) -> Optional[Dict[str, Optional[str]]]: + if isinstance(value, dict): + return value + elif isinstance(value, list): + result: Dict[str, Optional[str]] = {} + for item in value: + idx = item.find("=") + if idx == -1: + key, value = item, None + else: + key, value = item[:idx], item[idx + 1:] - # update with that dir's kiwi.yml - try: - result = result._update_from_file(os.path.join(directory, KIWI_CONF_NAME)) - except FileNotFoundError: - logging.info(f"No '{KIWI_CONF_NAME}' found at '{directory}'. Using defaults.") + result[key] = value - LoadedConfig.__instances[directory] = result - - # return singleton - return LoadedConfig.__instances[directory] + return result + else: + return None diff --git a/kiwi_scp/data/etc/kiwi_default.yml b/kiwi_scp/data/etc/kiwi_default.yml index 1404379..44766a7 100644 --- a/kiwi_scp/data/etc/kiwi_default.yml +++ b/kiwi_scp/data/etc/kiwi_default.yml @@ -1,12 +1,13 @@ -version: -runtime: - storage: /var/kiwi - shells: - - /bin/bash - env: null -markers: - project: .project - disabled: .disabled +version: 0.2.0 +shells: + - /bin/bash +projects: + - name: admin + enabled: true + - test: + - test2: false +storage: + directory: /var/local/kiwi network: name: kiwi_hub cidr: 10.22.46.0/24 diff --git a/kiwi_scp/scripts/kiwi_next.py b/kiwi_scp/scripts/kiwi_next.py new file mode 100644 index 0000000..22bb8a2 --- /dev/null +++ b/kiwi_scp/scripts/kiwi_next.py @@ -0,0 +1,14 @@ +from kiwi_scp.config import Config +import yaml + + +def main(): + with open("./kiwi_scp/data/etc/kiwi_default.yml") as kc: + yml = yaml.safe_load(kc) + kiwi = Config(**yml) + + print(repr(kiwi)) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 1cb8df1..c7fe7ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ authors = ["ldericher <40151420+ldericher@users.noreply.github.com>"] [tool.poetry.dependencies] python = "^3.6.1" PyYAML = "^5.4.1" +pydantic = "^1.8.2" [tool.poetry.dev-dependencies] virtualenv = "^20.8.1" From 370097e5c38043f876e8112c72aab95b213a9d7a Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 12 Oct 2021 19:06:49 +0200 Subject: [PATCH 003/141] config.py constraints and doc --- kiwi_scp/_constants.py | 12 ++++ kiwi_scp/config.py | 106 ++++++++++++++++++++--------- kiwi_scp/data/etc/kiwi_default.yml | 2 +- 3 files changed, 86 insertions(+), 34 deletions(-) diff --git a/kiwi_scp/_constants.py b/kiwi_scp/_constants.py index cd12c09..62a46f9 100644 --- a/kiwi_scp/_constants.py +++ b/kiwi_scp/_constants.py @@ -1,6 +1,18 @@ # system import os +############# +# REGEX PARTS + +# regex part for a number with no leading zeroes +_RE_NUMBER: str = r"[0-9]|[1-9][0-9]*" + +# regex for a semantic version string +RE_SEMVER = rf"^{_RE_NUMBER}(?:\.{_RE_NUMBER}(?:\.{_RE_NUMBER})?)?$" + +# regex for a lowercase variable name +RE_VARNAME = r"^[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9])$" + ############# # ENVIRONMENT diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index b276fa6..ea5ba12 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -1,48 +1,67 @@ -import re +from ipaddress import IPv4Network +from pathlib import Path from typing import Optional, Dict, List import pydantic +from ._constants import RE_SEMVER, RE_VARNAME + class _Storage(pydantic.BaseModel): """a storage subsection""" - directory: str + directory: Path class _Project(pydantic.BaseModel): """a project subsection""" - name: str + name: pydantic.constr( + regex=RE_VARNAME + ) enabled: bool = True - storage: Optional[_Storage] + override_storage: Optional[_Storage] @pydantic.root_validator(pre=True) @classmethod - def check_grammar(cls, values): - if isinstance(values, dict): - if "name" in values: - return values + def unify_project(cls, values): + """parse different project notations""" - elif len(values) == 1: - name, enabled = list(values.items())[0] - return {"name": name, "enabled": True if enabled is None else enabled} + if "name" in values: + # default format + return values - elif isinstance(values, str): - return {"name": values} + elif len(values) == 1: + # short format: + # - : + + name, enabled = list(values.items())[0] + return { + "name": name, + "enabled": True if enabled is None else enabled + } + + else: + # undefined format + raise ValueError class _Network(pydantic.BaseModel): """a network subsection""" - name: str - cidr: str + name: pydantic.constr( + to_lower=True, + regex=RE_VARNAME + ) + cidr: IPv4Network class Config(pydantic.BaseModel): """represents a kiwi.yml""" - version: str + version: pydantic.constr( + regex=RE_SEMVER + ) shells: Optional[List[str]] environment: Optional[Dict[str, Optional[str]]] @@ -50,30 +69,51 @@ class Config(pydantic.BaseModel): storage: _Storage network: _Network - @pydantic.validator("version") - @classmethod - def check_version(cls, value: str) -> str: - if not re.match(r"^[0-9]+(\.[0-9]+(\.[0-9]+)?)?$", value): - raise ValueError - - return value - @pydantic.validator("environment", pre=True) @classmethod - def unify_env(cls, value) -> Optional[Dict[str, Optional[str]]]: - if isinstance(value, dict): + def unify_environment(cls, value) -> Optional[Dict[str, Optional[str]]]: + """parse different environment notations""" + + def parse_str(var_val: str) -> (str, Optional[str]): + """parse a "=" string""" + + idx = var_val.find("=") + if idx == -1: + # don't split, just define the variable + return var_val, None + else: + # split string, set variable to value + return var_val[:idx], var_val[idx + 1:] + + if value is None: + # empty environment + return None + + elif isinstance(value, dict): + # native dict format return value + elif isinstance(value, list): + # list format (multiple strings) + result: Dict[str, Optional[str]] = {} for item in value: - idx = item.find("=") - if idx == -1: - key, value = item, None - else: - key, value = item[:idx], item[idx + 1:] - + key, value = parse_str(item) result[key] = value return result + + elif isinstance(value, str): + # string format (single variable): + # "=" + + key, value = parse_str(value) + return {key: value} + + elif isinstance(value, int): + # integer format (just define single oddly named variable) + return {str(value): None} + else: - return None + # undefined format + raise ValueError diff --git a/kiwi_scp/data/etc/kiwi_default.yml b/kiwi_scp/data/etc/kiwi_default.yml index 44766a7..bb8335f 100644 --- a/kiwi_scp/data/etc/kiwi_default.yml +++ b/kiwi_scp/data/etc/kiwi_default.yml @@ -4,7 +4,7 @@ shells: projects: - name: admin enabled: true - - test: + - Test: - test2: false storage: directory: /var/local/kiwi From af02c410e5d985f12b60d95f8f06e7539171a857 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 12 Oct 2021 19:08:35 +0200 Subject: [PATCH 004/141] config.py imports --- kiwi_scp/config.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index ea5ba12..9efe0e2 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -2,27 +2,27 @@ from ipaddress import IPv4Network from pathlib import Path from typing import Optional, Dict, List -import pydantic +from pydantic import BaseModel, constr, root_validator, validator from ._constants import RE_SEMVER, RE_VARNAME -class _Storage(pydantic.BaseModel): +class _Storage(BaseModel): """a storage subsection""" directory: Path -class _Project(pydantic.BaseModel): +class _Project(BaseModel): """a project subsection""" - name: pydantic.constr( + name: constr( regex=RE_VARNAME ) enabled: bool = True override_storage: Optional[_Storage] - @pydantic.root_validator(pre=True) + @root_validator(pre=True) @classmethod def unify_project(cls, values): """parse different project notations""" @@ -46,20 +46,20 @@ class _Project(pydantic.BaseModel): raise ValueError -class _Network(pydantic.BaseModel): +class _Network(BaseModel): """a network subsection""" - name: pydantic.constr( + name: constr( to_lower=True, regex=RE_VARNAME ) cidr: IPv4Network -class Config(pydantic.BaseModel): +class Config(BaseModel): """represents a kiwi.yml""" - version: pydantic.constr( + version: constr( regex=RE_SEMVER ) shells: Optional[List[str]] @@ -69,7 +69,7 @@ class Config(pydantic.BaseModel): storage: _Storage network: _Network - @pydantic.validator("environment", pre=True) + @validator("environment", pre=True) @classmethod def unify_environment(cls, value) -> Optional[Dict[str, Optional[str]]]: """parse different environment notations""" From e42b426f2e3e99a21fb55b67463680c880e80647 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 12 Oct 2021 19:17:28 +0200 Subject: [PATCH 005/141] remove need for kiwi_default.yml --- example/kiwi.yml | 16 +++++++-------- kiwi_scp/_constants.py | 2 +- kiwi_scp/config.py | 32 ++++++++++++++++++------------ kiwi_scp/data/etc/kiwi_default.yml | 13 ------------ kiwi_scp/scripts/kiwi_next.py | 5 ++++- 5 files changed, 32 insertions(+), 36 deletions(-) delete mode 100644 kiwi_scp/data/etc/kiwi_default.yml diff --git a/example/kiwi.yml b/example/kiwi.yml index 1eb6293..e40d97d 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -2,17 +2,17 @@ # kiwi-scp instance configuration # ################################### -version: '0.1.7' +version: '0.2.0' -runtime: - storage: /tmp/kiwi - shells: +shells: - /bin/bash - env: null -markers: - project: .project - disabled: .disabled +projects: + - name: hello-world.project + enabled: true + +storage: + directory: /var/local/kiwi network: name: kiwi_hub diff --git a/kiwi_scp/_constants.py b/kiwi_scp/_constants.py index 62a46f9..54b9607 100644 --- a/kiwi_scp/_constants.py +++ b/kiwi_scp/_constants.py @@ -11,7 +11,7 @@ _RE_NUMBER: str = r"[0-9]|[1-9][0-9]*" RE_SEMVER = rf"^{_RE_NUMBER}(?:\.{_RE_NUMBER}(?:\.{_RE_NUMBER})?)?$" # regex for a lowercase variable name -RE_VARNAME = r"^[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9])$" +RE_VARNAME = r"^[A-Za-z](?:[A-Za-z0-9\._-]*[A-Za-z0-9])$" ############# # ENVIRONMENT diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index 9efe0e2..792fd4c 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -49,29 +49,35 @@ class _Project(BaseModel): class _Network(BaseModel): """a network subsection""" - name: constr( - to_lower=True, - regex=RE_VARNAME - ) + name: constr(to_lower=True, regex=RE_VARNAME) cidr: IPv4Network class Config(BaseModel): """represents a kiwi.yml""" - version: constr( - regex=RE_SEMVER - ) - shells: Optional[List[str]] - environment: Optional[Dict[str, Optional[str]]] + version: constr(regex=RE_SEMVER) = "0.2.0" + + shells: Optional[List[str]] = [ + "/bin/bash", + ] + + environment: Dict[str, Optional[str]] = {} projects: Optional[List[_Project]] - storage: _Storage - network: _Network + + storage: _Storage = _Storage.parse_obj({ + "directory": "/var/local/kiwi", + }) + + network: _Network = _Network.parse_obj({ + "name": "kiwi_hub", + "cidr": "10.22.46.0/24", + }) @validator("environment", pre=True) @classmethod - def unify_environment(cls, value) -> Optional[Dict[str, Optional[str]]]: + def unify_environment(cls, value) -> Dict[str, Optional[str]]: """parse different environment notations""" def parse_str(var_val: str) -> (str, Optional[str]): @@ -87,7 +93,7 @@ class Config(BaseModel): if value is None: # empty environment - return None + return {} elif isinstance(value, dict): # native dict format diff --git a/kiwi_scp/data/etc/kiwi_default.yml b/kiwi_scp/data/etc/kiwi_default.yml deleted file mode 100644 index bb8335f..0000000 --- a/kiwi_scp/data/etc/kiwi_default.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: 0.2.0 -shells: - - /bin/bash -projects: - - name: admin - enabled: true - - Test: - - test2: false -storage: - directory: /var/local/kiwi -network: - name: kiwi_hub - cidr: 10.22.46.0/24 diff --git a/kiwi_scp/scripts/kiwi_next.py b/kiwi_scp/scripts/kiwi_next.py index 22bb8a2..3650070 100644 --- a/kiwi_scp/scripts/kiwi_next.py +++ b/kiwi_scp/scripts/kiwi_next.py @@ -3,12 +3,15 @@ import yaml def main(): - with open("./kiwi_scp/data/etc/kiwi_default.yml") as kc: + with open("./example/kiwi.yml") as kc: yml = yaml.safe_load(kc) kiwi = Config(**yml) print(repr(kiwi)) + kiwi = Config() + print(repr(kiwi)) + if __name__ == "__main__": main() From c08d44edb32805db891730d13806d5b12ecf9dcc Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Wed, 13 Oct 2021 02:16:09 +0200 Subject: [PATCH 006/141] errata --- dist/bump-version.sh | 7 ++++--- kiwi_scp/_constants.py | 2 +- kiwi_scp/config.py | 18 +++++++++--------- pyproject.toml | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/dist/bump-version.sh b/dist/bump-version.sh index 7c4b7b3..5d57040 100755 --- a/dist/bump-version.sh +++ b/dist/bump-version.sh @@ -6,7 +6,8 @@ this_dir="$(dirname "${this}")" git_branch="$(git rev-parse --abbrev-ref HEAD)" git_tag="$(git describe --abbrev=0)" version_str="${git_branch##*/}" +version_str="0.2.0" -echo "${version_str}" > "${this_dir}/kiwi_scp/data/etc/version_tag" -sed -ri "s/(version\s*:).*$/\1 '${version_str}'/" "${this_dir}/example/kiwi.yml" -sed -ri "s/(version\s*=\s*).*$/\1\"${version_str}\"/" "${this_dir}/pyproject.toml" +sed -ri "s/(version\s*:).*$/\1 '${version_str}'/" "${this_dir}/../example/kiwi.yml" +sed -ri "s/(version\s*=\s*).*$/\1\"${version_str}\"/" "${this_dir}/../pyproject.toml" +sed -ri "s/(version.*=\s*).*$/\1\"${version_str}\"/" "${this_dir}/../kiwi_scp/config.py" diff --git a/kiwi_scp/_constants.py b/kiwi_scp/_constants.py index 54b9607..e0f23c2 100644 --- a/kiwi_scp/_constants.py +++ b/kiwi_scp/_constants.py @@ -10,7 +10,7 @@ _RE_NUMBER: str = r"[0-9]|[1-9][0-9]*" # regex for a semantic version string RE_SEMVER = rf"^{_RE_NUMBER}(?:\.{_RE_NUMBER}(?:\.{_RE_NUMBER})?)?$" -# regex for a lowercase variable name +# regex for a variable name RE_VARNAME = r"^[A-Za-z](?:[A-Za-z0-9\._-]*[A-Za-z0-9])$" ############# diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index 792fd4c..37a020b 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -58,22 +58,22 @@ class Config(BaseModel): version: constr(regex=RE_SEMVER) = "0.2.0" - shells: Optional[List[str]] = [ - "/bin/bash", + shells: Optional[List[Path]] = [ + Path("/bin/bash"), ] environment: Dict[str, Optional[str]] = {} projects: Optional[List[_Project]] - storage: _Storage = _Storage.parse_obj({ - "directory": "/var/local/kiwi", - }) + storage: _Storage = _Storage( + directory="/var/local/kiwi", + ) - network: _Network = _Network.parse_obj({ - "name": "kiwi_hub", - "cidr": "10.22.46.0/24", - }) + network: _Network = _Network( + name="kiwi_hub", + cidr="10.22.46.0/24", + ) @validator("environment", pre=True) @classmethod diff --git a/pyproject.toml b/pyproject.toml index c7fe7ea..e1ab88b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "kiwi-scp" -version = "0.1.7" +version = "0.2.0" description = "kiwi is the simple tool for managing container servers." authors = ["ldericher <40151420+ldericher@users.noreply.github.com>"] From 175382184635303f74c9bb7b541945f090248fd2 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Wed, 13 Oct 2021 03:05:46 +0200 Subject: [PATCH 007/141] dump kiwi.yml --- kiwi_scp/config.py | 91 ++++++++++++++++++++++++++++++++--- kiwi_scp/scripts/kiwi_next.py | 4 +- 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index 37a020b..0cbf6d1 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -1,10 +1,18 @@ +import re from ipaddress import IPv4Network from pathlib import Path -from typing import Optional, Dict, List +from typing import Optional, Dict, List, Any +import yaml from pydantic import BaseModel, constr, root_validator, validator -from ._constants import RE_SEMVER, RE_VARNAME +from ._constants import RE_SEMVER, RE_VARNAME, HEADER_KIWI_CONF_NAME + + +# indent yaml lists +class _KiwiDumper(yaml.Dumper): + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) class _Storage(BaseModel): @@ -12,16 +20,32 @@ class _Storage(BaseModel): directory: Path + @property + def kiwi_dict(self) -> Dict[str, Any]: + """write this object as a dictionary of strings""" + + return {"directory": str(self.directory)} + class _Project(BaseModel): """a project subsection""" - name: constr( - regex=RE_VARNAME - ) + name: constr(regex=RE_VARNAME) enabled: bool = True override_storage: Optional[_Storage] + @property + def kiwi_dict(self) -> Dict[str, Any]: + """write this object as a dictionary of strings""" + + if self.override_storage is None: + return {self.name: self.enabled} + + else: + result = self.dict(exclude={"override_storage"}) + result["override_storage"] = self.override_storage.kiwi_dict + return result + @root_validator(pre=True) @classmethod def unify_project(cls, values): @@ -52,6 +76,15 @@ class _Network(BaseModel): name: constr(to_lower=True, regex=RE_VARNAME) cidr: IPv4Network + @property + def kiwi_dict(self) -> Dict[str, Any]: + """write this object as a dictionary of strings""" + + return { + "name": self.name, + "cidr": str(self.cidr) + } + class Config(BaseModel): """represents a kiwi.yml""" @@ -62,10 +95,10 @@ class Config(BaseModel): Path("/bin/bash"), ] - environment: Dict[str, Optional[str]] = {} - projects: Optional[List[_Project]] + environment: Dict[str, Optional[str]] = {} + storage: _Storage = _Storage( directory="/var/local/kiwi", ) @@ -75,6 +108,50 @@ class Config(BaseModel): cidr="10.22.46.0/24", ) + @property + def kiwi_dict(self) -> Dict[str, Any]: + """write this object as a dictionary of strings""" + + result = { + "version": self.version, + "shells": [str(shell) for shell in self.shells], + } + + if self.projects: + result["projects"] = [ + project.kiwi_dict + for project in self.projects + ] + + if self.environment: + result["environment"] = self.environment + + result["storage"] = self.storage.kiwi_dict + + result["network"] = self.network.kiwi_dict + + return result + + @property + def kiwi_yml(self) -> str: + """dump a kiwi.yml file""" + + yml_string = yaml.dump( + self.kiwi_dict, + Dumper=_KiwiDumper, + default_flow_style=False, + sort_keys=False, + ) + + # insert newline before every main key + yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) + + # load header comment from file + with open(HEADER_KIWI_CONF_NAME, 'r') as stream: + yml_string = stream.read() + yml_string + + return yml_string + @validator("environment", pre=True) @classmethod def unify_environment(cls, value) -> Dict[str, Optional[str]]: diff --git a/kiwi_scp/scripts/kiwi_next.py b/kiwi_scp/scripts/kiwi_next.py index 3650070..0885d92 100644 --- a/kiwi_scp/scripts/kiwi_next.py +++ b/kiwi_scp/scripts/kiwi_next.py @@ -8,9 +8,7 @@ def main(): kiwi = Config(**yml) print(repr(kiwi)) - - kiwi = Config() - print(repr(kiwi)) + print(kiwi.kiwi_yml) if __name__ == "__main__": From 734517719513a9b24d5429f96332c6a8f7e01dcd Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Thu, 14 Oct 2021 04:34:09 +0200 Subject: [PATCH 008/141] Some pytests for config.py --- .idea/runConfigurations/Tests.xml | 18 ++++ .idea/runConfigurations/kiwi_next.xml | 6 +- kiwi_scp/config.py | 16 ++-- pyproject.toml | 3 + tests/__init__.py | 0 tests/test_config.py | 118 ++++++++++++++++++++++++++ 6 files changed, 151 insertions(+), 10 deletions(-) create mode 100644 .idea/runConfigurations/Tests.xml create mode 100644 tests/__init__.py create mode 100644 tests/test_config.py diff --git a/.idea/runConfigurations/Tests.xml b/.idea/runConfigurations/Tests.xml new file mode 100644 index 0000000..17c8681 --- /dev/null +++ b/.idea/runConfigurations/Tests.xml @@ -0,0 +1,18 @@ + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/kiwi_next.xml b/.idea/runConfigurations/kiwi_next.xml index 9b3bd56..b4babb9 100644 --- a/.idea/runConfigurations/kiwi_next.xml +++ b/.idea/runConfigurations/kiwi_next.xml @@ -1,5 +1,5 @@ - + \ No newline at end of file diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index cb9a0de..cc7f895 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -13,7 +13,7 @@ from ._constants import RE_SEMVER, RE_VARNAME, HEADER_KIWI_CONF_NAME, KIWI_CONF_ # indent yaml lists class _KiwiDumper(yaml.Dumper): def increase_indent(self, flow=False, indentless=False): - return super().increase_indent(flow, False) + return super().increase_indent(flow, False) # pragma: no cover class _Storage(BaseModel): diff --git a/poetry.lock b/poetry.lock index c35b800..5c3cb7e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -55,6 +55,20 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "coverage" +version = "6.0.2" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + [[package]] name = "dataclasses" version = "0.8" @@ -187,11 +201,14 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pyparsing" -version = "2.4.7" +version = "3.0.1" description = "Python parsing module" category = "dev" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = ">=3.6" + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" @@ -215,6 +232,21 @@ toml = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-cov" +version = "3.0.0" +description = "Pytest plugin for measuring coverage." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] + [[package]] name = "pyyaml" version = "5.4.1" @@ -239,6 +271,14 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tomli" +version = "1.2.2" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.6" + [[package]] name = "typing-extensions" version = "3.10.0.2" @@ -249,7 +289,7 @@ python-versions = "*" [[package]] name = "virtualenv" -version = "20.8.1" +version = "20.9.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -258,7 +298,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] "backports.entry-points-selectable" = ">=1.0.4" distlib = ">=0.3.1,<1" -filelock = ">=3.0.0,<4" +filelock = ">=3.2,<4" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} importlib-resources = {version = ">=1.0", markers = "python_version < \"3.7\""} platformdirs = ">=2,<3" @@ -283,7 +323,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6.1" -content-hash = "872024f4d335f920d178ef6b631785f4f6908f8f9f87d970932bd54dee7fd297" +content-hash = "a5e47a9cdd079f42b70c323cf971290e0da9be66a15317b609a1f937a892fa7a" [metadata.files] atomicwrites = [ @@ -306,6 +346,41 @@ colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] +coverage = [ + {file = "coverage-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1549e1d08ce38259de2bc3e9a0d5f3642ff4a8f500ffc1b2df73fd621a6cdfc0"}, + {file = "coverage-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcae10fccb27ca2a5f456bf64d84110a5a74144be3136a5e598f9d9fb48c0caa"}, + {file = "coverage-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:53a294dc53cfb39c74758edaa6305193fb4258a30b1f6af24b360a6c8bd0ffa7"}, + {file = "coverage-6.0.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8251b37be1f2cd9c0e5ccd9ae0380909c24d2a5ed2162a41fcdbafaf59a85ebd"}, + {file = "coverage-6.0.2-cp310-cp310-win32.whl", hash = "sha256:db42baa892cba723326284490283a68d4de516bfb5aaba369b4e3b2787a778b7"}, + {file = "coverage-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:bbffde2a68398682623d9dd8c0ca3f46fda074709b26fcf08ae7a4c431a6ab2d"}, + {file = "coverage-6.0.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:60e51a3dd55540bec686d7fff61b05048ca31e804c1f32cbb44533e6372d9cc3"}, + {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a6a9409223a27d5ef3cca57dd7cd4dfcb64aadf2fad5c3b787830ac9223e01a"}, + {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4b34ae4f51bbfa5f96b758b55a163d502be3dcb24f505d0227858c2b3f94f5b9"}, + {file = "coverage-6.0.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3bbda1b550e70fa6ac40533d3f23acd4f4e9cb4e6e77251ce77fdf41b3309fb2"}, + {file = "coverage-6.0.2-cp36-cp36m-win32.whl", hash = "sha256:4e28d2a195c533b58fc94a12826f4431726d8eb029ac21d874345f943530c122"}, + {file = "coverage-6.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a82d79586a0a4f5fd1cf153e647464ced402938fbccb3ffc358c7babd4da1dd9"}, + {file = "coverage-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3be1206dc09fb6298de3fce70593e27436862331a85daee36270b6d0e1c251c4"}, + {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9cd3828bbe1a40070c11fe16a51df733fd2f0cb0d745fb83b7b5c1f05967df7"}, + {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d036dc1ed8e1388e995833c62325df3f996675779541f682677efc6af71e96cc"}, + {file = "coverage-6.0.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:04560539c19ec26995ecfb3d9307ff154fbb9a172cb57e3b3cfc4ced673103d1"}, + {file = "coverage-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:e4fb7ced4d9dec77d6cf533acfbf8e1415fe799430366affb18d69ee8a3c6330"}, + {file = "coverage-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:77b1da5767ed2f44611bc9bc019bc93c03fa495728ec389759b6e9e5039ac6b1"}, + {file = "coverage-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:61b598cbdbaae22d9e34e3f675997194342f866bb1d781da5d0be54783dce1ff"}, + {file = "coverage-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36e9040a43d2017f2787b28d365a4bb33fcd792c7ff46a047a04094dc0e2a30d"}, + {file = "coverage-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9f1627e162e3864a596486774876415a7410021f4b67fd2d9efdf93ade681afc"}, + {file = "coverage-6.0.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e7a0b42db2a47ecb488cde14e0f6c7679a2c5a9f44814393b162ff6397fcdfbb"}, + {file = "coverage-6.0.2-cp38-cp38-win32.whl", hash = "sha256:a1b73c7c4d2a42b9d37dd43199c5711d91424ff3c6c22681bc132db4a4afec6f"}, + {file = "coverage-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:1db67c497688fd4ba85b373b37cc52c50d437fd7267520ecd77bddbd89ea22c9"}, + {file = "coverage-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f2f184bf38e74f152eed7f87e345b51f3ab0b703842f447c22efe35e59942c24"}, + {file = "coverage-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1cf1deb3d5544bd942356364a2fdc8959bad2b6cf6eb17f47d301ea34ae822"}, + {file = "coverage-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ad9b8c1206ae41d46ec7380b78ba735ebb77758a650643e841dd3894966c31d0"}, + {file = "coverage-6.0.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:381d773d896cc7f8ba4ff3b92dee4ed740fb88dfe33b6e42efc5e8ab6dfa1cfe"}, + {file = "coverage-6.0.2-cp39-cp39-win32.whl", hash = "sha256:424c44f65e8be58b54e2b0bd1515e434b940679624b1b72726147cfc6a9fc7ce"}, + {file = "coverage-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:abbff240f77347d17306d3201e14431519bf64495648ca5a49571f988f88dee9"}, + {file = "coverage-6.0.2-pp36-none-any.whl", hash = "sha256:7092eab374346121805fb637572483270324407bf150c30a3b161fc0c4ca5164"}, + {file = "coverage-6.0.2-pp37-none-any.whl", hash = "sha256:30922626ce6f7a5a30bdba984ad21021529d3d05a68b4f71ea3b16bda35b8895"}, + {file = "coverage-6.0.2.tar.gz", hash = "sha256:6807947a09510dc31fa86f43595bf3a14017cd60bf633cc746d52141bfa6b149"}, +] dataclasses = [ {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, @@ -371,13 +446,17 @@ pydantic = [ {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, ] pyparsing = [ - {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, - {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, + {file = "pyparsing-3.0.1-py3-none-any.whl", hash = "sha256:fd93fc45c47893c300bd98f5dd1b41c0e783eaeb727e7cea210dcc09d64ce7c3"}, + {file = "pyparsing-3.0.1.tar.gz", hash = "sha256:84196357aa3566d64ad123d7a3c67b0e597a115c4934b097580e5ce220b91531"}, ] pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] +pytest-cov = [ + {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, + {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, +] pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, @@ -417,14 +496,18 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +tomli = [ + {file = "tomli-1.2.2-py3-none-any.whl", hash = "sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade"}, + {file = "tomli-1.2.2.tar.gz", hash = "sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee"}, +] typing-extensions = [ {file = "typing_extensions-3.10.0.2-py2-none-any.whl", hash = "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7"}, {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, ] virtualenv = [ - {file = "virtualenv-20.8.1-py2.py3-none-any.whl", hash = "sha256:10062e34c204b5e4ec5f62e6ef2473f8ba76513a9a617e873f1f8fb4a519d300"}, - {file = "virtualenv-20.8.1.tar.gz", hash = "sha256:bcc17f0b3a29670dd777d6f0755a4c04f28815395bca279cdcb213b97199a6b8"}, + {file = "virtualenv-20.9.0-py2.py3-none-any.whl", hash = "sha256:1d145deec2da86b29026be49c775cc5a9aab434f85f7efef98307fb3965165de"}, + {file = "virtualenv-20.9.0.tar.gz", hash = "sha256:bb55ace18de14593947354e5e6cd1be75fb32c3329651da62e92bf5d0aab7213"}, ] zipp = [ {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, diff --git a/pyproject.toml b/pyproject.toml index 50cd2a5..f65fb53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ PyYAML = "^5.4.1" [tool.poetry.dev-dependencies] pytest = "^6.2.5" +pytest-cov = "^3.0.0" toml = "^0.10.2" virtualenv = "^20.8.1" From 3c81021f1483251407e2be7d2f82e92af19e2f68 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 26 Oct 2021 12:19:02 +0200 Subject: [PATCH 033/141] pyyaml -> ruamel.yaml, 100% cov on config.py --- example/kiwi.yml | 2 +- kiwi_scp/commands/cmd_init.py | 2 +- kiwi_scp/config.py | 50 ++++++++---------- kiwi_scp/misc.py | 17 +++++- poetry.lock | 81 ++++++++++++++++------------- pyproject.toml | 2 +- tests/test_config.py | 98 +++++++++++++++++++++++++++++++---- 7 files changed, 175 insertions(+), 77 deletions(-) diff --git a/example/kiwi.yml b/example/kiwi.yml index e40d97d..581505a 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -2,7 +2,7 @@ # kiwi-scp instance configuration # ################################### -version: '0.2.0' +version: 0.2.0 shells: - /bin/bash diff --git a/kiwi_scp/commands/cmd_init.py b/kiwi_scp/commands/cmd_init.py index 65a3ca1..c8b68e7 100644 --- a/kiwi_scp/commands/cmd_init.py +++ b/kiwi_scp/commands/cmd_init.py @@ -72,4 +72,4 @@ def cmd(ctx: Instance, output: Path, force: bool, show: bool): # write out the new kiwi.yml with open(ctx.directory.joinpath(KIWI_CONF_NAME), "w") as file: - file.write(Config.parse_obj(kiwi_dict).kiwi_yml) + Config.parse_obj(kiwi_dict).dump_kiwi_yml(file) diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index cc7f895..e117ff6 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -1,19 +1,14 @@ import functools -import re +import io from ipaddress import IPv4Network from pathlib import Path -from typing import Optional, Dict, List, Any +from typing import Optional, Dict, List, Any, TextIO -import yaml +import ruamel.yaml from pydantic import BaseModel, constr, root_validator, validator -from ._constants import RE_SEMVER, RE_VARNAME, HEADER_KIWI_CONF_NAME, KIWI_CONF_NAME - - -# indent yaml lists -class _KiwiDumper(yaml.Dumper): - def increase_indent(self, flow=False, indentless=False): - return super().increase_indent(flow, False) # pragma: no cover +from ._constants import RE_SEMVER, RE_VARNAME, KIWI_CONF_NAME +from .misc import _format_kiwi_yml class _Storage(BaseModel): @@ -75,7 +70,8 @@ class _Project(BaseModel): return {"directory": value[0]} else: - raise ValueError("Invalid Storage Format") + # undefined format + return {} @root_validator(pre=True) @classmethod @@ -141,12 +137,12 @@ class Config(BaseModel): @classmethod @functools.lru_cache(maxsize=5) - def from_instance(cls, instance: Path): + def from_directory(cls, instance: Path): """parses an actual kiwi.yml from disk (cached)""" try: with open(instance.joinpath(KIWI_CONF_NAME)) as kc: - yml = yaml.safe_load(kc) + yml = ruamel.yaml.round_trip_load(kc) return cls.parse_obj(yml) except FileNotFoundError: @@ -184,25 +180,23 @@ class Config(BaseModel): return result - @property - def kiwi_yml(self) -> str: + def dump_kiwi_yml(self, stream: TextIO) -> None: """dump a kiwi.yml file""" - yml_string = yaml.dump( - self.kiwi_dict, - Dumper=_KiwiDumper, - default_flow_style=False, - sort_keys=False, - ) + yml = ruamel.yaml.YAML() + yml.indent(offset=2) + yml.dump(self.kiwi_dict, stream=stream, transform=_format_kiwi_yml) - # insert newline before every main key - yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) + @property + def kiwi_yml(self) -> str: + """get a kiwi.yml dump as a string""" - # load header comment from file - with open(HEADER_KIWI_CONF_NAME, 'r') as stream: - yml_string = stream.read() + yml_string + sio = io.StringIO() + self.dump_kiwi_yml(sio) + result: str = sio.getvalue() + sio.close() - return yml_string + return result @validator("shells", pre=True) @classmethod @@ -341,7 +335,7 @@ class Config(BaseModel): else: # undefined format - raise ValueError("Invalid Storage Format") + return {} @validator("network", pre=True) @classmethod diff --git a/kiwi_scp/misc.py b/kiwi_scp/misc.py index 580d5ef..845acf1 100644 --- a/kiwi_scp/misc.py +++ b/kiwi_scp/misc.py @@ -1,9 +1,12 @@ +import re from typing import Any, Type, List, Callable import attr import click from click.decorators import FC +from ._constants import HEADER_KIWI_CONF_NAME + @attr.s class _MultiDecorator: @@ -19,8 +22,7 @@ class _MultiDecorator: _project_arg = click.argument( "project", required=False, - type=click.Path(exists=True), - default=".", + type=str, ) _service_arg = click.argument( @@ -52,6 +54,17 @@ def user_query(description: str, default: Any, cast_to: Type[Any] = str): click.echo(f"Invalid input: {e}") +def _format_kiwi_yml(yml_string: str): + # insert newline before every main key + yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) + + # load header comment from file + with open(HEADER_KIWI_CONF_NAME, 'r') as stream: + yml_string = stream.read() + yml_string + + return yml_string + + def _surround(string, bang): midlane = f"{bang * 3} {string} {bang * 3}" sidelane = bang * len(midlane) diff --git a/poetry.lock b/poetry.lock index 5c3cb7e..eb51feb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -248,12 +248,27 @@ pytest = ">=4.6" testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] [[package]] -name = "pyyaml" -version = "5.4.1" -description = "YAML parser and emitter for Python" +name = "ruamel.yaml" +version = "0.17.16" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3" + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.1.2", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.10\""} + +[package.extras] +docs = ["ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel.yaml.clib" +version = "0.2.6" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +category = "main" +optional = false +python-versions = ">=3.5" [[package]] name = "six" @@ -323,7 +338,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6.1" -content-hash = "a5e47a9cdd079f42b70c323cf971290e0da9be66a15317b609a1f937a892fa7a" +content-hash = "631c4b1d21036305a2e9c891a631e66106549e1873b91264f481ba29207789fd" [metadata.files] atomicwrites = [ @@ -457,36 +472,32 @@ pytest-cov = [ {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, ] -pyyaml = [ - {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, - {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, - {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, - {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, - {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, - {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, - {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, - {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, - {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, +"ruamel.yaml" = [ + {file = "ruamel.yaml-0.17.16-py3-none-any.whl", hash = "sha256:ea21da1198c4b41b8e7a259301cc9710d3b972bf8ba52f06218478e6802dd1f1"}, + {file = "ruamel.yaml-0.17.16.tar.gz", hash = "sha256:1a771fc92d3823682b7f0893ad56cb5a5c87c48e62b5399d6f42c8759a583b33"}, +] +"ruamel.yaml.clib" = [ + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win32.whl", hash = "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5"}, + {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win_amd64.whl", hash = "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win32.whl", hash = "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94"}, + {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win_amd64.whl", hash = "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win32.whl", hash = "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb"}, + {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win32.whl", hash = "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b"}, + {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win32.whl", hash = "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104"}, + {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7"}, + {file = "ruamel.yaml.clib-0.2.6.tar.gz", hash = "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, diff --git a/pyproject.toml b/pyproject.toml index f65fb53..b9b0370 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ python = "^3.6.1" attrs = "^21.2.0" click = "^8.0.3" pydantic = "^1.8.2" -PyYAML = "^5.4.1" +"ruamel.yaml" = "^0.17.16" [tool.poetry.dev-dependencies] pytest = "^6.2.5" diff --git a/tests/test_config.py b/tests/test_config.py index 6871cd2..e6b6e42 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,7 +1,9 @@ +import io from ipaddress import IPv4Network from pathlib import Path import pytest +import ruamel.yaml from pydantic import ValidationError from kiwi_scp.config import Config @@ -31,6 +33,28 @@ def test_default(): assert c.network.name == "kiwi_hub" assert c.network.cidr == IPv4Network("10.22.46.0/24") + kiwi_dict = { + "version": version, + "shells": ["/bin/bash"], + "storage": {"directory": "/var/local/kiwi"}, + "network": { + "name": "kiwi_hub", + "cidr": "10.22.46.0/24", + }, + } + assert c.kiwi_dict == kiwi_dict + + yml = ruamel.yaml.YAML() + yml.indent(offset=2) + + sio = io.StringIO() + from kiwi_scp.misc import _format_kiwi_yml + yml.dump(kiwi_dict, stream=sio, transform=_format_kiwi_yml) + yml_string = sio.getvalue() + sio.close() + + assert c.kiwi_yml == yml_string + ######### # VERSION @@ -156,30 +180,79 @@ def test_proj_empty(): def test_proj_long(): - c = Config(projects=[{ + kiwi_dict = { "name": "project", "enabled": False, "override_storage": {"directory": "/test/directory"}, - }]) + } + c = Config(projects=[kiwi_dict]) assert len(c.projects) == 1 p = c.projects[0] assert p.name == "project" assert not p.enabled assert p.override_storage is not None - assert p.override_storage.directory == Path("/test/directory") + + assert c.kiwi_dict["projects"][0] == kiwi_dict + + +def test_proj_storage_str(): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": "/test/directory", + } + c = Config(projects=[kiwi_dict]) + + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert not p.enabled + assert p.override_storage is not None + + +def test_proj_storage_list(): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": ["/test/directory"], + } + c = Config(projects=[kiwi_dict]) + + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert not p.enabled + assert p.override_storage is not None + + +def test_proj_storage_invalid(): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": True, + } + with pytest.raises(ValidationError) as exc_info: + Config(projects=[kiwi_dict]) + + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Storage Format" + assert error["type"] == "value_error" def test_proj_short(): - c = Config(projects=[{ + kiwi_dict = { "project": False, - }]) + } + c = Config(projects=[kiwi_dict]) assert len(c.projects) == 1 p = c.projects[0] assert p.name == "project" assert not p.enabled assert p.override_storage is None + assert p.kiwi_dict == kiwi_dict def test_proj_dict(): @@ -252,12 +325,15 @@ def test_env_dict(): assert c.environment == {} - c = Config(environment={"variable": "value"}) + kiwi_dict = {"variable": "value"} + c = Config(environment=kiwi_dict) assert len(c.environment) == 1 assert "variable" in c.environment assert c.environment["variable"] == "value" + assert c.kiwi_dict["environment"] == kiwi_dict + def test_env_list(): c = Config(environment=[]) @@ -348,9 +424,11 @@ def test_storage_empty(): def test_storage_dict(): - c = Config(storage={"directory": "/test/directory"}) + kiwi_dict = {"directory": "/test/directory"} + c = Config(storage=kiwi_dict) assert c.storage.directory == Path("/test/directory") + assert c.storage.kiwi_dict == kiwi_dict def test_storage_invalid_dict(): @@ -400,10 +478,11 @@ def test_network_empty(): def test_network_dict(): - c = Config(network={ + kiwi_dict = { "name": "test_hub", "cidr": "1.2.3.4/32", - }) + } + c = Config(network=kiwi_dict) assert c == Config(network={ "name": "TEST_HUB", @@ -412,6 +491,7 @@ def test_network_dict(): assert c.network.name == "test_hub" assert c.network.cidr == IPv4Network("1.2.3.4/32") + assert c.network.kiwi_dict == kiwi_dict def test_network_invalid_dict(): From 7dc6e6789fbf83ed6881c98c4b5e02fb1504c9f1 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 26 Oct 2021 12:28:06 +0200 Subject: [PATCH 034/141] stray pyyaml import --- kiwi_scp/instance.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kiwi_scp/instance.py b/kiwi_scp/instance.py index edbe985..26f6eba 100644 --- a/kiwi_scp/instance.py +++ b/kiwi_scp/instance.py @@ -5,7 +5,7 @@ from typing import List, Dict, Any, Generator import attr import click -import yaml +import ruamel.yaml from ._constants import COMPOSE_FILE_NAME from .config import Config @@ -47,7 +47,7 @@ class Project: @functools.lru_cache(maxsize=10) def from_directory(cls, directory: Path): with open(directory.joinpath(COMPOSE_FILE_NAME), "r") as cf: - yml = yaml.safe_load(cf) + yml = ruamel.yaml.round_trip_load(cf) return cls( directory=directory, @@ -66,12 +66,15 @@ class Instance: def config(self) -> Config: """shorthand: get the current configuration""" - return Config.from_instance(self.directory) + return Config.from_directory(self.directory) + + def get_project(self, name: str) -> Project: + return Project.from_directory(self.directory.joinpath(name)) @property def projects(self) -> Generator[Project, None, None]: return ( - Project.from_directory(self.directory.joinpath(project.name)) + self.get_project(project.name) for project in self.config.projects ) From 8260ddef3cf7f44c70ce2e1b0b146cc0c3a7c52b Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 26 Oct 2021 15:56:58 +0200 Subject: [PATCH 035/141] Config -> KiwiConfig, _Storage -> StorageConfig, _Network -> NetworkConfig --- kiwi_scp/commands/cmd_init.py | 6 +- kiwi_scp/config.py | 27 ++++---- tests/test_config.py | 118 +++++++++++++++++----------------- 3 files changed, 75 insertions(+), 76 deletions(-) diff --git a/kiwi_scp/commands/cmd_init.py b/kiwi_scp/commands/cmd_init.py index c8b68e7..edae74e 100644 --- a/kiwi_scp/commands/cmd_init.py +++ b/kiwi_scp/commands/cmd_init.py @@ -6,7 +6,7 @@ from pathlib import Path import click from .._constants import KIWI_CONF_NAME -from ..config import Config +from ..config import KiwiConfig from ..instance import Instance, pass_instance from ..misc import user_query @@ -42,7 +42,7 @@ def cmd(ctx: Instance, output: Path, force: bool, show: bool): if output is not None: ctx.directory = output - current_config = Config() if force else ctx.config + current_config = KiwiConfig() if force else ctx.config if show: # just show the currently effective kiwi.yml @@ -72,4 +72,4 @@ def cmd(ctx: Instance, output: Path, force: bool, show: bool): # write out the new kiwi.yml with open(ctx.directory.joinpath(KIWI_CONF_NAME), "w") as file: - Config.parse_obj(kiwi_dict).dump_kiwi_yml(file) + KiwiConfig.parse_obj(kiwi_dict).dump_kiwi_yml(file) diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index e117ff6..1b8d868 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -4,14 +4,14 @@ from ipaddress import IPv4Network from pathlib import Path from typing import Optional, Dict, List, Any, TextIO -import ruamel.yaml from pydantic import BaseModel, constr, root_validator, validator +from ruamel.yaml import YAML from ._constants import RE_SEMVER, RE_VARNAME, KIWI_CONF_NAME from .misc import _format_kiwi_yml -class _Storage(BaseModel): +class StorageConfig(BaseModel): """a storage subsection""" directory: Path @@ -36,12 +36,12 @@ class _Storage(BaseModel): raise ValueError("Invalid Storage Format") -class _Project(BaseModel): +class ProjectConfig(BaseModel): """a project subsection""" name: constr(regex=RE_VARNAME) enabled: bool = True - override_storage: Optional[_Storage] + override_storage: Optional[StorageConfig] @property def kiwi_dict(self) -> Dict[str, Any]: @@ -97,7 +97,7 @@ class _Project(BaseModel): raise ValueError("Invalid Project Format") -class _Network(BaseModel): +class NetworkConfig(BaseModel): """a network subsection""" name: constr(to_lower=True, regex=RE_VARNAME) @@ -113,7 +113,7 @@ class _Network(BaseModel): } -class Config(BaseModel): +class KiwiConfig(BaseModel): """represents a kiwi.yml""" version: constr(regex=RE_SEMVER) = "0.2.0" @@ -122,28 +122,27 @@ class Config(BaseModel): Path("/bin/bash"), ] - projects: List[_Project] = [] + projects: List[ProjectConfig] = [] environment: Dict[str, Optional[str]] = {} - storage: _Storage = _Storage( + storage: StorageConfig = StorageConfig( directory="/var/local/kiwi", ) - network: _Network = _Network( + network: NetworkConfig = NetworkConfig( name="kiwi_hub", cidr="10.22.46.0/24", ) @classmethod @functools.lru_cache(maxsize=5) - def from_directory(cls, instance: Path): + def from_directory(cls, directory: Path): """parses an actual kiwi.yml from disk (cached)""" try: - with open(instance.joinpath(KIWI_CONF_NAME)) as kc: - yml = ruamel.yaml.round_trip_load(kc) - return cls.parse_obj(yml) + with open(directory.joinpath(KIWI_CONF_NAME)) as kc: + return cls.parse_obj(YAML().load(kc)) except FileNotFoundError: # return the defaults if no kiwi.yml found @@ -183,7 +182,7 @@ class Config(BaseModel): def dump_kiwi_yml(self, stream: TextIO) -> None: """dump a kiwi.yml file""" - yml = ruamel.yaml.YAML() + yml = YAML() yml.indent(offset=2) yml.dump(self.kiwi_dict, stream=stream, transform=_format_kiwi_yml) diff --git a/tests/test_config.py b/tests/test_config.py index e6b6e42..143b7b1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,7 +6,7 @@ import pytest import ruamel.yaml from pydantic import ValidationError -from kiwi_scp.config import Config +from kiwi_scp.config import KiwiConfig class UnCoercible: @@ -19,10 +19,10 @@ class UnCoercible: def test_default(): import toml - c = Config() + c = KiwiConfig() version = toml.load("./pyproject.toml")["tool"]["poetry"]["version"] - assert c == Config.from_default() + assert c == KiwiConfig.from_default() assert c.version == version assert len(c.shells) == 1 @@ -61,23 +61,23 @@ def test_default(): ######### def test_version_valid(): - c = Config(version="0.0.0") + c = KiwiConfig(version="0.0.0") assert c.version == "0.0.0" - c = Config(version="0.0") + c = KiwiConfig(version="0.0") assert c.version == "0.0" - c = Config(version="0") + c = KiwiConfig(version="0") assert c.version == "0" - c = Config(version=1.0) + c = KiwiConfig(version=1.0) assert c.version == "1.0" - c = Config(version=1) + c = KiwiConfig(version=1) assert c.version == "1" @@ -85,7 +85,7 @@ def test_version_valid(): def test_version_invalid(): # definitely not a version with pytest.raises(ValidationError) as exc_info: - Config(version="dnaf") + KiwiConfig(version="dnaf") assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -94,7 +94,7 @@ def test_version_invalid(): # barely a version with pytest.raises(ValidationError) as exc_info: - c = Config(version="0.0.0alpha") + c = KiwiConfig(version="0.0.0alpha") print(c.version) assert len(exc_info.value.errors()) == 1 @@ -108,42 +108,42 @@ def test_version_invalid(): ######## def test_shells_empty(): - c = Config(shells=None) + c = KiwiConfig(shells=None) - assert c == Config(shells=[]) + assert c == KiwiConfig(shells=[]) assert c.shells == [] def test_shells_list(): - c = Config(shells=["/bin/sh", "sh"]) + c = KiwiConfig(shells=["/bin/sh", "sh"]) assert len(c.shells) == 2 assert c.shells[0] == Path("/bin/sh") assert c.shells[1] == Path("sh") - c = Config(shells=["/bin/bash"]) + c = KiwiConfig(shells=["/bin/bash"]) assert len(c.shells) == 1 assert c.shells[0] == Path("/bin/bash") def test_shells_dict(): - c = Config(shells={"/bin/bash": None}) + c = KiwiConfig(shells={"/bin/bash": None}) assert len(c.shells) == 1 assert c.shells[0] == Path("/bin/bash") def test_shells_coercible(): - c = Config(shells="/bin/bash") + c = KiwiConfig(shells="/bin/bash") - assert c == Config(shells=Path("/bin/bash")) + assert c == KiwiConfig(shells=Path("/bin/bash")) assert len(c.shells) == 1 assert c.shells[0] == Path("/bin/bash") - c = Config(shells=123) + c = KiwiConfig(shells=123) assert len(c.shells) == 1 assert c.shells[0] == Path("123") @@ -151,7 +151,7 @@ def test_shells_coercible(): def test_shells_uncoercible(): with pytest.raises(ValidationError) as exc_info: - Config(shells=UnCoercible()) + KiwiConfig(shells=UnCoercible()) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -159,7 +159,7 @@ def test_shells_uncoercible(): assert error["type"] == "value_error" with pytest.raises(ValidationError) as exc_info: - Config(shells=["/bin/bash", UnCoercible()]) + KiwiConfig(shells=["/bin/bash", UnCoercible()]) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -172,9 +172,9 @@ def test_shells_uncoercible(): ########## def test_proj_empty(): - c = Config(projects=None) + c = KiwiConfig(projects=None) - assert c == Config(projects=[]) + assert c == KiwiConfig(projects=[]) assert c.projects == [] @@ -185,7 +185,7 @@ def test_proj_long(): "enabled": False, "override_storage": {"directory": "/test/directory"}, } - c = Config(projects=[kiwi_dict]) + c = KiwiConfig(projects=[kiwi_dict]) assert len(c.projects) == 1 p = c.projects[0] @@ -202,7 +202,7 @@ def test_proj_storage_str(): "enabled": False, "override_storage": "/test/directory", } - c = Config(projects=[kiwi_dict]) + c = KiwiConfig(projects=[kiwi_dict]) assert len(c.projects) == 1 p = c.projects[0] @@ -217,7 +217,7 @@ def test_proj_storage_list(): "enabled": False, "override_storage": ["/test/directory"], } - c = Config(projects=[kiwi_dict]) + c = KiwiConfig(projects=[kiwi_dict]) assert len(c.projects) == 1 p = c.projects[0] @@ -233,7 +233,7 @@ def test_proj_storage_invalid(): "override_storage": True, } with pytest.raises(ValidationError) as exc_info: - Config(projects=[kiwi_dict]) + KiwiConfig(projects=[kiwi_dict]) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -245,7 +245,7 @@ def test_proj_short(): kiwi_dict = { "project": False, } - c = Config(projects=[kiwi_dict]) + c = KiwiConfig(projects=[kiwi_dict]) assert len(c.projects) == 1 p = c.projects[0] @@ -256,9 +256,9 @@ def test_proj_short(): def test_proj_dict(): - c = Config(projects={"name": "project"}) + c = KiwiConfig(projects={"name": "project"}) - assert c == Config(projects=[{"name": "project"}]) + assert c == KiwiConfig(projects=[{"name": "project"}]) assert len(c.projects) == 1 p = c.projects[0] @@ -269,7 +269,7 @@ def test_proj_dict(): def test_proj_invalid_dict(): with pytest.raises(ValidationError) as exc_info: - Config(projects={ + KiwiConfig(projects={ "random key 1": "random value 1", "random key 2": "random value 2", }) @@ -281,9 +281,9 @@ def test_proj_invalid_dict(): def test_proj_coercible(): - c = Config(projects="project") + c = KiwiConfig(projects="project") - assert c == Config(projects=["project"]) + assert c == KiwiConfig(projects=["project"]) assert len(c.projects) == 1 p = c.projects[0] @@ -294,7 +294,7 @@ def test_proj_coercible(): def test_proj_uncoercible(): with pytest.raises(ValidationError) as exc_info: - Config(projects=["valid", UnCoercible()]) + KiwiConfig(projects=["valid", UnCoercible()]) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -302,7 +302,7 @@ def test_proj_uncoercible(): assert error["type"] == "value_error" with pytest.raises(ValidationError) as exc_info: - Config(projects=UnCoercible()) + KiwiConfig(projects=UnCoercible()) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -315,18 +315,18 @@ def test_proj_uncoercible(): ############# def test_env_empty(): - c = Config(environment=None) + c = KiwiConfig(environment=None) assert c.environment == {} def test_env_dict(): - c = Config(environment={}) + c = KiwiConfig(environment={}) assert c.environment == {} kiwi_dict = {"variable": "value"} - c = Config(environment=kiwi_dict) + c = KiwiConfig(environment=kiwi_dict) assert len(c.environment) == 1 assert "variable" in c.environment @@ -336,11 +336,11 @@ def test_env_dict(): def test_env_list(): - c = Config(environment=[]) + c = KiwiConfig(environment=[]) assert c.environment == {} - c = Config(environment=[ + c = KiwiConfig(environment=[ "variable=value", ]) @@ -348,7 +348,7 @@ def test_env_list(): assert "variable" in c.environment assert c.environment["variable"] == "value" - c = Config(environment=[ + c = KiwiConfig(environment=[ "variable", ]) @@ -356,7 +356,7 @@ def test_env_list(): assert "variable" in c.environment assert c.environment["variable"] is None - c = Config(environment=[ + c = KiwiConfig(environment=[ 123, ]) @@ -366,25 +366,25 @@ def test_env_list(): def test_env_coercible(): - c = Config(environment="variable=value") + c = KiwiConfig(environment="variable=value") assert len(c.environment) == 1 assert "variable" in c.environment assert c.environment["variable"] == "value" - c = Config(environment="variable") + c = KiwiConfig(environment="variable") assert len(c.environment) == 1 assert "variable" in c.environment assert c.environment["variable"] is None - c = Config(environment=123) + c = KiwiConfig(environment=123) assert len(c.environment) == 1 assert "123" in c.environment assert c.environment["123"] is None - c = Config(environment=123.4) + c = KiwiConfig(environment=123.4) assert len(c.environment) == 1 assert "123.4" in c.environment @@ -393,7 +393,7 @@ def test_env_coercible(): def test_env_uncoercible(): with pytest.raises(ValidationError) as exc_info: - Config(environment=UnCoercible()) + KiwiConfig(environment=UnCoercible()) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -401,7 +401,7 @@ def test_env_uncoercible(): assert error["type"] == "value_error" with pytest.raises(ValidationError) as exc_info: - Config(environment=["valid", UnCoercible()]) + KiwiConfig(environment=["valid", UnCoercible()]) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -415,7 +415,7 @@ def test_env_uncoercible(): def test_storage_empty(): with pytest.raises(ValidationError) as exc_info: - Config(storage=None) + KiwiConfig(storage=None) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -425,7 +425,7 @@ def test_storage_empty(): def test_storage_dict(): kiwi_dict = {"directory": "/test/directory"} - c = Config(storage=kiwi_dict) + c = KiwiConfig(storage=kiwi_dict) assert c.storage.directory == Path("/test/directory") assert c.storage.kiwi_dict == kiwi_dict @@ -433,7 +433,7 @@ def test_storage_dict(): def test_storage_invalid_dict(): with pytest.raises(ValidationError) as exc_info: - Config(storage={"random key": "random value"}) + KiwiConfig(storage={"random key": "random value"}) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -442,20 +442,20 @@ def test_storage_invalid_dict(): def test_storage_str(): - c = Config(storage="/test/directory") + c = KiwiConfig(storage="/test/directory") assert c.storage.directory == Path("/test/directory") def test_storage_list(): - c = Config(storage=["/test/directory"]) + c = KiwiConfig(storage=["/test/directory"]) assert c.storage.directory == Path("/test/directory") def test_storage_invalid(): with pytest.raises(ValidationError) as exc_info: - Config(storage=True) + KiwiConfig(storage=True) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -469,7 +469,7 @@ def test_storage_invalid(): def test_network_empty(): with pytest.raises(ValidationError) as exc_info: - Config(network=None) + KiwiConfig(network=None) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -482,9 +482,9 @@ def test_network_dict(): "name": "test_hub", "cidr": "1.2.3.4/32", } - c = Config(network=kiwi_dict) + c = KiwiConfig(network=kiwi_dict) - assert c == Config(network={ + assert c == KiwiConfig(network={ "name": "TEST_HUB", "cidr": "1.2.3.4/32", }) @@ -496,7 +496,7 @@ def test_network_dict(): def test_network_invalid_dict(): with pytest.raises(ValidationError) as exc_info: - Config(network={"name": "test_hub"}) + KiwiConfig(network={"name": "test_hub"}) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] @@ -504,7 +504,7 @@ def test_network_invalid_dict(): assert error["type"] == "value_error.missing" with pytest.raises(ValidationError) as exc_info: - Config(network={ + KiwiConfig(network={ "name": "test_hub", "cidr": "1.2.3.4/123", }) @@ -517,7 +517,7 @@ def test_network_invalid_dict(): def test_network_invalid(): with pytest.raises(ValidationError) as exc_info: - Config(network=True) + KiwiConfig(network=True) assert len(exc_info.value.errors()) == 1 error = exc_info.value.errors()[0] From 0ecee93241a38befc39068291e361d7b6c44789b Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 26 Oct 2021 15:59:38 +0200 Subject: [PATCH 036/141] Obsolete "Project" class --- kiwi_scp/config.py | 9 +++++++++ kiwi_scp/instance.py | 45 ++++++++++++++---------------------------- tests/test_instance.py | 11 +++++------ tests/test_project.py | 28 -------------------------- 4 files changed, 29 insertions(+), 64 deletions(-) delete mode 100644 tests/test_project.py diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index 1b8d868..3d79f0b 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -155,6 +155,15 @@ class KiwiConfig(BaseModel): return cls() + def get_project_config(self, name: str) -> ProjectConfig: + """returns the config of a project with a given name""" + + for project in self.projects: + if project.name == name: + return project + + raise ValueError("No Such Project") + @property def kiwi_dict(self) -> Dict[str, Any]: """write this object as a dictionary of strings""" diff --git a/kiwi_scp/instance.py b/kiwi_scp/instance.py index 26f6eba..c822133 100644 --- a/kiwi_scp/instance.py +++ b/kiwi_scp/instance.py @@ -5,10 +5,10 @@ from typing import List, Dict, Any, Generator import attr import click -import ruamel.yaml +from ruamel.yaml import YAML from ._constants import COMPOSE_FILE_NAME -from .config import Config +from .config import KiwiConfig, ProjectConfig _RE_CONFDIR = re.compile(r"^\s*\$(?:CONFDIR|{CONFDIR})/+(.*)$", flags=re.UNICODE) @@ -38,44 +38,29 @@ class Service: ) -@attr.s -class Project: - directory: Path = attr.ib() - services: List[Service] = attr.ib() - - @classmethod - @functools.lru_cache(maxsize=10) - def from_directory(cls, directory: Path): - with open(directory.joinpath(COMPOSE_FILE_NAME), "r") as cf: - yml = ruamel.yaml.round_trip_load(cf) - - return cls( - directory=directory, - services=[ - Service.from_description(name, description) - for name, description in yml["services"].items() - ], - ) - - @attr.s class Instance: directory: Path = attr.ib(default=Path('.')) @property - def config(self) -> Config: + def config(self) -> KiwiConfig: """shorthand: get the current configuration""" - return Config.from_directory(self.directory) + return KiwiConfig.from_directory(self.directory) - def get_project(self, name: str) -> Project: - return Project.from_directory(self.directory.joinpath(name)) + @classmethod + @functools.lru_cache(maxsize=10) + def _parse_compose_file(cls, directory: Path): + with open(directory.joinpath(COMPOSE_FILE_NAME), "r") as cf: + yml = YAML() + return yml.load(cf) + + def get_services(self, project_name: str) -> Generator[Service, None, None]: + yml = Instance._parse_compose_file(self.directory.joinpath(project_name)) - @property - def projects(self) -> Generator[Project, None, None]: return ( - self.get_project(project.name) - for project in self.config.projects + Service.from_description(name, description) + for name, description in yml["services"].items() ) diff --git a/tests/test_instance.py b/tests/test_instance.py index 60dd462..d008748 100644 --- a/tests/test_instance.py +++ b/tests/test_instance.py @@ -7,19 +7,18 @@ def test_example(): i = Instance(Path("example")) assert i.config is not None - assert len(list(i.projects)) == 1 + assert len(i.config.projects) == 1 - p = next(i.projects) + p = i.config.projects[0] - assert p.directory == Path("example/hello-world.project") + assert p.name == "hello-world.project" def test_empty(): i = Instance() - assert i.directory == Path(".") assert i.config is not None - assert len(list(i.projects)) == 0 + assert len(i.config.projects) == 0 def test_no_such_dir(): @@ -28,4 +27,4 @@ def test_no_such_dir(): assert i.directory == nonexistent_path assert i.config is not None - assert len(list(i.projects)) == 0 + assert len(i.config.projects) == 0 diff --git a/tests/test_project.py b/tests/test_project.py deleted file mode 100644 index cd0233d..0000000 --- a/tests/test_project.py +++ /dev/null @@ -1,28 +0,0 @@ -from pathlib import Path - -import pytest - -from kiwi_scp.instance import Project - - -def test_example(): - p = Project.from_directory(Path("example/hello-world.project")) - - assert p.directory == Path("example/hello-world.project") - assert p.services != [] - - -def test_caching(): - p = Project.from_directory(Path("example/hello-world.project")) - - assert p is Project.from_directory(Path("example/hello-world.project")) - - -def test_no_such_dir(): - nonexistent_path = Path("nonexistent") - - with pytest.raises(FileNotFoundError) as exc_info: - Project.from_directory(nonexistent_path) - - from kiwi_scp._constants import COMPOSE_FILE_NAME - assert exc_info.value.filename == str(nonexistent_path.joinpath(COMPOSE_FILE_NAME)) From 8d2c1e1facc2efa4002d16ee4dc2ea0ee769ecaa Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 26 Oct 2021 16:11:28 +0200 Subject: [PATCH 037/141] Test classes --- tests/test_config.py | 780 +++++++++++++++++++---------------------- tests/test_instance.py | 35 +- tests/test_service.py | 98 +++--- 3 files changed, 431 insertions(+), 482 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 143b7b1..459079c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -16,510 +16,462 @@ class UnCoercible: raise ValueError -def test_default(): - import toml +class TestDefault: + def test(self): + import toml - c = KiwiConfig() - version = toml.load("./pyproject.toml")["tool"]["poetry"]["version"] + c = KiwiConfig() + version = toml.load("./pyproject.toml")["tool"]["poetry"]["version"] - assert c == KiwiConfig.from_default() + assert c == KiwiConfig.from_default() - assert c.version == version - assert len(c.shells) == 1 - assert c.shells[0] == Path("/bin/bash") - assert c.projects == [] - assert c.environment == {} - assert c.storage.directory == Path("/var/local/kiwi") - assert c.network.name == "kiwi_hub" - assert c.network.cidr == IPv4Network("10.22.46.0/24") + assert c.version == version + assert len(c.shells) == 1 + assert c.shells[0] == Path("/bin/bash") + assert c.projects == [] + assert c.environment == {} + assert c.storage.directory == Path("/var/local/kiwi") + assert c.network.name == "kiwi_hub" + assert c.network.cidr == IPv4Network("10.22.46.0/24") - kiwi_dict = { - "version": version, - "shells": ["/bin/bash"], - "storage": {"directory": "/var/local/kiwi"}, - "network": { - "name": "kiwi_hub", - "cidr": "10.22.46.0/24", - }, - } - assert c.kiwi_dict == kiwi_dict + kiwi_dict = { + "version": version, + "shells": ["/bin/bash"], + "storage": {"directory": "/var/local/kiwi"}, + "network": { + "name": "kiwi_hub", + "cidr": "10.22.46.0/24", + }, + } + assert c.kiwi_dict == kiwi_dict - yml = ruamel.yaml.YAML() - yml.indent(offset=2) + yml = ruamel.yaml.YAML() + yml.indent(offset=2) - sio = io.StringIO() - from kiwi_scp.misc import _format_kiwi_yml - yml.dump(kiwi_dict, stream=sio, transform=_format_kiwi_yml) - yml_string = sio.getvalue() - sio.close() + sio = io.StringIO() + from kiwi_scp.misc import _format_kiwi_yml + yml.dump(kiwi_dict, stream=sio, transform=_format_kiwi_yml) + yml_string = sio.getvalue() + sio.close() - assert c.kiwi_yml == yml_string + assert c.kiwi_yml == yml_string -######### -# VERSION -######### +class TestVersion: + def test_valid(self): + c = KiwiConfig(version="0.0.0") + assert c.version == "0.0.0" -def test_version_valid(): - c = KiwiConfig(version="0.0.0") + c = KiwiConfig(version="0.0") + assert c.version == "0.0" - assert c.version == "0.0.0" + c = KiwiConfig(version="0") + assert c.version == "0" - c = KiwiConfig(version="0.0") + c = KiwiConfig(version=1.0) + assert c.version == "1.0" - assert c.version == "0.0" + c = KiwiConfig(version=1) + assert c.version == "1" - c = KiwiConfig(version="0") + def test_invalid(self): + # definitely not a version + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(version="dnaf") - assert c.version == "0" + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"].find("string does not match regex") != -1 + assert error["type"] == "value_error.str.regex" - c = KiwiConfig(version=1.0) + # almost a version + with pytest.raises(ValidationError) as exc_info: + c = KiwiConfig(version="0.0.0alpha") + print(c.version) - assert c.version == "1.0" + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"].find("string does not match regex") != -1 + assert error["type"] == "value_error.str.regex" - c = KiwiConfig(version=1) - assert c.version == "1" +class TestShells: + def test_empty(self): + c = KiwiConfig(shells=None) + assert c == KiwiConfig(shells=[]) -def test_version_invalid(): - # definitely not a version - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(version="dnaf") + assert c.shells == [] - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"].find("string does not match regex") != -1 - assert error["type"] == "value_error.str.regex" + def test_list(self): + c = KiwiConfig(shells=["/bin/sh", "sh"]) - # barely a version - with pytest.raises(ValidationError) as exc_info: - c = KiwiConfig(version="0.0.0alpha") - print(c.version) + assert len(c.shells) == 2 + assert c.shells[0] == Path("/bin/sh") + assert c.shells[1] == Path("sh") - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"].find("string does not match regex") != -1 - assert error["type"] == "value_error.str.regex" + c = KiwiConfig(shells=["/bin/bash"]) + + assert len(c.shells) == 1 + assert c.shells[0] == Path("/bin/bash") + def test_dict(self): + c = KiwiConfig(shells={"/bin/bash": None}) -######## -# SHELLS -######## + assert len(c.shells) == 1 + assert c.shells[0] == Path("/bin/bash") + + def test_coercible(self): + c = KiwiConfig(shells="/bin/bash") + + assert c == KiwiConfig(shells=Path("/bin/bash")) + + assert len(c.shells) == 1 + assert c.shells[0] == Path("/bin/bash") + + c = KiwiConfig(shells=123) + + assert len(c.shells) == 1 + assert c.shells[0] == Path("123") -def test_shells_empty(): - c = KiwiConfig(shells=None) + def test_uncoercible(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(shells=UnCoercible()) - assert c == KiwiConfig(shells=[]) + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Shells Format" + assert error["type"] == "value_error" - assert c.shells == [] + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(shells=["/bin/bash", UnCoercible()]) + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "value is not a valid path" + assert error["type"] == "type_error.path" -def test_shells_list(): - c = KiwiConfig(shells=["/bin/sh", "sh"]) - assert len(c.shells) == 2 - assert c.shells[0] == Path("/bin/sh") - assert c.shells[1] == Path("sh") +class TestProject: + def test_empty(self): + c = KiwiConfig(projects=None) - c = KiwiConfig(shells=["/bin/bash"]) + assert c == KiwiConfig(projects=[]) + + assert c.projects == [] - assert len(c.shells) == 1 - assert c.shells[0] == Path("/bin/bash") + def test_long(self): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": {"directory": "/test/directory"}, + } + c = KiwiConfig(projects=[kiwi_dict]) + + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert not p.enabled + assert p.override_storage is not None + + assert c.kiwi_dict["projects"][0] == kiwi_dict + + def test_storage_str(self): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": "/test/directory", + } + c = KiwiConfig(projects=[kiwi_dict]) + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert not p.enabled + assert p.override_storage is not None -def test_shells_dict(): - c = KiwiConfig(shells={"/bin/bash": None}) + def test_storage_list(self): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": ["/test/directory"], + } + c = KiwiConfig(projects=[kiwi_dict]) - assert len(c.shells) == 1 - assert c.shells[0] == Path("/bin/bash") + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert not p.enabled + assert p.override_storage is not None + def test_storage_invalid(self): + kiwi_dict = { + "name": "project", + "enabled": False, + "override_storage": True, + } + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(projects=[kiwi_dict]) -def test_shells_coercible(): - c = KiwiConfig(shells="/bin/bash") + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Storage Format" + assert error["type"] == "value_error" - assert c == KiwiConfig(shells=Path("/bin/bash")) + def test_short(self): + kiwi_dict = { + "project": False, + } + c = KiwiConfig(projects=[kiwi_dict]) - assert len(c.shells) == 1 - assert c.shells[0] == Path("/bin/bash") + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert not p.enabled + assert p.override_storage is None + assert p.kiwi_dict == kiwi_dict + + def test_dict(self): + c = KiwiConfig(projects={"name": "project"}) + + assert c == KiwiConfig(projects=[{"name": "project"}]) + + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert p.enabled + assert p.override_storage is None + + def test_invalid_dict(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(projects={ + "random key 1": "random value 1", + "random key 2": "random value 2", + }) + + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Project Format" + assert error["type"] == "value_error" + + def test_coercible(self): + c = KiwiConfig(projects="project") + + assert c == KiwiConfig(projects=["project"]) + + assert len(c.projects) == 1 + p = c.projects[0] + assert p.name == "project" + assert p.enabled + assert p.override_storage is None + + def test_uncoercible(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(projects=["valid", UnCoercible()]) + + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Projects Format" + assert error["type"] == "value_error" - c = KiwiConfig(shells=123) + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(projects=UnCoercible()) + + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Projects Format" + assert error["type"] == "value_error" - assert len(c.shells) == 1 - assert c.shells[0] == Path("123") +class TestEnvironment: + def test_empty(self): + c = KiwiConfig(environment=None) -def test_shells_uncoercible(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(shells=UnCoercible()) + assert c.environment == {} - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Shells Format" - assert error["type"] == "value_error" + def test_dict(self): + c = KiwiConfig(environment={}) - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(shells=["/bin/bash", UnCoercible()]) + assert c.environment == {} - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "value is not a valid path" - assert error["type"] == "type_error.path" + kiwi_dict = {"variable": "value"} + c = KiwiConfig(environment=kiwi_dict) + assert len(c.environment) == 1 + assert "variable" in c.environment + assert c.environment["variable"] == "value" -########## -# PROJECTS -########## + assert c.kiwi_dict["environment"] == kiwi_dict -def test_proj_empty(): - c = KiwiConfig(projects=None) + def test_list(self): + c = KiwiConfig(environment=[]) - assert c == KiwiConfig(projects=[]) + assert c.environment == {} - assert c.projects == [] + c = KiwiConfig(environment=[ + "variable=value", + ]) + assert len(c.environment) == 1 + assert "variable" in c.environment + assert c.environment["variable"] == "value" -def test_proj_long(): - kiwi_dict = { - "name": "project", - "enabled": False, - "override_storage": {"directory": "/test/directory"}, - } - c = KiwiConfig(projects=[kiwi_dict]) + c = KiwiConfig(environment=[ + "variable", + ]) - assert len(c.projects) == 1 - p = c.projects[0] - assert p.name == "project" - assert not p.enabled - assert p.override_storage is not None + assert len(c.environment) == 1 + assert "variable" in c.environment + assert c.environment["variable"] is None - assert c.kiwi_dict["projects"][0] == kiwi_dict + c = KiwiConfig(environment=[ + 123, + ]) + assert len(c.environment) == 1 + assert "123" in c.environment + assert c.environment["123"] is None -def test_proj_storage_str(): - kiwi_dict = { - "name": "project", - "enabled": False, - "override_storage": "/test/directory", - } - c = KiwiConfig(projects=[kiwi_dict]) + def test_coercible(self): + c = KiwiConfig(environment="variable=value") - assert len(c.projects) == 1 - p = c.projects[0] - assert p.name == "project" - assert not p.enabled - assert p.override_storage is not None + assert len(c.environment) == 1 + assert "variable" in c.environment + assert c.environment["variable"] == "value" + c = KiwiConfig(environment="variable") -def test_proj_storage_list(): - kiwi_dict = { - "name": "project", - "enabled": False, - "override_storage": ["/test/directory"], - } - c = KiwiConfig(projects=[kiwi_dict]) + assert len(c.environment) == 1 + assert "variable" in c.environment + assert c.environment["variable"] is None - assert len(c.projects) == 1 - p = c.projects[0] - assert p.name == "project" - assert not p.enabled - assert p.override_storage is not None + c = KiwiConfig(environment=123) + assert len(c.environment) == 1 + assert "123" in c.environment + assert c.environment["123"] is None -def test_proj_storage_invalid(): - kiwi_dict = { - "name": "project", - "enabled": False, - "override_storage": True, - } - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(projects=[kiwi_dict]) + c = KiwiConfig(environment=123.4) - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Storage Format" - assert error["type"] == "value_error" + assert len(c.environment) == 1 + assert "123.4" in c.environment + assert c.environment["123.4"] is None + def test_uncoercible(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(environment=UnCoercible()) -def test_proj_short(): - kiwi_dict = { - "project": False, - } - c = KiwiConfig(projects=[kiwi_dict]) + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Environment Format" + assert error["type"] == "value_error" - assert len(c.projects) == 1 - p = c.projects[0] - assert p.name == "project" - assert not p.enabled - assert p.override_storage is None - assert p.kiwi_dict == kiwi_dict + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(environment=["valid", UnCoercible()]) + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Environment Format" + assert error["type"] == "value_error" -def test_proj_dict(): - c = KiwiConfig(projects={"name": "project"}) - assert c == KiwiConfig(projects=[{"name": "project"}]) +class TestStorage: + def test_empty(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(storage=None) - assert len(c.projects) == 1 - p = c.projects[0] - assert p.name == "project" - assert p.enabled - assert p.override_storage is None + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "No Storage Given" + assert error["type"] == "value_error" + def test_dict(self): + kiwi_dict = {"directory": "/test/directory"} + c = KiwiConfig(storage=kiwi_dict) -def test_proj_invalid_dict(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(projects={ - "random key 1": "random value 1", - "random key 2": "random value 2", - }) + assert c.storage.directory == Path("/test/directory") + assert c.storage.kiwi_dict == kiwi_dict - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Project Format" - assert error["type"] == "value_error" + def test_invalid_dict(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(storage={"random key": "random value"}) + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Storage Format" + assert error["type"] == "value_error" -def test_proj_coercible(): - c = KiwiConfig(projects="project") + def test_str(self): + c = KiwiConfig(storage="/test/directory") - assert c == KiwiConfig(projects=["project"]) + assert c.storage.directory == Path("/test/directory") - assert len(c.projects) == 1 - p = c.projects[0] - assert p.name == "project" - assert p.enabled - assert p.override_storage is None + def test_list(self): + c = KiwiConfig(storage=["/test/directory"]) + assert c.storage.directory == Path("/test/directory") -def test_proj_uncoercible(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(projects=["valid", UnCoercible()]) + def test_invalid(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(storage=True) - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Projects Format" - assert error["type"] == "value_error" + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Storage Format" + assert error["type"] == "value_error" - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(projects=UnCoercible()) - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Projects Format" - assert error["type"] == "value_error" +class TestNetwork: + def test_empty(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(network=None) + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "No Network Given" + assert error["type"] == "value_error" -############# -# ENVIRONMENT -############# - -def test_env_empty(): - c = KiwiConfig(environment=None) - - assert c.environment == {} - - -def test_env_dict(): - c = KiwiConfig(environment={}) - - assert c.environment == {} - - kiwi_dict = {"variable": "value"} - c = KiwiConfig(environment=kiwi_dict) - - assert len(c.environment) == 1 - assert "variable" in c.environment - assert c.environment["variable"] == "value" - - assert c.kiwi_dict["environment"] == kiwi_dict - - -def test_env_list(): - c = KiwiConfig(environment=[]) - - assert c.environment == {} - - c = KiwiConfig(environment=[ - "variable=value", - ]) - - assert len(c.environment) == 1 - assert "variable" in c.environment - assert c.environment["variable"] == "value" - - c = KiwiConfig(environment=[ - "variable", - ]) - - assert len(c.environment) == 1 - assert "variable" in c.environment - assert c.environment["variable"] is None - - c = KiwiConfig(environment=[ - 123, - ]) - - assert len(c.environment) == 1 - assert "123" in c.environment - assert c.environment["123"] is None - - -def test_env_coercible(): - c = KiwiConfig(environment="variable=value") - - assert len(c.environment) == 1 - assert "variable" in c.environment - assert c.environment["variable"] == "value" - - c = KiwiConfig(environment="variable") - - assert len(c.environment) == 1 - assert "variable" in c.environment - assert c.environment["variable"] is None - - c = KiwiConfig(environment=123) - - assert len(c.environment) == 1 - assert "123" in c.environment - assert c.environment["123"] is None - - c = KiwiConfig(environment=123.4) - - assert len(c.environment) == 1 - assert "123.4" in c.environment - assert c.environment["123.4"] is None - - -def test_env_uncoercible(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(environment=UnCoercible()) - - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Environment Format" - assert error["type"] == "value_error" - - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(environment=["valid", UnCoercible()]) - - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Environment Format" - assert error["type"] == "value_error" - - -######### -# STORAGE -######### - -def test_storage_empty(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(storage=None) - - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "No Storage Given" - assert error["type"] == "value_error" - - -def test_storage_dict(): - kiwi_dict = {"directory": "/test/directory"} - c = KiwiConfig(storage=kiwi_dict) - - assert c.storage.directory == Path("/test/directory") - assert c.storage.kiwi_dict == kiwi_dict - - -def test_storage_invalid_dict(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(storage={"random key": "random value"}) - - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Storage Format" - assert error["type"] == "value_error" - - -def test_storage_str(): - c = KiwiConfig(storage="/test/directory") - - assert c.storage.directory == Path("/test/directory") - - -def test_storage_list(): - c = KiwiConfig(storage=["/test/directory"]) - - assert c.storage.directory == Path("/test/directory") - - -def test_storage_invalid(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(storage=True) - - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Storage Format" - assert error["type"] == "value_error" - - -######### -# NETWORK -######### - -def test_network_empty(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(network=None) - - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "No Network Given" - assert error["type"] == "value_error" - - -def test_network_dict(): - kiwi_dict = { - "name": "test_hub", - "cidr": "1.2.3.4/32", - } - c = KiwiConfig(network=kiwi_dict) - - assert c == KiwiConfig(network={ - "name": "TEST_HUB", - "cidr": "1.2.3.4/32", - }) - - assert c.network.name == "test_hub" - assert c.network.cidr == IPv4Network("1.2.3.4/32") - assert c.network.kiwi_dict == kiwi_dict - - -def test_network_invalid_dict(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(network={"name": "test_hub"}) - - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "field required" - assert error["type"] == "value_error.missing" - - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(network={ + def test_dict(self): + kiwi_dict = { "name": "test_hub", - "cidr": "1.2.3.4/123", + "cidr": "1.2.3.4/32", + } + c = KiwiConfig(network=kiwi_dict) + + assert c == KiwiConfig(network={ + "name": "TEST_HUB", + "cidr": "1.2.3.4/32", }) - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "value is not a valid IPv4 network" - assert error["type"] == "value_error.ipv4network" + assert c.network.name == "test_hub" + assert c.network.cidr == IPv4Network("1.2.3.4/32") + assert c.network.kiwi_dict == kiwi_dict + def test_invalid_dict(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(network={"name": "test_hub"}) -def test_network_invalid(): - with pytest.raises(ValidationError) as exc_info: - KiwiConfig(network=True) + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "field required" + assert error["type"] == "value_error.missing" - assert len(exc_info.value.errors()) == 1 - error = exc_info.value.errors()[0] - assert error["msg"] == "Invalid Network Format" - assert error["type"] == "value_error" + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(network={ + "name": "test_hub", + "cidr": "1.2.3.4/123", + }) + + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "value is not a valid IPv4 network" + assert error["type"] == "value_error.ipv4network" + + def test_invalid(self): + with pytest.raises(ValidationError) as exc_info: + KiwiConfig(network=True) + + assert len(exc_info.value.errors()) == 1 + error = exc_info.value.errors()[0] + assert error["msg"] == "Invalid Network Format" + assert error["type"] == "value_error" diff --git a/tests/test_instance.py b/tests/test_instance.py index d008748..68f4280 100644 --- a/tests/test_instance.py +++ b/tests/test_instance.py @@ -3,28 +3,27 @@ from pathlib import Path from kiwi_scp.instance import Instance -def test_example(): - i = Instance(Path("example")) +class TestDefault: + def test_example(self): + i = Instance(Path("example")) - assert i.config is not None - assert len(i.config.projects) == 1 + assert i.config is not None + assert len(i.config.projects) == 1 - p = i.config.projects[0] + p = i.config.projects[0] - assert p.name == "hello-world.project" + assert p.name == "hello-world.project" + def test_empty(self): + i = Instance() -def test_empty(): - i = Instance() + assert i.config is not None + assert len(i.config.projects) == 0 - assert i.config is not None - assert len(i.config.projects) == 0 + def test_no_such_dir(self): + nonexistent_path = Path("nonexistent") + i = Instance(nonexistent_path) - -def test_no_such_dir(): - nonexistent_path = Path("nonexistent") - i = Instance(nonexistent_path) - - assert i.directory == nonexistent_path - assert i.config is not None - assert len(i.config.projects) == 0 + assert i.directory == nonexistent_path + assert i.config is not None + assert len(i.config.projects) == 0 diff --git a/tests/test_service.py b/tests/test_service.py index b44bba8..d0198a7 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -3,60 +3,58 @@ from pathlib import Path from kiwi_scp.instance import Service -def test_no_description(): - s = Service.from_description( - name="s", - description={}, - ) +class TestDefault: + def test_no_description(self): + s = Service.from_description( + name="s", + description={}, + ) - assert s.name == "s" - assert s.configs == [] + assert s.name == "s" + assert s.configs == [] + def test_no_configs(self): + s = Service.from_description( + name="s", + description={ + "image": "repo/image:tag", + }, + ) -def test_no_configs(): - s = Service.from_description( - name="s", - description={ - "image": "repo/image:tag", - }, - ) + assert s.name == "s" + assert s.configs == [] - assert s.name == "s" - assert s.configs == [] + def test_no_configs_in_volumes(self): + s = Service.from_description( + name="s", + description={ + "image": "repo/image:tag", + "volumes": [ + "docker_volume/third/dir:/path/to/third/mountpoint", + "${TARGETDIR}/some/dir:/path/to/some/mountpoint", + "$TARGETDIR/other/dir:/path/to/other/mountpoint", + ] + }, + ) + assert s.name == "s" + assert s.configs == [] -def test_no_configs_in_volumes(): - s = Service.from_description( - name="s", - description={ - "image": "repo/image:tag", - "volumes": [ - "docker_volume/third/dir:/path/to/third/mountpoint", - "${TARGETDIR}/some/dir:/path/to/some/mountpoint", - "$TARGETDIR/other/dir:/path/to/other/mountpoint", - ] - }, - ) + def test_with_configs(self): + s = Service.from_description( + name="s", + description={ + "image": "repo/image:tag", + "volumes": [ + "${CONFDIR}/some/config:/path/to/some/config", + "$CONFDIR/other/config:/path/to/other/config", + ] + }, + ) - assert s.name == "s" - assert s.configs == [] - - -def test_with_configs(): - s = Service.from_description( - name="s", - description={ - "image": "repo/image:tag", - "volumes": [ - "${CONFDIR}/some/config:/path/to/some/config", - "$CONFDIR/other/config:/path/to/other/config", - ] - }, - ) - - assert s.name == "s" - assert len(s.configs) == 2 - assert s.configs == [ - Path("some/config"), - Path("other/config"), - ] + assert s.name == "s" + assert len(s.configs) == 2 + assert s.configs == [ + Path("some/config"), + Path("other/config"), + ] From b8027777d95350d6b90a3996bfb5967a63dae1f0 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Tue, 26 Oct 2021 16:19:33 +0200 Subject: [PATCH 038/141] Test coverage --- tests/test_config.py | 7 ++++++- tests/test_instance.py | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 459079c..26fdd1d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -157,9 +157,13 @@ class TestProject: c = KiwiConfig(projects=None) assert c == KiwiConfig(projects=[]) - assert c.projects == [] + with pytest.raises(ValueError) as exc_info: + c.get_project_config("invalid") + + assert str(exc_info.value) == "No Such Project" + def test_long(self): kiwi_dict = { "name": "project", @@ -171,6 +175,7 @@ class TestProject: assert len(c.projects) == 1 p = c.projects[0] assert p.name == "project" + assert p == c.get_project_config("project") assert not p.enabled assert p.override_storage is not None diff --git a/tests/test_instance.py b/tests/test_instance.py index 68f4280..68799a5 100644 --- a/tests/test_instance.py +++ b/tests/test_instance.py @@ -14,6 +14,10 @@ class TestDefault: assert p.name == "hello-world.project" + s = list(i.get_services(p.name)) + + assert len(s) == 5 + def test_empty(self): i = Instance() From c331e77060457b6cec0f175a3d15f41f35121b17 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Wed, 27 Oct 2021 13:33:26 +0200 Subject: [PATCH 039/141] Added kiwi_scp.misc.YAML --- kiwi_scp/config.py | 7 ++----- kiwi_scp/instance.py | 7 +++---- kiwi_scp/misc.py | 7 +++++++ tests/test_config.py | 7 ++----- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/kiwi_scp/config.py b/kiwi_scp/config.py index 3d79f0b..e4bf1b9 100644 --- a/kiwi_scp/config.py +++ b/kiwi_scp/config.py @@ -5,10 +5,9 @@ from pathlib import Path from typing import Optional, Dict, List, Any, TextIO from pydantic import BaseModel, constr, root_validator, validator -from ruamel.yaml import YAML from ._constants import RE_SEMVER, RE_VARNAME, KIWI_CONF_NAME -from .misc import _format_kiwi_yml +from .misc import YAML, _format_kiwi_yml class StorageConfig(BaseModel): @@ -191,9 +190,7 @@ class KiwiConfig(BaseModel): def dump_kiwi_yml(self, stream: TextIO) -> None: """dump a kiwi.yml file""" - yml = YAML() - yml.indent(offset=2) - yml.dump(self.kiwi_dict, stream=stream, transform=_format_kiwi_yml) + YAML().dump(self.kiwi_dict, stream=stream, transform=_format_kiwi_yml) @property def kiwi_yml(self) -> str: diff --git a/kiwi_scp/instance.py b/kiwi_scp/instance.py index c822133..bcc2c6f 100644 --- a/kiwi_scp/instance.py +++ b/kiwi_scp/instance.py @@ -5,10 +5,10 @@ from typing import List, Dict, Any, Generator import attr import click -from ruamel.yaml import YAML from ._constants import COMPOSE_FILE_NAME -from .config import KiwiConfig, ProjectConfig +from .config import KiwiConfig +from .misc import YAML _RE_CONFDIR = re.compile(r"^\s*\$(?:CONFDIR|{CONFDIR})/+(.*)$", flags=re.UNICODE) @@ -52,8 +52,7 @@ class Instance: @functools.lru_cache(maxsize=10) def _parse_compose_file(cls, directory: Path): with open(directory.joinpath(COMPOSE_FILE_NAME), "r") as cf: - yml = YAML() - return yml.load(cf) + return YAML().load(cf) def get_services(self, project_name: str) -> Generator[Service, None, None]: yml = Instance._parse_compose_file(self.directory.joinpath(project_name)) diff --git a/kiwi_scp/misc.py b/kiwi_scp/misc.py index 845acf1..72d5d8c 100644 --- a/kiwi_scp/misc.py +++ b/kiwi_scp/misc.py @@ -3,6 +3,7 @@ from typing import Any, Type, List, Callable import attr import click +import ruamel.yaml from click.decorators import FC from ._constants import HEADER_KIWI_CONF_NAME @@ -54,6 +55,12 @@ def user_query(description: str, default: Any, cast_to: Type[Any] = str): click.echo(f"Invalid input: {e}") +class YAML(ruamel.yaml.YAML): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.indent(offset=2) + + def _format_kiwi_yml(yml_string: str): # insert newline before every main key yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) diff --git a/tests/test_config.py b/tests/test_config.py index 26fdd1d..1d09e05 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,10 +3,10 @@ from ipaddress import IPv4Network from pathlib import Path import pytest -import ruamel.yaml from pydantic import ValidationError from kiwi_scp.config import KiwiConfig +from kiwi_scp.misc import YAML class UnCoercible: @@ -45,12 +45,9 @@ class TestDefault: } assert c.kiwi_dict == kiwi_dict - yml = ruamel.yaml.YAML() - yml.indent(offset=2) - sio = io.StringIO() from kiwi_scp.misc import _format_kiwi_yml - yml.dump(kiwi_dict, stream=sio, transform=_format_kiwi_yml) + YAML(typ="safe").dump(kiwi_dict, stream=sio, transform=_format_kiwi_yml) yml_string = sio.getvalue() sio.close() From 34988e58943f67d71e3d5f63daf66fa74d9dfb26 Mon Sep 17 00:00:00 2001 From: ldericher <40151420+ldericher@users.noreply.github.com> Date: Thu, 28 Oct 2021 13:51:53 +0200 Subject: [PATCH 040/141] Test run configs --- .../{Tests.xml => Test_Coverage.xml} | 2 +- .idea/runConfigurations/Tests__Debuggable_.xml | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) rename .idea/runConfigurations/{Tests.xml => Test_Coverage.xml} (91%) create mode 100644 .idea/runConfigurations/Tests__Debuggable_.xml diff --git a/.idea/runConfigurations/Tests.xml b/.idea/runConfigurations/Test_Coverage.xml similarity index 91% rename from .idea/runConfigurations/Tests.xml rename to .idea/runConfigurations/Test_Coverage.xml index 532689b..6278cf1 100644 --- a/.idea/runConfigurations/Tests.xml +++ b/.idea/runConfigurations/Test_Coverage.xml @@ -1,5 +1,5 @@ - +