From 746f28d65ed89e5745970f0861ad096d61e6e55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Thu, 6 Dec 2018 14:43:26 +0100 Subject: [PATCH 001/118] internal networking --- Makefile | 24 ++++++++++++++++++------ base.conf | 2 ++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 8c9de11..62932cc 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,11 @@ ifeq ($(CONF_DOCKERNET),) $(error DOCKERNET not set in $(CONF_WILDC)) endif +CONF_DOCKERCIDR:=$(call confvalue,DOCKERCIDR) +ifeq ($(CONF_DOCKERNET),) +$(error DOCKERCIDR not set in $(CONF_WILDC)) +endif + # persistent data directory CONF_TARGETROOT:=$(call confvalue,TARGETROOT) ifeq ($(CONF_TARGETROOT),) @@ -62,18 +67,18 @@ all: purge-conf up ######### # manage the docker network (container name local DNS) $(FILE_DOCKERNET): - sudo docker network create "$(CONF_DOCKERNET)" ||: + sudo docker network create --driver bridge --internal --subnet "$(CONF_DOCKERCIDR)" "$(CONF_DOCKERNET)" ||: sudo mkdir -p "$(CONF_TARGETROOT)" sudo chmod 700 "$(CONF_TARGETROOT)" - sudo docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$(CONF_DOCKERNET)" | sudo tee "$@" + sudo echo "$(CONF_DOCKERCIDR)" | sudo tee "$@" .PHONY: net-up net-up: $(FILE_DOCKERNET) .PHONY: net-down net-down: down - sudo docker network rm $(CONF_DOCKERNET) - sudo rm $(FILE_DOCKERNET) + sudo docker network rm "$(CONF_DOCKERNET)" + sudo rm "$(FILE_DOCKERNET)" ######### # sync project config directory to variable folder @@ -160,10 +165,14 @@ s?=bash # default compose file define COMPOSEFILE -version: "3" +version: "2" networks: - default: + # reachable from outside + default: + driver: bridge + # interconnects projects + gassi: external: name: $$DOCKERNET @@ -171,5 +180,8 @@ services: something: image: maintainer/repo:tag restart: unless-stopped + networks: + - default + - gassi [...] endef diff --git a/base.conf b/base.conf index 49c1b20..7d392e4 100644 --- a/base.conf +++ b/base.conf @@ -2,4 +2,6 @@ export SUFFIX_PROJECT=.project export SUFFIX_DOWN=.down export DOCKERNET=kiwinet +export DOCKERCIDR=10.13.37.0/24 + export TARGETROOT=/var/kiwi From 7f6b8952cb937636512082bbfdd1a62b3c09e72d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Thu, 6 Dec 2018 15:01:37 +0100 Subject: [PATCH 002/118] tabs and spaces --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 62932cc..64fed4b 100644 --- a/Makefile +++ b/Makefile @@ -168,10 +168,10 @@ define COMPOSEFILE version: "2" networks: - # reachable from outside - default: - driver: bridge - # interconnects projects + # reachable from outside + default: + driver: bridge + # interconnects projects gassi: external: name: $$DOCKERNET From d3434557fc0160d146d7889977fb77f3b7c95769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Tue, 15 Oct 2019 13:28:43 +0200 Subject: [PATCH 003/118] General QoL --- Makefile | 135 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 99 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 64fed4b..1478c89 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,22 @@ +########## +# COMMANDS + +DOCKER:=docker +DOCKER_COMPOSE:=docker-compose +docker_bash=bash -c '$(1)' + +# Check if needs root privileges? +PRIVGROUP:=docker +ifneq ($(findstring $(PRIVGROUP),$(shell groups)),$(PRIVGROUP)) +DOCKER:=sudo $(DOCKER) +docker_bash=sudo $(docker_bash) +endif + ######### # CONFIGS CONF_WILDC:=$(wildcard $(PWD)/*.conf) +# apply source to *all* configs! CONF_SOURCE:=$(patsubst %,. %;,$(CONF_WILDC)) # extraction of env variables from *.conf files @@ -13,6 +28,7 @@ ifeq ($(CONF_DOCKERNET),) $(error DOCKERNET not set in $(CONF_WILDC)) endif +# docker network CIDR CONF_DOCKERCIDR:=$(call confvalue,DOCKERCIDR) ifeq ($(CONF_DOCKERNET),) $(error DOCKERCIDR not set in $(CONF_WILDC)) @@ -49,13 +65,17 @@ PROJ_NAMES:=$(basename $(PROJ_WILDC)) ######### # FUNCTIONS -# different complexities of commands with root privileges -# - in project directory -projsudo=cd "$<"; sudo bash -c "$(1)" -# - additionally with sourced *.conf files -confprojsudo=$(call projsudo,$(CONF_SOURCE) $(1)) -# - only for compose: additionally with COMPOSE_PROJECT_NAME, CONFDIR and TARGETDIR set -sudocompose=$(call confprojsudo,COMPOSE_PROJECT_NAME="$(basename $<)" CONFDIR="$(CONF_TARGETROOT)/conf" TARGETDIR="$(CONF_TARGETROOT)/$<" docker-compose $(1)) +# run DOCKER_COMPOSE: +# - in project directory +# - with sourced *.conf files +# - with COMPOSE_PROJECT_NAME, CONFDIR and TARGETDIR set +kiwicompose=$(call docker_bash,\ + cd "$(<)"; \ + $(CONF_SOURCE) \ + COMPOSE_PROJECT_NAME="$(patsubst %$(PROJ_SUFFX),%,$<)" \ + CONFDIR="$(CONF_TARGETROOT)/conf" \ + TARGETDIR="$(CONF_TARGETROOT)/$<" \ + $(DOCKER_COMPOSE) $(1)) ######### # TARGETS @@ -67,75 +87,118 @@ all: purge-conf up ######### # manage the docker network (container name local DNS) $(FILE_DOCKERNET): - sudo docker network create --driver bridge --internal --subnet "$(CONF_DOCKERCIDR)" "$(CONF_DOCKERNET)" ||: - sudo mkdir -p "$(CONF_TARGETROOT)" - sudo chmod 700 "$(CONF_TARGETROOT)" - sudo echo "$(CONF_DOCKERCIDR)" | sudo tee "$@" + -$(DOCKER) network create \ + --driver bridge \ + --internal \ + --subnet "$(CONF_DOCKERCIDR)" \ + "$(CONF_DOCKERNET)" + @echo "Creating canary $(FILE_DOCKERNET) ..." + @$(DOCKER) run --rm \ + -v "/:/mnt" -u root alpine:latest \ + ash -c '\ + mkdir -p "$(addprefix /mnt, $(CONF_TARGETROOT))"; \ + echo "$(CONF_DOCKERCIDR)" > "$(addprefix /mnt, $(FILE_DOCKERNET))"; \ + ' .PHONY: net-up net-up: $(FILE_DOCKERNET) .PHONY: net-down net-down: down - sudo docker network rm "$(CONF_DOCKERNET)" - sudo rm "$(FILE_DOCKERNET)" + $(DOCKER) network rm "$(CONF_DOCKERNET)" + @echo "Removing canary $(FILE_DOCKERNET) ..." + @$(DOCKER) run --rm \ + -v "/:/mnt" -u root alpine:latest \ + ash -c '\ + rm -f "$(addprefix /mnt, $(FILE_DOCKERNET))"; \ + ' ######### # sync project config directory to variable folder -.PHONY: %-copyconf -%-copyconf: %$(PROJ_SUFFX) - @if [ -d "$ /dev/null + + $(eval sources:=$(wildcard *${PROJ_SUFFX}/conf)) + @echo "Syncing $(sources) to $(CONF_TARGETROOT) ..." + + $(eval sources:=$(realpath $(sources))) + $(eval sources:=$(addprefix /mnt, $(sources))) + $(eval sources:=$(patsubst %,'%',$(sources))) + $(eval dest:='$(addprefix /mnt, $(CONF_TARGETROOT))') + + @$(DOCKER) run --rm \ + -v "/:/mnt" -u root ldericher/kiwi-config:rsync \ + ash -c '\ + rsync -r $(sources) $(dest); \ + ' +endif .PHONY: purge-conf purge-conf: - sudo rm -rf "$(CONF_TARGETROOT)/conf" + @echo "Emptying $(CONF_TARGETROOT)/conf ..." + @$(DOCKER) run --rm \ + -v "/:/mnt" -u root alpine:latest \ + ash -c '\ + rm -rf "$(addprefix /mnt, $(CONF_TARGETROOT)/conf)"; \ + ' ######### # manage all projects .PHONY: up down update -up: net-up $(patsubst %,%-copyconf,$(PROJ_NAMES)) $(patsubst %,%-up,$(PROJ_NAMES)) +up: net-up copy-conf $(patsubst %,%-up,$(PROJ_NAMES)) down: $(patsubst %,%-down,$(PROJ_NAMES)) update: $(patsubst %,%-update,$(PROJ_NAMES)) ######### # manage single project .PHONY: %-up -%-up: %$(PROJ_SUFFX) - $(call sudocompose,up -d $(x)) +%-up: %$(PROJ_SUFFX) net-up + $(call kiwicompose,up -d $(x)) .PHONY: %-down ifeq ($(x),) %-down: %$(PROJ_SUFFX) - $(call sudocompose,down) + $(call kiwicompose,down) else %-down: %$(PROJ_SUFFX) - $(call sudocompose,stop $(x)) - $(call sudocompose,rm -f $(x)) + $(call kiwicompose,stop $(x)) + $(call kiwicompose,rm -f $(x)) endif .PHONY: %-pull %-pull: %$(PROJ_SUFFX) - $(call sudocompose,pull $(x)) + $(call kiwicompose,pull --ignore-pull-failures $(x)) .PHONY: %-build %-build: %$(PROJ_SUFFX) - $(call sudocompose,build --pull $(x)) + $(call kiwicompose,build --pull $(x)) .PHONY: %-logs %-logs: %$(PROJ_SUFFX) - $(call sudocompose,logs -t $(x)) 2>/dev/null | less -R +G + $(call kiwicompose,logs -t $(x)) 2>/dev/null | less -R +G .PHONY: %-logf %-logf: %$(PROJ_SUFFX) - $(call sudocompose,logs -tf --tail=10 $(x)) ||: + $(call kiwicompose,logs -tf --tail=10 $(x)) ||: +ifneq ($(x),) s?=bash .PHONY: %-sh %-sh: %$(PROJ_SUFFX) - $(call sudocompose,exec $(x) $(s)) ||: + $(call kiwicompose,exec $(x) /bin/sh -c "[ -e /bin/$(s) ] && /bin/$(s) || /bin/sh") +endif # enabling and disabling .PHONY: %-enable %-disable @@ -152,7 +215,7 @@ s?=bash # Arbitrary compose command .PHONY: %-cmd %-cmd: %$(PROJ_SUFFX) - $(call sudocompose,$(x)) + $(call kiwicompose,$(x)) ######### # project creation @@ -161,7 +224,7 @@ s?=bash $(eval proj_dir:=$(patsubst %-new,%$(PROJ_SUFFX)$(DOWN_SUFFX),$@)) mkdir $(proj_dir) $(eval export COMPOSEFILE) - echo -e "$$COMPOSEFILE" > $(proj_dir)/docker-compose.yml + echo -e "$${COMPOSEFILE}" > $(proj_dir)/docker-compose.yml # default compose file define COMPOSEFILE @@ -172,7 +235,7 @@ networks: default: driver: bridge # interconnects projects - gassi: + hubnet: external: name: $$DOCKERNET @@ -182,6 +245,6 @@ services: restart: unless-stopped networks: - default - - gassi + - hubnet [...] -endef +endef \ No newline at end of file From 9910619bf127b4db817027826fe041061a0f9dd2 Mon Sep 17 00:00:00 2001 From: JMM <40151420+ldericher@users.noreply.github.com> Date: Tue, 15 Oct 2019 21:53:41 +0200 Subject: [PATCH 004/118] Update Makefile Remove unused `bash` call, correct suffix removal --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 1478c89..4d7ff2b 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,13 @@ DOCKER:=docker DOCKER_COMPOSE:=docker-compose -docker_bash=bash -c '$(1)' +docker_bash=$(1) # Check if needs root privileges? PRIVGROUP:=docker ifneq ($(findstring $(PRIVGROUP),$(shell groups)),$(PRIVGROUP)) DOCKER:=sudo $(DOCKER) -docker_bash=sudo $(docker_bash) +docker_bash=sudo bash -c '$(1)' endif ######### @@ -60,7 +60,7 @@ FILE_DOCKERNET:=$(CONF_TARGETROOT)/up-$(CONF_DOCKERNET) # project directory handling PROJ_WILDC:=$(wildcard *$(PROJ_SUFFX)) -PROJ_NAMES:=$(basename $(PROJ_WILDC)) +PROJ_NAMES:=$(patsubst %$(PROJ_SUFFX),%,$(PROJ_WILDC)) ######### # FUNCTIONS @@ -70,7 +70,7 @@ PROJ_NAMES:=$(basename $(PROJ_WILDC)) # - with sourced *.conf files # - with COMPOSE_PROJECT_NAME, CONFDIR and TARGETDIR set kiwicompose=$(call docker_bash,\ - cd "$(<)"; \ + cd "$<"; \ $(CONF_SOURCE) \ COMPOSE_PROJECT_NAME="$(patsubst %$(PROJ_SUFFX),%,$<)" \ CONFDIR="$(CONF_TARGETROOT)/conf" \ @@ -247,4 +247,4 @@ services: - default - hubnet [...] -endef \ No newline at end of file +endef From 53a2380f948493875094cdfc1ed1c884292804a4 Mon Sep 17 00:00:00 2001 From: JMM <40151420+ldericher@users.noreply.github.com> Date: Thu, 17 Oct 2019 13:45:48 +0200 Subject: [PATCH 005/118] Update Makefile correct "basename", %-update command --- Makefile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4d7ff2b..c10d92b 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,14 @@ endif # file to store docker network cidr FILE_DOCKERNET:=$(CONF_TARGETROOT)/up-$(CONF_DOCKERNET) +# remove any suffix $2 from $1 +rmsuffix=$(patsubst %$2,%,$1) +# remove project suffix from $1 +projname=$(call rmsuffix,$1,$(PROJ_SUFFX)) + # project directory handling PROJ_WILDC:=$(wildcard *$(PROJ_SUFFX)) -PROJ_NAMES:=$(patsubst %$(PROJ_SUFFX),%,$(PROJ_WILDC)) +PROJ_NAMES:=$(call projname,$(PROJ_WILDC)) ######### # FUNCTIONS @@ -72,7 +77,7 @@ PROJ_NAMES:=$(patsubst %$(PROJ_SUFFX),%,$(PROJ_WILDC)) kiwicompose=$(call docker_bash,\ cd "$<"; \ $(CONF_SOURCE) \ - COMPOSE_PROJECT_NAME="$(patsubst %$(PROJ_SUFFX),%,$<)" \ + COMPOSE_PROJECT_NAME="$(call projname,$<)" \ CONFDIR="$(CONF_TARGETROOT)/conf" \ TARGETDIR="$(CONF_TARGETROOT)/$<" \ $(DOCKER_COMPOSE) $(1)) @@ -116,7 +121,7 @@ net-down: down ######### # sync project config directory to variable folder -# Dockerfile for +# Dockerfile for running rsync as root define DOCKERFILE_RSYNC FROM alpine:latest RUN apk --no-cache add rsync @@ -203,14 +208,15 @@ endif # enabling and disabling .PHONY: %-enable %-disable %-enable: %$(PROJ_SUFFX)$(DOWN_SUFFX) - mv "$<" "$(basename $<)" + mv "$<" "$(call projname,$(call rmsuffix,$<,$(DOWN_SUFFX)))$(PROJ_SUFFX)" %-disable: %$(PROJ_SUFFX) mv "$<" "$<$(DOWN_SUFFX)" # Combinations .PHONY: %-update -%-update: %$(PROJ_SUFFX) %-build %-pull - $(MAKE) $(basename $<)-up +%-update: %$(PROJ_SUFFX) %-build %-pull copy-conf + $(MAKE) $(call projname,$<)-cmd x="restart $(x)" + $(MAKE) $(call projname,$<)-up # Arbitrary compose command .PHONY: %-cmd From b92adfaf4b31abb4f1a4a69f18a7b14954bc2130 Mon Sep 17 00:00:00 2001 From: JMM <40151420+ldericher@users.noreply.github.com> Date: Thu, 17 Oct 2019 15:11:32 +0200 Subject: [PATCH 006/118] Update Makefile full restart on container update --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c10d92b..b464b4f 100644 --- a/Makefile +++ b/Makefile @@ -215,7 +215,7 @@ endif # Combinations .PHONY: %-update %-update: %$(PROJ_SUFFX) %-build %-pull copy-conf - $(MAKE) $(call projname,$<)-cmd x="restart $(x)" + $(MAKE) $(call projname,$<)-down $(MAKE) $(call projname,$<)-up # Arbitrary compose command From bd71a6dea48a3ac32d6cec310f2d1355f479afa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= Date: Tue, 4 Aug 2020 02:56:29 +0200 Subject: [PATCH 007/118] silent multiversion installer --- bin/main.sh | 13 ++++++++++ kiwi | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++ kiwi.conf | 1 + version-tag | 1 + 4 files changed, 85 insertions(+) create mode 100755 bin/main.sh create mode 100755 kiwi create mode 100644 kiwi.conf create mode 100644 version-tag diff --git a/bin/main.sh b/bin/main.sh new file mode 100755 index 0000000..ece7b24 --- /dev/null +++ b/bin/main.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo "Hello World!" + +echo "This is ${0}." + +echo "A.K.A. $(readlink -f ${0})." + +echo "Arguments are:" + +for an_arg in "${@}" ; do + echo "- ${an_arg}" +done \ No newline at end of file diff --git a/kiwi b/kiwi new file mode 100755 index 0000000..1b330e9 --- /dev/null +++ b/kiwi @@ -0,0 +1,70 @@ +#!/bin/bash + +############# +# CONSTANTS # +############# + +# base config filename (constant) +KIWI_CONF_NAME="kiwi.conf" +# base install dir +KIWI_BASEDIR="${HOME}/.cache/kiwi-config-bin" +# repository uri +KIWI_REPO="https://github.com/ldericher/kiwi-config" + +######## +# MAIN # +######## + +# use latest version by default +version="master" + +# check if pwd is a kiwi folder +if [ -f "./${KIWI_CONF_NAME}" ]; then + # determine needed kiwi-config version + version=$(source "./${KIWI_CONF_NAME}" && echo "v${VERSION}") +fi + +# install if kiwi-config not found +if [ ! -x "${KIWI_BASEDIR}/${version}/main.sh" ]; then + echo -n "Installing kiwi-config ${version} into ${KIWI_BASEDIR} ... " + + ### production version ### + + # # create temp dir + # tmpdir=$(mktemp -d) + + # ( + # cd "${tmpdir}" + + # # download archive + # wget "${KIWI_REPO}/archive/${version}.zip" + # unzip "${version}.zip" + + # # read archive version tag + # cd "kiwi-config-${version}" + # version=$(cat ./version-tag) + + # # install archive + # mkdir -p "${KIWI_BASEDIR}" + # mv ./bin "${KIWI_BASEDIR}/${version}" + # ) + + # # discard temp dir + # rm -rf "${tmpdir}" + + ### development version ### + + ( + # read this version tag + version=$(cat ./version-tag) + + # install this + mkdir -p "${KIWI_BASEDIR}" + ln -s "$(readlink -f ./bin)" "${KIWI_BASEDIR}/${version}" + ) + + echo "OK" +fi + +# run main script +exec "${KIWI_BASEDIR}/${version}/main.sh" "${@}" \ No newline at end of file diff --git a/kiwi.conf b/kiwi.conf new file mode 100644 index 0000000..a5b8ea7 --- /dev/null +++ b/kiwi.conf @@ -0,0 +1 @@ +VERSION="0.1" \ No newline at end of file diff --git a/version-tag b/version-tag new file mode 100644 index 0000000..aa33868 --- /dev/null +++ b/version-tag @@ -0,0 +1 @@ +v0.1 \ No newline at end of file From d2be89ee454e709951d36b41969743f6017fdc0f Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 4 Aug 2020 16:38:51 +0200 Subject: [PATCH 008/118] Some unstable shellscript work --- bin/main.sh | 13 --------- kiwi | 57 +++++++++++++++++++------------------ src/bin/init.sh | 59 +++++++++++++++++++++++++++++++++++++++ src/bin/main.sh | 14 ++++++++++ src/etc/default.kiwi.conf | 7 +++++ src/inc/functions.sh | 15 ++++++++++ version-tag | 2 +- 7 files changed, 124 insertions(+), 43 deletions(-) delete mode 100755 bin/main.sh create mode 100755 src/bin/init.sh create mode 100755 src/bin/main.sh create mode 100644 src/etc/default.kiwi.conf create mode 100644 src/inc/functions.sh diff --git a/bin/main.sh b/bin/main.sh deleted file mode 100755 index ece7b24..0000000 --- a/bin/main.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -echo "Hello World!" - -echo "This is ${0}." - -echo "A.K.A. $(readlink -f ${0})." - -echo "Arguments are:" - -for an_arg in "${@}" ; do - echo "- ${an_arg}" -done \ No newline at end of file diff --git a/kiwi b/kiwi index 1b330e9..0e66857 100755 --- a/kiwi +++ b/kiwi @@ -4,10 +4,10 @@ # CONSTANTS # ############# -# base config filename (constant) -KIWI_CONF_NAME="kiwi.conf" +# base config filename +export KIWI_CONF_NAME="kiwi.conf" # base install dir -KIWI_BASEDIR="${HOME}/.cache/kiwi-config-bin" +KIWI_BASEDIR="${HOME}/.cache/kiwi-config" # repository uri KIWI_REPO="https://github.com/ldericher/kiwi-config" @@ -16,55 +16,54 @@ KIWI_REPO="https://github.com/ldericher/kiwi-config" ######## # use latest version by default -version="master" +export KIWI_VERSION="master" # check if pwd is a kiwi folder if [ -f "./${KIWI_CONF_NAME}" ]; then # determine needed kiwi-config version - version=$(source "./${KIWI_CONF_NAME}" && echo "v${VERSION}") + export KIWI_VERSION=$(source "./${KIWI_CONF_NAME}" && echo "${VERSION}") fi # install if kiwi-config not found -if [ ! -x "${KIWI_BASEDIR}/${version}/main.sh" ]; then - echo -n "Installing kiwi-config ${version} into ${KIWI_BASEDIR} ... " +if [ ! -x "${KIWI_BASEDIR}/${KIWI_VERSION}/bin/main.sh" ]; then + echo -n "Installing kiwi-config v${KIWI_VERSION} into ${KIWI_BASEDIR} ... " ### production version ### - # # create temp dir + # # switch to temp dir + # workdir=$(pwd) # tmpdir=$(mktemp -d) + # cd "${tmpdir}" - # ( - # cd "${tmpdir}" + # # download archive + # wget "${KIWI_REPO}/archive/${KIWI_VERSION}.zip" + # unzip "${KIWI_VERSION}.zip" - # # download archive - # wget "${KIWI_REPO}/archive/${version}.zip" - # unzip "${version}.zip" + # # read archive version tag + # cd "kiwi-config-${KIWI_VERSION}" + # export KIWI_VERSION=$(cat ./version-tag) - # # read archive version tag - # cd "kiwi-config-${version}" - # version=$(cat ./version-tag) - - # # install archive - # mkdir -p "${KIWI_BASEDIR}" - # mv ./bin "${KIWI_BASEDIR}/${version}" - # ) + # # install archive + # mkdir -p "${KIWI_BASEDIR}" + # mv ./src "${KIWI_BASEDIR}/${KIWI_VERSION}" # # discard temp dir + # cd "${workdir}" # rm -rf "${tmpdir}" ### development version ### - ( - # read this version tag - version=$(cat ./version-tag) + # read this version tag + export KIWI_VERSION=$(cat ./version-tag) - # install this - mkdir -p "${KIWI_BASEDIR}" - ln -s "$(readlink -f ./bin)" "${KIWI_BASEDIR}/${version}" - ) + # install this + mkdir -p "${KIWI_BASEDIR}" + ln -s "$(readlink -f ./src)" "${KIWI_BASEDIR}/${KIWI_VERSION}" echo "OK" fi +export KIWI_ROOT="${KIWI_BASEDIR}/${KIWI_VERSION}" + # run main script -exec "${KIWI_BASEDIR}/${version}/main.sh" "${@}" \ No newline at end of file +exec "${KIWI_ROOT}/bin/main.sh" "${@}" \ No newline at end of file diff --git a/src/bin/init.sh b/src/bin/init.sh new file mode 100755 index 0000000..561de07 --- /dev/null +++ b/src/bin/init.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +source "${KIWI_ROOT}/inc/functions.sh" + +# read default baseconfig +conf_VERSION="${KIWI_VERSION}" +read_kiwi_config "${KIWI_ROOT}/etc/default.${KIWI_CONF_NAME}" + +# if pwd is a kiwi folder, read local baseconfig +if [ -f "./${KIWI_CONF_NAME}" ]; then + echo "[WARN] Overwriting existing '${KIWI_CONF_NAME}'" + read_kiwi_config "./${KIWI_CONF_NAME}" +fi + +function user_input() { + local prompt="${1}" + local varname="${2}" + + local input + read -p "${prompt} [Default '${!varname}']: " input + + eval "${varname}='${input:-${!varname}}'" +} + +declare -A config_explain +config_explain=( + [VERSION]="kiwi-config version" + [SUFFIX_PROJECT]="suffix for project directories" + [SUFFIX_DOWN]="suffix for disabled projects" +) + +for varname in "${!config_explain[@]}"; do + echo "${varname}" + user_input "Enter ${config_explain[${varname}]}" conf_${varname} +done + +exit 0 + +user_input "Choose kiwi-config version" conf_VERSION + +user_input "Enter suffix for project directories" conf_SUFFIX_PROJECT +user_input "Enter suffix for disabled projects" conf_SUFFIX_DOWN + +user_input "Enter suffix for disabled projects" conf_SUFFIX_DOWN +user_input "Enter suffix for disabled projects" conf_SUFFIX_DOWN + +user_input "Enter suffix for disabled projects" conf_SUFFIX_DOWN + +echo "conf_VERSION: ${conf_VERSION}" +echo "conf_SUFFIX_PROJECT: ${conf_SUFFIX_PROJECT}" + +exit 0 + +read -p "Choose kiwi-config version [Default '${conf_version}']: " kiwi_VERSION +conf_VERSION=${kiwi_VERSION:-${conf_VERSION}} + +read -p "suffix for kiwi-config version to use [Default '${conf_version}']: " kiwi_suffix_project + +echo "conf_VERSION: ${conf_VERSION}" \ No newline at end of file diff --git a/src/bin/main.sh b/src/bin/main.sh new file mode 100755 index 0000000..5a442fa --- /dev/null +++ b/src/bin/main.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +command="${1}" +shift 1 + +case "${command}" in + "init") + exec "${KIWI_ROOT}/bin/${command}.sh" "${@}" + ;; + *) + echo "Unknown kiwi command '${command}'." + exit 1 + ;; +esac \ No newline at end of file diff --git a/src/etc/default.kiwi.conf b/src/etc/default.kiwi.conf new file mode 100644 index 0000000..4e9712d --- /dev/null +++ b/src/etc/default.kiwi.conf @@ -0,0 +1,7 @@ +SUFFIX_PROJECT=.project +SUFFIX_DOWN=.down + +DOCKERNET=kiwinet +DOCKERCIDR=10.22.46.0/24 + +TARGETROOT=/var/kiwi diff --git a/src/inc/functions.sh b/src/inc/functions.sh new file mode 100644 index 0000000..6f88a53 --- /dev/null +++ b/src/inc/functions.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +function read_kiwi_config() { + local conf_file="${1}" + local conf_prefix="${2:-conf_}" + + local conf_file_content=$(sed -r 's/^\s*(\S+)/'${conf_prefix}'\1/g' "${conf_file}") + eval "${conf_file_content}" +} + +function write_kiwi_config() { + local conf_prefix="${2:-conf_}" + + +} \ No newline at end of file diff --git a/version-tag b/version-tag index aa33868..ceab6e1 100644 --- a/version-tag +++ b/version-tag @@ -1 +1 @@ -v0.1 \ No newline at end of file +0.1 \ No newline at end of file From b4e9e94f2c6dafa32dee7d70adee2c53ab24ab3d Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 4 Aug 2020 16:52:30 +0200 Subject: [PATCH 009/118] begin work on python implementation --- .gitignore | 1 + .idea/.gitignore | 4 ++++ .../inspectionProfiles/profiles_settings.xml | 6 ++++++ .idea/kiwi-config.iml | 10 ++++++++++ .idea/misc.xml | 4 ++++ .idea/modules.xml | 8 ++++++++ .idea/vcs.xml | 6 ++++++ src/etc/default.kiwi.yml | 12 +++++++++++ src/kiwi-config.py | 20 +++++++++++++++++++ src/kiwi/__init__.py | 1 + src/kiwi/config.py | 15 ++++++++++++++ 11 files changed, 87 insertions(+) create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/kiwi-config.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 src/etc/default.kiwi.yml create mode 100644 src/kiwi-config.py create mode 100644 src/kiwi/__init__.py create mode 100644 src/kiwi/config.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..351c96d --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,4 @@ +# Default ignored files +/shelf/ +/workspace.xml + diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/kiwi-config.iml b/.idea/kiwi-config.iml new file mode 100644 index 0000000..15c0cd1 --- /dev/null +++ b/.idea/kiwi-config.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d4fc832 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..b44487f --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/etc/default.kiwi.yml b/src/etc/default.kiwi.yml new file mode 100644 index 0000000..ec93e07 --- /dev/null +++ b/src/etc/default.kiwi.yml @@ -0,0 +1,12 @@ +version: 0.1 + +suffix: + project: .project + down: .down + +docker: + net: kiwinet + cidr: 10.22.46.0/24 + +target: + root: /var/kiwi diff --git a/src/kiwi-config.py b/src/kiwi-config.py new file mode 100644 index 0000000..5fae606 --- /dev/null +++ b/src/kiwi-config.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import argparse +from kiwi import * + + +def main(): + parser = argparse.ArgumentParser(description='kiwi-config') + parser.add_argument('cmd', metavar='command', type=str, help='subcommand to execute') + + args = parser.parse_args() + print(args.cmd) + + cf = config.Config.default() + + pass + + +if __name__ == "__main__": + main() diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py new file mode 100644 index 0000000..7c281ff --- /dev/null +++ b/src/kiwi/__init__.py @@ -0,0 +1 @@ +__all__ = ['config'] diff --git a/src/kiwi/config.py b/src/kiwi/config.py new file mode 100644 index 0000000..abff981 --- /dev/null +++ b/src/kiwi/config.py @@ -0,0 +1,15 @@ +class Config: + # version, suffix_project, suffix_down, docker_net, docker_cidr, target_root = None + + def __init__(self, filename): + import yaml + + with open(filename, 'r') as stream: + try: + print(yaml.safe_load(stream)) + except yaml.YAMLError as exc: + print(exc) + + @classmethod + def default(cls): + return cls("./etc/default.kiwi.yml") From 6067c30eddbbff899632428bc098b64469544b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= Date: Tue, 4 Aug 2020 20:24:19 +0200 Subject: [PATCH 010/118] Crude init function --- src/{etc => }/default.kiwi.yml | 2 +- src/kiwi-config.py | 17 ++++++++++--- src/kiwi/config.py | 46 ++++++++++++++++++++++++++++++---- version-tag => src/version-tag | 0 4 files changed, 55 insertions(+), 10 deletions(-) rename src/{etc => }/default.kiwi.yml (89%) mode change 100644 => 100755 src/kiwi-config.py rename version-tag => src/version-tag (100%) diff --git a/src/etc/default.kiwi.yml b/src/default.kiwi.yml similarity index 89% rename from src/etc/default.kiwi.yml rename to src/default.kiwi.yml index ec93e07..f3bd13b 100644 --- a/src/etc/default.kiwi.yml +++ b/src/default.kiwi.yml @@ -1,4 +1,4 @@ -version: 0.1 +version: suffix: project: .project diff --git a/src/kiwi-config.py b/src/kiwi-config.py old mode 100644 new mode 100755 index 5fae606..1f38814 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -1,17 +1,26 @@ #!/usr/bin/env python3 import argparse -from kiwi import * +from kiwi.config import * def main(): parser = argparse.ArgumentParser(description='kiwi-config') - parser.add_argument('cmd', metavar='command', type=str, help='subcommand to execute') + + subs = parser.add_subparsers() + subs.required = True + subs.dest = 'command' + + subs.add_parser('init', help="Create new kiwi-config instance") + subs.add_parser('add', help="Add a project to kiwi-config") + + # parser.add_argument('cmd', metavar='command', type=str, help='subcommand to execute') args = parser.parse_args() - print(args.cmd) + print(args.command) - cf = config.Config.default() + cf = Config.default() + cf.init() pass diff --git a/src/kiwi/config.py b/src/kiwi/config.py index abff981..07f0e69 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -1,15 +1,51 @@ +import os +import yaml + + + class Config: - # version, suffix_project, suffix_down, docker_net, docker_cidr, target_root = None + __content = None def __init__(self, filename): - import yaml - with open(filename, 'r') as stream: try: - print(yaml.safe_load(stream)) + self.__content = yaml.safe_load(stream) + print(self.__content) + except yaml.YAMLError as exc: print(exc) @classmethod def default(cls): - return cls("./etc/default.kiwi.yml") + kiwi_root = os.environ.get('KIWI_ROOT') + + cfg = cls(kiwi_root + "/default.kiwi.yml") + with open(kiwi_root + "/version-tag", 'r') as stream: + cfg.__content["version"] = stream.read().strip() + + return cfg + + def __user_input(self, path, key, prompt): + content = self.__content + + for step in path: + content = content[step] + + try: + result = input("{} [Default: {}] ".format(prompt, content[key])).strip() + except: + result = None + + if result: + content[key] = result + + def init(self): + self.__user_input([], "version", "Choose kiwi-config version") + + self.__user_input(["suffix"], "project", "Enter suffix for project directories") + self.__user_input(["suffix"], "down", "Enter suffix for disabled projects") + + self.__user_input(["docker"], "net", "Enter ") + self.__user_input(["docker"], "cidr", "Enter ") + + self.__user_input(["target"], "root", "Enter ") diff --git a/version-tag b/src/version-tag similarity index 100% rename from version-tag rename to src/version-tag From 9444597aaee2d8006943f953fd585c639e970b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= Date: Wed, 5 Aug 2020 02:08:55 +0200 Subject: [PATCH 011/118] Crude init function --- .idea/kiwi-config.iml | 3 +- .idea/misc.xml | 2 +- src/default.kiwi.yml | 13 +++---- src/kiwi-config.py | 2 +- src/kiwi/config.py | 53 ++++++++++++++------------ src/kiwi/init.py | 0 src/venv/bin/activate | 76 ++++++++++++++++++++++++++++++++++++++ src/venv/bin/activate.csh | 37 +++++++++++++++++++ src/venv/bin/activate.fish | 75 +++++++++++++++++++++++++++++++++++++ src/venv/bin/python | 1 + src/venv/bin/python3 | 1 + src/venv/lib64 | 1 + src/venv/pyvenv.cfg | 3 ++ 13 files changed, 233 insertions(+), 34 deletions(-) create mode 100644 src/kiwi/init.py create mode 100644 src/venv/bin/activate create mode 100644 src/venv/bin/activate.csh create mode 100644 src/venv/bin/activate.fish create mode 120000 src/venv/bin/python create mode 120000 src/venv/bin/python3 create mode 120000 src/venv/lib64 create mode 100644 src/venv/pyvenv.cfg diff --git a/.idea/kiwi-config.iml b/.idea/kiwi-config.iml index 15c0cd1..48e8b4f 100644 --- a/.idea/kiwi-config.iml +++ b/.idea/kiwi-config.iml @@ -3,8 +3,9 @@ + - + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index d4fc832..b2ea69b 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/src/default.kiwi.yml b/src/default.kiwi.yml index f3bd13b..b1edc12 100644 --- a/src/default.kiwi.yml +++ b/src/default.kiwi.yml @@ -1,12 +1,9 @@ version: - -suffix: +suffixes: project: .project down: .down - -docker: - net: kiwinet +network: + name: kiwinet cidr: 10.22.46.0/24 - -target: - root: /var/kiwi +storage: + location: /var/kiwi diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 1f38814..ab1183c 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -20,7 +20,7 @@ def main(): print(args.command) cf = Config.default() - cf.init() + cf.user_input() pass diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 07f0e69..3ba490e 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -2,50 +2,57 @@ import os import yaml - class Config: - __content = None + KIWI_ROOT = os.getenv('KIWI_ROOT', '.') + __ymlContent = None - def __init__(self, filename): + def __init__(self): + + + @classmethod + def __from_file(cls, filename): with open(filename, 'r') as stream: try: - self.__content = yaml.safe_load(stream) - print(self.__content) + self.__ymlContent = yaml.safe_load(stream) except yaml.YAMLError as exc: print(exc) @classmethod def default(cls): - kiwi_root = os.environ.get('KIWI_ROOT') + result = cls.__from_file(cls.KIWI_ROOT + "/default.kiwi.yml") - cfg = cls(kiwi_root + "/default.kiwi.yml") - with open(kiwi_root + "/version-tag", 'r') as stream: - cfg.__content["version"] = stream.read().strip() + with open(cls.KIWI_ROOT + "/version-tag", 'r') as stream: + result.__ymlContent["version"] = stream.read().strip() - return cfg + return result - def __user_input(self, path, key, prompt): - content = self.__content + def __user_input(self, key, prompt): + """""" + # "a:b:c" => path = ['a', 'b'], key = 'c' + path = key.split(':') + (path, key) = (path[:-1], path[-1]) + + # resolve path + content = self.__ymlContent for step in path: content = content[step] - try: - result = input("{} [Default: {}] ".format(prompt, content[key])).strip() - except: - result = None + # prompt user as per argument + result = input("{} [Default: {}] ".format(prompt, content[key])).strip() + # if result: content[key] = result - def init(self): - self.__user_input([], "version", "Choose kiwi-config version") + def user_input(self): + self.__user_input("version", "Choose kiwi-config version") - self.__user_input(["suffix"], "project", "Enter suffix for project directories") - self.__user_input(["suffix"], "down", "Enter suffix for disabled projects") + self.__user_input("suffixes:project", "Enter suffix for project directories") + self.__user_input("suffixes:down", "Enter suffix for disabled projects") - self.__user_input(["docker"], "net", "Enter ") - self.__user_input(["docker"], "cidr", "Enter ") + self.__user_input("network:name", "Enter name for docker network") + self.__user_input("network:cidr", "Enter ") - self.__user_input(["target"], "root", "Enter ") + self.__user_input("storage:location", "Enter ") diff --git a/src/kiwi/init.py b/src/kiwi/init.py new file mode 100644 index 0000000..e69de29 diff --git a/src/venv/bin/activate b/src/venv/bin/activate new file mode 100644 index 0000000..7acdce2 --- /dev/null +++ b/src/venv/bin/activate @@ -0,0 +1,76 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + if [ ! "$1" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="/home/jmm/Dokumente/git/dev/kiwi-config/src/venv" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + if [ "x(venv) " != x ] ; then + PS1="(venv) ${PS1:-}" + else + if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" + else + PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" + fi + fi + export PS1 +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r +fi diff --git a/src/venv/bin/activate.csh b/src/venv/bin/activate.csh new file mode 100644 index 0000000..7fa116f --- /dev/null +++ b/src/venv/bin/activate.csh @@ -0,0 +1,37 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/home/jmm/Dokumente/git/dev/kiwi-config/src/venv" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + if ("venv" != "") then + set env_name = "venv" + else + if (`basename "VIRTUAL_ENV"` == "__") then + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` + else + set env_name = `basename "$VIRTUAL_ENV"` + endif + endif + set prompt = "[$env_name] $prompt" + unset env_name +endif + +alias pydoc python -m pydoc + +rehash diff --git a/src/venv/bin/activate.fish b/src/venv/bin/activate.fish new file mode 100644 index 0000000..2819498 --- /dev/null +++ b/src/venv/bin/activate.fish @@ -0,0 +1,75 @@ +# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) +# you cannot run it directly + +function deactivate -d "Exit virtualenv and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + functions -e fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + + set -e VIRTUAL_ENV + if test "$argv[1]" != "nondestructive" + # Self destruct! + functions -e deactivate + end +end + +# unset irrelevant variables +deactivate nondestructive + +set -gx VIRTUAL_ENV "/home/jmm/Dokumente/git/dev/kiwi-config/src/venv" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# unset PYTHONHOME if set +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # save the current fish_prompt function as the function _old_fish_prompt + functions -c fish_prompt _old_fish_prompt + + # with the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command + set -l old_status $status + + # Prompt override? + if test -n "(venv) " + printf "%s%s" "(venv) " (set_color normal) + else + # ...Otherwise, prepend env + set -l _checkbase (basename "$VIRTUAL_ENV") + if test $_checkbase = "__" + # special case for Aspen magic directories + # see http://www.zetadev.com/software/aspen/ + printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) + else + printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) + end + end + + # Restore the return status of the previous command. + echo "exit $old_status" | . + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" +end diff --git a/src/venv/bin/python b/src/venv/bin/python new file mode 120000 index 0000000..b8a0adb --- /dev/null +++ b/src/venv/bin/python @@ -0,0 +1 @@ +python3 \ No newline at end of file diff --git a/src/venv/bin/python3 b/src/venv/bin/python3 new file mode 120000 index 0000000..ae65fda --- /dev/null +++ b/src/venv/bin/python3 @@ -0,0 +1 @@ +/usr/bin/python3 \ No newline at end of file diff --git a/src/venv/lib64 b/src/venv/lib64 new file mode 120000 index 0000000..7951405 --- /dev/null +++ b/src/venv/lib64 @@ -0,0 +1 @@ +lib \ No newline at end of file diff --git a/src/venv/pyvenv.cfg b/src/venv/pyvenv.cfg new file mode 100644 index 0000000..d411f1c --- /dev/null +++ b/src/venv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /usr/bin +include-system-site-packages = false +version = 3.6.9 From e44c0a86f8c484aa7256a21bfe67467df4fe0889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= Date: Thu, 6 Aug 2020 03:45:12 +0200 Subject: [PATCH 012/118] write out kiwi.yml --- src/default.kiwi.yml | 14 +++++++--- src/kiwi-config.py | 3 ++- src/kiwi/config.py | 63 +++++++++++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 25 deletions(-) diff --git a/src/default.kiwi.yml b/src/default.kiwi.yml index b1edc12..0001927 100644 --- a/src/default.kiwi.yml +++ b/src/default.kiwi.yml @@ -1,9 +1,17 @@ +###################################### +# kiwi-config instance configuration # +###################################### + version: -suffixes: + +markers: project: .project down: .down + network: name: kiwinet cidr: 10.22.46.0/24 -storage: - location: /var/kiwi + +runtime: + storage: /var/kiwi + env: null \ No newline at end of file diff --git a/src/kiwi-config.py b/src/kiwi-config.py index ab1183c..e4c41b7 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -20,7 +20,8 @@ def main(): print(args.command) cf = Config.default() - cf.user_input() + # cf.user_input() + cf.dump() pass diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 3ba490e..36321aa 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -1,58 +1,79 @@ import os +import re import yaml class Config: KIWI_ROOT = os.getenv('KIWI_ROOT', '.') - __ymlContent = None - - def __init__(self): - + __yml_content = None @classmethod def __from_file(cls, filename): + result = cls() + with open(filename, 'r') as stream: try: - self.__ymlContent = yaml.safe_load(stream) - + result.__yml_content = yaml.safe_load(stream) except yaml.YAMLError as exc: print(exc) + return result + @classmethod def default(cls): result = cls.__from_file(cls.KIWI_ROOT + "/default.kiwi.yml") with open(cls.KIWI_ROOT + "/version-tag", 'r') as stream: - result.__ymlContent["version"] = stream.read().strip() + result.__yml_content["version"] = stream.read().strip() return result - def __user_input(self, key, prompt): - """""" - + def __yml_resolve(self, key): # "a:b:c" => path = ['a', 'b'], key = 'c' path = key.split(':') - (path, key) = (path[:-1], path[-1]) + path, key = path[:-1], path[-1] # resolve path - content = self.__ymlContent + content = self.__yml_content for step in path: content = content[step] - # prompt user as per argument - result = input("{} [Default: {}] ".format(prompt, content[key])).strip() + return content, key - # + def __yml_get(self, key): + content, key = self.__yml_resolve(key) + return content[key] + + def __yml_set(self, key, value): + content, key = self.__yml_resolve(key) + content[key] = value + + def __user_input(self, key, prompt): + # prompt user as per argument + result = input("{} [Current: {}] ".format(prompt, self.__yml_get(key))).strip() + + # store result if present if result: - content[key] = result + self.__yml_set(key, result) def user_input(self): self.__user_input("version", "Choose kiwi-config version") - self.__user_input("suffixes:project", "Enter suffix for project directories") - self.__user_input("suffixes:down", "Enter suffix for disabled projects") + self.__user_input("markers:project", "Enter marker string for project directories") + self.__user_input("markers:down", "Enter marker string for disabled projects") - self.__user_input("network:name", "Enter name for docker network") - self.__user_input("network:cidr", "Enter ") + self.__user_input("network:name", "Enter name for local docker network") + self.__user_input("network:cidr", "Enter CIDR block for local docker network") - self.__user_input("storage:location", "Enter ") + self.__user_input("runtime:storage", "Enter main directory for local data") + + def dump(self): + yml_string = yaml.dump(self.__yml_content, default_flow_style=False, sort_keys=False) + yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) + + with open(Config.KIWI_ROOT + "/default.kiwi.yml", 'r') as stream: + yml_header = stream.read().strip() + yml_header = re.sub(r'^[^#].*', r'', yml_header, flags=re.MULTILINE).strip() + yml_string = "{}\n{}".format(yml_header, yml_string) + + print(yml_string) From 26280f1108a1c4ef8ec36eafcc46588893a0144d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= Date: Thu, 6 Aug 2020 03:49:26 +0200 Subject: [PATCH 013/118] gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index c18dd8d..425bba8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ __pycache__/ +**/venv/lib +**/venv/bin/easy_install* +**/venv/bin/pip* From d108ae2183f2a814b7c38db060410e7e2360994a Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 6 Aug 2020 11:10:23 +0200 Subject: [PATCH 014/118] switch to pipenv --- .gitignore | 3 - .../inspectionProfiles/profiles_settings.xml | 1 + .idea/kiwi-config.iml | 2 +- .idea/misc.xml | 2 +- Pipfile | 12 +++ Pipfile.lock | 38 ++++++++++ src/default.kiwi.yml | 6 +- src/venv/bin/activate | 76 ------------------- src/venv/bin/activate.csh | 37 --------- src/venv/bin/activate.fish | 75 ------------------ src/venv/bin/python | 1 - src/venv/bin/python3 | 1 - src/venv/lib64 | 1 - src/venv/pyvenv.cfg | 3 - 14 files changed, 54 insertions(+), 204 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock delete mode 100644 src/venv/bin/activate delete mode 100644 src/venv/bin/activate.csh delete mode 100644 src/venv/bin/activate.fish delete mode 120000 src/venv/bin/python delete mode 120000 src/venv/bin/python3 delete mode 120000 src/venv/lib64 delete mode 100644 src/venv/pyvenv.cfg diff --git a/.gitignore b/.gitignore index 425bba8..c18dd8d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1 @@ __pycache__/ -**/venv/lib -**/venv/bin/easy_install* -**/venv/bin/pip* diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml index 105ce2d..dd4c951 100644 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -1,5 +1,6 @@ + diff --git a/.idea/kiwi-config.iml b/.idea/kiwi-config.iml index 48e8b4f..d3e64c5 100644 --- a/.idea/kiwi-config.iml +++ b/.idea/kiwi-config.iml @@ -5,7 +5,7 @@ - + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index b2ea69b..6d2139f 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..38477a7 --- /dev/null +++ b/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +pyyaml = "*" + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..94a7e7e --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,38 @@ +{ + "_meta": { + "hash": { + "sha256": "d518a36ed441568acff15b0a3c4b536738a55fb68801cdd682045be04d29954a" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "pyyaml": { + "hashes": [ + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" + ], + "index": "pypi", + "version": "==5.3.1" + } + }, + "develop": {} +} diff --git a/src/default.kiwi.yml b/src/default.kiwi.yml index 0001927..9689074 100644 --- a/src/default.kiwi.yml +++ b/src/default.kiwi.yml @@ -1,17 +1,13 @@ ###################################### # kiwi-config instance configuration # ###################################### - version: - markers: project: .project down: .down - network: name: kiwinet cidr: 10.22.46.0/24 - runtime: storage: /var/kiwi - env: null \ No newline at end of file + env: diff --git a/src/venv/bin/activate b/src/venv/bin/activate deleted file mode 100644 index 7acdce2..0000000 --- a/src/venv/bin/activate +++ /dev/null @@ -1,76 +0,0 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - # reset old environment variables - if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then - PATH="${_OLD_VIRTUAL_PATH:-}" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then - PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r - fi - - if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then - PS1="${_OLD_VIRTUAL_PS1:-}" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - if [ ! "$1" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelevant variables -deactivate nondestructive - -VIRTUAL_ENV="/home/jmm/Dokumente/git/dev/kiwi-config/src/venv" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/bin:$PATH" -export PATH - -# unset PYTHONHOME if set -# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) -# could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "${PYTHONHOME:-}" ] ; then - _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" - unset PYTHONHOME -fi - -if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then - _OLD_VIRTUAL_PS1="${PS1:-}" - if [ "x(venv) " != x ] ; then - PS1="(venv) ${PS1:-}" - else - if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" - else - PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" - fi - fi - export PS1 -fi - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r -fi diff --git a/src/venv/bin/activate.csh b/src/venv/bin/activate.csh deleted file mode 100644 index 7fa116f..0000000 --- a/src/venv/bin/activate.csh +++ /dev/null @@ -1,37 +0,0 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi . -# Ported to Python 3.3 venv by Andrew Svetlov - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' - -# Unset irrelevant variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "/home/jmm/Dokumente/git/dev/kiwi-config/src/venv" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/bin:$PATH" - - -set _OLD_VIRTUAL_PROMPT="$prompt" - -if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - if ("venv" != "") then - set env_name = "venv" - else - if (`basename "VIRTUAL_ENV"` == "__") then - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` - else - set env_name = `basename "$VIRTUAL_ENV"` - endif - endif - set prompt = "[$env_name] $prompt" - unset env_name -endif - -alias pydoc python -m pydoc - -rehash diff --git a/src/venv/bin/activate.fish b/src/venv/bin/activate.fish deleted file mode 100644 index 2819498..0000000 --- a/src/venv/bin/activate.fish +++ /dev/null @@ -1,75 +0,0 @@ -# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) -# you cannot run it directly - -function deactivate -d "Exit virtualenv and return to normal shell environment" - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt - set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt - end - - set -e VIRTUAL_ENV - if test "$argv[1]" != "nondestructive" - # Self destruct! - functions -e deactivate - end -end - -# unset irrelevant variables -deactivate nondestructive - -set -gx VIRTUAL_ENV "/home/jmm/Dokumente/git/dev/kiwi-config/src/venv" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/bin" $PATH - -# unset PYTHONHOME if set -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # fish uses a function instead of an env var to generate the prompt. - - # save the current fish_prompt function as the function _old_fish_prompt - functions -c fish_prompt _old_fish_prompt - - # with the original prompt function renamed, we can override with our own. - function fish_prompt - # Save the return status of the last command - set -l old_status $status - - # Prompt override? - if test -n "(venv) " - printf "%s%s" "(venv) " (set_color normal) - else - # ...Otherwise, prepend env - set -l _checkbase (basename "$VIRTUAL_ENV") - if test $_checkbase = "__" - # special case for Aspen magic directories - # see http://www.zetadev.com/software/aspen/ - printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) - else - printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) - end - end - - # Restore the return status of the previous command. - echo "exit $old_status" | . - _old_fish_prompt - end - - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" -end diff --git a/src/venv/bin/python b/src/venv/bin/python deleted file mode 120000 index b8a0adb..0000000 --- a/src/venv/bin/python +++ /dev/null @@ -1 +0,0 @@ -python3 \ No newline at end of file diff --git a/src/venv/bin/python3 b/src/venv/bin/python3 deleted file mode 120000 index ae65fda..0000000 --- a/src/venv/bin/python3 +++ /dev/null @@ -1 +0,0 @@ -/usr/bin/python3 \ No newline at end of file diff --git a/src/venv/lib64 b/src/venv/lib64 deleted file mode 120000 index 7951405..0000000 --- a/src/venv/lib64 +++ /dev/null @@ -1 +0,0 @@ -lib \ No newline at end of file diff --git a/src/venv/pyvenv.cfg b/src/venv/pyvenv.cfg deleted file mode 100644 index d411f1c..0000000 --- a/src/venv/pyvenv.cfg +++ /dev/null @@ -1,3 +0,0 @@ -home = /usr/bin -include-system-site-packages = false -version = 3.6.9 From 1929a1369ec99c32f56b5cbc614cfc43b66cbc6e Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 6 Aug 2020 13:43:45 +0200 Subject: [PATCH 015/118] init routine in subcommand class --- kiwi.conf | 1 - kiwi.yml | 1 + src/default.kiwi.yml | 1 + src/kiwi-config.py | 23 +++------- src/kiwi/__init__.py | 5 ++- src/kiwi/cmd_init.py | 40 +++++++++++++++++ src/kiwi/config.py | 103 +++++++++++++++++++++---------------------- src/kiwi/core.py | 56 +++++++++++++++++++++++ src/kiwi/init.py | 0 9 files changed, 158 insertions(+), 72 deletions(-) delete mode 100644 kiwi.conf create mode 100644 kiwi.yml create mode 100644 src/kiwi/cmd_init.py create mode 100644 src/kiwi/core.py delete mode 100644 src/kiwi/init.py diff --git a/kiwi.conf b/kiwi.conf deleted file mode 100644 index a5b8ea7..0000000 --- a/kiwi.conf +++ /dev/null @@ -1 +0,0 @@ -VERSION="0.1" \ No newline at end of file diff --git a/kiwi.yml b/kiwi.yml new file mode 100644 index 0000000..71e7332 --- /dev/null +++ b/kiwi.yml @@ -0,0 +1 @@ +version: 0.1 \ No newline at end of file diff --git a/src/default.kiwi.yml b/src/default.kiwi.yml index 9689074..24013a4 100644 --- a/src/default.kiwi.yml +++ b/src/default.kiwi.yml @@ -11,3 +11,4 @@ network: runtime: storage: /var/kiwi env: + HELLO_KIWI: Hello World! diff --git a/src/kiwi-config.py b/src/kiwi-config.py index e4c41b7..d5dea8c 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -1,27 +1,16 @@ #!/usr/bin/env python3 -import argparse -from kiwi.config import * - +import kiwi +from kiwi.cmd_init import InitSubCommand def main(): - parser = argparse.ArgumentParser(description='kiwi-config') + isc = InitSubCommand() + isc.setup() - subs = parser.add_subparsers() - subs.required = True - subs.dest = 'command' - - subs.add_parser('init', help="Create new kiwi-config instance") - subs.add_parser('add', help="Add a project to kiwi-config") - - # parser.add_argument('cmd', metavar='command', type=str, help='subcommand to execute') - - args = parser.parse_args() + args = kiwi.Parser.get_args() print(args.command) - cf = Config.default() - # cf.user_input() - cf.dump() + isc.run() pass diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py index 7c281ff..06b43a2 100644 --- a/src/kiwi/__init__.py +++ b/src/kiwi/__init__.py @@ -1 +1,4 @@ -__all__ = ['config'] +from .core import Parser +from .config import Config + +__all__ = ['Parser', 'Config'] diff --git a/src/kiwi/cmd_init.py b/src/kiwi/cmd_init.py new file mode 100644 index 0000000..c3a1b31 --- /dev/null +++ b/src/kiwi/cmd_init.py @@ -0,0 +1,40 @@ +import logging +import os + +from .core import KIWI_CONF_NAME, Parser, SubCommand +from .config import Config + + +class InitSubCommand(SubCommand): + def __init__(self): + super(InitSubCommand, self).__init__('init') + + def setup(self): + init_parser = Parser.get_subparsers().add_parser(str(self), help="Create new kiwi-config instance") + # init_parser.add_argument('cmd', metavar='command', type=str, help='subcommand to execute') + + def __user_input(self, config, key, prompt): + # prompt user as per argument + result = input("{} [Current: {}] ".format(prompt, config[key])).strip() + + # store result if present + if result: + config[key] = result + + def run(self): + config = Config.default() + + if os.path.isfile(KIWI_CONF_NAME): + logging.warning("Overwriting existing '%s'!", KIWI_CONF_NAME) + + self.__user_input(config, 'version', "Choose kiwi-config version") + + self.__user_input(config, 'markers:project', "Enter marker string for project directories") + self.__user_input(config, 'markers:down', "Enter marker string for disabled projects") + + self.__user_input(config, 'network:name', "Enter name for local docker network") + self.__user_input(config, 'network:cidr', "Enter CIDR block for local docker network") + + self.__user_input(config, 'runtime:storage', "Enter main directory for local data") + + print(str(config)) diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 36321aa..fbb3c2b 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -1,79 +1,76 @@ -import os +import logging import re import yaml +from .core import KIWI_ROOT, KIWI_CONF_NAME + +########### +# CONSTANTS + +DEFAULT_KIWI_CONF_NAME = KIWI_ROOT + "/default.kiwi.yml" + class Config: - KIWI_ROOT = os.getenv('KIWI_ROOT', '.') - __yml_content = None + __yml_content = {} - @classmethod - def __from_file(cls, filename): - result = cls() + def __str__(self): + # dump yml content + yml_string = yaml.dump(self.__yml_content, default_flow_style=False, sort_keys=False) - with open(filename, 'r') as stream: - try: - result.__yml_content = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) + # insert newline before every main key + yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) - return result + # extract header comment from default config + with open(DEFAULT_KIWI_CONF_NAME, 'r') as stream: + yml_header = stream.read().strip() + yml_header = re.sub(r'^[^#].*', r'', yml_header, flags=re.MULTILINE).strip() + yml_string = "{}\n{}".format(yml_header, yml_string) - @classmethod - def default(cls): - result = cls.__from_file(cls.KIWI_ROOT + "/default.kiwi.yml") + return yml_string - with open(cls.KIWI_ROOT + "/version-tag", 'r') as stream: - result.__yml_content["version"] = stream.read().strip() - - return result - - def __yml_resolve(self, key): + def __key_resolve(self, key): # "a:b:c" => path = ['a', 'b'], key = 'c' path = key.split(':') path, key = path[:-1], path[-1] # resolve path - content = self.__yml_content + container = self.__yml_content for step in path: - content = content[step] + container = container[step] - return content, key + return container, key - def __yml_get(self, key): - content, key = self.__yml_resolve(key) - return content[key] + def __setitem__(self, key, value): + container, key = self.__key_resolve(key) + container[key] = value - def __yml_set(self, key, value): - content, key = self.__yml_resolve(key) - content[key] = value + def __getitem__(self, key): + container, key = self.__key_resolve(key) + return container[key] - def __user_input(self, key, prompt): - # prompt user as per argument - result = input("{} [Current: {}] ".format(prompt, self.__yml_get(key))).strip() + def __update_from_file(self, filename): + with open(filename, 'r') as stream: + try: + self.__yml_content.update(yaml.safe_load(stream)) + except yaml.YAMLError as exc: + logging.error(exc) - # store result if present - if result: - self.__yml_set(key, result) + @classmethod + def default(cls): + result = cls() + result.__update_from_file(DEFAULT_KIWI_CONF_NAME) - def user_input(self): - self.__user_input("version", "Choose kiwi-config version") + with open(KIWI_ROOT + "/version-tag", 'r') as stream: + result.__yml_content["version"] = stream.read().strip() - self.__user_input("markers:project", "Enter marker string for project directories") - self.__user_input("markers:down", "Enter marker string for disabled projects") + return result - self.__user_input("network:name", "Enter name for local docker network") - self.__user_input("network:cidr", "Enter CIDR block for local docker network") + @classmethod + def load(cls): + result = cls.default() + result.__update_from_file(KIWI_CONF_NAME) - self.__user_input("runtime:storage", "Enter main directory for local data") + return result - def dump(self): - yml_string = yaml.dump(self.__yml_content, default_flow_style=False, sort_keys=False) - yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) - - with open(Config.KIWI_ROOT + "/default.kiwi.yml", 'r') as stream: - yml_header = stream.read().strip() - yml_header = re.sub(r'^[^#].*', r'', yml_header, flags=re.MULTILINE).strip() - yml_string = "{}\n{}".format(yml_header, yml_string) - - print(yml_string) + def save(self): + pass diff --git a/src/kiwi/core.py b/src/kiwi/core.py new file mode 100644 index 0000000..f426548 --- /dev/null +++ b/src/kiwi/core.py @@ -0,0 +1,56 @@ +import argparse +import os + +########### +# CONSTANTS + +KIWI_ROOT = os.getenv('KIWI_ROOT', ".") +KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") + + +class Parser: + __instance = None + __subparsers = None + __args = None + + @classmethod + def __init_instance(cls): + if not cls.__instance: + cls.__instance = argparse.ArgumentParser(description='kiwi-config') + + cls.__subparsers = Parser.__instance.add_subparsers() + cls.__subparsers.required = True + cls.__subparsers.dest = 'command' + + @classmethod + def get_instance(cls): + cls.__init_instance() + return cls.__instance + + @classmethod + def get_subparsers(cls): + cls.__init_instance() + return cls.__subparsers + + @classmethod + def get_args(cls): + if not cls.__args: + cls.__args = cls.get_instance().parse_args() + + return cls.__args + + +class SubCommand: + __cmd = None + + def __init__(self, cmd): + self.__cmd = cmd + + def __str__(self): + return self.__cmd + + def setup(self): + pass + + def run(self): + pass diff --git a/src/kiwi/init.py b/src/kiwi/init.py deleted file mode 100644 index e69de29..0000000 From 69dc1fd213c467b4c9e51d3a06548b6e51d1b5e9 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 6 Aug 2020 14:34:25 +0200 Subject: [PATCH 016/118] Runner for subcommands --- src/kiwi-config.py | 14 ++------ src/kiwi/__init__.py | 5 ++- src/kiwi/cmd_init.py | 40 ----------------------- src/kiwi/config.py | 51 +++++++++++++++++------------- src/kiwi/core.py | 16 ---------- src/kiwi/runner.py | 24 ++++++++++++++ src/kiwi/subcommands/__init__.py | 4 +++ src/kiwi/subcommands/init.py | 48 ++++++++++++++++++++++++++++ src/kiwi/subcommands/subcommand.py | 12 +++++++ 9 files changed, 122 insertions(+), 92 deletions(-) delete mode 100644 src/kiwi/cmd_init.py create mode 100644 src/kiwi/runner.py create mode 100644 src/kiwi/subcommands/__init__.py create mode 100644 src/kiwi/subcommands/init.py create mode 100644 src/kiwi/subcommands/subcommand.py diff --git a/src/kiwi-config.py b/src/kiwi-config.py index d5dea8c..25eb590 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -1,18 +1,10 @@ #!/usr/bin/env python3 - import kiwi -from kiwi.cmd_init import InitSubCommand + def main(): - isc = InitSubCommand() - isc.setup() - - args = kiwi.Parser.get_args() - print(args.command) - - isc.run() - - pass + kiwi.Runner.setup_all() + kiwi.Runner.run() if __name__ == "__main__": diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py index 06b43a2..247ace4 100644 --- a/src/kiwi/__init__.py +++ b/src/kiwi/__init__.py @@ -1,4 +1,3 @@ -from .core import Parser -from .config import Config +from .runner import Runner -__all__ = ['Parser', 'Config'] +__all__ = ['Runner'] \ No newline at end of file diff --git a/src/kiwi/cmd_init.py b/src/kiwi/cmd_init.py deleted file mode 100644 index c3a1b31..0000000 --- a/src/kiwi/cmd_init.py +++ /dev/null @@ -1,40 +0,0 @@ -import logging -import os - -from .core import KIWI_CONF_NAME, Parser, SubCommand -from .config import Config - - -class InitSubCommand(SubCommand): - def __init__(self): - super(InitSubCommand, self).__init__('init') - - def setup(self): - init_parser = Parser.get_subparsers().add_parser(str(self), help="Create new kiwi-config instance") - # init_parser.add_argument('cmd', metavar='command', type=str, help='subcommand to execute') - - def __user_input(self, config, key, prompt): - # prompt user as per argument - result = input("{} [Current: {}] ".format(prompt, config[key])).strip() - - # store result if present - if result: - config[key] = result - - def run(self): - config = Config.default() - - if os.path.isfile(KIWI_CONF_NAME): - logging.warning("Overwriting existing '%s'!", KIWI_CONF_NAME) - - self.__user_input(config, 'version', "Choose kiwi-config version") - - self.__user_input(config, 'markers:project', "Enter marker string for project directories") - self.__user_input(config, 'markers:down', "Enter marker string for disabled projects") - - self.__user_input(config, 'network:name', "Enter name for local docker network") - self.__user_input(config, 'network:cidr', "Enter CIDR block for local docker network") - - self.__user_input(config, 'runtime:storage', "Enter main directory for local data") - - print(str(config)) diff --git a/src/kiwi/config.py b/src/kiwi/config.py index fbb3c2b..7f32b09 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -1,5 +1,6 @@ import logging import re +import os import yaml from .core import KIWI_ROOT, KIWI_CONF_NAME @@ -13,6 +14,26 @@ DEFAULT_KIWI_CONF_NAME = KIWI_ROOT + "/default.kiwi.yml" class Config: __yml_content = {} + def __key_resolve(self, key): + # "a:b:c" => path = ['a', 'b'], key = 'c' + path = key.split(':') + path, key = path[:-1], path[-1] + + # resolve path + container = self.__yml_content + for step in path: + container = container[step] + + return container, key + + def __getitem__(self, key): + container, key = self.__key_resolve(key) + return container[key] + + def __setitem__(self, key, value): + container, key = self.__key_resolve(key) + container[key] = value + def __str__(self): # dump yml content yml_string = yaml.dump(self.__yml_content, default_flow_style=False, sort_keys=False) @@ -28,26 +49,6 @@ class Config: return yml_string - def __key_resolve(self, key): - # "a:b:c" => path = ['a', 'b'], key = 'c' - path = key.split(':') - path, key = path[:-1], path[-1] - - # resolve path - container = self.__yml_content - for step in path: - container = container[step] - - return container, key - - def __setitem__(self, key, value): - container, key = self.__key_resolve(key) - container[key] = value - - def __getitem__(self, key): - container, key = self.__key_resolve(key) - return container[key] - def __update_from_file(self, filename): with open(filename, 'r') as stream: try: @@ -55,6 +56,10 @@ class Config: except yaml.YAMLError as exc: logging.error(exc) + def __save_to_file(self, filename): + with open(filename, 'w') as stream: + stream.write(str(self)) + @classmethod def default(cls): result = cls() @@ -68,9 +73,11 @@ class Config: @classmethod def load(cls): result = cls.default() - result.__update_from_file(KIWI_CONF_NAME) + + if os.path.isfile(KIWI_CONF_NAME): + result.__update_from_file(KIWI_CONF_NAME) return result def save(self): - pass + self.__save_to_file(KIWI_CONF_NAME) diff --git a/src/kiwi/core.py b/src/kiwi/core.py index f426548..4316fcc 100644 --- a/src/kiwi/core.py +++ b/src/kiwi/core.py @@ -38,19 +38,3 @@ class Parser: cls.__args = cls.get_instance().parse_args() return cls.__args - - -class SubCommand: - __cmd = None - - def __init__(self, cmd): - self.__cmd = cmd - - def __str__(self): - return self.__cmd - - def setup(self): - pass - - def run(self): - pass diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py new file mode 100644 index 0000000..ecb9e22 --- /dev/null +++ b/src/kiwi/runner.py @@ -0,0 +1,24 @@ +from typing import List + +from .core import Parser +from .subcommands import * + + +class Runner: + __commands: List[SubCommand] = [ + InitCommand + ] + + @classmethod + def setup_all(cls): + for cmd in cls.__commands: + cmd.setup() + + @classmethod + def run(cls): + args = Parser.get_args() + + for cmd in cls.__commands: + if cmd.get_cmd() == args.command: + cmd.run() + return diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py new file mode 100644 index 0000000..cdf8394 --- /dev/null +++ b/src/kiwi/subcommands/__init__.py @@ -0,0 +1,4 @@ +from .subcommand import SubCommand +from .init import InitCommand + +__all__ = ['InitCommand', 'SubCommand'] \ No newline at end of file diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py new file mode 100644 index 0000000..3e84936 --- /dev/null +++ b/src/kiwi/subcommands/init.py @@ -0,0 +1,48 @@ +import logging +import os + +from kiwi.core import KIWI_CONF_NAME, Parser +from kiwi.config import Config + +from .subcommand import SubCommand + + +def user_input(config, key, prompt): + # prompt user as per argument + result = input("{} [Current: {}] ".format(prompt, config[key])).strip() + + # store result if present + if result: + config[key] = result + + +class InitCommand(SubCommand): + __parser = None + + @classmethod + def get_cmd(cls): + return 'init' + + @classmethod + def setup(cls): + cls.__parser = Parser.get_subparsers().add_parser(cls.get_cmd(), help="Create new kiwi-config instance") + # cls.__parser.add_argument('cmd', metavar='command', type=str, help='subcommand to execute') + + @classmethod + def run(cls): + config = Config.default() + + if os.path.isfile(KIWI_CONF_NAME): + logging.warning("Overwriting existing '%s'!", KIWI_CONF_NAME) + + user_input(config, 'version', "Choose kiwi-config version") + + user_input(config, 'markers:project', "Enter marker string for project directories") + user_input(config, 'markers:down', "Enter marker string for disabled projects") + + user_input(config, 'network:name', "Enter name for local docker network") + user_input(config, 'network:cidr', "Enter CIDR block for local docker network") + + user_input(config, 'runtime:storage', "Enter main directory for local data") + + config.save() diff --git a/src/kiwi/subcommands/subcommand.py b/src/kiwi/subcommands/subcommand.py new file mode 100644 index 0000000..21535e5 --- /dev/null +++ b/src/kiwi/subcommands/subcommand.py @@ -0,0 +1,12 @@ +class SubCommand: + @classmethod + def get_cmd(cls): + pass + + @classmethod + def setup(cls): + pass + + @classmethod + def run(cls): + pass From 26cac6c07969ec3d2527a4e00e7ae48a54495b3d Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 6 Aug 2020 14:39:12 +0200 Subject: [PATCH 017/118] deleted crude bash version --- src/bin/init.sh | 59 --------------------------------------- src/bin/main.sh | 14 ---------- src/etc/default.kiwi.conf | 7 ----- src/inc/functions.sh | 15 ---------- 4 files changed, 95 deletions(-) delete mode 100755 src/bin/init.sh delete mode 100755 src/bin/main.sh delete mode 100644 src/etc/default.kiwi.conf delete mode 100644 src/inc/functions.sh diff --git a/src/bin/init.sh b/src/bin/init.sh deleted file mode 100755 index 561de07..0000000 --- a/src/bin/init.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -source "${KIWI_ROOT}/inc/functions.sh" - -# read default baseconfig -conf_VERSION="${KIWI_VERSION}" -read_kiwi_config "${KIWI_ROOT}/etc/default.${KIWI_CONF_NAME}" - -# if pwd is a kiwi folder, read local baseconfig -if [ -f "./${KIWI_CONF_NAME}" ]; then - echo "[WARN] Overwriting existing '${KIWI_CONF_NAME}'" - read_kiwi_config "./${KIWI_CONF_NAME}" -fi - -function user_input() { - local prompt="${1}" - local varname="${2}" - - local input - read -p "${prompt} [Default '${!varname}']: " input - - eval "${varname}='${input:-${!varname}}'" -} - -declare -A config_explain -config_explain=( - [VERSION]="kiwi-config version" - [SUFFIX_PROJECT]="suffix for project directories" - [SUFFIX_DOWN]="suffix for disabled projects" -) - -for varname in "${!config_explain[@]}"; do - echo "${varname}" - user_input "Enter ${config_explain[${varname}]}" conf_${varname} -done - -exit 0 - -user_input "Choose kiwi-config version" conf_VERSION - -user_input "Enter suffix for project directories" conf_SUFFIX_PROJECT -user_input "Enter suffix for disabled projects" conf_SUFFIX_DOWN - -user_input "Enter suffix for disabled projects" conf_SUFFIX_DOWN -user_input "Enter suffix for disabled projects" conf_SUFFIX_DOWN - -user_input "Enter suffix for disabled projects" conf_SUFFIX_DOWN - -echo "conf_VERSION: ${conf_VERSION}" -echo "conf_SUFFIX_PROJECT: ${conf_SUFFIX_PROJECT}" - -exit 0 - -read -p "Choose kiwi-config version [Default '${conf_version}']: " kiwi_VERSION -conf_VERSION=${kiwi_VERSION:-${conf_VERSION}} - -read -p "suffix for kiwi-config version to use [Default '${conf_version}']: " kiwi_suffix_project - -echo "conf_VERSION: ${conf_VERSION}" \ No newline at end of file diff --git a/src/bin/main.sh b/src/bin/main.sh deleted file mode 100755 index 5a442fa..0000000 --- a/src/bin/main.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -command="${1}" -shift 1 - -case "${command}" in - "init") - exec "${KIWI_ROOT}/bin/${command}.sh" "${@}" - ;; - *) - echo "Unknown kiwi command '${command}'." - exit 1 - ;; -esac \ No newline at end of file diff --git a/src/etc/default.kiwi.conf b/src/etc/default.kiwi.conf deleted file mode 100644 index 4e9712d..0000000 --- a/src/etc/default.kiwi.conf +++ /dev/null @@ -1,7 +0,0 @@ -SUFFIX_PROJECT=.project -SUFFIX_DOWN=.down - -DOCKERNET=kiwinet -DOCKERCIDR=10.22.46.0/24 - -TARGETROOT=/var/kiwi diff --git a/src/inc/functions.sh b/src/inc/functions.sh deleted file mode 100644 index 6f88a53..0000000 --- a/src/inc/functions.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -function read_kiwi_config() { - local conf_file="${1}" - local conf_prefix="${2:-conf_}" - - local conf_file_content=$(sed -r 's/^\s*(\S+)/'${conf_prefix}'\1/g' "${conf_file}") - eval "${conf_file_content}" -} - -function write_kiwi_config() { - local conf_prefix="${2:-conf_}" - - -} \ No newline at end of file From 9139032a0eb3d823e08e339a0255621237f1760f Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 6 Aug 2020 14:55:27 +0200 Subject: [PATCH 018/118] "kiwi show" --- kiwi.yml | 2 +- src/default.kiwi.yml | 3 +-- src/kiwi/runner.py | 3 ++- src/kiwi/subcommands/__init__.py | 6 ++++-- src/kiwi/subcommands/show.py | 21 +++++++++++++++++++++ 5 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 src/kiwi/subcommands/show.py diff --git a/kiwi.yml b/kiwi.yml index 71e7332..57bf16d 100644 --- a/kiwi.yml +++ b/kiwi.yml @@ -1 +1 @@ -version: 0.1 \ No newline at end of file +version: '0.1' \ No newline at end of file diff --git a/src/default.kiwi.yml b/src/default.kiwi.yml index 24013a4..b642fda 100644 --- a/src/default.kiwi.yml +++ b/src/default.kiwi.yml @@ -10,5 +10,4 @@ network: cidr: 10.22.46.0/24 runtime: storage: /var/kiwi - env: - HELLO_KIWI: Hello World! + env: null diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index ecb9e22..c51606a 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -6,7 +6,8 @@ from .subcommands import * class Runner: __commands: List[SubCommand] = [ - InitCommand + InitCommand, + ShowCommand ] @classmethod diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index cdf8394..a8cc797 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,4 +1,6 @@ from .subcommand import SubCommand -from .init import InitCommand -__all__ = ['InitCommand', 'SubCommand'] \ No newline at end of file +from .init import InitCommand +from .show import ShowCommand + +__all__ = ['SubCommand', 'InitCommand', 'ShowCommand'] \ No newline at end of file diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py new file mode 100644 index 0000000..34b7df9 --- /dev/null +++ b/src/kiwi/subcommands/show.py @@ -0,0 +1,21 @@ +from kiwi.config import Config +from kiwi.core import Parser + +from .subcommand import SubCommand + + +class ShowCommand(SubCommand): + __parser = None + + @classmethod + def get_cmd(cls): + return 'show' + + @classmethod + def setup(cls): + cls.__parser = Parser.get_subparsers().add_parser(cls.get_cmd(), help="Show effective kiwi.yml") + + @classmethod + def run(cls): + config = Config.load() + print(config) From 1214137005936602ef0ddb11b717acb80919ce5a Mon Sep 17 00:00:00 2001 From: ldericher Date: Fri, 7 Aug 2020 02:01:54 +0200 Subject: [PATCH 019/118] "Current" => "Default" --- src/kiwi/subcommands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 3e84936..d8485d6 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -9,7 +9,7 @@ from .subcommand import SubCommand def user_input(config, key, prompt): # prompt user as per argument - result = input("{} [Current: {}] ".format(prompt, config[key])).strip() + result = input("{} [Default: {}] ".format(prompt, config[key])).strip() # store result if present if result: From 3e1d9510dc5c2714c2f5fb5ed80db571d81ba149 Mon Sep 17 00:00:00 2001 From: ldericher Date: Sat, 8 Aug 2020 19:41:11 +0200 Subject: [PATCH 020/118] YML executable paths, "logs" subcommand --- example/Makefile | 1 + base.conf => example/base.conf | 2 +- .../hello-world.project/docker-compose.yml | 15 ++++++ example/kiwi.yml | 22 ++++++++ kiwi.yml | 1 - src/default.kiwi.yml | 10 ++-- src/kiwi/config.py | 39 ++++++++------ src/kiwi/runner.py | 3 +- src/kiwi/subcommands/__init__.py | 10 +++- src/kiwi/subcommands/_utils.py | 53 +++++++++++++++++++ src/kiwi/subcommands/init.py | 33 ++++++++++-- src/kiwi/subcommands/logs.py | 19 +++++++ src/kiwi/subcommands/show.py | 8 +-- src/kiwi/subcommands/subcommand.py | 12 ----- 14 files changed, 182 insertions(+), 46 deletions(-) create mode 120000 example/Makefile rename base.conf => example/base.conf (77%) create mode 100644 example/hello-world.project/docker-compose.yml create mode 100644 example/kiwi.yml delete mode 100644 kiwi.yml create mode 100644 src/kiwi/subcommands/_utils.py create mode 100644 src/kiwi/subcommands/logs.py delete mode 100644 src/kiwi/subcommands/subcommand.py diff --git a/example/Makefile b/example/Makefile new file mode 120000 index 0000000..d0b0e8e --- /dev/null +++ b/example/Makefile @@ -0,0 +1 @@ +../Makefile \ No newline at end of file diff --git a/base.conf b/example/base.conf similarity index 77% rename from base.conf rename to example/base.conf index 7d392e4..0c8a157 100644 --- a/base.conf +++ b/example/base.conf @@ -2,6 +2,6 @@ export SUFFIX_PROJECT=.project export SUFFIX_DOWN=.down export DOCKERNET=kiwinet -export DOCKERCIDR=10.13.37.0/24 +export DOCKERCIDR=10.22.46.0/24 export TARGETROOT=/var/kiwi diff --git a/example/hello-world.project/docker-compose.yml b/example/hello-world.project/docker-compose.yml new file mode 100644 index 0000000..201bd21 --- /dev/null +++ b/example/hello-world.project/docker-compose.yml @@ -0,0 +1,15 @@ +version: "2" + +networks: + # reachable from outside + default: + driver: bridge + # interconnects projects + kiwihub: + external: + name: $DOCKERNET + +services: + hello-world: + image: alpine:latest + command: sh -c 'while :; do echo Hello World; sleep 10; done' diff --git a/example/kiwi.yml b/example/kiwi.yml new file mode 100644 index 0000000..4a18c26 --- /dev/null +++ b/example/kiwi.yml @@ -0,0 +1,22 @@ +###################################### +# kiwi-config instance configuration # +###################################### + +version: '0.1' + +runtime: + storage: /var/kiwi + env: null + +markers: + project: .project + down: .down + +network: + name: kiwinet + cidr: 10.22.46.0/24 + +executables: + docker: /usr/bin/docker + docker-compose: /usr/local/bin/docker-compose + sudo: /usr/bin/sudo diff --git a/kiwi.yml b/kiwi.yml deleted file mode 100644 index 57bf16d..0000000 --- a/kiwi.yml +++ /dev/null @@ -1 +0,0 @@ -version: '0.1' \ No newline at end of file diff --git a/src/default.kiwi.yml b/src/default.kiwi.yml index b642fda..06384df 100644 --- a/src/default.kiwi.yml +++ b/src/default.kiwi.yml @@ -2,12 +2,16 @@ # kiwi-config instance configuration # ###################################### version: +runtime: + storage: /var/kiwi + env: null markers: project: .project down: .down network: name: kiwinet cidr: 10.22.46.0/24 -runtime: - storage: /var/kiwi - env: null +executables: + docker: null + docker-compose: null + sudo: null diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 7f32b09..2d23d99 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -9,6 +9,7 @@ from .core import KIWI_ROOT, KIWI_CONF_NAME # CONSTANTS DEFAULT_KIWI_CONF_NAME = KIWI_ROOT + "/default.kiwi.yml" +VERSION_TAG_NAME = KIWI_ROOT + "/version-tag" class Config: @@ -49,35 +50,39 @@ class Config: return yml_string - def __update_from_file(self, filename): + def _update_from_file(self, filename): with open(filename, 'r') as stream: try: self.__yml_content.update(yaml.safe_load(stream)) except yaml.YAMLError as exc: logging.error(exc) - def __save_to_file(self, filename): - with open(filename, 'w') as stream: + def save(self): + with open(KIWI_CONF_NAME, 'w') as stream: stream.write(str(self)) - @classmethod - def default(cls): - result = cls() - result.__update_from_file(DEFAULT_KIWI_CONF_NAME) - with open(KIWI_ROOT + "/version-tag", 'r') as stream: - result.__yml_content["version"] = stream.read().strip() - - return result +class DefaultConfig(Config): + __instance = None @classmethod - def load(cls): - result = cls.default() + def get(cls): + if cls.__instance is None: + cls.__instance = cls() + cls.__instance._update_from_file(DEFAULT_KIWI_CONF_NAME) + + with open(VERSION_TAG_NAME, 'r') as stream: + cls.__instance["version"] = stream.read().strip() + + return cls.__instance + + +class LoadedConfig(Config): + @classmethod + def get(cls): + result = DefaultConfig.get() if os.path.isfile(KIWI_CONF_NAME): - result.__update_from_file(KIWI_CONF_NAME) + result._update_from_file(KIWI_CONF_NAME) return result - - def save(self): - self.__save_to_file(KIWI_CONF_NAME) diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index c51606a..553f497 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -7,7 +7,8 @@ from .subcommands import * class Runner: __commands: List[SubCommand] = [ InitCommand, - ShowCommand + ShowCommand, + LogsCommand ] @classmethod diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index a8cc797..d1a8d83 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,6 +1,12 @@ -from .subcommand import SubCommand +from ._utils import SubCommand from .init import InitCommand from .show import ShowCommand +from .logs import LogsCommand -__all__ = ['SubCommand', 'InitCommand', 'ShowCommand'] \ No newline at end of file +__all__ = [ + 'SubCommand', + 'InitCommand', + 'ShowCommand', + 'LogsCommand' +] \ No newline at end of file diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py new file mode 100644 index 0000000..c985e57 --- /dev/null +++ b/src/kiwi/subcommands/_utils.py @@ -0,0 +1,53 @@ +import subprocess + +from ..config import LoadedConfig + + +class SubCommand: + @classmethod + def get_cmd(cls): + pass + + @classmethod + def setup(cls): + pass + + @classmethod + def run(cls): + pass + + +class Docker: + __requires_root = None + + @classmethod + def __check_requires_root(cls): + if cls.__requires_root is None: + try: + config = LoadedConfig.get() + subprocess.run( + [config['executables:docker'], 'ps'], + check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + cls.__requires_root = False + except subprocess.CalledProcessError: + cls.__requires_root = True + + return cls.__requires_root + + @classmethod + def run_command(cls, program, args, cwd=None, env=None): + config = LoadedConfig.get() + cmd = [config['executables:' + program], *args] + + if cls.__check_requires_root(): + cmd = [config['executables:sudo'], *cmd] + + print(cmd) + return subprocess.run( + cmd, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE, + cwd=cwd, env=env + ) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index d8485d6..4e342d6 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -1,10 +1,10 @@ import logging import os -from kiwi.core import KIWI_CONF_NAME, Parser -from kiwi.config import Config +from ..core import KIWI_CONF_NAME, Parser +from ..config import DefaultConfig -from .subcommand import SubCommand +from ._utils import SubCommand def user_input(config, key, prompt): @@ -16,6 +16,25 @@ def user_input(config, key, prompt): config[key] = result +def find_exe(program_name): + for path in os.environ["PATH"].split(os.pathsep): + exe_file = os.path.join(path, program_name) + if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): + return exe_file + + return None + + +def user_input_exe(config, program_name): + exe_file = find_exe(program_name) + key = 'executables:' + program_name + + if exe_file is not None: + config[key] = exe_file + else: + user_input(config, key, "Enter path to '{}' executable".format(program_name)) + + class InitCommand(SubCommand): __parser = None @@ -30,19 +49,23 @@ class InitCommand(SubCommand): @classmethod def run(cls): - config = Config.default() + config = DefaultConfig.get() if os.path.isfile(KIWI_CONF_NAME): logging.warning("Overwriting existing '%s'!", KIWI_CONF_NAME) user_input(config, 'version', "Choose kiwi-config version") + user_input(config, 'runtime:storage', "Enter main directory for local data") + user_input(config, 'markers:project', "Enter marker string for project directories") user_input(config, 'markers:down', "Enter marker string for disabled projects") user_input(config, 'network:name', "Enter name for local docker network") user_input(config, 'network:cidr', "Enter CIDR block for local docker network") - user_input(config, 'runtime:storage', "Enter main directory for local data") + user_input_exe(config, 'docker') + user_input_exe(config, 'docker-compose') + user_input_exe(config, 'sudo') config.save() diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py new file mode 100644 index 0000000..f9bf622 --- /dev/null +++ b/src/kiwi/subcommands/logs.py @@ -0,0 +1,19 @@ +from ..core import Parser + +from ._utils import SubCommand, Docker + + +class LogsCommand(SubCommand): + __parser = None + + @classmethod + def get_cmd(cls): + return 'logs' + + @classmethod + def setup(cls): + cls.__parser = Parser.get_subparsers().add_parser(cls.get_cmd(), help="Show logs of a project") + + @classmethod + def run(cls): + print(Docker.run_command('docker-compose', ['logs', '-tf', '--tail=10'], cwd='hello-world.project', env={'COMPOSE_PROJECT_NAME': 'hello-world'})) diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 34b7df9..593a949 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -1,7 +1,7 @@ -from kiwi.config import Config -from kiwi.core import Parser +from ..config import LoadedConfig +from ..core import Parser -from .subcommand import SubCommand +from ._utils import SubCommand class ShowCommand(SubCommand): @@ -17,5 +17,5 @@ class ShowCommand(SubCommand): @classmethod def run(cls): - config = Config.load() + config = LoadedConfig.get() print(config) diff --git a/src/kiwi/subcommands/subcommand.py b/src/kiwi/subcommands/subcommand.py deleted file mode 100644 index 21535e5..0000000 --- a/src/kiwi/subcommands/subcommand.py +++ /dev/null @@ -1,12 +0,0 @@ -class SubCommand: - @classmethod - def get_cmd(cls): - pass - - @classmethod - def setup(cls): - pass - - @classmethod - def run(cls): - pass From f188d05e566a17e71641cb357440ee7e323672d6 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 11:24:39 +0200 Subject: [PATCH 021/118] rework "SubCommand" class members --- example/base.conf | 2 +- example/kiwi.yml | 2 +- src/kiwi/runner.py | 2 +- src/kiwi/subcommands/_utils.py | 4 +--- src/kiwi/subcommands/init.py | 33 ++++++++++++++++++++------------- src/kiwi/subcommands/logs.py | 11 +++++------ src/kiwi/subcommands/show.py | 11 +++++------ 7 files changed, 34 insertions(+), 31 deletions(-) diff --git a/example/base.conf b/example/base.conf index 0c8a157..ac5fcfc 100644 --- a/example/base.conf +++ b/example/base.conf @@ -4,4 +4,4 @@ export SUFFIX_DOWN=.down export DOCKERNET=kiwinet export DOCKERCIDR=10.22.46.0/24 -export TARGETROOT=/var/kiwi +export TARGETROOT=/tmp/kiwi diff --git a/example/kiwi.yml b/example/kiwi.yml index 4a18c26..a41a7a9 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -5,7 +5,7 @@ version: '0.1' runtime: - storage: /var/kiwi + storage: /tmp/kiwi env: null markers: diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 553f497..a0da00f 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -21,6 +21,6 @@ class Runner: args = Parser.get_args() for cmd in cls.__commands: - if cmd.get_cmd() == args.command: + if cmd.command == args.command: cmd.run() return diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py index c985e57..2501b52 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/_utils.py @@ -4,9 +4,7 @@ from ..config import LoadedConfig class SubCommand: - @classmethod - def get_cmd(cls): - pass + command = None @classmethod def setup(cls): diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 4e342d6..2791784 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -2,14 +2,14 @@ import logging import os from ..core import KIWI_CONF_NAME, Parser -from ..config import DefaultConfig +from ..config import DefaultConfig, LoadedConfig from ._utils import SubCommand def user_input(config, key, prompt): # prompt user as per argument - result = input("{} [Default: {}] ".format(prompt, config[key])).strip() + result = input("{} [{}] ".format(prompt, config[key])).strip() # store result if present if result: @@ -36,27 +36,34 @@ def user_input_exe(config, program_name): class InitCommand(SubCommand): - __parser = None - - @classmethod - def get_cmd(cls): - return 'init' + command = 'init' @classmethod def setup(cls): - cls.__parser = Parser.get_subparsers().add_parser(cls.get_cmd(), help="Create new kiwi-config instance") - # cls.__parser.add_argument('cmd', metavar='command', type=str, help='subcommand to execute') + parser = Parser.get_subparsers().add_parser( + cls.command, + description="Create a new kiwi-config instance" + ) + + parser.add_argument( + '-f', '--force', + action='store_true', + help="Use default values even if {} is present".format(KIWI_CONF_NAME) + ) @classmethod def run(cls): - config = DefaultConfig.get() + logging.info("Initializing kiwi-config instance in '%s'", os.getcwd()) - if os.path.isfile(KIWI_CONF_NAME): + if Parser.get_args().force and os.path.isfile(KIWI_CONF_NAME): logging.warning("Overwriting existing '%s'!", KIWI_CONF_NAME) + config = DefaultConfig.get() + else: + config = LoadedConfig.get() - user_input(config, 'version', "Choose kiwi-config version") + user_input(config, 'version', "Enter kiwi-config version for this instance") - user_input(config, 'runtime:storage', "Enter main directory for local data") + user_input(config, 'runtime:storage', "Enter local directory for service data") user_input(config, 'markers:project', "Enter marker string for project directories") user_input(config, 'markers:down', "Enter marker string for disabled projects") diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index f9bf622..5494a30 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -4,15 +4,14 @@ from ._utils import SubCommand, Docker class LogsCommand(SubCommand): - __parser = None - - @classmethod - def get_cmd(cls): - return 'logs' + command = 'logs' @classmethod def setup(cls): - cls.__parser = Parser.get_subparsers().add_parser(cls.get_cmd(), help="Show logs of a project") + parser = Parser.get_subparsers().add_parser( + cls.command, + description="Show logs of a project" + ) @classmethod def run(cls): diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 593a949..0ae5eb9 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -5,15 +5,14 @@ from ._utils import SubCommand class ShowCommand(SubCommand): - __parser = None - - @classmethod - def get_cmd(cls): - return 'show' + command = 'show' @classmethod def setup(cls): - cls.__parser = Parser.get_subparsers().add_parser(cls.get_cmd(), help="Show effective kiwi.yml") + parser = Parser.get_subparsers().add_parser( + cls.command, + description="Show effective kiwi.yml" + ) @classmethod def run(cls): From 93643de8fbf769468c15792cd8918156d3db10e5 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 14:05:19 +0200 Subject: [PATCH 022/118] "Runner" into main function, rework Parser and InitCommand --- src/kiwi-config.py | 34 +++++++++++++++++++++++-- src/kiwi/__init__.py | 6 +++-- src/kiwi/core.py | 31 ++++++++++++----------- src/kiwi/runner.py | 26 ------------------- src/kiwi/subcommands/__init__.py | 5 +--- src/kiwi/subcommands/init.py | 43 ++++++++++++++++++++++---------- 6 files changed, 84 insertions(+), 61 deletions(-) delete mode 100644 src/kiwi/runner.py diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 25eb590..f0da7c1 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -1,10 +1,40 @@ #!/usr/bin/env python3 +import logging + import kiwi +from kiwi.subcommands import * def main(): - kiwi.Runner.setup_all() - kiwi.Runner.run() + logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=logging.NOTSET + ) + + commands = [ + InitCommand, + ShowCommand, + LogsCommand + ] + + for cmd in commands: + cmd.setup() + + args = kiwi.Parser.get_args() + + if args.verbose >= 2: + log_level = logging.DEBUG + elif args.verbose >= 1: + log_level = logging.INFO + else: + log_level = logging.WARNING + + logging.getLogger().setLevel(log_level) + + for cmd in commands: + if cmd.command == args.command: + cmd.run() + return if __name__ == "__main__": diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py index 247ace4..ded6eea 100644 --- a/src/kiwi/__init__.py +++ b/src/kiwi/__init__.py @@ -1,3 +1,5 @@ -from .runner import Runner +from .core import Parser -__all__ = ['Runner'] \ No newline at end of file +__all__ = [ + 'Parser' +] diff --git a/src/kiwi/core.py b/src/kiwi/core.py index 4316fcc..1ded8e5 100644 --- a/src/kiwi/core.py +++ b/src/kiwi/core.py @@ -9,32 +9,35 @@ KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") class Parser: - __instance = None + __parser = None __subparsers = None __args = None @classmethod - def __init_instance(cls): - if not cls.__instance: - cls.__instance = argparse.ArgumentParser(description='kiwi-config') + def get_parser(cls): + if cls.__parser is None: + cls.__parser = argparse.ArgumentParser(description='kiwi-config') - cls.__subparsers = Parser.__instance.add_subparsers() - cls.__subparsers.required = True - cls.__subparsers.dest = 'command' + cls.__parser.add_argument( + '-v', '--verbose', + action='count', default=0 + ) - @classmethod - def get_instance(cls): - cls.__init_instance() - return cls.__instance + return cls.__parser @classmethod def get_subparsers(cls): - cls.__init_instance() + if cls.__subparsers is None: + cls.__subparsers = cls.get_parser().add_subparsers() + cls.__subparsers.required = True + cls.__subparsers.dest = 'command' + return cls.__subparsers @classmethod def get_args(cls): - if not cls.__args: - cls.__args = cls.get_instance().parse_args() + if cls.__args is None: + cls.__args = cls.get_parser().parse_args() return cls.__args + diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py deleted file mode 100644 index a0da00f..0000000 --- a/src/kiwi/runner.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing import List - -from .core import Parser -from .subcommands import * - - -class Runner: - __commands: List[SubCommand] = [ - InitCommand, - ShowCommand, - LogsCommand - ] - - @classmethod - def setup_all(cls): - for cmd in cls.__commands: - cmd.setup() - - @classmethod - def run(cls): - args = Parser.get_args() - - for cmd in cls.__commands: - if cmd.command == args.command: - cmd.run() - return diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index d1a8d83..5219579 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,12 +1,9 @@ -from ._utils import SubCommand - from .init import InitCommand from .show import ShowCommand from .logs import LogsCommand __all__ = [ - 'SubCommand', 'InitCommand', 'ShowCommand', 'LogsCommand' -] \ No newline at end of file +] diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 2791784..429a073 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -16,23 +16,35 @@ def user_input(config, key, prompt): config[key] = result +def is_executable(filename): + if filename is None: + return False + + return os.path.isfile(filename) and os.access(filename, os.X_OK) + + def find_exe(program_name): for path in os.environ["PATH"].split(os.pathsep): exe_file = os.path.join(path, program_name) - if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): + if is_executable(exe_file): return exe_file return None -def user_input_exe(config, program_name): - exe_file = find_exe(program_name) - key = 'executables:' + program_name +def user_input_exe(config, key): + exe_file = config[key] + program_name = key.split(':')[1] - if exe_file is not None: - config[key] = exe_file - else: - user_input(config, key, "Enter path to '{}' executable".format(program_name)) + if not is_executable(exe_file): + logging.warning("Reconfiguring '%s' executable path.", program_name) + exe_file = find_exe(program_name) + + if exe_file is not None: + logging.info("Found executable at '%s'.", exe_file) + config[key] = exe_file + else: + user_input(config, key, f"Enter path to '{program_name}' executable") class InitCommand(SubCommand): @@ -48,12 +60,12 @@ class InitCommand(SubCommand): parser.add_argument( '-f', '--force', action='store_true', - help="Use default values even if {} is present".format(KIWI_CONF_NAME) + help=f"Use default values even if {KIWI_CONF_NAME} is present" ) @classmethod def run(cls): - logging.info("Initializing kiwi-config instance in '%s'", os.getcwd()) + logging.info(f"Initializing kiwi-config instance in '{os.getcwd()}'") if Parser.get_args().force and os.path.isfile(KIWI_CONF_NAME): logging.warning("Overwriting existing '%s'!", KIWI_CONF_NAME) @@ -61,18 +73,23 @@ class InitCommand(SubCommand): else: config = LoadedConfig.get() + # version user_input(config, 'version', "Enter kiwi-config version for this instance") + # runtime user_input(config, 'runtime:storage', "Enter local directory for service data") + # markers user_input(config, 'markers:project', "Enter marker string for project directories") user_input(config, 'markers:down', "Enter marker string for disabled projects") + # network user_input(config, 'network:name', "Enter name for local docker network") user_input(config, 'network:cidr', "Enter CIDR block for local docker network") - user_input_exe(config, 'docker') - user_input_exe(config, 'docker-compose') - user_input_exe(config, 'sudo') + # executables + user_input_exe(config, 'executables:docker') + user_input_exe(config, 'executables:docker-compose') + user_input_exe(config, 'executables:sudo') config.save() From 41274c3239da60af44a6f31482bfca12006fe659 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 14:39:28 +0200 Subject: [PATCH 023/118] logging verbosity --- src/kiwi-config.py | 30 ++++++++++++++++++------------ src/kiwi/core.py | 3 +-- src/kiwi/subcommands/init.py | 4 ++-- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/kiwi-config.py b/src/kiwi-config.py index f0da7c1..3e2c589 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -5,12 +5,22 @@ import kiwi from kiwi.subcommands import * -def main(): - logging.basicConfig( - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - level=logging.NOTSET - ) +def set_verbosity(logger, handler, verbosity): + if verbosity >= 2: + log_level = logging.DEBUG + log_format = "[%(asctime)s] %(levelname)s @ %(filename)s:%(funcName)s:%(lineno)d: %(message)s" + elif verbosity >= 1: + log_level = logging.INFO + log_format = "[%(asctime)s] %(levelname)s: %(message)s" + else: + log_level = logging.WARNING + log_format = "%(levelname)s: %(message)s" + logger.setLevel(log_level) + handler.setFormatter(logging.Formatter(log_format)) + + +def main(): commands = [ InitCommand, ShowCommand, @@ -22,14 +32,10 @@ def main(): args = kiwi.Parser.get_args() - if args.verbose >= 2: - log_level = logging.DEBUG - elif args.verbose >= 1: - log_level = logging.INFO - else: - log_level = logging.WARNING + log_handler = logging.StreamHandler() + logging.getLogger().addHandler(log_handler) - logging.getLogger().setLevel(log_level) + set_verbosity(logging.getLogger(), log_handler, args.verbosity) for cmd in commands: if cmd.command == args.command: diff --git a/src/kiwi/core.py b/src/kiwi/core.py index 1ded8e5..8de4feb 100644 --- a/src/kiwi/core.py +++ b/src/kiwi/core.py @@ -19,7 +19,7 @@ class Parser: cls.__parser = argparse.ArgumentParser(description='kiwi-config') cls.__parser.add_argument( - '-v', '--verbose', + '-v', '--verbosity', action='count', default=0 ) @@ -40,4 +40,3 @@ class Parser: cls.__args = cls.get_parser().parse_args() return cls.__args - diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 429a073..b377545 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -37,11 +37,11 @@ def user_input_exe(config, key): program_name = key.split(':')[1] if not is_executable(exe_file): - logging.warning("Reconfiguring '%s' executable path.", program_name) + logging.info("Reconfiguring '%s' executable path.", program_name) exe_file = find_exe(program_name) if exe_file is not None: - logging.info("Found executable at '%s'.", exe_file) + logging.debug("Found executable at '%s'.", exe_file) config[key] = exe_file else: user_input(config, key, f"Enter path to '{program_name}' executable") From e43dadd4fe31a1709e7d2d2b41ed05fe40eb6357 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 14:50:47 +0200 Subject: [PATCH 024/118] "executables:*" key handling in _util --- src/kiwi/subcommands/_utils.py | 32 ++++++++++++++++++++++++---- src/kiwi/subcommands/init.py | 38 ++++++++++------------------------ 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py index 2501b52..adf5b45 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/_utils.py @@ -1,7 +1,31 @@ +import os import subprocess from ..config import LoadedConfig +########### +# CONSTANTS + + +def is_executable(filename): + if filename is None: + return False + + return os.path.isfile(filename) and os.access(filename, os.X_OK) + + +def find_exe_file(exe_name): + for path in os.environ['PATH'].split(os.pathsep): + exe_file = os.path.join(path, exe_name) + if is_executable(exe_file): + return exe_file + + return None + + +def get_exe_key(exe_name): + return f'executables:{exe_name}' + class SubCommand: command = None @@ -24,7 +48,7 @@ class Docker: try: config = LoadedConfig.get() subprocess.run( - [config['executables:docker'], 'ps'], + [config[get_exe_key('docker')], 'ps'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) @@ -35,12 +59,12 @@ class Docker: return cls.__requires_root @classmethod - def run_command(cls, program, args, cwd=None, env=None): + def run_command(cls, exe_key, args, cwd=None, env=None): config = LoadedConfig.get() - cmd = [config['executables:' + program], *args] + cmd = [config[get_exe_key(exe_key)], *args] if cls.__check_requires_root(): - cmd = [config['executables:sudo'], *cmd] + cmd = [config[get_exe_key('sudo')], *cmd] print(cmd) return subprocess.run( diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index b377545..f95408e 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -4,7 +4,7 @@ import os from ..core import KIWI_CONF_NAME, Parser from ..config import DefaultConfig, LoadedConfig -from ._utils import SubCommand +from ._utils import SubCommand, is_executable, find_exe_file, get_exe_key def user_input(config, key, prompt): @@ -16,35 +16,19 @@ def user_input(config, key, prompt): config[key] = result -def is_executable(filename): - if filename is None: - return False - - return os.path.isfile(filename) and os.access(filename, os.X_OK) - - -def find_exe(program_name): - for path in os.environ["PATH"].split(os.pathsep): - exe_file = os.path.join(path, program_name) - if is_executable(exe_file): - return exe_file - - return None - - -def user_input_exe(config, key): +def user_input_exe(config, exe_name): + key = get_exe_key(exe_name) exe_file = config[key] - program_name = key.split(':')[1] if not is_executable(exe_file): - logging.info("Reconfiguring '%s' executable path.", program_name) - exe_file = find_exe(program_name) + logging.info(f"Reconfiguring '{exe_name}' executable path.") + exe_file = find_exe_file(exe_name) if exe_file is not None: - logging.debug("Found executable at '%s'.", exe_file) + logging.debug(f"Found executable at '{exe_file}'.") config[key] = exe_file else: - user_input(config, key, f"Enter path to '{program_name}' executable") + user_input(config, key, f"Enter path to '{exe_name}' executable") class InitCommand(SubCommand): @@ -68,7 +52,7 @@ class InitCommand(SubCommand): logging.info(f"Initializing kiwi-config instance in '{os.getcwd()}'") if Parser.get_args().force and os.path.isfile(KIWI_CONF_NAME): - logging.warning("Overwriting existing '%s'!", KIWI_CONF_NAME) + logging.warning(f"Overwriting existing '{KIWI_CONF_NAME}'!") config = DefaultConfig.get() else: config = LoadedConfig.get() @@ -88,8 +72,8 @@ class InitCommand(SubCommand): user_input(config, 'network:cidr', "Enter CIDR block for local docker network") # executables - user_input_exe(config, 'executables:docker') - user_input_exe(config, 'executables:docker-compose') - user_input_exe(config, 'executables:sudo') + user_input_exe(config, 'docker') + user_input_exe(config, 'docker-compose') + user_input_exe(config, 'sudo') config.save() From 65fcb91bd0afe322bcd38ca24ec852841594edb3 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 15:21:39 +0200 Subject: [PATCH 025/118] Parser singleton --- src/kiwi-config.py | 2 +- src/kiwi/core.py | 48 ++++++++++++++++++++---------------- src/kiwi/subcommands/init.py | 4 +-- src/kiwi/subcommands/logs.py | 2 +- src/kiwi/subcommands/show.py | 2 +- 5 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 3e2c589..61fa52a 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -30,7 +30,7 @@ def main(): for cmd in commands: cmd.setup() - args = kiwi.Parser.get_args() + args = kiwi.Parser().get_args() log_handler = logging.StreamHandler() logging.getLogger().addHandler(log_handler) diff --git a/src/kiwi/core.py b/src/kiwi/core.py index 8de4feb..86ccff6 100644 --- a/src/kiwi/core.py +++ b/src/kiwi/core.py @@ -9,34 +9,40 @@ KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") class Parser: - __parser = None - __subparsers = None - __args = None + class __Parser: + __parser = None + __subparsers = None + __args = None - @classmethod - def get_parser(cls): - if cls.__parser is None: - cls.__parser = argparse.ArgumentParser(description='kiwi-config') + def __init__(self): + self.__parser = argparse.ArgumentParser(description='kiwi-config') - cls.__parser.add_argument( + self.__parser.add_argument( '-v', '--verbosity', action='count', default=0 ) - return cls.__parser + self.__subparsers = self.__parser.add_subparsers() + self.__subparsers.required = True + self.__subparsers.dest = 'command' - @classmethod - def get_subparsers(cls): - if cls.__subparsers is None: - cls.__subparsers = cls.get_parser().add_subparsers() - cls.__subparsers.required = True - cls.__subparsers.dest = 'command' + def get_parser(self): + return self.__parser - return cls.__subparsers + def get_subparsers(self): + return self.__subparsers - @classmethod - def get_args(cls): - if cls.__args is None: - cls.__args = cls.get_parser().parse_args() + def get_args(self): + if self.__args is None: + self.__args = self.__parser.parse_args() - return cls.__args + return self.__args + + __instance = None + + def __init__(self): + if Parser.__instance is None: + Parser.__instance = Parser.__Parser() + + def __getattr__(self, item): + return getattr(self.__instance, item) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index f95408e..00c6f9a 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -36,7 +36,7 @@ class InitCommand(SubCommand): @classmethod def setup(cls): - parser = Parser.get_subparsers().add_parser( + parser = Parser().get_subparsers().add_parser( cls.command, description="Create a new kiwi-config instance" ) @@ -51,7 +51,7 @@ class InitCommand(SubCommand): def run(cls): logging.info(f"Initializing kiwi-config instance in '{os.getcwd()}'") - if Parser.get_args().force and os.path.isfile(KIWI_CONF_NAME): + if Parser().get_args().force and os.path.isfile(KIWI_CONF_NAME): logging.warning(f"Overwriting existing '{KIWI_CONF_NAME}'!") config = DefaultConfig.get() else: diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 5494a30..824d735 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -8,7 +8,7 @@ class LogsCommand(SubCommand): @classmethod def setup(cls): - parser = Parser.get_subparsers().add_parser( + _ = Parser().get_subparsers().add_parser( cls.command, description="Show logs of a project" ) diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 0ae5eb9..30d582e 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -9,7 +9,7 @@ class ShowCommand(SubCommand): @classmethod def setup(cls): - parser = Parser.get_subparsers().add_parser( + _ = Parser().get_subparsers().add_parser( cls.command, description="Show effective kiwi.yml" ) From 3cb195fe1aaab29f9c42853d009bc2a9d69d0006 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 15:48:15 +0200 Subject: [PATCH 026/118] Runner logic back into package, as singleton --- src/kiwi-config.py | 15 +------------ src/kiwi/__init__.py | 4 +++- src/kiwi/runner.py | 40 ++++++++++++++++++++++++++++++++++ src/kiwi/subcommands/_utils.py | 22 +++++++++++-------- src/kiwi/subcommands/init.py | 14 +++++------- src/kiwi/subcommands/logs.py | 12 ++++------ src/kiwi/subcommands/show.py | 14 +++++------- 7 files changed, 71 insertions(+), 50 deletions(-) create mode 100644 src/kiwi/runner.py diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 61fa52a..55effc2 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -21,26 +21,13 @@ def set_verbosity(logger, handler, verbosity): def main(): - commands = [ - InitCommand, - ShowCommand, - LogsCommand - ] - - for cmd in commands: - cmd.setup() - args = kiwi.Parser().get_args() log_handler = logging.StreamHandler() logging.getLogger().addHandler(log_handler) - set_verbosity(logging.getLogger(), log_handler, args.verbosity) - for cmd in commands: - if cmd.command == args.command: - cmd.run() - return + kiwi.Runner().run(args.command) if __name__ == "__main__": diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py index ded6eea..f5decab 100644 --- a/src/kiwi/__init__.py +++ b/src/kiwi/__init__.py @@ -1,5 +1,7 @@ from .core import Parser +from .runner import Runner __all__ = [ - 'Parser' + 'Parser', + 'Runner' ] diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py new file mode 100644 index 0000000..981529a --- /dev/null +++ b/src/kiwi/runner.py @@ -0,0 +1,40 @@ +from .subcommands import * + +########### +# CONSTANTS + +SUBCOMMANDS = [ + InitCommand, + ShowCommand, + LogsCommand +] + + +class Runner: + class __Runner: + __commands = [] + + def __init__(self): + for cmd in SUBCOMMANDS: + self.__commands.append(cmd()) + + def run(self, command_name): + for cmd in self.__commands: + if str(cmd) == command_name: + cmd.run() + return True + + return False + + __instance = None + + def __init__(self): + if Runner.__instance is None: + Runner.__instance = Runner.__Runner() + + def __getattr__(self, item): + return getattr(self.__instance, item) + + +if __name__ == 'kiwi.runner': + Runner().setup_all() diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py index adf5b45..6123945 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/_utils.py @@ -1,11 +1,9 @@ import os import subprocess +from ..core import Parser from ..config import LoadedConfig -########### -# CONSTANTS - def is_executable(filename): if filename is None: @@ -28,14 +26,20 @@ def get_exe_key(exe_name): class SubCommand: - command = None + __name = None + __parser = None - @classmethod - def setup(cls): - pass + def __init__(self, name, **kwargs): + self.__name = name + self.__parser = Parser().get_subparsers().add_parser(name, **kwargs) - @classmethod - def run(cls): + def __str__(self): + return self.__name + + def get_parser(self): + return self.__parser + + def run(self): pass diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 00c6f9a..f7924d8 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -32,23 +32,19 @@ def user_input_exe(config, exe_name): class InitCommand(SubCommand): - command = 'init' - - @classmethod - def setup(cls): - parser = Parser().get_subparsers().add_parser( - cls.command, + def __init__(self): + super().__init__( + 'init', description="Create a new kiwi-config instance" ) - parser.add_argument( + self.get_parser().add_argument( '-f', '--force', action='store_true', help=f"Use default values even if {KIWI_CONF_NAME} is present" ) - @classmethod - def run(cls): + def run(self): logging.info(f"Initializing kiwi-config instance in '{os.getcwd()}'") if Parser().get_args().force and os.path.isfile(KIWI_CONF_NAME): diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 824d735..6f2fb2e 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -4,15 +4,11 @@ from ._utils import SubCommand, Docker class LogsCommand(SubCommand): - command = 'logs' - - @classmethod - def setup(cls): - _ = Parser().get_subparsers().add_parser( - cls.command, + def __init__(self): + super().__init__( + 'logs', description="Show logs of a project" ) - @classmethod - def run(cls): + def run(self): print(Docker.run_command('docker-compose', ['logs', '-tf', '--tail=10'], cwd='hello-world.project', env={'COMPOSE_PROJECT_NAME': 'hello-world'})) diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 30d582e..1819d0d 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -5,16 +5,12 @@ from ._utils import SubCommand class ShowCommand(SubCommand): - command = 'show' - - @classmethod - def setup(cls): - _ = Parser().get_subparsers().add_parser( - cls.command, - description="Show effective kiwi.yml" + def __init__(self): + super().__init__( + 'logs', + description="Show logs of a project" ) - @classmethod - def run(cls): + def run(self): config = LoadedConfig.get() print(config) From 81127096cd00246188673af884adb1d99d9173f7 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 16:15:10 +0200 Subject: [PATCH 027/118] Typos --- src/kiwi/runner.py | 2 +- src/kiwi/subcommands/show.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 981529a..b65d4f7 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -37,4 +37,4 @@ class Runner: if __name__ == 'kiwi.runner': - Runner().setup_all() + _ = Runner() diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 1819d0d..6785485 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -1,5 +1,4 @@ from ..config import LoadedConfig -from ..core import Parser from ._utils import SubCommand @@ -7,8 +6,8 @@ from ._utils import SubCommand class ShowCommand(SubCommand): def __init__(self): super().__init__( - 'logs', - description="Show logs of a project" + 'show', + description="Show effective kiwi.yml" ) def run(self): From 90bfb32289a5f4205f846e00c25b1df72f9f30b5 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 16:28:42 +0200 Subject: [PATCH 028/118] DockerProgram Singleton dictionary and simple logs implementation --- src/kiwi/subcommands/_utils.py | 51 +++++++++++++++++++--------------- src/kiwi/subcommands/logs.py | 10 ++++--- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py index 6123945..42d1e11 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/_utils.py @@ -43,37 +43,42 @@ class SubCommand: pass -class Docker: +class DockerProgram: + class __DockerProgram: + __cmd = [] + + def __init__(self, exe_name): + config = LoadedConfig.get() + self.__cmd = [config[get_exe_key(exe_name)]] + + if DockerProgram.__requires_root: + self.__cmd = [config[get_exe_key("sudo")], *self.__cmd] + + def run(self, args, **kwargs): + cmd = [*self.__cmd, *args] + print(cmd) + return subprocess.run(cmd, **kwargs) + + __exe_name = None + __instances = {} __requires_root = None - @classmethod - def __check_requires_root(cls): - if cls.__requires_root is None: + def __init__(self, exe_name): + if DockerProgram.__requires_root is None: try: config = LoadedConfig.get() subprocess.run( [config[get_exe_key('docker')], 'ps'], - check=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) - cls.__requires_root = False + DockerProgram.__requires_root = False except subprocess.CalledProcessError: - cls.__requires_root = True + DockerProgram.__requires_root = True - return cls.__requires_root + self.__exe_name = exe_name - @classmethod - def run_command(cls, exe_key, args, cwd=None, env=None): - config = LoadedConfig.get() - cmd = [config[get_exe_key(exe_key)], *args] + if exe_name not in DockerProgram.__instances: + DockerProgram.__instances[exe_name] = DockerProgram.__DockerProgram(exe_name) - if cls.__check_requires_root(): - cmd = [config[get_exe_key('sudo')], *cmd] - - print(cmd) - return subprocess.run( - cmd, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE, - cwd=cwd, env=env - ) + def __getattr__(self, item): + return getattr(self.__instances[self.__exe_name], item) diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 6f2fb2e..699f297 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,6 +1,4 @@ -from ..core import Parser - -from ._utils import SubCommand, Docker +from ._utils import SubCommand, DockerProgram class LogsCommand(SubCommand): @@ -11,4 +9,8 @@ class LogsCommand(SubCommand): ) def run(self): - print(Docker.run_command('docker-compose', ['logs', '-tf', '--tail=10'], cwd='hello-world.project', env={'COMPOSE_PROJECT_NAME': 'hello-world'})) + DockerProgram('docker-compose').run( + ['logs', '-tf', '--tail=10'], + cwd='hello-world.project', + env={'COMPOSE_PROJECT_NAME': 'hello-world'} + ) From 717e1ec81e20ac955f332a8c1ffba67309819157 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 16:35:38 +0200 Subject: [PATCH 029/118] Typos --- src/kiwi/subcommands/init.py | 2 +- src/kiwi/subcommands/show.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index f7924d8..530e77a 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -41,7 +41,7 @@ class InitCommand(SubCommand): self.get_parser().add_argument( '-f', '--force', action='store_true', - help=f"Use default values even if {KIWI_CONF_NAME} is present" + help=f"use default values even if {KIWI_CONF_NAME} is present" ) def run(self): diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 6785485..f15b220 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -7,7 +7,7 @@ class ShowCommand(SubCommand): def __init__(self): super().__init__( 'show', - description="Show effective kiwi.yml" + description="show effective kiwi.yml" ) def run(self): From 02c2945ecf1f52f78dda4fb7c70fffdfcaa155de Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 16:36:05 +0200 Subject: [PATCH 030/118] "logs" command with "-f" switch --- src/kiwi/subcommands/logs.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 699f297..5ee7935 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,3 +1,5 @@ +from ..core import Parser + from ._utils import SubCommand, DockerProgram @@ -5,12 +7,22 @@ class LogsCommand(SubCommand): def __init__(self): super().__init__( 'logs', - description="Show logs of a project" + description="Show logs of a project or service" + ) + + self.get_parser().add_argument( + '-f', '--follow', + action='store_true', + help="output appended data as log grows" ) def run(self): + args = ['logs', '-t'] + if Parser().get_args().follow: + args = [*args, '-f', '--tail=10'] + DockerProgram('docker-compose').run( - ['logs', '-tf', '--tail=10'], + args, cwd='hello-world.project', env={'COMPOSE_PROJECT_NAME': 'hello-world'} ) From 90caad4aba6a01013d5071090fbc2227eac89560 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 17:38:02 +0200 Subject: [PATCH 031/118] "hubnet" -> "kiwihub" --- Makefile | 4 ++-- example/base.conf | 2 +- example/kiwi.yml | 2 +- src/default.kiwi.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b464b4f..2b13b76 100644 --- a/Makefile +++ b/Makefile @@ -241,7 +241,7 @@ networks: default: driver: bridge # interconnects projects - hubnet: + kiwihub: external: name: $$DOCKERNET @@ -251,6 +251,6 @@ services: restart: unless-stopped networks: - default - - hubnet + - kiwihub [...] endef diff --git a/example/base.conf b/example/base.conf index ac5fcfc..e012a32 100644 --- a/example/base.conf +++ b/example/base.conf @@ -1,7 +1,7 @@ export SUFFIX_PROJECT=.project export SUFFIX_DOWN=.down -export DOCKERNET=kiwinet +export DOCKERNET=kiwihub export DOCKERCIDR=10.22.46.0/24 export TARGETROOT=/tmp/kiwi diff --git a/example/kiwi.yml b/example/kiwi.yml index a41a7a9..e83688b 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -13,7 +13,7 @@ markers: down: .down network: - name: kiwinet + name: kiwihub cidr: 10.22.46.0/24 executables: diff --git a/src/default.kiwi.yml b/src/default.kiwi.yml index 06384df..b729709 100644 --- a/src/default.kiwi.yml +++ b/src/default.kiwi.yml @@ -9,7 +9,7 @@ markers: project: .project down: .down network: - name: kiwinet + name: kiwihub cidr: 10.22.46.0/24 executables: docker: null From ceb4bc043005480e3e70d06d6eabadac634cf9b2 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 17:39:15 +0200 Subject: [PATCH 032/118] "-v" switch in main program --- src/kiwi-config.py | 6 +++++- src/kiwi/core.py | 5 ----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 55effc2..737fe98 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -2,7 +2,6 @@ import logging import kiwi -from kiwi.subcommands import * def set_verbosity(logger, handler, verbosity): @@ -21,6 +20,11 @@ def set_verbosity(logger, handler, verbosity): def main(): + kiwi.Parser().get_parser().add_argument( + '-v', '--verbosity', + action='count', default=0 + ) + args = kiwi.Parser().get_args() log_handler = logging.StreamHandler() diff --git a/src/kiwi/core.py b/src/kiwi/core.py index 86ccff6..9f06fa7 100644 --- a/src/kiwi/core.py +++ b/src/kiwi/core.py @@ -17,11 +17,6 @@ class Parser: def __init__(self): self.__parser = argparse.ArgumentParser(description='kiwi-config') - self.__parser.add_argument( - '-v', '--verbosity', - action='count', default=0 - ) - self.__subparsers = self.__parser.add_subparsers() self.__subparsers.required = True self.__subparsers.dest = 'command' From e667c00b04d59fc85d5e4d938d03935a30a77872 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 17:39:33 +0200 Subject: [PATCH 033/118] f-Strings consistency --- src/kiwi/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 2d23d99..1f98ef1 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -8,8 +8,8 @@ from .core import KIWI_ROOT, KIWI_CONF_NAME ########### # CONSTANTS -DEFAULT_KIWI_CONF_NAME = KIWI_ROOT + "/default.kiwi.yml" -VERSION_TAG_NAME = KIWI_ROOT + "/version-tag" +DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/default.kiwi.yml" +VERSION_TAG_NAME = f"{KIWI_ROOT}/version-tag" class Config: From 6ee528f4cb55227b33764fe27d0dead71bb408c8 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 10 Aug 2020 17:40:56 +0200 Subject: [PATCH 034/118] DockerCommand: __build_cmd, run_less --- src/kiwi/subcommands/_utils.py | 44 +++++++++++++++++++++++++--------- src/kiwi/subcommands/logs.py | 41 ++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 19 deletions(-) diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py index 42d1e11..7b615d5 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/_utils.py @@ -1,3 +1,4 @@ +import logging import os import subprocess @@ -43,42 +44,63 @@ class SubCommand: pass -class DockerProgram: - class __DockerProgram: +class DockerCommand: + class __DockerCommand: __cmd = [] def __init__(self, exe_name): config = LoadedConfig.get() self.__cmd = [config[get_exe_key(exe_name)]] - if DockerProgram.__requires_root: + if DockerCommand.__requires_root: self.__cmd = [config[get_exe_key("sudo")], *self.__cmd] - def run(self, args, **kwargs): + def __build_cmd(self, args): cmd = [*self.__cmd, *args] - print(cmd) - return subprocess.run(cmd, **kwargs) + logging.debug(f"DockerProgram: {cmd}") + return cmd + + def run(self, args, **kwargs): + return subprocess.run( + self.__build_cmd(args), + **kwargs + ) + + def run_less(self, args, **kwargs): + process = subprocess.Popen( + self.__build_cmd(args), + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + **kwargs + ) + + less_process = subprocess.run( + ['less', '-R', '+G'], + stdin=process.stdout + ) + + process.communicate() + return less_process __exe_name = None __instances = {} __requires_root = None def __init__(self, exe_name): - if DockerProgram.__requires_root is None: + if DockerCommand.__requires_root is None: try: config = LoadedConfig.get() subprocess.run( [config[get_exe_key('docker')], 'ps'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) - DockerProgram.__requires_root = False + DockerCommand.__requires_root = False except subprocess.CalledProcessError: - DockerProgram.__requires_root = True + DockerCommand.__requires_root = True self.__exe_name = exe_name - if exe_name not in DockerProgram.__instances: - DockerProgram.__instances[exe_name] = DockerProgram.__DockerProgram(exe_name) + if exe_name not in DockerCommand.__instances: + DockerCommand.__instances[exe_name] = DockerCommand.__DockerCommand(exe_name) def __getattr__(self, item): return getattr(self.__instances[self.__exe_name], item) diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 5ee7935..e7fb766 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,6 +1,7 @@ +from ..config import LoadedConfig from ..core import Parser -from ._utils import SubCommand, DockerProgram +from ._utils import SubCommand, DockerCommand class LogsCommand(SubCommand): @@ -11,18 +12,42 @@ class LogsCommand(SubCommand): ) self.get_parser().add_argument( - '-f', '--follow', - action='store_true', + '-f', '--follow', action='store_true', help="output appended data as log grows" ) + self.get_parser().add_argument( + 'project', + help="narf" + ) + + self.get_parser().add_argument( + 'service', nargs='?', + help="narf" + ) + def run(self): + config = LoadedConfig.get() + + project_name = Parser().get_args().project + project_marker = config['markers:project'] + project_dir = f'{project_name}{project_marker}' + + environment = { + 'DOCKERNET': config['network:name'], + 'COMPOSE_PROJECT_NAME': project_name + } + args = ['logs', '-t'] if Parser().get_args().follow: args = [*args, '-f', '--tail=10'] - DockerProgram('docker-compose').run( - args, - cwd='hello-world.project', - env={'COMPOSE_PROJECT_NAME': 'hello-world'} - ) + DockerCommand('docker-compose').run( + args, + cwd=project_dir, env=environment + ) + else: + DockerCommand('docker-compose').run_less( + args, + cwd=project_dir, env=environment + ) From 3522d6bba54ce11d969ec79231e326a3a7672af2 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 11 Aug 2020 11:20:08 +0200 Subject: [PATCH 035/118] $DOCKERNET -> $KIWI_HUB_NAME, $DOCKERCIDR -> $KIWI_HUB_CIDR --- Makefile | 34 +++++++++---------- example/base.conf | 4 +-- .../hello-world.project/docker-compose.yml | 4 +-- src/kiwi/subcommands/logs.py | 2 +- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 2b13b76..388f4c4 100644 --- a/Makefile +++ b/Makefile @@ -23,15 +23,15 @@ CONF_SOURCE:=$(patsubst %,. %;,$(CONF_WILDC)) confvalue=$(shell $(CONF_SOURCE) echo -n $${$(1)}) # docker network name -CONF_DOCKERNET:=$(call confvalue,DOCKERNET) -ifeq ($(CONF_DOCKERNET),) -$(error DOCKERNET not set in $(CONF_WILDC)) +CONF_KIWI_HUB_NAME:=$(call confvalue,KIWI_HUB_NAME) +ifeq ($(CONF_KIWI_HUB_NAME),) +$(error KIWI_HUB_NAME not set in $(CONF_WILDC)) endif # docker network CIDR -CONF_DOCKERCIDR:=$(call confvalue,DOCKERCIDR) -ifeq ($(CONF_DOCKERNET),) -$(error DOCKERCIDR not set in $(CONF_WILDC)) +CONF_KIWI_HUB_CIDR:=$(call confvalue,KIWI_HUB_CIDR) +ifeq ($(CONF_KIWI_HUB_CIDR),) +$(error KIWI_HUB_CIDR not set in $(CONF_WILDC)) endif # persistent data directory @@ -56,7 +56,7 @@ endif # CONSTANTS # file to store docker network cidr -FILE_DOCKERNET:=$(CONF_TARGETROOT)/up-$(CONF_DOCKERNET) +KIWI_HUB_FILE:=$(CONF_TARGETROOT)/up-$(CONF_KIWI_HUB_NAME) # remove any suffix $2 from $1 rmsuffix=$(patsubst %$2,%,$1) @@ -91,31 +91,31 @@ all: purge-conf up ######### # manage the docker network (container name local DNS) -$(FILE_DOCKERNET): +$(KIWI_HUB_FILE): -$(DOCKER) network create \ --driver bridge \ --internal \ - --subnet "$(CONF_DOCKERCIDR)" \ - "$(CONF_DOCKERNET)" - @echo "Creating canary $(FILE_DOCKERNET) ..." + --subnet "$(CONF_KIWI_HUB_CIDR)" \ + "$(CONF_KIWI_HUB_NAME)" + @echo "Creating canary $(KIWI_HUB_FILE) ..." @$(DOCKER) run --rm \ -v "/:/mnt" -u root alpine:latest \ ash -c '\ mkdir -p "$(addprefix /mnt, $(CONF_TARGETROOT))"; \ - echo "$(CONF_DOCKERCIDR)" > "$(addprefix /mnt, $(FILE_DOCKERNET))"; \ + echo "$(CONF_KIWI_HUB_CIDR)" > "$(addprefix /mnt, $(KIWI_HUB_FILE))"; \ ' .PHONY: net-up -net-up: $(FILE_DOCKERNET) +net-up: $(KIWI_HUB_FILE) .PHONY: net-down net-down: down - $(DOCKER) network rm "$(CONF_DOCKERNET)" - @echo "Removing canary $(FILE_DOCKERNET) ..." + $(DOCKER) network rm "$(CONF_KIWI_HUB_NAME)" + @echo "Removing canary $(KIWI_HUB_FILE) ..." @$(DOCKER) run --rm \ -v "/:/mnt" -u root alpine:latest \ ash -c '\ - rm -f "$(addprefix /mnt, $(FILE_DOCKERNET))"; \ + rm -f "$(addprefix /mnt, $(KIWI_HUB_FILE))"; \ ' ######### @@ -243,7 +243,7 @@ networks: # interconnects projects kiwihub: external: - name: $$DOCKERNET + name: $$KIWI_HUB_NAME services: something: diff --git a/example/base.conf b/example/base.conf index e012a32..80f2b32 100644 --- a/example/base.conf +++ b/example/base.conf @@ -1,7 +1,7 @@ export SUFFIX_PROJECT=.project export SUFFIX_DOWN=.down -export DOCKERNET=kiwihub -export DOCKERCIDR=10.22.46.0/24 +export KIWI_HUB_NAME=kiwihub +export KIWI_HUB_CIDR=10.22.46.0/24 export TARGETROOT=/tmp/kiwi diff --git a/example/hello-world.project/docker-compose.yml b/example/hello-world.project/docker-compose.yml index 201bd21..e8398b7 100644 --- a/example/hello-world.project/docker-compose.yml +++ b/example/hello-world.project/docker-compose.yml @@ -7,9 +7,9 @@ networks: # interconnects projects kiwihub: external: - name: $DOCKERNET + name: $KIWI_HUB_NAME services: hello-world: image: alpine:latest - command: sh -c 'while :; do echo Hello World; sleep 10; done' + command: sh -c 'while :; do echo Hello World "$$RANDOM"; sleep 10; done' diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index e7fb766..eea6985 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -34,7 +34,7 @@ class LogsCommand(SubCommand): project_dir = f'{project_name}{project_marker}' environment = { - 'DOCKERNET': config['network:name'], + 'KIWI_NET_NAME': config['network:name'], 'COMPOSE_PROJECT_NAME': project_name } From 0299e6a86f3f402d3367ee091071cec1c8ebe783 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 11 Aug 2020 12:08:03 +0200 Subject: [PATCH 036/118] logging improve --- src/kiwi-config.py | 2 ++ src/kiwi/subcommands/_utils.py | 8 ++++---- src/kiwi/subcommands/show.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 737fe98..87789e7 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -31,6 +31,8 @@ def main(): logging.getLogger().addHandler(log_handler) set_verbosity(logging.getLogger(), log_handler, args.verbosity) + logging.debug(f"main CLI args: {args}") + kiwi.Runner().run(args.command) diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py index 7b615d5..e0e4f77 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/_utils.py @@ -55,20 +55,20 @@ class DockerCommand: if DockerCommand.__requires_root: self.__cmd = [config[get_exe_key("sudo")], *self.__cmd] - def __build_cmd(self, args): + def __build_cmd(self, args, **kwargs): cmd = [*self.__cmd, *args] - logging.debug(f"DockerProgram: {cmd}") + logging.debug(f"DockerProgram cmd{cmd}, kwargs{kwargs}") return cmd def run(self, args, **kwargs): return subprocess.run( - self.__build_cmd(args), + self.__build_cmd(args, **kwargs), **kwargs ) def run_less(self, args, **kwargs): process = subprocess.Popen( - self.__build_cmd(args), + self.__build_cmd(args, **kwargs), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, **kwargs ) diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index f15b220..6785485 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -7,7 +7,7 @@ class ShowCommand(SubCommand): def __init__(self): super().__init__( 'show', - description="show effective kiwi.yml" + description="Show effective kiwi.yml" ) def run(self): From f94db364d73cc21c84999f01f831130e55a48b8f Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 11 Aug 2020 12:33:21 +0200 Subject: [PATCH 037/118] "logs" command finished --- .../hello-world.project/docker-compose.yml | 5 +- example/kiwi | 1 + src/kiwi/subcommands/_utils.py | 7 +-- src/kiwi/subcommands/init.py | 2 +- src/kiwi/subcommands/logs.py | 51 +++++++++++-------- 5 files changed, 39 insertions(+), 27 deletions(-) create mode 120000 example/kiwi diff --git a/example/hello-world.project/docker-compose.yml b/example/hello-world.project/docker-compose.yml index e8398b7..f49c63f 100644 --- a/example/hello-world.project/docker-compose.yml +++ b/example/hello-world.project/docker-compose.yml @@ -12,4 +12,7 @@ networks: services: hello-world: image: alpine:latest - command: sh -c 'while :; do echo Hello World "$$RANDOM"; sleep 10; done' + command: sh -c 'LOOP=1; while :; do echo Hello World "$$LOOP"; LOOP=$$(($$LOOP + 1)); sleep 10; done' + foo-bar: + image: alpine:latest + command: sh -c 'LOOP=1; while :; do echo Foo Bar "$$LOOP"; LOOP=$$(($$LOOP + 1)); sleep 20; done' diff --git a/example/kiwi b/example/kiwi new file mode 120000 index 0000000..d7c36e5 --- /dev/null +++ b/example/kiwi @@ -0,0 +1 @@ +../src/kiwi-config.py \ No newline at end of file diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py index e0e4f77..47458ab 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/_utils.py @@ -28,18 +28,15 @@ def get_exe_key(exe_name): class SubCommand: __name = None - __parser = None + _parser = None def __init__(self, name, **kwargs): self.__name = name - self.__parser = Parser().get_subparsers().add_parser(name, **kwargs) + self._parser = Parser().get_subparsers().add_parser(name, **kwargs) def __str__(self): return self.__name - def get_parser(self): - return self.__parser - def run(self): pass diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 530e77a..00b0554 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -38,7 +38,7 @@ class InitCommand(SubCommand): description="Create a new kiwi-config instance" ) - self.get_parser().add_argument( + self._parser.add_argument( '-f', '--force', action='store_true', help=f"use default values even if {KIWI_CONF_NAME} is present" diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index eea6985..7228cef 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,3 +1,5 @@ +import logging + from ..config import LoadedConfig from ..core import Parser @@ -8,46 +10,55 @@ class LogsCommand(SubCommand): def __init__(self): super().__init__( 'logs', - description="Show logs of a project or service" + description="Show logs of a project or service(s) of a project" ) - self.get_parser().add_argument( + self._parser.add_argument( '-f', '--follow', action='store_true', help="output appended data as log grows" ) - self.get_parser().add_argument( - 'project', - help="narf" + self._parser.add_argument( + 'project', type=str, + help="select a project in this instance" ) - self.get_parser().add_argument( - 'service', nargs='?', - help="narf" + self._parser.add_argument( + 'services', metavar='service', nargs='*', type=str, + help="select service(s) in a project" ) def run(self): config = LoadedConfig.get() + cli_args = Parser().get_args() - project_name = Parser().get_args().project + project_name = cli_args.project project_marker = config['markers:project'] project_dir = f'{project_name}{project_marker}' environment = { - 'KIWI_NET_NAME': config['network:name'], + 'KIWI_HUB_NAME': config['network:name'], 'COMPOSE_PROJECT_NAME': project_name } args = ['logs', '-t'] - if Parser().get_args().follow: + if cli_args.follow: args = [*args, '-f', '--tail=10'] - DockerCommand('docker-compose').run( - args, - cwd=project_dir, env=environment - ) - else: - DockerCommand('docker-compose').run_less( - args, - cwd=project_dir, env=environment - ) + if cli_args.services: + args = [*args, *cli_args.services] + + try: + if cli_args.follow: + DockerCommand('docker-compose').run( + args, + cwd=project_dir, env=environment + ) + else: + DockerCommand('docker-compose').run_less( + args, + cwd=project_dir, env=environment + ) + except KeyboardInterrupt: + logging.debug("Subprocess aborted.") + print() From 9da09d56cfb11fd536364f87c2ae9b9a91431dea Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 11 Aug 2020 14:03:00 +0200 Subject: [PATCH 038/118] Massive QoL reworks --- src/kiwi-config.py | 13 +----- src/kiwi/__init__.py | 16 +++++-- src/kiwi/_constants.py | 4 ++ src/kiwi/config.py | 13 +++--- src/kiwi/{core.py => parser.py} | 14 +++--- src/kiwi/runner.py | 18 +++++--- src/kiwi/subcommands/_utils.py | 8 ++-- src/kiwi/subcommands/init.py | 53 ++++++++++++---------- src/kiwi/subcommands/logs.py | 32 ++++++------- src/kiwi/subcommands/show.py | 5 +- src/{default.kiwi.yml => kiwi_default.yml} | 3 -- src/kiwi_header.yml | 3 ++ 12 files changed, 93 insertions(+), 89 deletions(-) create mode 100644 src/kiwi/_constants.py rename src/kiwi/{core.py => parser.py} (82%) rename src/{default.kiwi.yml => kiwi_default.yml} (63%) create mode 100644 src/kiwi_header.yml diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 87789e7..c399ab0 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -20,20 +20,11 @@ def set_verbosity(logger, handler, verbosity): def main(): - kiwi.Parser().get_parser().add_argument( - '-v', '--verbosity', - action='count', default=0 - ) - - args = kiwi.Parser().get_args() - log_handler = logging.StreamHandler() logging.getLogger().addHandler(log_handler) - set_verbosity(logging.getLogger(), log_handler, args.verbosity) + set_verbosity(logging.getLogger(), log_handler, kiwi.verbosity()) - logging.debug(f"main CLI args: {args}") - - kiwi.Runner().run(args.command) + kiwi.run() if __name__ == "__main__": diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py index f5decab..2f4a363 100644 --- a/src/kiwi/__init__.py +++ b/src/kiwi/__init__.py @@ -1,7 +1,17 @@ -from .core import Parser +from .parser import Parser from .runner import Runner + +def verbosity(): + _ = Runner() + return Parser().get_args().verbosity + + +def run(): + Runner().run() + + __all__ = [ - 'Parser', - 'Runner' + 'verbosity', + 'run' ] diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py new file mode 100644 index 0000000..e16de2c --- /dev/null +++ b/src/kiwi/_constants.py @@ -0,0 +1,4 @@ +import os + +KIWI_ROOT = os.getenv('KIWI_ROOT', ".") +KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 1f98ef1..7e467cf 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -3,12 +3,13 @@ import re import os import yaml -from .core import KIWI_ROOT, KIWI_CONF_NAME +from ._constants import KIWI_ROOT, KIWI_CONF_NAME ########### # CONSTANTS -DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/default.kiwi.yml" +HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_header.yml" +DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_default.yml" VERSION_TAG_NAME = f"{KIWI_ROOT}/version-tag" @@ -42,11 +43,9 @@ class Config: # insert newline before every main key yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) - # extract header comment from default config - with open(DEFAULT_KIWI_CONF_NAME, 'r') as stream: - yml_header = stream.read().strip() - yml_header = re.sub(r'^[^#].*', r'', yml_header, flags=re.MULTILINE).strip() - yml_string = "{}\n{}".format(yml_header, yml_string) + # load header comment from file + with open(HEADER_KIWI_CONF_NAME, 'r') as stream: + yml_string = stream.read() + yml_string return yml_string diff --git a/src/kiwi/core.py b/src/kiwi/parser.py similarity index 82% rename from src/kiwi/core.py rename to src/kiwi/parser.py index 9f06fa7..858fed9 100644 --- a/src/kiwi/core.py +++ b/src/kiwi/parser.py @@ -1,11 +1,4 @@ import argparse -import os - -########### -# CONSTANTS - -KIWI_ROOT = os.getenv('KIWI_ROOT', ".") -KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") class Parser: @@ -17,6 +10,11 @@ class Parser: def __init__(self): self.__parser = argparse.ArgumentParser(description='kiwi-config') + self.__parser.add_argument( + '-v', '--verbosity', + action='count', default=0 + ) + self.__subparsers = self.__parser.add_subparsers() self.__subparsers.required = True self.__subparsers.dest = 'command' @@ -40,4 +38,4 @@ class Parser: Parser.__instance = Parser.__Parser() def __getattr__(self, item): - return getattr(self.__instance, item) + return getattr(self.__instance, item) \ No newline at end of file diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index b65d4f7..75cc6bc 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -1,3 +1,7 @@ +import logging + +from .config import LoadedConfig +from .parser import Parser from .subcommands import * ########### @@ -18,10 +22,14 @@ class Runner: for cmd in SUBCOMMANDS: self.__commands.append(cmd()) - def run(self, command_name): + def run(self): + config = LoadedConfig.get() + args = Parser().get_args() + for cmd in self.__commands: - if str(cmd) == command_name: - cmd.run() + if str(cmd) == args.command: + logging.debug(f"Running '{cmd}' with args: {args}") + cmd.run(config, args) return True return False @@ -34,7 +42,3 @@ class Runner: def __getattr__(self, item): return getattr(self.__instance, item) - - -if __name__ == 'kiwi.runner': - _ = Runner() diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/_utils.py index 47458ab..6de617b 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/_utils.py @@ -2,7 +2,7 @@ import logging import os import subprocess -from ..core import Parser +from ..parser import Parser from ..config import LoadedConfig @@ -28,16 +28,16 @@ def get_exe_key(exe_name): class SubCommand: __name = None - _parser = None + _sub_parser = None def __init__(self, name, **kwargs): self.__name = name - self._parser = Parser().get_subparsers().add_parser(name, **kwargs) + self._sub_parser = Parser().get_subparsers().add_parser(name, **kwargs) def __str__(self): return self.__name - def run(self): + def run(self, config, args): pass diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 00b0554..bba50be 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -1,15 +1,19 @@ import logging import os -from ..core import KIWI_CONF_NAME, Parser -from ..config import DefaultConfig, LoadedConfig +from .._constants import KIWI_CONF_NAME +from ..config import DefaultConfig from ._utils import SubCommand, is_executable, find_exe_file, get_exe_key def user_input(config, key, prompt): # prompt user as per argument - result = input("{} [{}] ".format(prompt, config[key])).strip() + try: + result = input("{} [{}] ".format(prompt, config[key])).strip() + except EOFError: + print() + result = None # store result if present if result: @@ -38,38 +42,41 @@ class InitCommand(SubCommand): description="Create a new kiwi-config instance" ) - self._parser.add_argument( + self._sub_parser.add_argument( '-f', '--force', action='store_true', help=f"use default values even if {KIWI_CONF_NAME} is present" ) - def run(self): + def run(self, config, args): logging.info(f"Initializing kiwi-config instance in '{os.getcwd()}'") - if Parser().get_args().force and os.path.isfile(KIWI_CONF_NAME): + if args.force and os.path.isfile(KIWI_CONF_NAME): logging.warning(f"Overwriting existing '{KIWI_CONF_NAME}'!") config = DefaultConfig.get() - else: - config = LoadedConfig.get() - # version - user_input(config, 'version', "Enter kiwi-config version for this instance") + try: + # version + user_input(config, 'version', "Enter kiwi-config version for this instance") - # runtime - user_input(config, 'runtime:storage', "Enter local directory for service data") + # runtime + user_input(config, 'runtime:storage', "Enter local directory for service data") - # markers - user_input(config, 'markers:project', "Enter marker string for project directories") - user_input(config, 'markers:down', "Enter marker string for disabled projects") + # markers + user_input(config, 'markers:project', "Enter marker string for project directories") + user_input(config, 'markers:down', "Enter marker string for disabled projects") - # network - user_input(config, 'network:name', "Enter name for local docker network") - user_input(config, 'network:cidr', "Enter CIDR block for local docker network") + # network + user_input(config, 'network:name', "Enter name for local docker network") + user_input(config, 'network:cidr', "Enter CIDR block for local docker network") - # executables - user_input_exe(config, 'docker') - user_input_exe(config, 'docker-compose') - user_input_exe(config, 'sudo') + # executables + user_input_exe(config, 'docker') + user_input_exe(config, 'docker-compose') + user_input_exe(config, 'sudo') - config.save() + config.save() + + except KeyboardInterrupt: + print() + logging.warning(f"'{self}' aborted, input discarded.") diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 7228cef..5ae932a 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,8 +1,5 @@ import logging -from ..config import LoadedConfig -from ..core import Parser - from ._utils import SubCommand, DockerCommand @@ -13,26 +10,23 @@ class LogsCommand(SubCommand): description="Show logs of a project or service(s) of a project" ) - self._parser.add_argument( + self._sub_parser.add_argument( '-f', '--follow', action='store_true', help="output appended data as log grows" ) - self._parser.add_argument( + self._sub_parser.add_argument( 'project', type=str, help="select a project in this instance" ) - self._parser.add_argument( + self._sub_parser.add_argument( 'services', metavar='service', nargs='*', type=str, help="select service(s) in a project" ) - def run(self): - config = LoadedConfig.get() - cli_args = Parser().get_args() - - project_name = cli_args.project + def run(self, config, args): + project_name = args.project project_marker = config['markers:project'] project_dir = f'{project_name}{project_marker}' @@ -41,22 +35,22 @@ class LogsCommand(SubCommand): 'COMPOSE_PROJECT_NAME': project_name } - args = ['logs', '-t'] - if cli_args.follow: - args = [*args, '-f', '--tail=10'] + process_args = ['logs', '-t'] + if args.follow: + process_args = [*process_args, '-f', '--tail=10'] - if cli_args.services: - args = [*args, *cli_args.services] + if args.services: + process_args = [*process_args, *args.services] try: - if cli_args.follow: + if args.follow: DockerCommand('docker-compose').run( - args, + process_args, cwd=project_dir, env=environment ) else: DockerCommand('docker-compose').run_less( - args, + process_args, cwd=project_dir, env=environment ) except KeyboardInterrupt: diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 6785485..1deb830 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -1,5 +1,3 @@ -from ..config import LoadedConfig - from ._utils import SubCommand @@ -10,6 +8,5 @@ class ShowCommand(SubCommand): description="Show effective kiwi.yml" ) - def run(self): - config = LoadedConfig.get() + def run(self, config, args): print(config) diff --git a/src/default.kiwi.yml b/src/kiwi_default.yml similarity index 63% rename from src/default.kiwi.yml rename to src/kiwi_default.yml index b729709..9bfce99 100644 --- a/src/default.kiwi.yml +++ b/src/kiwi_default.yml @@ -1,6 +1,3 @@ -###################################### -# kiwi-config instance configuration # -###################################### version: runtime: storage: /var/kiwi diff --git a/src/kiwi_header.yml b/src/kiwi_header.yml new file mode 100644 index 0000000..95d3b8f --- /dev/null +++ b/src/kiwi_header.yml @@ -0,0 +1,3 @@ +###################################### +# kiwi-config instance configuration # +###################################### From a602076291572826154abd8cb38c30c71f522f90 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 11 Aug 2020 17:23:24 +0200 Subject: [PATCH 039/118] clone default config before loading --- src/kiwi/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 7e467cf..22cf12d 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -1,3 +1,4 @@ +import copy import logging import re import os @@ -56,6 +57,12 @@ class Config: except yaml.YAMLError as exc: logging.error(exc) + def clone(self): + result = Config() + result.__yml_content = copy.deepcopy(self.__yml_content) + + return result + def save(self): with open(KIWI_CONF_NAME, 'w') as stream: stream.write(str(self)) @@ -79,7 +86,7 @@ class DefaultConfig(Config): class LoadedConfig(Config): @classmethod def get(cls): - result = DefaultConfig.get() + result = DefaultConfig.get().clone() if os.path.isfile(KIWI_CONF_NAME): result._update_from_file(KIWI_CONF_NAME) From daced4dd7dcd951ec1dda4f09ac999524918e26e Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 12 Aug 2020 16:43:13 +0200 Subject: [PATCH 040/118] Package subcommands.utils --- src/kiwi/subcommands/_subcommand.py | 18 ++++++++ src/kiwi/subcommands/init.py | 3 +- src/kiwi/subcommands/logs.py | 3 +- src/kiwi/subcommands/show.py | 2 +- src/kiwi/subcommands/utils/__init__.py | 1 + .../{_utils.py => utils/dockercommand.py} | 42 ++----------------- src/kiwi/subcommands/utils/executable.py | 21 ++++++++++ 7 files changed, 48 insertions(+), 42 deletions(-) create mode 100644 src/kiwi/subcommands/_subcommand.py create mode 100644 src/kiwi/subcommands/utils/__init__.py rename src/kiwi/subcommands/{_utils.py => utils/dockercommand.py} (68%) create mode 100644 src/kiwi/subcommands/utils/executable.py diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py new file mode 100644 index 0000000..d6c890d --- /dev/null +++ b/src/kiwi/subcommands/_subcommand.py @@ -0,0 +1,18 @@ +from ..parser import Parser + + +class SubCommand: + __name = None + _sub_parser = None + + def __init__(self, name, **kwargs): + self.__name = name + self._sub_parser = Parser().get_subparsers().add_parser(name, **kwargs) + + def __str__(self): + return self.__name + + def run(self, config, args): + pass + + diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index bba50be..c2e5469 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -4,7 +4,8 @@ import os from .._constants import KIWI_CONF_NAME from ..config import DefaultConfig -from ._utils import SubCommand, is_executable, find_exe_file, get_exe_key +from ._subcommand import SubCommand +from .utils.executable import get_exe_key, is_executable, find_exe_file def user_input(config, key, prompt): diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 5ae932a..5755611 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,6 +1,7 @@ import logging -from ._utils import SubCommand, DockerCommand +from ._subcommand import SubCommand +from .utils.dockercommand import DockerCommand class LogsCommand(SubCommand): diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 1deb830..3cb9069 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -1,4 +1,4 @@ -from ._utils import SubCommand +from ._subcommand import SubCommand class ShowCommand(SubCommand): diff --git a/src/kiwi/subcommands/utils/__init__.py b/src/kiwi/subcommands/utils/__init__.py new file mode 100644 index 0000000..5a65bd6 --- /dev/null +++ b/src/kiwi/subcommands/utils/__init__.py @@ -0,0 +1 @@ +# git keep diff --git a/src/kiwi/subcommands/_utils.py b/src/kiwi/subcommands/utils/dockercommand.py similarity index 68% rename from src/kiwi/subcommands/_utils.py rename to src/kiwi/subcommands/utils/dockercommand.py index 6de617b..725ac5d 100644 --- a/src/kiwi/subcommands/_utils.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -1,44 +1,8 @@ import logging -import os import subprocess -from ..parser import Parser -from ..config import LoadedConfig - - -def is_executable(filename): - if filename is None: - return False - - return os.path.isfile(filename) and os.access(filename, os.X_OK) - - -def find_exe_file(exe_name): - for path in os.environ['PATH'].split(os.pathsep): - exe_file = os.path.join(path, exe_name) - if is_executable(exe_file): - return exe_file - - return None - - -def get_exe_key(exe_name): - return f'executables:{exe_name}' - - -class SubCommand: - __name = None - _sub_parser = None - - def __init__(self, name, **kwargs): - self.__name = name - self._sub_parser = Parser().get_subparsers().add_parser(name, **kwargs) - - def __str__(self): - return self.__name - - def run(self, config, args): - pass +from ...config import LoadedConfig +from .executable import get_exe_key class DockerCommand: @@ -100,4 +64,4 @@ class DockerCommand: DockerCommand.__instances[exe_name] = DockerCommand.__DockerCommand(exe_name) def __getattr__(self, item): - return getattr(self.__instances[self.__exe_name], item) + return getattr(self.__instances[self.__exe_name], item) \ No newline at end of file diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py new file mode 100644 index 0000000..58e2d1e --- /dev/null +++ b/src/kiwi/subcommands/utils/executable.py @@ -0,0 +1,21 @@ +import os + + +def get_exe_key(exe_name): + return f'executables:{exe_name}' + + +def is_executable(filename): + if filename is None: + return False + + return os.path.isfile(filename) and os.access(filename, os.X_OK) + + +def find_exe_file(exe_name): + for path in os.environ['PATH'].split(os.pathsep): + exe_file = os.path.join(path, exe_name) + if is_executable(exe_file): + return exe_file + + return None \ No newline at end of file From 8c2cdbda3a4533d158c9dfa331678af232aa1d3f Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 12 Aug 2020 17:18:20 +0200 Subject: [PATCH 041/118] utils.executable based on PATH instead of YML --- example/kiwi.yml | 5 -- src/kiwi/subcommands/init.py | 21 ------- src/kiwi/subcommands/utils/dockercommand.py | 69 ++++++--------------- src/kiwi/subcommands/utils/executable.py | 48 ++++++++++++-- src/kiwi_default.yml | 4 -- 5 files changed, 62 insertions(+), 85 deletions(-) diff --git a/example/kiwi.yml b/example/kiwi.yml index e83688b..139c3c4 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -15,8 +15,3 @@ markers: network: name: kiwihub cidr: 10.22.46.0/24 - -executables: - docker: /usr/bin/docker - docker-compose: /usr/local/bin/docker-compose - sudo: /usr/bin/sudo diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index c2e5469..12cb9f8 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -5,7 +5,6 @@ from .._constants import KIWI_CONF_NAME from ..config import DefaultConfig from ._subcommand import SubCommand -from .utils.executable import get_exe_key, is_executable, find_exe_file def user_input(config, key, prompt): @@ -21,21 +20,6 @@ def user_input(config, key, prompt): config[key] = result -def user_input_exe(config, exe_name): - key = get_exe_key(exe_name) - exe_file = config[key] - - if not is_executable(exe_file): - logging.info(f"Reconfiguring '{exe_name}' executable path.") - exe_file = find_exe_file(exe_name) - - if exe_file is not None: - logging.debug(f"Found executable at '{exe_file}'.") - config[key] = exe_file - else: - user_input(config, key, f"Enter path to '{exe_name}' executable") - - class InitCommand(SubCommand): def __init__(self): super().__init__( @@ -71,11 +55,6 @@ class InitCommand(SubCommand): user_input(config, 'network:name', "Enter name for local docker network") user_input(config, 'network:cidr', "Enter CIDR block for local docker network") - # executables - user_input_exe(config, 'docker') - user_input_exe(config, 'docker-compose') - user_input_exe(config, 'sudo') - config.save() except KeyboardInterrupt: diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 725ac5d..c063930 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -1,67 +1,36 @@ -import logging import subprocess -from ...config import LoadedConfig -from .executable import get_exe_key +from .executable import Executable class DockerCommand: - class __DockerCommand: - __cmd = [] - - def __init__(self, exe_name): - config = LoadedConfig.get() - self.__cmd = [config[get_exe_key(exe_name)]] - - if DockerCommand.__requires_root: - self.__cmd = [config[get_exe_key("sudo")], *self.__cmd] - - def __build_cmd(self, args, **kwargs): - cmd = [*self.__cmd, *args] - logging.debug(f"DockerProgram cmd{cmd}, kwargs{kwargs}") - return cmd - - def run(self, args, **kwargs): - return subprocess.run( - self.__build_cmd(args, **kwargs), - **kwargs - ) - - def run_less(self, args, **kwargs): - process = subprocess.Popen( - self.__build_cmd(args, **kwargs), - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - **kwargs - ) - - less_process = subprocess.run( - ['less', '-R', '+G'], - stdin=process.stdout - ) - - process.communicate() - return less_process - - __exe_name = None - __instances = {} __requires_root = None + __exe = None def __init__(self, exe_name): if DockerCommand.__requires_root is None: try: - config = LoadedConfig.get() - subprocess.run( - [config[get_exe_key('docker')], 'ps'], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + Executable('docker').run( + ['ps'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) DockerCommand.__requires_root = False except subprocess.CalledProcessError: DockerCommand.__requires_root = True - self.__exe_name = exe_name - - if exe_name not in DockerCommand.__instances: - DockerCommand.__instances[exe_name] = DockerCommand.__DockerCommand(exe_name) + self.__exe = Executable(exe_name, DockerCommand.__requires_root) def __getattr__(self, item): - return getattr(self.__instances[self.__exe_name], item) \ No newline at end of file + return getattr(self.__exe, item) + + def run_less(self, args, **kwargs): + process = self.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + **kwargs + ) + + less_process = Executable('less').run( + ['-R', '+G'], stdin=process.stdout + ) + + process.communicate() + return less_process diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py index 58e2d1e..8d793d9 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/subcommands/utils/executable.py @@ -1,8 +1,6 @@ import os - - -def get_exe_key(exe_name): - return f'executables:{exe_name}' +import logging +import subprocess def is_executable(filename): @@ -18,4 +16,44 @@ def find_exe_file(exe_name): if is_executable(exe_file): return exe_file - return None \ No newline at end of file + raise FileNotFoundError(f"Executable '{exe_name}' not found in $PATH!") + + +class Executable: + __exe_name = None + __instances = {} + + class __Executable: + __cmd = None + + def __init__(self, exe_name, requires_root=False): + self.__cmd = [find_exe_file(exe_name)] + + if requires_root: + self.__cmd = [find_exe_file("sudo"), *self.__cmd] + + def __build_cmd(self, args, **kwargs): + cmd = [*self.__cmd, *args] + logging.debug(f"Executable cmd{cmd}, kwargs{kwargs}") + return cmd + + def run(self, args, **kwargs): + return subprocess.run( + self.__build_cmd(args, **kwargs), + **kwargs + ) + + def Popen(self, args, **kwargs): + return subprocess.Popen( + self.__build_cmd(args, **kwargs), + **kwargs + ) + + def __init__(self, exe_name, requires_root=False): + self.__exe_name = exe_name + + if exe_name not in Executable.__instances: + Executable.__instances[exe_name] = Executable.__Executable(exe_name, requires_root) + + def __getattr__(self, item): + return getattr(self.__instances[self.__exe_name], item) diff --git a/src/kiwi_default.yml b/src/kiwi_default.yml index 9bfce99..0174e5f 100644 --- a/src/kiwi_default.yml +++ b/src/kiwi_default.yml @@ -8,7 +8,3 @@ markers: network: name: kiwihub cidr: 10.22.46.0/24 -executables: - docker: null - docker-compose: null - sudo: null From 70b04f039a637bb30a9b8b3d566e941372007687 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 12 Aug 2020 17:39:55 +0200 Subject: [PATCH 042/118] DockerCommand now is an Executable --- src/kiwi/subcommands/utils/dockercommand.py | 24 ++++++++++++-------- src/kiwi/subcommands/utils/executable.py | 25 +++++++++++---------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index c063930..0a52b4e 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -3,33 +3,39 @@ import subprocess from .executable import Executable -class DockerCommand: +class DockerCommand(Executable): __requires_root = None - __exe = None def __init__(self, exe_name): + super().__init__(exe_name) + if DockerCommand.__requires_root is None: try: Executable('docker').run( - ['ps'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ['ps'], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) DockerCommand.__requires_root = False except subprocess.CalledProcessError: DockerCommand.__requires_root = True - self.__exe = Executable(exe_name, DockerCommand.__requires_root) - - def __getattr__(self, item): - return getattr(self.__exe, item) + def run(self, args, **kwargs): + # equivalent to 'super().run' but agnostic of nested class construct + super().__getattr__("run")( + args, DockerCommand.__requires_root, + **kwargs + ) def run_less(self, args, **kwargs): process = self.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + args, DockerCommand.__requires_root, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, **kwargs ) less_process = Executable('less').run( - ['-R', '+G'], stdin=process.stdout + ['-R', '+G'], + stdin=process.stdout ) process.communicate() diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py index 8d793d9..f71802b 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/subcommands/utils/executable.py @@ -20,35 +20,36 @@ def find_exe_file(exe_name): class Executable: - __exe_name = None - __instances = {} - class __Executable: - __cmd = None + __exe_path = None def __init__(self, exe_name, requires_root=False): - self.__cmd = [find_exe_file(exe_name)] + self.__exe_path = find_exe_file(exe_name) + + def __build_cmd(self, args, requires_root=False, **kwargs): + cmd = [self.__exe_path, *args] if requires_root: - self.__cmd = [find_exe_file("sudo"), *self.__cmd] + self.__exe_path = [find_exe_file("sudo"), self.__exe_path] - def __build_cmd(self, args, **kwargs): - cmd = [*self.__cmd, *args] logging.debug(f"Executable cmd{cmd}, kwargs{kwargs}") return cmd - def run(self, args, **kwargs): + def run(self, args, requires_root=False, **kwargs): return subprocess.run( - self.__build_cmd(args, **kwargs), + self.__build_cmd(args, requires_root, **kwargs), **kwargs ) - def Popen(self, args, **kwargs): + def Popen(self, args, requires_root=False, **kwargs): return subprocess.Popen( - self.__build_cmd(args, **kwargs), + self.__build_cmd(args, requires_root, **kwargs), **kwargs ) + __exe_name = None + __instances = {} + def __init__(self, exe_name, requires_root=False): self.__exe_name = exe_name From 1bd547587bc9e8bcb1d090b1a11407a40faae7ea Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 12 Aug 2020 17:46:50 +0200 Subject: [PATCH 043/118] organized imports --- src/kiwi-config.py | 3 +++ src/kiwi/__init__.py | 1 + src/kiwi/_constants.py | 1 + src/kiwi/config.py | 4 +++- src/kiwi/parser.py | 1 + src/kiwi/runner.py | 2 ++ src/kiwi/subcommands/__init__.py | 7 ++++--- src/kiwi/subcommands/_subcommand.py | 3 +-- src/kiwi/subcommands/init.py | 6 +++++- src/kiwi/subcommands/logs.py | 2 ++ src/kiwi/subcommands/show.py | 1 + src/kiwi/subcommands/utils/dockercommand.py | 2 ++ src/kiwi/subcommands/utils/executable.py | 3 ++- 13 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/kiwi-config.py b/src/kiwi-config.py index c399ab0..0b47176 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 + +# system import logging +# local import kiwi diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py index 2f4a363..f53a9c1 100644 --- a/src/kiwi/__init__.py +++ b/src/kiwi/__init__.py @@ -1,3 +1,4 @@ +# local from .parser import Parser from .runner import Runner diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py index e16de2c..8524233 100644 --- a/src/kiwi/_constants.py +++ b/src/kiwi/_constants.py @@ -1,3 +1,4 @@ +# system import os KIWI_ROOT = os.getenv('KIWI_ROOT', ".") diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 22cf12d..c238766 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -1,9 +1,11 @@ +# system import copy import logging -import re import os +import re import yaml +# local from ._constants import KIWI_ROOT, KIWI_CONF_NAME ########### diff --git a/src/kiwi/parser.py b/src/kiwi/parser.py index 858fed9..497e520 100644 --- a/src/kiwi/parser.py +++ b/src/kiwi/parser.py @@ -1,3 +1,4 @@ +# system import argparse diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 75cc6bc..37b08da 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -1,5 +1,7 @@ +# system import logging +# local from .config import LoadedConfig from .parser import Parser from .subcommands import * diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 5219579..63c7d4c 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,9 +1,10 @@ +# local from .init import InitCommand -from .show import ShowCommand from .logs import LogsCommand +from .show import ShowCommand __all__ = [ 'InitCommand', - 'ShowCommand', - 'LogsCommand' + 'LogsCommand', + 'ShowCommand' ] diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index d6c890d..f8d6dd7 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -1,3 +1,4 @@ +# parent from ..parser import Parser @@ -14,5 +15,3 @@ class SubCommand: def run(self, config, args): pass - - diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 12cb9f8..d1545c5 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -1,9 +1,11 @@ +# system import logging import os +# parent (display purposes only) from .._constants import KIWI_CONF_NAME -from ..config import DefaultConfig +# local from ._subcommand import SubCommand @@ -37,6 +39,8 @@ class InitCommand(SubCommand): logging.info(f"Initializing kiwi-config instance in '{os.getcwd()}'") if args.force and os.path.isfile(KIWI_CONF_NAME): + from ..config import DefaultConfig + logging.warning(f"Overwriting existing '{KIWI_CONF_NAME}'!") config = DefaultConfig.get() diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 5755611..1928233 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,5 +1,7 @@ +# system import logging +# local from ._subcommand import SubCommand from .utils.dockercommand import DockerCommand diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 3cb9069..672df0a 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -1,3 +1,4 @@ +# local from ._subcommand import SubCommand diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 0a52b4e..5d5a28d 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -1,5 +1,7 @@ +# system import subprocess +# local from .executable import Executable diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py index f71802b..2c1c56f 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/subcommands/utils/executable.py @@ -1,5 +1,6 @@ -import os +# system import logging +import os import subprocess From 0b2b9a0a75ca151277c0adf3441f0ecb4d7a59b4 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 13 Aug 2020 10:48:01 +0200 Subject: [PATCH 044/118] doc --- src/kiwi-config.py | 4 ++ src/kiwi/__init__.py | 2 + src/kiwi/_constants.py | 2 + src/kiwi/config.py | 68 +++++++++++++++++++++++------ src/kiwi/parser.py | 20 ++++++--- src/kiwi/runner.py | 18 +++++++- src/kiwi/subcommands/_subcommand.py | 10 ++++- src/kiwi/subcommands/init.py | 7 ++- 8 files changed, 108 insertions(+), 23 deletions(-) diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 0b47176..4c5acad 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -8,6 +8,8 @@ import kiwi def set_verbosity(logger, handler, verbosity): + """set logging default verbosity level and format""" + if verbosity >= 2: log_level = logging.DEBUG log_format = "[%(asctime)s] %(levelname)s @ %(filename)s:%(funcName)s:%(lineno)d: %(message)s" @@ -23,10 +25,12 @@ def set_verbosity(logger, handler, verbosity): def main(): + # add a new handler (needed to set the level) log_handler = logging.StreamHandler() logging.getLogger().addHandler(log_handler) set_verbosity(logging.getLogger(), log_handler, kiwi.verbosity()) + # run the app kiwi.run() diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py index f53a9c1..8a4a6db 100644 --- a/src/kiwi/__init__.py +++ b/src/kiwi/__init__.py @@ -4,11 +4,13 @@ from .runner import Runner def verbosity(): + # ensure singleton is instantiated: runs subcommand setup routines _ = Runner() return Parser().get_args().verbosity def run(): + # pass down Runner().run() diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py index 8524233..836bedc 100644 --- a/src/kiwi/_constants.py +++ b/src/kiwi/_constants.py @@ -1,5 +1,7 @@ # system import os +# location of "src" directory to use KIWI_ROOT = os.getenv('KIWI_ROOT', ".") +# default name of kiwi-config file KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") diff --git a/src/kiwi/config.py b/src/kiwi/config.py index c238766..dcbe94a 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -11,15 +11,25 @@ from ._constants import KIWI_ROOT, KIWI_CONF_NAME ########### # CONSTANTS +# text files inside kiwi-config "src" directory HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_header.yml" DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_default.yml" VERSION_TAG_NAME = f"{KIWI_ROOT}/version-tag" class Config: + """represents a kiwi.yml""" + __yml_content = {} def __key_resolve(self, key): + """ + Resolve nested dictionaries + + If __yml_content is {'a': {'b': {'c': "val"}}} and key is 'a:b:c', + this returns a single dict {'c': "val"} and the direct key 'c' + """ + # "a:b:c" => path = ['a', 'b'], key = 'c' path = key.split(':') path, key = path[:-1], path[-1] @@ -32,14 +42,20 @@ class Config: return container, key def __getitem__(self, key): + """array-like read access to __yml_content""" + container, key = self.__key_resolve(key) return container[key] def __setitem__(self, key, value): + """array-like write access to __yml_content""" + container, key = self.__key_resolve(key) container[key] = value def __str__(self): + """dump into textual representation""" + # dump yml content yml_string = yaml.dump(self.__yml_content, default_flow_style=False, sort_keys=False) @@ -53,44 +69,68 @@ class Config: return yml_string def _update_from_file(self, filename): + """return a copy updated using a kiwi.yml file""" + with open(filename, 'r') as stream: try: - self.__yml_content.update(yaml.safe_load(stream)) + # create copy + result = Config() + result.__yml_content = copy.deepcopy(self.__yml_content) + + # read file + logging.debug(f"Reading '{filename}' into '{id(result.__yml_content)}'") + result.__yml_content.update(yaml.safe_load(stream)) + + return result except yaml.YAMLError as exc: logging.error(exc) - def clone(self): - result = Config() - result.__yml_content = copy.deepcopy(self.__yml_content) - - return result - def save(self): + """save current yml representation in current directory's kiwi.yml""" + with open(KIWI_CONF_NAME, 'w') as stream: stream.write(str(self)) class DefaultConfig(Config): + """Singleton: The default kiwi.yml file""" + __instance = None @classmethod def get(cls): if cls.__instance is None: - cls.__instance = cls() - cls.__instance._update_from_file(DEFAULT_KIWI_CONF_NAME) + # create singleton + cls.__instance = cls()._update_from_file(DEFAULT_KIWI_CONF_NAME) + # add version data from separate file (keeps default config cleaner) with open(VERSION_TAG_NAME, 'r') as stream: - cls.__instance["version"] = stream.read().strip() + cls.__instance['version'] = stream.read().strip() + # return singleton return cls.__instance class LoadedConfig(Config): + """Singleton collection: kiwi.yml files by path""" + + __instances = {} + @classmethod def get(cls): - result = DefaultConfig.get().clone() + cwd = os.getcwd() - if os.path.isfile(KIWI_CONF_NAME): - result._update_from_file(KIWI_CONF_NAME) + if cwd not in LoadedConfig.__instances: + # create singleton for new path + result = DefaultConfig.get() - return result + # update with current dir's kiwi.yml + try: + result = result._update_from_file(KIWI_CONF_NAME) + except FileNotFoundError: + logging.info(f"No '{KIWI_CONF_NAME}' found at '{cwd}'. Using defaults.") + + LoadedConfig.__instances[cwd] = result + + # return singleton + return LoadedConfig.__instances[cwd] diff --git a/src/kiwi/parser.py b/src/kiwi/parser.py index 497e520..2814c4c 100644 --- a/src/kiwi/parser.py +++ b/src/kiwi/parser.py @@ -3,31 +3,39 @@ import argparse class Parser: + """Singleton: Main CLI arguments parser""" + class __Parser: + """Singleton type""" + + # argparse objects __parser = None __subparsers = None __args = None def __init__(self): - self.__parser = argparse.ArgumentParser(description='kiwi-config') + # create main parsers + self.__parser = argparse.ArgumentParser( + description='kiwi-config' + ) + # main arguments self.__parser.add_argument( '-v', '--verbosity', action='count', default=0 ) + # attach subparsers self.__subparsers = self.__parser.add_subparsers() self.__subparsers.required = True self.__subparsers.dest = 'command' - def get_parser(self): - return self.__parser - def get_subparsers(self): return self.__subparsers def get_args(self): if self.__args is None: + # parse args if needed self.__args = self.__parser.parse_args() return self.__args @@ -36,7 +44,9 @@ class Parser: def __init__(self): if Parser.__instance is None: + # create singleton Parser.__instance = Parser.__Parser() def __getattr__(self, item): - return getattr(self.__instance, item) \ No newline at end of file + """Inner singleton direct access""" + return getattr(self.__instance, item) diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 37b08da..1561c30 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -9,6 +9,7 @@ from .subcommands import * ########### # CONSTANTS +# all available subcommands SUBCOMMANDS = [ InitCommand, ShowCommand, @@ -17,30 +18,43 @@ SUBCOMMANDS = [ class Runner: + """Singleton: Subcommands setup and run""" + class __Runner: + """Singleton type""" + + __parser = None __commands = [] def __init__(self): + # setup all subcommands for cmd in SUBCOMMANDS: self.__commands.append(cmd()) def run(self): - config = LoadedConfig.get() + """run the desired subcommand""" + args = Parser().get_args() for cmd in self.__commands: if str(cmd) == args.command: + # command found logging.debug(f"Running '{cmd}' with args: {args}") - cmd.run(config, args) + cmd.run(LoadedConfig.get(), args) return True + # command not found + logging.error(f"kiwi command '{args.command}' unknown") return False __instance = None def __init__(self): if Runner.__instance is None: + # create singleton Runner.__instance = Runner.__Runner() def __getattr__(self, item): + """Inner singleton direct access""" + return getattr(self.__instance, item) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index f8d6dd7..ef3833f 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -3,15 +3,23 @@ from ..parser import Parser class SubCommand: + """represents kiwi [anything] command""" + + # actual command string __name = None + # command parser _sub_parser = None def __init__(self, name, **kwargs): self.__name = name - self._sub_parser = Parser().get_subparsers().add_parser(name, **kwargs) + self._sub_parser = Parser().get_subparsers().add_parser( + name, + **kwargs + ) def __str__(self): return self.__name def run(self, config, args): + """actually run command with this dir's config and parsed CLI args""" pass diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index d1545c5..8d2c2b1 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -10,6 +10,8 @@ from ._subcommand import SubCommand def user_input(config, key, prompt): + """query user for new config value""" + # prompt user as per argument try: result = input("{} [{}] ".format(prompt, config[key])).strip() @@ -23,12 +25,15 @@ def user_input(config, key, prompt): class InitCommand(SubCommand): + """kiwi init""" + def __init__(self): super().__init__( 'init', description="Create a new kiwi-config instance" ) + # -f switch: Initialize with default config self._sub_parser.add_argument( '-f', '--force', action='store_true', @@ -36,7 +41,7 @@ class InitCommand(SubCommand): ) def run(self, config, args): - logging.info(f"Initializing kiwi-config instance in '{os.getcwd()}'") + logging.info(f"Initializing '{KIWI_CONF_NAME}' in '{os.getcwd()}'") if args.force and os.path.isfile(KIWI_CONF_NAME): from ..config import DefaultConfig From d67b04897f6ef173fd564ba91ecad9d1f88d9a8b Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 13 Aug 2020 11:31:41 +0200 Subject: [PATCH 045/118] "TARGETROOT" -> "TARGET_ROOT", "kiwihub" -> "kiwi_hub" --- Makefile | 22 +++++++++++----------- example/base.conf | 4 ++-- example/kiwi.yml | 2 +- src/kiwi_default.yml | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 388f4c4..449d921 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ $(error KIWI_HUB_CIDR not set in $(CONF_WILDC)) endif # persistent data directory -CONF_TARGETROOT:=$(call confvalue,TARGETROOT) -ifeq ($(CONF_TARGETROOT),) -$(error TARGETROOT not set in $(CONF_WILDC)) +CONF_TARGET_ROOT:=$(call confvalue,TARGET_ROOT) +ifeq ($(CONF_TARGET_ROOT),) +$(error TARGET_ROOT not set in $(CONF_WILDC)) endif # suffix for project directories @@ -56,7 +56,7 @@ endif # CONSTANTS # file to store docker network cidr -KIWI_HUB_FILE:=$(CONF_TARGETROOT)/up-$(CONF_KIWI_HUB_NAME) +KIWI_HUB_FILE:=$(CONF_TARGET_ROOT)/up-$(CONF_KIWI_HUB_NAME) # remove any suffix $2 from $1 rmsuffix=$(patsubst %$2,%,$1) @@ -78,8 +78,8 @@ kiwicompose=$(call docker_bash,\ cd "$<"; \ $(CONF_SOURCE) \ COMPOSE_PROJECT_NAME="$(call projname,$<)" \ - CONFDIR="$(CONF_TARGETROOT)/conf" \ - TARGETDIR="$(CONF_TARGETROOT)/$<" \ + CONFDIR="$(CONF_TARGET_ROOT)/conf" \ + TARGETDIR="$(CONF_TARGET_ROOT)/$<" \ $(DOCKER_COMPOSE) $(1)) ######### @@ -101,7 +101,7 @@ $(KIWI_HUB_FILE): @$(DOCKER) run --rm \ -v "/:/mnt" -u root alpine:latest \ ash -c '\ - mkdir -p "$(addprefix /mnt, $(CONF_TARGETROOT))"; \ + mkdir -p "$(addprefix /mnt, $(CONF_TARGET_ROOT))"; \ echo "$(CONF_KIWI_HUB_CIDR)" > "$(addprefix /mnt, $(KIWI_HUB_FILE))"; \ ' @@ -136,12 +136,12 @@ ifneq ($(wildcard *${PROJ_SUFFX}/conf),) $(DOCKER) build -t ldericher/kiwi-config:rsync . -f- &> /dev/null $(eval sources:=$(wildcard *${PROJ_SUFFX}/conf)) - @echo "Syncing $(sources) to $(CONF_TARGETROOT) ..." + @echo "Syncing $(sources) to $(CONF_TARGET_ROOT) ..." $(eval sources:=$(realpath $(sources))) $(eval sources:=$(addprefix /mnt, $(sources))) $(eval sources:=$(patsubst %,'%',$(sources))) - $(eval dest:='$(addprefix /mnt, $(CONF_TARGETROOT))') + $(eval dest:='$(addprefix /mnt, $(CONF_TARGET_ROOT))') @$(DOCKER) run --rm \ -v "/:/mnt" -u root ldericher/kiwi-config:rsync \ @@ -152,11 +152,11 @@ endif .PHONY: purge-conf purge-conf: - @echo "Emptying $(CONF_TARGETROOT)/conf ..." + @echo "Emptying $(CONF_TARGET_ROOT)/conf ..." @$(DOCKER) run --rm \ -v "/:/mnt" -u root alpine:latest \ ash -c '\ - rm -rf "$(addprefix /mnt, $(CONF_TARGETROOT)/conf)"; \ + rm -rf "$(addprefix /mnt, $(CONF_TARGET_ROOT)/conf)"; \ ' ######### diff --git a/example/base.conf b/example/base.conf index 80f2b32..3b2f803 100644 --- a/example/base.conf +++ b/example/base.conf @@ -1,7 +1,7 @@ export SUFFIX_PROJECT=.project export SUFFIX_DOWN=.down -export KIWI_HUB_NAME=kiwihub +export KIWI_HUB_NAME=kiwi_hub export KIWI_HUB_CIDR=10.22.46.0/24 -export TARGETROOT=/tmp/kiwi +export TARGET_ROOT=/tmp/kiwi diff --git a/example/kiwi.yml b/example/kiwi.yml index 139c3c4..a78bc97 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -13,5 +13,5 @@ markers: down: .down network: - name: kiwihub + name: kiwi_hub cidr: 10.22.46.0/24 diff --git a/src/kiwi_default.yml b/src/kiwi_default.yml index 0174e5f..2e5cd0c 100644 --- a/src/kiwi_default.yml +++ b/src/kiwi_default.yml @@ -6,5 +6,5 @@ markers: project: .project down: .down network: - name: kiwihub + name: kiwi_hub cidr: 10.22.46.0/24 From 082c5f220cb09dc0ded9fc0218e18b174d5a04dc Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 13 Aug 2020 11:32:54 +0200 Subject: [PATCH 046/118] ProjectCommand, ServiceCommand aux classes --- src/kiwi/subcommands/_subcommand.py | 30 +++++++++++++++++++++++++++++ src/kiwi/subcommands/logs.py | 15 +++------------ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index ef3833f..cd84a26 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -23,3 +23,33 @@ class SubCommand: def run(self, config, args): """actually run command with this dir's config and parsed CLI args""" pass + + +class ProjectCommand(SubCommand): + """this command concerns a project in current instance""" + + def __init__(self, name, **kwargs): + super().__init__( + name, + **kwargs + ) + + self._sub_parser.add_argument( + 'project', type=str, + help="select a project in this instance" + ) + + +class ServiceCommand(ProjectCommand): + """this command concerns services in a project""" + + def __init__(self, name, **kwargs): + super().__init__( + name, + **kwargs + ) + + self._sub_parser.add_argument( + 'services', metavar='service', nargs='*', type=str, + help="select service(s) in a project" + ) diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 1928233..abcf354 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -2,32 +2,23 @@ import logging # local -from ._subcommand import SubCommand +from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand -class LogsCommand(SubCommand): +class LogsCommand(ServiceCommand): def __init__(self): super().__init__( 'logs', description="Show logs of a project or service(s) of a project" ) + # -f switch: Follow logs self._sub_parser.add_argument( '-f', '--follow', action='store_true', help="output appended data as log grows" ) - self._sub_parser.add_argument( - 'project', type=str, - help="select a project in this instance" - ) - - self._sub_parser.add_argument( - 'services', metavar='service', nargs='*', type=str, - help="select service(s) in a project" - ) - def run(self, config, args): project_name = args.project project_marker = config['markers:project'] From feda828c06dfd2fe1bf62df20b5d792f56ce1685 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 13 Aug 2020 11:34:30 +0200 Subject: [PATCH 047/118] DockerCommand is now project-sensitive --- src/kiwi/subcommands/logs.py | 26 +++-------- src/kiwi/subcommands/utils/dockercommand.py | 48 +++++++++++++++------ src/kiwi/subcommands/utils/executable.py | 25 +++++++++-- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index abcf354..c38a635 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -20,33 +20,19 @@ class LogsCommand(ServiceCommand): ) def run(self, config, args): - project_name = args.project - project_marker = config['markers:project'] - project_dir = f'{project_name}{project_marker}' - - environment = { - 'KIWI_HUB_NAME': config['network:name'], - 'COMPOSE_PROJECT_NAME': project_name - } - - process_args = ['logs', '-t'] + compose_args = ['logs', '-t'] if args.follow: - process_args = [*process_args, '-f', '--tail=10'] + compose_args = [*compose_args, '-f', '--tail=10'] if args.services: - process_args = [*process_args, *args.services] + compose_args = [*compose_args, *args.services] try: if args.follow: - DockerCommand('docker-compose').run( - process_args, - cwd=project_dir, env=environment - ) + DockerCommand('docker-compose').run(config, args, compose_args) else: - DockerCommand('docker-compose').run_less( - process_args, - cwd=project_dir, env=environment - ) + DockerCommand('docker-compose').run_less(config, args, compose_args) + except KeyboardInterrupt: logging.debug("Subprocess aborted.") print() diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 5d5a28d..00a2f46 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -1,10 +1,35 @@ # system +import logging +import os import subprocess # local from .executable import Executable +def _update_kwargs(config, args, **kwargs): + project_name = args.project + project_marker = config['markers:project'] + project_dir = f'{project_name}{project_marker}' + kwargs['cwd'] = project_dir + + if 'env' not in kwargs: + kwargs['env'] = {} + + if config['runtime:env'] is not None: + kwargs['env'].update(config['runtime:env']) + + kwargs['env'].update({ + 'KIWI_HUB_NAME': config['network:name'], + 'COMPOSE_PROJECT_NAME': project_name, + 'CONFDIR': os.path.join(config['runtime:storage'], 'conf'), + 'TARGETDIR': os.path.join(config['runtime:storage'], project_dir) + }) + + logging.debug(f"kwargs updated: {kwargs}") + return kwargs + + class DockerCommand(Executable): __requires_root = None @@ -21,24 +46,19 @@ class DockerCommand(Executable): except subprocess.CalledProcessError: DockerCommand.__requires_root = True - def run(self, args, **kwargs): + def run(self, config, args, process_args, **kwargs): + kwargs = _update_kwargs(config, args, **kwargs) + # equivalent to 'super().run' but agnostic of nested class construct super().__getattr__("run")( - args, DockerCommand.__requires_root, + process_args, DockerCommand.__requires_root, **kwargs ) - def run_less(self, args, **kwargs): - process = self.Popen( - args, DockerCommand.__requires_root, - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + def run_less(self, config, args, process_args, **kwargs): + kwargs = _update_kwargs(config, args, **kwargs) + + super().__getattr__("run_less")( + process_args, DockerCommand.__requires_root, **kwargs ) - - less_process = Executable('less').run( - ['-R', '+G'], - stdin=process.stdout - ) - - process.communicate() - return less_process diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py index 2c1c56f..7ec5f65 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/subcommands/utils/executable.py @@ -36,18 +36,35 @@ class Executable: logging.debug(f"Executable cmd{cmd}, kwargs{kwargs}") return cmd - def run(self, args, requires_root=False, **kwargs): + def run(self, process_args, requires_root=False, **kwargs): return subprocess.run( - self.__build_cmd(args, requires_root, **kwargs), + self.__build_cmd(process_args, requires_root, **kwargs), **kwargs ) - def Popen(self, args, requires_root=False, **kwargs): + def Popen(self, process_args, requires_root=False, **kwargs): return subprocess.Popen( - self.__build_cmd(args, requires_root, **kwargs), + self.__build_cmd(process_args, requires_root, **kwargs), **kwargs ) + def run_less(self, process_args, requires_root=False, **kwargs): + kwargs['stdout'] = subprocess.PIPE + kwargs['stderr'] = subprocess.DEVNULL + + process = self.Popen( + process_args, requires_root, + **kwargs + ) + + less_process = Executable('less').run( + ['-R', '+G'], + stdin=process.stdout + ) + + process.communicate() + return less_process + __exe_name = None __instances = {} From fc7d1d5ec478f39a7cfffee6c692ec9203d49559 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 13 Aug 2020 13:00:32 +0200 Subject: [PATCH 048/118] Commands "cmd" and "sh" --- src/kiwi/runner.py | 4 +- src/kiwi/subcommands/__init__.py | 6 ++- src/kiwi/subcommands/_subcommand.py | 4 +- src/kiwi/subcommands/cmd.py | 29 +++++++++++++++ src/kiwi/subcommands/shell.py | 57 +++++++++++++++++++++++++++++ 5 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 src/kiwi/subcommands/cmd.py create mode 100644 src/kiwi/subcommands/shell.py diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 1561c30..a783382 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -13,7 +13,9 @@ from .subcommands import * SUBCOMMANDS = [ InitCommand, ShowCommand, - LogsCommand + LogsCommand, + CmdCommand, + ShellCommand ] diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 63c7d4c..9b026d7 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -2,9 +2,13 @@ from .init import InitCommand from .logs import LogsCommand from .show import ShowCommand +from .cmd import CmdCommand +from .shell import ShellCommand __all__ = [ 'InitCommand', 'LogsCommand', - 'ShowCommand' + 'ShowCommand', + 'CmdCommand', + 'ShellCommand' ] diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index cd84a26..653c9fe 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -43,13 +43,13 @@ class ProjectCommand(SubCommand): class ServiceCommand(ProjectCommand): """this command concerns services in a project""" - def __init__(self, name, **kwargs): + def __init__(self, name, nargs='*', **kwargs): super().__init__( name, **kwargs ) self._sub_parser.add_argument( - 'services', metavar='service', nargs='*', type=str, + 'services', metavar='service', nargs=nargs, type=str, help="select service(s) in a project" ) diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py new file mode 100644 index 0000000..52cb6ff --- /dev/null +++ b/src/kiwi/subcommands/cmd.py @@ -0,0 +1,29 @@ +# system +import logging + +# local +from ._subcommand import ProjectCommand +from .utils.dockercommand import DockerCommand + + +class CmdCommand(ProjectCommand): + def __init__(self): + super().__init__( + 'cmd', + description="Run raw docker-compose command in a project" + ) + + # arguments for docker-compose command + self._sub_parser.add_argument( + 'compose_cmd', metavar='cmd', type=str, + help="runs `docker-compose " + ) + + def run(self, config, args): + try: + import shlex + DockerCommand('docker-compose').run(config, args, shlex.split(args.compose_cmd)) + + except KeyboardInterrupt: + logging.debug("Subprocess aborted.") + print() diff --git a/src/kiwi/subcommands/shell.py b/src/kiwi/subcommands/shell.py new file mode 100644 index 0000000..8622c89 --- /dev/null +++ b/src/kiwi/subcommands/shell.py @@ -0,0 +1,57 @@ +# system +import logging +import subprocess + +# local +from ._subcommand import ServiceCommand +from .utils.dockercommand import DockerCommand + + +def _service_has_shell(config, args, exec_service, try_shell): + try: + # test if desired shell exists + DockerCommand('docker-compose').run( + config, args, [*exec_service, '/bin/sh', '-c', f"which {try_shell}"], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + return True + + except subprocess.CalledProcessError: + # fallback + logging.info(f"Shell '{try_shell}' not found in container") + return False + + +class ShellCommand(ServiceCommand): + def __init__(self): + super().__init__( + 'sh', 1, + description="Spawn shell inside project's service" + ) + + # -s switch: Select shell + self._sub_parser.add_argument( + '-s', '--shell', type=str, default="/bin/bash", + help="shell to spawn" + ) + + def run(self, config, args): + try: + exec_service = ['exec', args.services[0]] + exec_shell = args.shell + + if not _service_has_shell(config, args, exec_service, exec_shell): + # fallback + exec_shell = '/bin/bash' + + if not _service_has_shell(config, args, exec_service, exec_shell): + exec_shell = '/bin/sh' + + # spawn shell + DockerCommand('docker-compose').run( + config, args, [*exec_service, exec_shell] + ) + + except KeyboardInterrupt: + logging.debug("Subprocess aborted.") + print() From 7aa257f26ad30f5528224a0bd5bfc612f954623f Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 13 Aug 2020 14:26:09 +0200 Subject: [PATCH 049/118] Typo --- Makefile | 2 +- example/hello-world.project/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 449d921..2480bcf 100644 --- a/Makefile +++ b/Makefile @@ -241,7 +241,7 @@ networks: default: driver: bridge # interconnects projects - kiwihub: + kiwi_hub: external: name: $$KIWI_HUB_NAME diff --git a/example/hello-world.project/docker-compose.yml b/example/hello-world.project/docker-compose.yml index f49c63f..6d66369 100644 --- a/example/hello-world.project/docker-compose.yml +++ b/example/hello-world.project/docker-compose.yml @@ -5,7 +5,7 @@ networks: default: driver: bridge # interconnects projects - kiwihub: + kiwi_hub: external: name: $KIWI_HUB_NAME From a471ac4f51ee30c26bea8049cc2862adef1629aa Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 13 Aug 2020 14:26:49 +0200 Subject: [PATCH 050/118] doc + error handling --- src/kiwi/runner.py | 9 ++++++- src/kiwi/subcommands/_subcommand.py | 11 ++++++--- src/kiwi/subcommands/cmd.py | 15 ++++++------ src/kiwi/subcommands/init.py | 28 +++++++++------------ src/kiwi/subcommands/logs.py | 26 ++++++++++---------- src/kiwi/subcommands/shell.py | 38 +++++++++++++---------------- 6 files changed, 65 insertions(+), 62 deletions(-) diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index a783382..f772372 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -42,7 +42,14 @@ class Runner: if str(cmd) == args.command: # command found logging.debug(f"Running '{cmd}' with args: {args}") - cmd.run(LoadedConfig.get(), args) + + try: + cmd.run(LoadedConfig.get(), args) + + except KeyboardInterrupt: + print() + logging.warning(f"'{cmd}' aborted, inputs may have been discarded.") + return True # command not found diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index 653c9fe..b886d92 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -41,15 +41,20 @@ class ProjectCommand(SubCommand): class ServiceCommand(ProjectCommand): - """this command concerns services in a project""" + """this command concerns service(s) in a project""" - def __init__(self, name, nargs='*', **kwargs): + def __init__(self, name, nargs=1, **kwargs): super().__init__( name, **kwargs ) + services = "service" + + if not nargs == 1: + services = "services" + self._sub_parser.add_argument( 'services', metavar='service', nargs=nargs, type=str, - help="select service(s) in a project" + help=f"select {services} in a project" ) diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index 52cb6ff..56cd5f4 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -7,23 +7,22 @@ from .utils.dockercommand import DockerCommand class CmdCommand(ProjectCommand): + """kiwi cmd""" + def __init__(self): super().__init__( 'cmd', description="Run raw docker-compose command in a project" ) - # arguments for docker-compose command + # command string after docker-compose self._sub_parser.add_argument( 'compose_cmd', metavar='cmd', type=str, - help="runs `docker-compose " + help="runs `docker-compose `" ) def run(self, config, args): - try: - import shlex - DockerCommand('docker-compose').run(config, args, shlex.split(args.compose_cmd)) + import shlex - except KeyboardInterrupt: - logging.debug("Subprocess aborted.") - print() + # run with split compose_cmd argument + DockerCommand('docker-compose').run(config, args, shlex.split(args.compose_cmd)) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 8d2c2b1..af54bcd 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -43,29 +43,25 @@ class InitCommand(SubCommand): def run(self, config, args): logging.info(f"Initializing '{KIWI_CONF_NAME}' in '{os.getcwd()}'") + # check force switch if args.force and os.path.isfile(KIWI_CONF_NAME): from ..config import DefaultConfig logging.warning(f"Overwriting existing '{KIWI_CONF_NAME}'!") config = DefaultConfig.get() - try: - # version - user_input(config, 'version', "Enter kiwi-config version for this instance") + # version + user_input(config, 'version', "Enter kiwi-config version for this instance") - # runtime - user_input(config, 'runtime:storage', "Enter local directory for service data") + # runtime + user_input(config, 'runtime:storage', "Enter local directory for service data") - # markers - user_input(config, 'markers:project', "Enter marker string for project directories") - user_input(config, 'markers:down', "Enter marker string for disabled projects") + # markers + user_input(config, 'markers:project', "Enter marker string for project directories") + user_input(config, 'markers:down', "Enter marker string for disabled projects") - # network - user_input(config, 'network:name', "Enter name for local docker network") - user_input(config, 'network:cidr', "Enter CIDR block for local docker network") + # network + user_input(config, 'network:name', "Enter name for local docker network") + user_input(config, 'network:cidr', "Enter CIDR block for local docker network") - config.save() - - except KeyboardInterrupt: - print() - logging.warning(f"'{self}' aborted, input discarded.") + config.save() diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index c38a635..8909f8f 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -9,7 +9,7 @@ from .utils.dockercommand import DockerCommand class LogsCommand(ServiceCommand): def __init__(self): super().__init__( - 'logs', + 'logs', nargs='*', description="Show logs of a project or service(s) of a project" ) @@ -20,19 +20,19 @@ class LogsCommand(ServiceCommand): ) def run(self, config, args): - compose_args = ['logs', '-t'] + # include timestamps + compose_cmd = ['logs', '-t'] + + # handle following the log output if args.follow: - compose_args = [*compose_args, '-f', '--tail=10'] + compose_cmd = [*compose_cmd, '-f', '--tail=10'] + # append if one or more services are given if args.services: - compose_args = [*compose_args, *args.services] + compose_cmd = [*compose_cmd, *args.services] - try: - if args.follow: - DockerCommand('docker-compose').run(config, args, compose_args) - else: - DockerCommand('docker-compose').run_less(config, args, compose_args) - - except KeyboardInterrupt: - logging.debug("Subprocess aborted.") - print() + # use 'less' viewer if output will be static + if args.follow: + DockerCommand('docker-compose').run(config, args, compose_cmd) + else: + DockerCommand('docker-compose').run_less(config, args, compose_cmd) diff --git a/src/kiwi/subcommands/shell.py b/src/kiwi/subcommands/shell.py index 8622c89..7967b3e 100644 --- a/src/kiwi/subcommands/shell.py +++ b/src/kiwi/subcommands/shell.py @@ -7,26 +7,26 @@ from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand -def _service_has_shell(config, args, exec_service, try_shell): +def _service_has_shell(config, args, compose_cmd, shell): try: # test if desired shell exists DockerCommand('docker-compose').run( - config, args, [*exec_service, '/bin/sh', '-c', f"which {try_shell}"], + config, args, [*compose_cmd, '/bin/sh', '-c', f"which {shell}"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) return True except subprocess.CalledProcessError: # fallback - logging.info(f"Shell '{try_shell}' not found in container") + logging.info(f"Shell '{shell}' not found in container") return False class ShellCommand(ServiceCommand): def __init__(self): super().__init__( - 'sh', 1, - description="Spawn shell inside project's service" + 'sh', + description="Spawn shell inside a project's service" ) # -s switch: Select shell @@ -36,22 +36,18 @@ class ShellCommand(ServiceCommand): ) def run(self, config, args): - try: - exec_service = ['exec', args.services[0]] - exec_shell = args.shell + compose_cmd = ['exec', args.services[0]] + shell = args.shell - if not _service_has_shell(config, args, exec_service, exec_shell): - # fallback - exec_shell = '/bin/bash' + if not _service_has_shell(config, args, compose_cmd, shell): + # fallback + shell = '/bin/bash' - if not _service_has_shell(config, args, exec_service, exec_shell): - exec_shell = '/bin/sh' + if not _service_has_shell(config, args, compose_cmd, shell): + # safe fallback + shell = '/bin/sh' - # spawn shell - DockerCommand('docker-compose').run( - config, args, [*exec_service, exec_shell] - ) - - except KeyboardInterrupt: - logging.debug("Subprocess aborted.") - print() + # spawn shell + DockerCommand('docker-compose').run( + config, args, [*compose_cmd, shell] + ) From 156318295ff7c706c87ff17e8ea50ece62b487fb Mon Sep 17 00:00:00 2001 From: ldericher Date: Sat, 15 Aug 2020 02:06:55 +0200 Subject: [PATCH 051/118] "shell" -> "sh", redundant SUBCOMMANDS array --- src/kiwi/runner.py | 17 +++-------------- src/kiwi/subcommands/__init__.py | 4 ++-- src/kiwi/subcommands/{shell.py => sh.py} | 2 +- 3 files changed, 6 insertions(+), 17 deletions(-) rename src/kiwi/subcommands/{shell.py => sh.py} (97%) diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index f772372..b62cbd9 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -2,21 +2,9 @@ import logging # local +from . import subcommands from .config import LoadedConfig from .parser import Parser -from .subcommands import * - -########### -# CONSTANTS - -# all available subcommands -SUBCOMMANDS = [ - InitCommand, - ShowCommand, - LogsCommand, - CmdCommand, - ShellCommand -] class Runner: @@ -30,7 +18,8 @@ class Runner: def __init__(self): # setup all subcommands - for cmd in SUBCOMMANDS: + for className in subcommands.__all__: + cmd = getattr(subcommands, className) self.__commands.append(cmd()) def run(self): diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 9b026d7..7cef98e 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -3,12 +3,12 @@ from .init import InitCommand from .logs import LogsCommand from .show import ShowCommand from .cmd import CmdCommand -from .shell import ShellCommand +from .sh import ShCommand __all__ = [ 'InitCommand', 'LogsCommand', 'ShowCommand', 'CmdCommand', - 'ShellCommand' + 'ShCommand' ] diff --git a/src/kiwi/subcommands/shell.py b/src/kiwi/subcommands/sh.py similarity index 97% rename from src/kiwi/subcommands/shell.py rename to src/kiwi/subcommands/sh.py index 7967b3e..00f5d48 100644 --- a/src/kiwi/subcommands/shell.py +++ b/src/kiwi/subcommands/sh.py @@ -22,7 +22,7 @@ def _service_has_shell(config, args, compose_cmd, shell): return False -class ShellCommand(ServiceCommand): +class ShCommand(ServiceCommand): def __init__(self): super().__init__( 'sh', From 2dd524fbad61d700d36fca7e8d2013895dc68e90 Mon Sep 17 00:00:00 2001 From: ldericher Date: Sat, 15 Aug 2020 04:06:25 +0200 Subject: [PATCH 052/118] better logging for "sh", 'runtime:shells' config key --- example/kiwi.yml | 2 ++ src/kiwi/subcommands/sh.py | 51 +++++++++++++++++++++++++++----------- src/kiwi_default.yml | 2 ++ 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/example/kiwi.yml b/example/kiwi.yml index a78bc97..323d023 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -6,6 +6,8 @@ version: '0.1' runtime: storage: /tmp/kiwi + shells: + - /bin/bash env: null markers: diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index 00f5d48..19ba909 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -18,10 +18,38 @@ def _service_has_shell(config, args, compose_cmd, shell): except subprocess.CalledProcessError: # fallback - logging.info(f"Shell '{shell}' not found in container") return False +def _find_shell(config, args, compose_cmd): + # as a last resort, fallback to "sh" + shells = ['/bin/sh', 'sh'] + + # load favorite shells from config + if config['runtime:shells']: + shells = [*config['runtime:shells'], *shells] + + # consider shell from args + if args.shell: + shells = [args.shell, *shells] + + logging.debug(f"Shells priority: {shells}") + + # actually try shells + for i, shell in enumerate(shells): + if _service_has_shell(config, args, compose_cmd, shell): + # shell found + logging.debug(f"Using shell '{shell}'") + return shell + elif i + 1 < len(shells): + # not found, try next + logging.info(f"Shell '{shell}' not found in container, trying '{shells[i+1]}'") + else: + # not found, search exhausted + logging.error(f"None of the shells {shells} found in container, please provide -s SHELL!") + return None + + class ShCommand(ServiceCommand): def __init__(self): super().__init__( @@ -31,23 +59,16 @@ class ShCommand(ServiceCommand): # -s switch: Select shell self._sub_parser.add_argument( - '-s', '--shell', type=str, default="/bin/bash", + '-s', '--shell', type=str, help="shell to spawn" ) def run(self, config, args): compose_cmd = ['exec', args.services[0]] - shell = args.shell + shell = _find_shell(config, args, compose_cmd) - if not _service_has_shell(config, args, compose_cmd, shell): - # fallback - shell = '/bin/bash' - - if not _service_has_shell(config, args, compose_cmd, shell): - # safe fallback - shell = '/bin/sh' - - # spawn shell - DockerCommand('docker-compose').run( - config, args, [*compose_cmd, shell] - ) + if shell: + # spawn shell + DockerCommand('docker-compose').run( + config, args, [*compose_cmd, shell] + ) diff --git a/src/kiwi_default.yml b/src/kiwi_default.yml index 2e5cd0c..8f477ee 100644 --- a/src/kiwi_default.yml +++ b/src/kiwi_default.yml @@ -1,6 +1,8 @@ version: runtime: storage: /var/kiwi + shells: + - /bin/bash env: null markers: project: .project From 971e26794391bf7fb903a78b4b401257c42d26b7 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 10:08:29 +0200 Subject: [PATCH 053/118] doc --- src/kiwi/subcommands/logs.py | 2 ++ src/kiwi/subcommands/sh.py | 37 +++++++++++++++++++++++++++--------- src/kiwi/subcommands/show.py | 2 ++ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 8909f8f..a704de2 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -7,6 +7,8 @@ from .utils.dockercommand import DockerCommand class LogsCommand(ServiceCommand): + """kiwi logs""" + def __init__(self): super().__init__( 'logs', nargs='*', diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index 19ba909..3dd6a40 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -7,22 +7,29 @@ from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand -def _service_has_shell(config, args, compose_cmd, shell): +def _service_has_executable(config, args, compose_cmd, exe_name): + """ + Test if container (as of compose_cmd array) has an executable exe_name in its PATH. + Requires /bin/sh and which. + """ + try: # test if desired shell exists DockerCommand('docker-compose').run( - config, args, [*compose_cmd, '/bin/sh', '-c', f"which {shell}"], + config, args, [*compose_cmd, '/bin/sh', '-c', f"which {exe_name}"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) return True - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as e: # fallback return False def _find_shell(config, args, compose_cmd): - # as a last resort, fallback to "sh" + """find first working shell (provided by config and args) in container (as of compose_cmd array)""" + + # builtin shells: as a last resort, fallback to '/bin/sh' and 'sh' shells = ['/bin/sh', 'sh'] # load favorite shells from config @@ -37,20 +44,32 @@ def _find_shell(config, args, compose_cmd): # actually try shells for i, shell in enumerate(shells): - if _service_has_shell(config, args, compose_cmd, shell): - # shell found + if _service_has_executable(config, args, compose_cmd, shell): + # found working shell logging.debug(f"Using shell '{shell}'") return shell + elif i + 1 < len(shells): - # not found, try next + # try next in list logging.info(f"Shell '{shell}' not found in container, trying '{shells[i+1]}'") + + elif args.shell: + # not found, user suggestion provided + logging.warning(f"Could not find any working shell in this container. " + f"Launching provided '{args.shell}' nevertheless. " + f"Don't get mad if this fails!") + return args.shell + else: # not found, search exhausted - logging.error(f"None of the shells {shells} found in container, please provide -s SHELL!") + logging.error(f"Could not find any working shell among '{shells}' in this container. " + f"Please suggest a shell using the '-s SHELL' command line option!") return None class ShCommand(ServiceCommand): + """kiwi sh""" + def __init__(self): super().__init__( 'sh', @@ -67,7 +86,7 @@ class ShCommand(ServiceCommand): compose_cmd = ['exec', args.services[0]] shell = _find_shell(config, args, compose_cmd) - if shell: + if shell is not None: # spawn shell DockerCommand('docker-compose').run( config, args, [*compose_cmd, shell] diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 672df0a..7921af5 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -3,6 +3,8 @@ from ._subcommand import SubCommand class ShowCommand(SubCommand): + """kiwi show""" + def __init__(self): super().__init__( 'show', From 387b7406b597ea848ea7236dfe2ae8593253b4c0 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 10:55:08 +0200 Subject: [PATCH 054/118] format --- src/kiwi/runner.py | 9 ++++++--- src/kiwi/subcommands/__init__.py | 8 ++++---- src/kiwi/subcommands/cmd.py | 3 --- src/kiwi/subcommands/logs.py | 3 --- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index b62cbd9..f18bdb4 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -22,13 +22,16 @@ class Runner: cmd = getattr(subcommands, className) self.__commands.append(cmd()) - def run(self): + def run(self, command=None): """run the desired subcommand""" args = Parser().get_args() + if command is None: + command = args.command + for cmd in self.__commands: - if str(cmd) == args.command: + if str(cmd) == command: # command found logging.debug(f"Running '{cmd}' with args: {args}") @@ -42,7 +45,7 @@ class Runner: return True # command not found - logging.error(f"kiwi command '{args.command}' unknown") + logging.error(f"kiwi command '{command}' unknown") return False __instance = None diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 7cef98e..e582594 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,14 +1,14 @@ # local +from .cmd import CmdCommand from .init import InitCommand from .logs import LogsCommand -from .show import ShowCommand -from .cmd import CmdCommand from .sh import ShCommand +from .show import ShowCommand __all__ = [ + 'CmdCommand', 'InitCommand', 'LogsCommand', + 'ShCommand', 'ShowCommand', - 'CmdCommand', - 'ShCommand' ] diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index 56cd5f4..d89f6f8 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -1,6 +1,3 @@ -# system -import logging - # local from ._subcommand import ProjectCommand from .utils.dockercommand import DockerCommand diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index a704de2..b63bc21 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,6 +1,3 @@ -# system -import logging - # local from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand From 4c70b7b2c1bbef129c13f860def17b2aeb049c0a Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 10:57:45 +0200 Subject: [PATCH 055/118] simple command dependencies --- src/kiwi/runner.py | 2 +- src/kiwi/subcommands/_subcommand.py | 2 +- src/kiwi/subcommands/cmd.py | 2 +- src/kiwi/subcommands/logs.py | 2 +- src/kiwi/subcommands/sh.py | 2 +- src/kiwi/subcommands/show.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index f18bdb4..186029b 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -36,7 +36,7 @@ class Runner: logging.debug(f"Running '{cmd}' with args: {args}") try: - cmd.run(LoadedConfig.get(), args) + cmd.run(self, LoadedConfig.get(), args) except KeyboardInterrupt: print() diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index b886d92..59b04f8 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -20,7 +20,7 @@ class SubCommand: def __str__(self): return self.__name - def run(self, config, args): + def run(self, runner, config, args): """actually run command with this dir's config and parsed CLI args""" pass diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index d89f6f8..4aba79c 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -18,7 +18,7 @@ class CmdCommand(ProjectCommand): help="runs `docker-compose `" ) - def run(self, config, args): + def run(self, runner, config, args): import shlex # run with split compose_cmd argument diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index b63bc21..178713e 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -18,7 +18,7 @@ class LogsCommand(ServiceCommand): help="output appended data as log grows" ) - def run(self, config, args): + def run(self, runner, config, args): # include timestamps compose_cmd = ['logs', '-t'] diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index 3dd6a40..9c2b1df 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -82,7 +82,7 @@ class ShCommand(ServiceCommand): help="shell to spawn" ) - def run(self, config, args): + def run(self, runner, config, args): compose_cmd = ['exec', args.services[0]] shell = _find_shell(config, args, compose_cmd) diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 7921af5..9b65da4 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -11,5 +11,5 @@ class ShowCommand(SubCommand): description="Show effective kiwi.yml" ) - def run(self, config, args): + def run(self, runner, config, args): print(config) From 20f6131cfc9fb4c1b69d4559858531737a755692 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 11:57:08 +0200 Subject: [PATCH 056/118] subprocess environment consistency --- src/kiwi/subcommands/utils/dockercommand.py | 38 ++++++++--------- src/kiwi/subcommands/utils/executable.py | 45 +++++++++++++++------ 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 00a2f46..1fd06e0 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -8,25 +8,25 @@ from .executable import Executable def _update_kwargs(config, args, **kwargs): - project_name = args.project - project_marker = config['markers:project'] - project_dir = f'{project_name}{project_marker}' - kwargs['cwd'] = project_dir + if 'project' in args: + # command affects a project in this instance - if 'env' not in kwargs: - kwargs['env'] = {} + project_name = args.project + project_marker = config['markers:project'] + project_dir = f'{project_name}{project_marker}' + kwargs['cwd'] = project_dir - if config['runtime:env'] is not None: - kwargs['env'].update(config['runtime:env']) + if 'env' not in kwargs: + kwargs['env'] = {} - kwargs['env'].update({ - 'KIWI_HUB_NAME': config['network:name'], - 'COMPOSE_PROJECT_NAME': project_name, - 'CONFDIR': os.path.join(config['runtime:storage'], 'conf'), - 'TARGETDIR': os.path.join(config['runtime:storage'], project_dir) - }) + kwargs['env'].update({ + 'COMPOSE_PROJECT_NAME': project_name, + 'CONFDIR': os.path.join(config['runtime:storage'], 'conf'), + 'TARGETDIR': os.path.join(config['runtime:storage'], project_dir) + }) + + logging.debug(f"kwargs updated: {kwargs}") - logging.debug(f"kwargs updated: {kwargs}") return kwargs @@ -50,15 +50,15 @@ class DockerCommand(Executable): kwargs = _update_kwargs(config, args, **kwargs) # equivalent to 'super().run' but agnostic of nested class construct - super().__getattr__("run")( - process_args, DockerCommand.__requires_root, + return super().__getattr__("run")( + process_args, config, DockerCommand.__requires_root, **kwargs ) def run_less(self, config, args, process_args, **kwargs): kwargs = _update_kwargs(config, args, **kwargs) - super().__getattr__("run_less")( - process_args, DockerCommand.__requires_root, + return super().__getattr__("run_less")( + process_args, config, DockerCommand.__requires_root, **kwargs ) diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py index 7ec5f65..5c04fc8 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/subcommands/utils/executable.py @@ -4,17 +4,34 @@ import os import subprocess -def is_executable(filename): +def _update_kwargs(config, **kwargs): + if config is not None: + if config['runtime:env'] is not None: + kwargs['env'].update(config['runtime:env']) + + if 'env' not in kwargs: + kwargs['env'] = {} + + kwargs['env'].update({ + 'KIWI_HUB_NAME': config['network:name'] + }) + + logging.debug(f"kwargs updated: {kwargs}") + + return kwargs + + +def _is_executable(filename): if filename is None: return False return os.path.isfile(filename) and os.access(filename, os.X_OK) -def find_exe_file(exe_name): +def _find_exe_file(exe_name): for path in os.environ['PATH'].split(os.pathsep): exe_file = os.path.join(path, exe_name) - if is_executable(exe_file): + if _is_executable(exe_file): return exe_file raise FileNotFoundError(f"Executable '{exe_name}' not found in $PATH!") @@ -24,36 +41,40 @@ class Executable: class __Executable: __exe_path = None - def __init__(self, exe_name, requires_root=False): - self.__exe_path = find_exe_file(exe_name) + def __init__(self, exe_name): + self.__exe_path = _find_exe_file(exe_name) def __build_cmd(self, args, requires_root=False, **kwargs): cmd = [self.__exe_path, *args] if requires_root: - self.__exe_path = [find_exe_file("sudo"), self.__exe_path] + self.__exe_path = [_find_exe_file("sudo"), self.__exe_path] logging.debug(f"Executable cmd{cmd}, kwargs{kwargs}") return cmd - def run(self, process_args, requires_root=False, **kwargs): + def run(self, process_args, config=None, requires_root=False, **kwargs): + kwargs = _update_kwargs(config, **kwargs) + return subprocess.run( self.__build_cmd(process_args, requires_root, **kwargs), **kwargs ) - def Popen(self, process_args, requires_root=False, **kwargs): + def Popen(self, process_args, config=None, requires_root=False, **kwargs): + kwargs = _update_kwargs(config, **kwargs) + return subprocess.Popen( self.__build_cmd(process_args, requires_root, **kwargs), **kwargs ) - def run_less(self, process_args, requires_root=False, **kwargs): + def run_less(self, process_args, config=None, requires_root=False, **kwargs): kwargs['stdout'] = subprocess.PIPE kwargs['stderr'] = subprocess.DEVNULL process = self.Popen( - process_args, requires_root, + process_args, config, requires_root, **kwargs ) @@ -68,11 +89,11 @@ class Executable: __exe_name = None __instances = {} - def __init__(self, exe_name, requires_root=False): + def __init__(self, exe_name): self.__exe_name = exe_name if exe_name not in Executable.__instances: - Executable.__instances[exe_name] = Executable.__Executable(exe_name, requires_root) + Executable.__instances[exe_name] = Executable.__Executable(exe_name) def __getattr__(self, item): return getattr(self.__instances[self.__exe_name], item) From fe094fbecb737928ac085efdd4b00599a6bfd6ef Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 12:08:42 +0200 Subject: [PATCH 057/118] subcommand "net-up" --- src/kiwi/subcommands/__init__.py | 2 ++ src/kiwi/subcommands/net_up.py | 46 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 src/kiwi/subcommands/net_up.py diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index e582594..67861bd 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -2,6 +2,7 @@ from .cmd import CmdCommand from .init import InitCommand from .logs import LogsCommand +from .net_up import NetUpCommand from .sh import ShCommand from .show import ShowCommand @@ -9,6 +10,7 @@ __all__ = [ 'CmdCommand', 'InitCommand', 'LogsCommand', + 'NetUpCommand', 'ShCommand', 'ShowCommand', ] diff --git a/src/kiwi/subcommands/net_up.py b/src/kiwi/subcommands/net_up.py new file mode 100644 index 0000000..339768e --- /dev/null +++ b/src/kiwi/subcommands/net_up.py @@ -0,0 +1,46 @@ +# system +import logging +import subprocess + +# local +from ._subcommand import SubCommand +from .utils.dockercommand import DockerCommand + + +class NetUpCommand(SubCommand): + """kiwi net-up""" + + def __init__(self): + super().__init__( + 'net-up', + description="Create the local network hub for this instance" + ) + + def run(self, runner, config, args): + ps = DockerCommand('docker').run( + config, args, ['network', 'ls', '--filter', f"name={config['network:name']}", '--format', '{{.Name}}'], + stdout=subprocess.PIPE + ) + + net_found = str(ps.stdout, 'utf-8').strip() + + if net_found == config['network:name']: + logging.info(f"Network '{config['network:name']}' already exists") + return + + try: + DockerCommand('docker').run( + config, args, + [ + 'network', 'create', + '--driver', 'bridge', + '--internal', + '--subnet', config['network:cidr'], + config['network:name'] + ], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + logging.info(f"Network '{config['network:name']}' created") + + except subprocess.CalledProcessError: + logging.error(f"Error creating network '{config['network:name']}'") From 8adbb5b2a0e85a3115d193b3a6a2eff774a3d565 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 12:20:38 +0200 Subject: [PATCH 058/118] subcommand "net-down" --- src/kiwi/subcommands/__init__.py | 3 +- src/kiwi/subcommands/net.py | 76 ++++++++++++++++++++++++++++++++ src/kiwi/subcommands/net_up.py | 46 ------------------- 3 files changed, 78 insertions(+), 47 deletions(-) create mode 100644 src/kiwi/subcommands/net.py delete mode 100644 src/kiwi/subcommands/net_up.py diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 67861bd..e3b6ed4 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -2,7 +2,7 @@ from .cmd import CmdCommand from .init import InitCommand from .logs import LogsCommand -from .net_up import NetUpCommand +from .net import NetUpCommand, NetDownCommand from .sh import ShCommand from .show import ShowCommand @@ -11,6 +11,7 @@ __all__ = [ 'InitCommand', 'LogsCommand', 'NetUpCommand', + 'NetDownCommand', 'ShCommand', 'ShowCommand', ] diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py new file mode 100644 index 0000000..6908285 --- /dev/null +++ b/src/kiwi/subcommands/net.py @@ -0,0 +1,76 @@ +# system +import logging +import subprocess + +# local +from ._subcommand import SubCommand +from .utils.dockercommand import DockerCommand + + +def _find_net(config, args): + ps = DockerCommand('docker').run( + config, args, ['network', 'ls', '--filter', f"name={config['network:name']}", '--format', '{{.Name}}'], + stdout=subprocess.PIPE + ) + + net_found = str(ps.stdout, 'utf-8').strip() + + return net_found == config['network:name'] + + +class NetUpCommand(SubCommand): + """kiwi net-up""" + + def __init__(self): + super().__init__( + 'net-up', + description="Create the local network hub for this instance" + ) + + def run(self, runner, config, args): + if _find_net(config, args): + logging.info(f"Network '{config['network:name']}' already exists") + return + + try: + DockerCommand('docker').run( + config, args, + [ + 'network', 'create', + '--driver', 'bridge', + '--internal', + '--subnet', config['network:cidr'], + config['network:name'] + ], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + logging.info(f"Network '{config['network:name']}' created") + + except subprocess.CalledProcessError: + logging.error(f"Error creating network '{config['network:name']}'") + + +class NetDownCommand(SubCommand): + """kiwi net-down""" + + def __init__(self): + super().__init__( + 'net-down', + description="Remove the local network hub for this instance" + ) + + def run(self, runner, config, args): + if not _find_net(config, args): + logging.info(f"Network '{config['network:name']}' already removed") + return + + try: + DockerCommand('docker').run( + config, args, + ['network', 'rm', config['network:name']], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + logging.info(f"Network '{config['network:name']}' removed") + + except subprocess.CalledProcessError: + logging.error(f"Error removing network '{config['network:name']}'") \ No newline at end of file diff --git a/src/kiwi/subcommands/net_up.py b/src/kiwi/subcommands/net_up.py deleted file mode 100644 index 339768e..0000000 --- a/src/kiwi/subcommands/net_up.py +++ /dev/null @@ -1,46 +0,0 @@ -# system -import logging -import subprocess - -# local -from ._subcommand import SubCommand -from .utils.dockercommand import DockerCommand - - -class NetUpCommand(SubCommand): - """kiwi net-up""" - - def __init__(self): - super().__init__( - 'net-up', - description="Create the local network hub for this instance" - ) - - def run(self, runner, config, args): - ps = DockerCommand('docker').run( - config, args, ['network', 'ls', '--filter', f"name={config['network:name']}", '--format', '{{.Name}}'], - stdout=subprocess.PIPE - ) - - net_found = str(ps.stdout, 'utf-8').strip() - - if net_found == config['network:name']: - logging.info(f"Network '{config['network:name']}' already exists") - return - - try: - DockerCommand('docker').run( - config, args, - [ - 'network', 'create', - '--driver', 'bridge', - '--internal', - '--subnet', config['network:cidr'], - config['network:name'] - ], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - logging.info(f"Network '{config['network:name']}' created") - - except subprocess.CalledProcessError: - logging.error(f"Error creating network '{config['network:name']}'") From 1cf3b5ca55dd37f14fd1cddc9460df2a7b68b014 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 12:58:18 +0200 Subject: [PATCH 059/118] nargs for ProjectCommand and ServiceCommand --- src/kiwi/subcommands/_subcommand.py | 21 +++++++++++++-------- src/kiwi/subcommands/cmd.py | 2 +- src/kiwi/subcommands/logs.py | 2 +- src/kiwi/subcommands/sh.py | 4 ++-- src/kiwi/subcommands/utils/dockercommand.py | 4 ++-- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index 59b04f8..e7c0274 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -28,33 +28,38 @@ class SubCommand: class ProjectCommand(SubCommand): """this command concerns a project in current instance""" - def __init__(self, name, **kwargs): + def __init__(self, name, num_projects, **kwargs): super().__init__( name, **kwargs ) + projects = "a project" + + if not str(num_projects) == '1': + projects = "projects" + self._sub_parser.add_argument( - 'project', type=str, - help="select a project in this instance" + 'projects', metavar='project', nargs=num_projects, type=str, + help=f"select {projects} in this instance" ) class ServiceCommand(ProjectCommand): """this command concerns service(s) in a project""" - def __init__(self, name, nargs=1, **kwargs): + def __init__(self, name, num_projects, num_services, **kwargs): super().__init__( - name, + name, num_projects=num_projects, **kwargs ) - services = "service" + services = "a service" - if not nargs == 1: + if not str(num_services) == '1': services = "services" self._sub_parser.add_argument( - 'services', metavar='service', nargs=nargs, type=str, + 'services', metavar='service', nargs=num_services, type=str, help=f"select {services} in a project" ) diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index 4aba79c..a935880 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -8,7 +8,7 @@ class CmdCommand(ProjectCommand): def __init__(self): super().__init__( - 'cmd', + 'cmd', num_projects=1, description="Run raw docker-compose command in a project" ) diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 178713e..5a22912 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -8,7 +8,7 @@ class LogsCommand(ServiceCommand): def __init__(self): super().__init__( - 'logs', nargs='*', + 'logs', num_projects=1, num_services='*', description="Show logs of a project or service(s) of a project" ) diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index 9c2b1df..17d5ca8 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -72,11 +72,11 @@ class ShCommand(ServiceCommand): def __init__(self): super().__init__( - 'sh', + 'sh', num_projects=1, num_services=1, description="Spawn shell inside a project's service" ) - # -s switch: Select shell + # -s argument: Select shell self._sub_parser.add_argument( '-s', '--shell', type=str, help="shell to spawn" diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 1fd06e0..1fca7b6 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -8,10 +8,10 @@ from .executable import Executable def _update_kwargs(config, args, **kwargs): - if 'project' in args: + if 'projects' in args: # command affects a project in this instance - project_name = args.project + project_name = args.projects[0] project_marker = config['markers:project'] project_dir = f'{project_name}{project_marker}' kwargs['cwd'] = project_dir From f222786015ac9cdd2a0ad322dc00bfbe08a9e917 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 13:14:54 +0200 Subject: [PATCH 060/118] Handle num_projects='?' --- src/kiwi/subcommands/utils/dockercommand.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 1fca7b6..b2738e0 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -8,10 +8,13 @@ from .executable import Executable def _update_kwargs(config, args, **kwargs): - if 'projects' in args: + if args is not None and 'projects' in args and args.projects is not None: # command affects a project in this instance - project_name = args.projects[0] + project_name = args.projects + if isinstance(project_name, list) and len(project_name) > 0: + project_name = project_name[0] + project_marker = config['markers:project'] project_dir = f'{project_name}{project_marker}' kwargs['cwd'] = project_dir From 34507b9b8cbbac2c2c8b76ae42e1174287b6425c Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 14:12:33 +0200 Subject: [PATCH 061/118] Subcommand 'up' --- src/kiwi/subcommands/__init__.py | 2 ++ src/kiwi/subcommands/up.py | 33 ++++++++++++++++++++++++ src/kiwi/subcommands/utils/project.py | 37 +++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 src/kiwi/subcommands/up.py create mode 100644 src/kiwi/subcommands/utils/project.py diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index e3b6ed4..b265b46 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -5,6 +5,7 @@ from .logs import LogsCommand from .net import NetUpCommand, NetDownCommand from .sh import ShCommand from .show import ShowCommand +from .up import UpCommand __all__ = [ 'CmdCommand', @@ -14,4 +15,5 @@ __all__ = [ 'NetDownCommand', 'ShCommand', 'ShowCommand', + 'UpCommand', ] diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py new file mode 100644 index 0000000..f3e3753 --- /dev/null +++ b/src/kiwi/subcommands/up.py @@ -0,0 +1,33 @@ +# system +import logging + +# local +from ._subcommand import ServiceCommand +from .utils.dockercommand import DockerCommand +from .utils.project import list_projects + + +class UpCommand(ServiceCommand): + """kiwi up""" + + def __init__(self): + super().__init__( + 'up', num_projects='?', num_services='*', + description="Start the whole instance, a project or service(s) inside a project" + ) + + def run(self, runner, config, args): + if args.projects is None: + # "up" for all projects + for project_name in list_projects(config): + logging.info(f"Bringing up project '{project_name}'") + args.projects = project_name + + runner.run('up') + + return + + runner.run('net-up') + DockerCommand('docker-compose').run( + config, args, ['up', '-d', *args.services] + ) diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/subcommands/utils/project.py new file mode 100644 index 0000000..ae02225 --- /dev/null +++ b/src/kiwi/subcommands/utils/project.py @@ -0,0 +1,37 @@ +import os + + +def get_project_name(args): + """get project name from CLI args""" + + if args is not None and 'projects' in args: + if isinstance(args.projects, list) and len(args.projects) > 0: + return args.projects[0] + else: + return args.projects + + return None + + +def get_project_dir(config, project_name): + """get project directory""" + + return f"{project_name}{config['markers:project']}" + + +def list_projects(config): + """list projects in current instance""" + + # complete dir listing + content = os.listdir() + + # filter directories + dirs = [d for d in content if os.path.isdir(d)] + + # filter suffix + project_dirs = [p for p in dirs if p.endswith(config['markers:project'])] + + # remove suffix + projects = [p[:-len(config['markers:project'])] for p in project_dirs] + + return projects From d44da6eabd66441dfc251e6bc4ef299f31cb5c64 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 14:13:09 +0200 Subject: [PATCH 062/118] typo --- src/kiwi/subcommands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index af54bcd..cec8dd3 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -40,7 +40,7 @@ class InitCommand(SubCommand): help=f"use default values even if {KIWI_CONF_NAME} is present" ) - def run(self, config, args): + def run(self, runner, config, args): logging.info(f"Initializing '{KIWI_CONF_NAME}' in '{os.getcwd()}'") # check force switch From 4ad2b7c45853adc812a90180efd14399781b46d1 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 14:13:42 +0200 Subject: [PATCH 063/118] use new utils.project routines --- src/kiwi/subcommands/utils/dockercommand.py | 21 ++++++++++----------- src/kiwi/subcommands/utils/executable.py | 10 ++++------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index b2738e0..34cf42a 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -5,27 +5,26 @@ import subprocess # local from .executable import Executable +from .project import * def _update_kwargs(config, args, **kwargs): - if args is not None and 'projects' in args and args.projects is not None: - # command affects a project in this instance - - project_name = args.projects - if isinstance(project_name, list) and len(project_name) > 0: - project_name = project_name[0] - - project_marker = config['markers:project'] - project_dir = f'{project_name}{project_marker}' - kwargs['cwd'] = project_dir + # project given in args: command affects a project in this instance + project_name = get_project_name(args) + if project_name is not None: + # execute command in project directory + kwargs['cwd'] = get_project_dir(config, project_name) + # ensure there is an environment if 'env' not in kwargs: kwargs['env'] = {} + # create environment variables for docker commands kwargs['env'].update({ 'COMPOSE_PROJECT_NAME': project_name, + 'KIWI_HUB_NAME': config['network:name'], 'CONFDIR': os.path.join(config['runtime:storage'], 'conf'), - 'TARGETDIR': os.path.join(config['runtime:storage'], project_dir) + 'TARGETDIR': os.path.join(config['runtime:storage'], get_project_dir(config, project_name)) }) logging.debug(f"kwargs updated: {kwargs}") diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py index 5c04fc8..e1116ad 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/subcommands/utils/executable.py @@ -6,15 +6,13 @@ import subprocess def _update_kwargs(config, **kwargs): if config is not None: - if config['runtime:env'] is not None: - kwargs['env'].update(config['runtime:env']) - + # ensure there is an environment if 'env' not in kwargs: kwargs['env'] = {} - kwargs['env'].update({ - 'KIWI_HUB_NAME': config['network:name'] - }) + # add common environment from config + if config['runtime:env'] is not None: + kwargs['env'].update(config['runtime:env']) logging.debug(f"kwargs updated: {kwargs}") From 79520c2a63e937bf4c745c337dff4da94738f432 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 14:51:11 +0200 Subject: [PATCH 064/118] subcommand "down" --- src/kiwi/subcommands/__init__.py | 2 ++ src/kiwi/subcommands/down.py | 46 ++++++++++++++++++++++++ src/kiwi/subcommands/utils/user_input.py | 23 ++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 src/kiwi/subcommands/down.py create mode 100644 src/kiwi/subcommands/utils/user_input.py diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index b265b46..69d75d3 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,5 +1,6 @@ # local from .cmd import CmdCommand +from .down import DownCommand from .init import InitCommand from .logs import LogsCommand from .net import NetUpCommand, NetDownCommand @@ -9,6 +10,7 @@ from .up import UpCommand __all__ = [ 'CmdCommand', + 'DownCommand', 'InitCommand', 'LogsCommand', 'NetUpCommand', diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py new file mode 100644 index 0000000..d931cc3 --- /dev/null +++ b/src/kiwi/subcommands/down.py @@ -0,0 +1,46 @@ +# system +import logging + +# local +from ._subcommand import ServiceCommand +from .utils.dockercommand import DockerCommand +from .utils.project import get_project_name, list_projects +from .utils.user_input import are_you_sure + + +class DownCommand(ServiceCommand): + """kiwi down""" + + def __init__(self): + super().__init__( + 'down', num_projects='?', num_services='*', + description="Bring down the whole instance, a project or service(s) inside a project" + ) + + def run(self, runner, config, args): + if 'projects' not in args or args.projects is None: + # "down" for all projects + if are_you_sure("This will bring down the entire instance."): + for project_name in list_projects(config): + args.projects = project_name + runner.run('down') + return + + if 'services' in args and args.services: + # "down" for service(s) inside project + logging.info(f"Bringing down services {args.services} in project '{get_project_name(args)}'") + + DockerCommand('docker-compose').run( + config, args, ['stop', *args.services] + ) + DockerCommand('docker-compose').run( + config, args, ['rm', '-f', *args.services] + ) + + else: + # "down" for project + logging.info(f"Bringing down project '{get_project_name(args)}'") + + DockerCommand('docker-compose').run( + config, args, ['down'] + ) diff --git a/src/kiwi/subcommands/utils/user_input.py b/src/kiwi/subcommands/utils/user_input.py new file mode 100644 index 0000000..39fff73 --- /dev/null +++ b/src/kiwi/subcommands/utils/user_input.py @@ -0,0 +1,23 @@ +def are_you_sure(prompt, default="no"): + if default.lower() == 'yes': + suffix = "[YES|no]" + else: + suffix = "[yes|NO]" + + answer = input( + "!!!!!!!!!!!!!!!!!!\n" + "!!! BE CAREFUL !!!\n" + "!!!!!!!!!!!!!!!!!!\n" + "\n" + f"{prompt}\n" + "\n" + f"Are you sure you want to proceed? {suffix} " + ).strip().lower() + + if answer == '': + answer = default + + while answer not in ['yes', 'no']: + answer = input("Please type 'yes' or 'no' explicitly: ").strip().lower() + + return answer == 'yes' From 2acbc1c534f34c898d2edb59086c90fe433dc6d6 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 14:51:43 +0200 Subject: [PATCH 065/118] dependencies --- src/kiwi/subcommands/net.py | 16 ++++++++++------ src/kiwi/subcommands/up.py | 13 ++++++++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 6908285..63718b6 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -5,6 +5,7 @@ import subprocess # local from ._subcommand import SubCommand from .utils.dockercommand import DockerCommand +from .utils.user_input import are_you_sure def _find_net(config, args): @@ -65,12 +66,15 @@ class NetDownCommand(SubCommand): return try: - DockerCommand('docker').run( - config, args, - ['network', 'rm', config['network:name']], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - logging.info(f"Network '{config['network:name']}' removed") + if are_you_sure("This will bring down this instance's hub network!"): + runner.run('down') + + DockerCommand('docker').run( + config, args, + ['network', 'rm', config['network:name']], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + logging.info(f"Network '{config['network:name']}' removed") except subprocess.CalledProcessError: logging.error(f"Error removing network '{config['network:name']}'") \ No newline at end of file diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index f3e3753..adc50d2 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -4,7 +4,7 @@ import logging # local from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand -from .utils.project import list_projects +from .utils.project import get_project_name, list_projects class UpCommand(ServiceCommand): @@ -13,20 +13,23 @@ class UpCommand(ServiceCommand): def __init__(self): super().__init__( 'up', num_projects='?', num_services='*', - description="Start the whole instance, a project or service(s) inside a project" + description="Bring up the whole instance, a project or service(s) inside a project" ) def run(self, runner, config, args): - if args.projects is None: + if 'projects' not in args or args.projects is None: # "up" for all projects for project_name in list_projects(config): - logging.info(f"Bringing up project '{project_name}'") args.projects = project_name - runner.run('up') return + if 'services' in args and args.services: + logging.info(f"Bringing up services {args.services} in project '{get_project_name(args)}'") + else: + logging.info(f"Bringing up project '{get_project_name(args)}'") + runner.run('net-up') DockerCommand('docker-compose').run( config, args, ['up', '-d', *args.services] From 11dafb0db62e7eac43c33286b2c9fa5d09165f07 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 15:00:05 +0200 Subject: [PATCH 066/118] return values --- src/kiwi/runner.py | 5 +++-- src/kiwi/subcommands/cmd.py | 2 ++ src/kiwi/subcommands/down.py | 7 +++++-- src/kiwi/subcommands/init.py | 1 + src/kiwi/subcommands/logs.py | 2 ++ src/kiwi/subcommands/net.py | 29 ++++++++++++++++++----------- src/kiwi/subcommands/sh.py | 3 +++ src/kiwi/subcommands/show.py | 1 + src/kiwi/subcommands/up.py | 11 +++++++---- 9 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 186029b..d84cca1 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -36,13 +36,14 @@ class Runner: logging.debug(f"Running '{cmd}' with args: {args}") try: - cmd.run(self, LoadedConfig.get(), args) + result = cmd.run(self, LoadedConfig.get(), args) except KeyboardInterrupt: print() logging.warning(f"'{cmd}' aborted, inputs may have been discarded.") + result = False - return True + return result # command not found logging.error(f"kiwi command '{command}' unknown") diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index a935880..3182f11 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -23,3 +23,5 @@ class CmdCommand(ProjectCommand): # run with split compose_cmd argument DockerCommand('docker-compose').run(config, args, shlex.split(args.compose_cmd)) + + return True diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index d931cc3..790f3bb 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -24,9 +24,10 @@ class DownCommand(ServiceCommand): for project_name in list_projects(config): args.projects = project_name runner.run('down') - return + else: + return False - if 'services' in args and args.services: + elif 'services' in args and args.services: # "down" for service(s) inside project logging.info(f"Bringing down services {args.services} in project '{get_project_name(args)}'") @@ -44,3 +45,5 @@ class DownCommand(ServiceCommand): DockerCommand('docker-compose').run( config, args, ['down'] ) + + return True diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index cec8dd3..e82240e 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -65,3 +65,4 @@ class InitCommand(SubCommand): user_input(config, 'network:cidr', "Enter CIDR block for local docker network") config.save() + return True diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 5a22912..a60122f 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -35,3 +35,5 @@ class LogsCommand(ServiceCommand): DockerCommand('docker-compose').run(config, args, compose_cmd) else: DockerCommand('docker-compose').run_less(config, args, compose_cmd) + + return True diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 63718b6..3eb801e 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -31,7 +31,7 @@ class NetUpCommand(SubCommand): def run(self, runner, config, args): if _find_net(config, args): logging.info(f"Network '{config['network:name']}' already exists") - return + return True try: DockerCommand('docker').run( @@ -49,6 +49,9 @@ class NetUpCommand(SubCommand): except subprocess.CalledProcessError: logging.error(f"Error creating network '{config['network:name']}'") + return False + + return True class NetDownCommand(SubCommand): @@ -63,18 +66,22 @@ class NetDownCommand(SubCommand): def run(self, runner, config, args): if not _find_net(config, args): logging.info(f"Network '{config['network:name']}' already removed") - return + return True try: if are_you_sure("This will bring down this instance's hub network!"): - runner.run('down') - - DockerCommand('docker').run( - config, args, - ['network', 'rm', config['network:name']], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - logging.info(f"Network '{config['network:name']}' removed") + if runner.run('down'): + DockerCommand('docker').run( + config, args, + ['network', 'rm', config['network:name']], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + logging.info(f"Network '{config['network:name']}' removed") + else: + return False except subprocess.CalledProcessError: - logging.error(f"Error removing network '{config['network:name']}'") \ No newline at end of file + logging.error(f"Error removing network '{config['network:name']}'") + return False + + return True \ No newline at end of file diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index 17d5ca8..7e3d31e 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -91,3 +91,6 @@ class ShCommand(ServiceCommand): DockerCommand('docker-compose').run( config, args, [*compose_cmd, shell] ) + return True + + return False diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 9b65da4..4448734 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -13,3 +13,4 @@ class ShowCommand(SubCommand): def run(self, runner, config, args): print(config) + return True diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index adc50d2..0681d83 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -30,7 +30,10 @@ class UpCommand(ServiceCommand): else: logging.info(f"Bringing up project '{get_project_name(args)}'") - runner.run('net-up') - DockerCommand('docker-compose').run( - config, args, ['up', '-d', *args.services] - ) + if runner.run('net-up'): + DockerCommand('docker-compose').run( + config, args, ['up', '-d', *args.services] + ) + return True + + return False From 322033ca92a21db4130c213f1f95926c84f9de85 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 15:03:25 +0200 Subject: [PATCH 067/118] f strings init --- src/kiwi/subcommands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index e82240e..a8ecee6 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -14,7 +14,7 @@ def user_input(config, key, prompt): # prompt user as per argument try: - result = input("{} [{}] ".format(prompt, config[key])).strip() + result = input(f"{prompt} [{config[key]}] ").strip() except EOFError: print() result = None From 0433daa748c0212c532c888aeb130f31de56b7e3 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 15:21:42 +0200 Subject: [PATCH 068/118] neat prompt --- src/kiwi/subcommands/net.py | 4 ++-- src/kiwi/subcommands/utils/user_input.py | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 3eb801e..4a5e488 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -65,7 +65,7 @@ class NetDownCommand(SubCommand): def run(self, runner, config, args): if not _find_net(config, args): - logging.info(f"Network '{config['network:name']}' already removed") + logging.info(f"Network '{config['network:name']}' does not exist") return True try: @@ -84,4 +84,4 @@ class NetDownCommand(SubCommand): logging.error(f"Error removing network '{config['network:name']}'") return False - return True \ No newline at end of file + return True diff --git a/src/kiwi/subcommands/utils/user_input.py b/src/kiwi/subcommands/utils/user_input.py index 39fff73..51b71e5 100644 --- a/src/kiwi/subcommands/utils/user_input.py +++ b/src/kiwi/subcommands/utils/user_input.py @@ -1,3 +1,11 @@ + +def _surround(string, bang): + midlane = f"{bang * 3} {string} {bang * 3}" + sidelane = bang*len(midlane) + + return f"{sidelane}\n{midlane}\n{sidelane}" + + def are_you_sure(prompt, default="no"): if default.lower() == 'yes': suffix = "[YES|no]" @@ -5,12 +13,10 @@ def are_you_sure(prompt, default="no"): suffix = "[yes|NO]" answer = input( - "!!!!!!!!!!!!!!!!!!\n" - "!!! BE CAREFUL !!!\n" - "!!!!!!!!!!!!!!!!!!\n" - "\n" - f"{prompt}\n" - "\n" + f"{_surround('CAREFULING IN PROGRESS', '!')}\n" + f"\n" + f">>> {prompt} <<<\n" + f"\n" f"Are you sure you want to proceed? {suffix} " ).strip().lower() From 500652c9d6ba7f1592827e4bcef95e31f6961e24 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 16:47:08 +0200 Subject: [PATCH 069/118] subcommand "copy-conf" --- example/hello-world.project/conf/test.txt | 1 + src/images/rsync.Dockerfile | 2 + src/kiwi/subcommands/__init__.py | 2 + src/kiwi/subcommands/copy_conf.py | 70 +++++++++++++++++++++++ src/kiwi/subcommands/utils/project.py | 6 ++ 5 files changed, 81 insertions(+) create mode 100644 example/hello-world.project/conf/test.txt create mode 100644 src/images/rsync.Dockerfile create mode 100644 src/kiwi/subcommands/copy_conf.py diff --git a/example/hello-world.project/conf/test.txt b/example/hello-world.project/conf/test.txt new file mode 100644 index 0000000..484ba93 --- /dev/null +++ b/example/hello-world.project/conf/test.txt @@ -0,0 +1 @@ +This is a test. diff --git a/src/images/rsync.Dockerfile b/src/images/rsync.Dockerfile new file mode 100644 index 0000000..47c262e --- /dev/null +++ b/src/images/rsync.Dockerfile @@ -0,0 +1,2 @@ +FROM alpine:latest +RUN apk --no-cache add rsync \ No newline at end of file diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 69d75d3..e31437b 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,5 +1,6 @@ # local from .cmd import CmdCommand +from .copy_conf import CopyConfCommand from .down import DownCommand from .init import InitCommand from .logs import LogsCommand @@ -10,6 +11,7 @@ from .up import UpCommand __all__ = [ 'CmdCommand', + 'CopyConfCommand', 'DownCommand', 'InitCommand', 'LogsCommand', diff --git a/src/kiwi/subcommands/copy_conf.py b/src/kiwi/subcommands/copy_conf.py new file mode 100644 index 0000000..805640b --- /dev/null +++ b/src/kiwi/subcommands/copy_conf.py @@ -0,0 +1,70 @@ +# system +import logging +import os +import subprocess + +# parent +from .._constants import KIWI_ROOT + +# local +from ._subcommand import SubCommand +from .utils.dockercommand import DockerCommand +from .utils.project import list_projects, get_project_dir + + +def _add_prefix(prefix, path): + abs_path = os.path.abspath(path) + return os.path.realpath(prefix + '/' + abs_path) + + +class CopyConfCommand(SubCommand): + """kiwi copy-conf""" + + def __init__(self): + super().__init__( + 'copy-conf', + description="Synchronize all config files to target directory" + ) + + def run(self, runner, config, args): + logging.info("Building image kiwi-config/auxiliary:rsync") + DockerCommand('docker').run( + config, args, + [ + 'build', + '-t', 'kiwi-config/auxiliary:rsync', + '-f', f"{KIWI_ROOT}/images/rsync.Dockerfile", + f"{KIWI_ROOT}/images" + ], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + + conf_sources = [] + + for project_name in list_projects(config): + project_conf = f"{get_project_dir(config, project_name)}/conf" + + if os.path.isdir(project_conf): + conf_sources.append(project_conf) + + if conf_sources: + print(f"Syncing {conf_sources} to '{config['runtime:storage']}'") + conf_sources = [f"'{_add_prefix('/mnt', src)}'" for src in conf_sources] + conf_sources = ' '.join(conf_sources) + + conf_target = f"'{_add_prefix('/mnt', config['runtime:storage'])}'" + logging.debug(f"Config sources {conf_sources}, Config target {conf_target}") + + DockerCommand('docker').run( + config, args, + [ + 'run', '--rm', + '-v', '/:/mnt', + '-u', 'root', + 'kiwi-config/auxiliary:rsync', + 'ash', '-c', f"rsync -r {conf_sources} {conf_target}" + ], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + + return True diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/subcommands/utils/project.py index ae02225..12f9a9a 100644 --- a/src/kiwi/subcommands/utils/project.py +++ b/src/kiwi/subcommands/utils/project.py @@ -19,6 +19,12 @@ def get_project_dir(config, project_name): return f"{project_name}{config['markers:project']}" +def get_target_dir(config, project_name): + """get project's target directory""" + + return f"{config['runtime:storage']}{get_project_dir(config, project_name)}" + + def list_projects(config): """list projects in current instance""" From 9907e722447b0251ba324710e9e67dcc55fc19c8 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 16:58:12 +0200 Subject: [PATCH 070/118] cleaner way of getting root access --- src/kiwi/subcommands/utils/dockercommand.py | 13 +++++++------ src/kiwi/subcommands/utils/executable.py | 17 +++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 34cf42a..d0426bd 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -33,27 +33,28 @@ def _update_kwargs(config, args, **kwargs): class DockerCommand(Executable): - __requires_root = None + __has_tried = False def __init__(self, exe_name): super().__init__(exe_name) - if DockerCommand.__requires_root is None: + if not DockerCommand.__has_tried: try: Executable('docker').run( ['ps'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) - DockerCommand.__requires_root = False + DockerCommand.__has_tried = True + except subprocess.CalledProcessError: - DockerCommand.__requires_root = True + raise PermissionError("Cannot access docker, please get into the docker group or run as root!") def run(self, config, args, process_args, **kwargs): kwargs = _update_kwargs(config, args, **kwargs) # equivalent to 'super().run' but agnostic of nested class construct return super().__getattr__("run")( - process_args, config, DockerCommand.__requires_root, + process_args, config, **kwargs ) @@ -61,6 +62,6 @@ class DockerCommand(Executable): kwargs = _update_kwargs(config, args, **kwargs) return super().__getattr__("run_less")( - process_args, config, DockerCommand.__requires_root, + process_args, config, **kwargs ) diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py index e1116ad..4b7fecb 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/subcommands/utils/executable.py @@ -42,37 +42,34 @@ class Executable: def __init__(self, exe_name): self.__exe_path = _find_exe_file(exe_name) - def __build_cmd(self, args, requires_root=False, **kwargs): + def __build_cmd(self, args, **kwargs): cmd = [self.__exe_path, *args] - if requires_root: - self.__exe_path = [_find_exe_file("sudo"), self.__exe_path] - logging.debug(f"Executable cmd{cmd}, kwargs{kwargs}") return cmd - def run(self, process_args, config=None, requires_root=False, **kwargs): + def run(self, process_args, config=None, **kwargs): kwargs = _update_kwargs(config, **kwargs) return subprocess.run( - self.__build_cmd(process_args, requires_root, **kwargs), + self.__build_cmd(process_args, **kwargs), **kwargs ) - def Popen(self, process_args, config=None, requires_root=False, **kwargs): + def Popen(self, process_args, config=None, **kwargs): kwargs = _update_kwargs(config, **kwargs) return subprocess.Popen( - self.__build_cmd(process_args, requires_root, **kwargs), + self.__build_cmd(process_args, **kwargs), **kwargs ) - def run_less(self, process_args, config=None, requires_root=False, **kwargs): + def run_less(self, process_args, config=None, **kwargs): kwargs['stdout'] = subprocess.PIPE kwargs['stderr'] = subprocess.DEVNULL process = self.Popen( - process_args, config, requires_root, + process_args, config, **kwargs ) From 366ed9b4fe8759a1b5c76fd73bb9a67fd71b396b Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 17 Aug 2020 17:56:29 +0200 Subject: [PATCH 071/118] Rootkit implementation for copyconf (and other commands as root) --- src/kiwi/subcommands/copy_conf.py | 39 +++-------- src/kiwi/subcommands/utils/rootkit.py | 99 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 30 deletions(-) create mode 100644 src/kiwi/subcommands/utils/rootkit.py diff --git a/src/kiwi/subcommands/copy_conf.py b/src/kiwi/subcommands/copy_conf.py index 805640b..db694ea 100644 --- a/src/kiwi/subcommands/copy_conf.py +++ b/src/kiwi/subcommands/copy_conf.py @@ -10,6 +10,7 @@ from .._constants import KIWI_ROOT from ._subcommand import SubCommand from .utils.dockercommand import DockerCommand from .utils.project import list_projects, get_project_dir +from .utils.rootkit import Rootkit, prefix_path def _add_prefix(prefix, path): @@ -27,43 +28,21 @@ class CopyConfCommand(SubCommand): ) def run(self, runner, config, args): - logging.info("Building image kiwi-config/auxiliary:rsync") - DockerCommand('docker').run( - config, args, - [ - 'build', - '-t', 'kiwi-config/auxiliary:rsync', - '-f', f"{KIWI_ROOT}/images/rsync.Dockerfile", - f"{KIWI_ROOT}/images" - ], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - - conf_sources = [] + conf_dirs = [] for project_name in list_projects(config): project_conf = f"{get_project_dir(config, project_name)}/conf" if os.path.isdir(project_conf): - conf_sources.append(project_conf) + conf_dirs.append(project_conf) - if conf_sources: - print(f"Syncing {conf_sources} to '{config['runtime:storage']}'") - conf_sources = [f"'{_add_prefix('/mnt', src)}'" for src in conf_sources] - conf_sources = ' '.join(conf_sources) + if conf_dirs: + # add target directory + conf_dirs.append(config['runtime:storage']) + logging.info(f"Sync directories: {conf_dirs}") - conf_target = f"'{_add_prefix('/mnt', config['runtime:storage'])}'" - logging.debug(f"Config sources {conf_sources}, Config target {conf_target}") - - DockerCommand('docker').run( - config, args, - [ - 'run', '--rm', - '-v', '/:/mnt', - '-u', 'root', - 'kiwi-config/auxiliary:rsync', - 'ash', '-c', f"rsync -r {conf_sources} {conf_target}" - ], + Rootkit('rsync').run( + config, args, ['rsync', '-r', *prefix_path(conf_dirs)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) diff --git a/src/kiwi/subcommands/utils/rootkit.py b/src/kiwi/subcommands/utils/rootkit.py new file mode 100644 index 0000000..c18e5ac --- /dev/null +++ b/src/kiwi/subcommands/utils/rootkit.py @@ -0,0 +1,99 @@ +# system +import logging +import os +import subprocess + +# parent +from ..._constants import KIWI_ROOT + +# local +from .dockercommand import DockerCommand + + +def _prefix_path(prefix, path): + abs_path = os.path.abspath(path) + return os.path.realpath(prefix + '/' + abs_path) + + +def prefix_path(path): + if isinstance(path, str): + return _prefix_path('/mnt/', path) + elif isinstance(path, list): + return [_prefix_path('/mnt/', p) for p in path] + + +def _image_name(image_tag): + if image_tag is not None: + return f"kiwi-config/auxiliary:{image_tag}" + else: + return "alpine:latest" + + +class Rootkit: + class __Rootkit: + __image_tag = None + + def __init__(self, image_tag=None): + self.__image_tag = image_tag + + def __exists(self, config, args): + ps = DockerCommand('docker').run( + config, args, [ + 'images', + '--filter', f"reference={_image_name(self.__image_tag)}", + '--format', '{{.Repository}}:{{.Tag}}' + ], + stdout=subprocess.PIPE + ) + + return str(ps.stdout, 'utf-8').strip() == _image_name(self.__image_tag) + + def __build_image(self, config, args): + if self.__exists(config, args): + logging.info(f"Using image {_image_name(self.__image_tag)}") + else: + if self.__image_tag is None: + logging.info(f"Pulling image {_image_name(self.__image_tag)}") + DockerCommand('docker').run( + config, args, ['pull', _image_name(self.__image_tag)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + + else: + logging.info(f"Building image {_image_name(self.__image_tag)}") + DockerCommand('docker').run( + config, args, + [ + 'build', + '-t', _image_name(self.__image_tag), + '-f', f"{KIWI_ROOT}/images/{self.__image_tag}.Dockerfile", + f"{KIWI_ROOT}/images" + ], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + + def run(self, config, args, process_args, **kwargs): + self.__build_image(config, args) + DockerCommand('docker').run( + config, args, + [ + 'run', '--rm', + '-v', '/:/mnt', + '-u', 'root', + _image_name(self.__image_tag), + *process_args + ], + **kwargs + ) + + __image_tag = None + __instances = {} + + def __init__(self, image_tag=None): + self.__image_tag = image_tag + + if _image_name(self.__image_tag) not in Rootkit.__instances: + Rootkit.__instances[_image_name(self.__image_tag)] = Rootkit.__Rootkit(image_tag) + + def __getattr__(self, item): + return getattr(self.__instances[_image_name(self.__image_tag)], item) From 5b489d039279641f0fdbbf0f0ee473a8b94b9166 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 12:24:55 +0200 Subject: [PATCH 072/118] "copy-conf" -> "conf-copy", subcommand "conf-purge", constants centralization --- src/kiwi/_constants.py | 26 ++++++++++++ src/kiwi/config.py | 10 +---- src/kiwi/subcommands/__init__.py | 5 ++- .../subcommands/{copy_conf.py => conf.py} | 41 +++++++++++++------ src/kiwi/subcommands/utils/dockercommand.py | 4 +- src/kiwi/subcommands/utils/rootkit.py | 24 +++++------ 6 files changed, 72 insertions(+), 38 deletions(-) rename src/kiwi/subcommands/{copy_conf.py => conf.py} (50%) diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py index 836bedc..ade6c87 100644 --- a/src/kiwi/_constants.py +++ b/src/kiwi/_constants.py @@ -1,7 +1,33 @@ # system import os + +############# +# ENVIRONMENT + # location of "src" directory to use KIWI_ROOT = os.getenv('KIWI_ROOT', ".") # default name of kiwi-config file KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") + + +############ +# FILE NAMES + +# text files inside kiwi-config "src" directory +HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_header.yml" +DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_default.yml" +VERSION_TAG_NAME = f"{KIWI_ROOT}/version-tag" + +# special config directory in projects +CONF_DIRECTORY_NAME = 'conf' +# location for auxiliary Dockerfiles +IMAGES_DIRECTORY_NAME = f"{KIWI_ROOT}/images" + + +#################### +# DOCKER IMAGE NAMES + +# name for auxiliary docker images +LOCAL_IMAGES_NAME = 'localhost/kiwi-config/auxiliary' +DEFAULT_IMAGE_NAME = 'alpine:latest' diff --git a/src/kiwi/config.py b/src/kiwi/config.py index dcbe94a..f557c10 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -6,15 +6,7 @@ import re import yaml # local -from ._constants import KIWI_ROOT, KIWI_CONF_NAME - -########### -# CONSTANTS - -# text files inside kiwi-config "src" directory -HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_header.yml" -DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_default.yml" -VERSION_TAG_NAME = f"{KIWI_ROOT}/version-tag" +from ._constants import KIWI_CONF_NAME, HEADER_KIWI_CONF_NAME, DEFAULT_KIWI_CONF_NAME, VERSION_TAG_NAME class Config: diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index e31437b..6b37a7e 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,6 +1,6 @@ # local from .cmd import CmdCommand -from .copy_conf import CopyConfCommand +from .conf import ConfCopyCommand, ConfPurgeCommand from .down import DownCommand from .init import InitCommand from .logs import LogsCommand @@ -11,7 +11,8 @@ from .up import UpCommand __all__ = [ 'CmdCommand', - 'CopyConfCommand', + 'ConfCopyCommand', + 'ConfPurgeCommand', 'DownCommand', 'InitCommand', 'LogsCommand', diff --git a/src/kiwi/subcommands/copy_conf.py b/src/kiwi/subcommands/conf.py similarity index 50% rename from src/kiwi/subcommands/copy_conf.py rename to src/kiwi/subcommands/conf.py index db694ea..98c8bf2 100644 --- a/src/kiwi/subcommands/copy_conf.py +++ b/src/kiwi/subcommands/conf.py @@ -4,26 +4,20 @@ import os import subprocess # parent -from .._constants import KIWI_ROOT +from .._constants import CONF_DIRECTORY_NAME # local from ._subcommand import SubCommand -from .utils.dockercommand import DockerCommand from .utils.project import list_projects, get_project_dir -from .utils.rootkit import Rootkit, prefix_path +from .utils.rootkit import Rootkit, _prefix_path_mnt -def _add_prefix(prefix, path): - abs_path = os.path.abspath(path) - return os.path.realpath(prefix + '/' + abs_path) - - -class CopyConfCommand(SubCommand): - """kiwi copy-conf""" +class ConfCopyCommand(SubCommand): + """kiwi conf-copy""" def __init__(self): super().__init__( - 'copy-conf', + 'conf-copy', description="Synchronize all config files to target directory" ) @@ -31,7 +25,7 @@ class CopyConfCommand(SubCommand): conf_dirs = [] for project_name in list_projects(config): - project_conf = f"{get_project_dir(config, project_name)}/conf" + project_conf = f"{get_project_dir(config, project_name)}/{CONF_DIRECTORY_NAME}" if os.path.isdir(project_conf): conf_dirs.append(project_conf) @@ -42,8 +36,29 @@ class CopyConfCommand(SubCommand): logging.info(f"Sync directories: {conf_dirs}") Rootkit('rsync').run( - config, args, ['rsync', '-r', *prefix_path(conf_dirs)], + config, args, ['rsync', '-r', *_prefix_path_mnt(conf_dirs)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) return True + + +class ConfPurgeCommand(SubCommand): + """kiwi conf-purge""" + + def __init__(self): + super().__init__( + 'conf-purge', + description="Remove all config files in target directory" + ) + + def run(self, runner, config, args): + conf_target = f"{config['runtime:storage']}/{CONF_DIRECTORY_NAME}" + logging.info(f"Purging directories: {conf_target}") + + Rootkit().run( + config, args, ['rm', '-rf', _prefix_path_mnt(conf_target)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + + return True diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index d0426bd..eb631c7 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -1,9 +1,9 @@ # system import logging -import os import subprocess # local +from ..._constants import CONF_DIRECTORY_NAME from .executable import Executable from .project import * @@ -23,7 +23,7 @@ def _update_kwargs(config, args, **kwargs): kwargs['env'].update({ 'COMPOSE_PROJECT_NAME': project_name, 'KIWI_HUB_NAME': config['network:name'], - 'CONFDIR': os.path.join(config['runtime:storage'], 'conf'), + 'CONFDIR': os.path.join(config['runtime:storage'], CONF_DIRECTORY_NAME), 'TARGETDIR': os.path.join(config['runtime:storage'], get_project_dir(config, project_name)) }) diff --git a/src/kiwi/subcommands/utils/rootkit.py b/src/kiwi/subcommands/utils/rootkit.py index c18e5ac..c9cbf96 100644 --- a/src/kiwi/subcommands/utils/rootkit.py +++ b/src/kiwi/subcommands/utils/rootkit.py @@ -4,29 +4,29 @@ import os import subprocess # parent -from ..._constants import KIWI_ROOT +from ..._constants import IMAGES_DIRECTORY_NAME, LOCAL_IMAGES_NAME, DEFAULT_IMAGE_NAME # local from .dockercommand import DockerCommand def _prefix_path(prefix, path): - abs_path = os.path.abspath(path) - return os.path.realpath(prefix + '/' + abs_path) - - -def prefix_path(path): if isinstance(path, str): - return _prefix_path('/mnt/', path) + abs_path = os.path.abspath(path) + return os.path.realpath(f"{prefix}/{abs_path}") elif isinstance(path, list): - return [_prefix_path('/mnt/', p) for p in path] + return [_prefix_path(prefix, p) for p in path] + + +def _prefix_path_mnt(path): + return _prefix_path('/mnt/', path) def _image_name(image_tag): if image_tag is not None: - return f"kiwi-config/auxiliary:{image_tag}" + return f"{LOCAL_IMAGES_NAME}:{image_tag}" else: - return "alpine:latest" + return DEFAULT_IMAGE_NAME class Rootkit: @@ -66,8 +66,8 @@ class Rootkit: [ 'build', '-t', _image_name(self.__image_tag), - '-f', f"{KIWI_ROOT}/images/{self.__image_tag}.Dockerfile", - f"{KIWI_ROOT}/images" + '-f', f"{IMAGES_DIRECTORY_NAME}/{self.__image_tag}.Dockerfile", + f"{IMAGES_DIRECTORY_NAME}" ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) From 1fda26ef4f3612ff8128336919d22e8bee3874dc Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 12:28:59 +0200 Subject: [PATCH 073/118] imports --- src/kiwi/subcommands/conf.py | 12 ++++++------ src/kiwi/subcommands/init.py | 6 +++--- src/kiwi/subcommands/utils/dockercommand.py | 4 +++- src/kiwi/subcommands/utils/rootkit.py | 8 ++++---- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index 98c8bf2..9c0682d 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -3,13 +3,13 @@ import logging import os import subprocess -# parent -from .._constants import CONF_DIRECTORY_NAME - # local from ._subcommand import SubCommand from .utils.project import list_projects, get_project_dir -from .utils.rootkit import Rootkit, _prefix_path_mnt +from .utils.rootkit import Rootkit, prefix_path_mnt + +# parent +from .._constants import CONF_DIRECTORY_NAME class ConfCopyCommand(SubCommand): @@ -36,7 +36,7 @@ class ConfCopyCommand(SubCommand): logging.info(f"Sync directories: {conf_dirs}") Rootkit('rsync').run( - config, args, ['rsync', '-r', *_prefix_path_mnt(conf_dirs)], + config, args, ['rsync', '-r', *prefix_path_mnt(conf_dirs)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) @@ -57,7 +57,7 @@ class ConfPurgeCommand(SubCommand): logging.info(f"Purging directories: {conf_target}") Rootkit().run( - config, args, ['rm', '-rf', _prefix_path_mnt(conf_target)], + config, args, ['rm', '-rf', prefix_path_mnt(conf_target)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index a8ecee6..1f3714b 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -2,12 +2,12 @@ import logging import os -# parent (display purposes only) -from .._constants import KIWI_CONF_NAME - # local from ._subcommand import SubCommand +# parent (display purposes only) +from .._constants import KIWI_CONF_NAME + def user_input(config, key, prompt): """query user for new config value""" diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index eb631c7..6ef40b6 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -3,10 +3,12 @@ import logging import subprocess # local -from ..._constants import CONF_DIRECTORY_NAME from .executable import Executable from .project import * +# parent +from ..._constants import CONF_DIRECTORY_NAME + def _update_kwargs(config, args, **kwargs): # project given in args: command affects a project in this instance diff --git a/src/kiwi/subcommands/utils/rootkit.py b/src/kiwi/subcommands/utils/rootkit.py index c9cbf96..b3686c7 100644 --- a/src/kiwi/subcommands/utils/rootkit.py +++ b/src/kiwi/subcommands/utils/rootkit.py @@ -3,12 +3,12 @@ import logging import os import subprocess -# parent -from ..._constants import IMAGES_DIRECTORY_NAME, LOCAL_IMAGES_NAME, DEFAULT_IMAGE_NAME - # local from .dockercommand import DockerCommand +# parent +from ..._constants import IMAGES_DIRECTORY_NAME, LOCAL_IMAGES_NAME, DEFAULT_IMAGE_NAME + def _prefix_path(prefix, path): if isinstance(path, str): @@ -18,7 +18,7 @@ def _prefix_path(prefix, path): return [_prefix_path(prefix, p) for p in path] -def _prefix_path_mnt(path): +def prefix_path_mnt(path): return _prefix_path('/mnt/', path) From 79ecca2322bad16844c19a4a73b9e55ddedecbea Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 12:34:09 +0200 Subject: [PATCH 074/118] dir structure --- src/{ => etc}/kiwi_default.yml | 0 src/{ => etc}/kiwi_header.yml | 0 src/{version-tag => etc/version_tag} | 0 src/kiwi/_constants.py | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename src/{ => etc}/kiwi_default.yml (100%) rename src/{ => etc}/kiwi_header.yml (100%) rename src/{version-tag => etc/version_tag} (100%) diff --git a/src/kiwi_default.yml b/src/etc/kiwi_default.yml similarity index 100% rename from src/kiwi_default.yml rename to src/etc/kiwi_default.yml diff --git a/src/kiwi_header.yml b/src/etc/kiwi_header.yml similarity index 100% rename from src/kiwi_header.yml rename to src/etc/kiwi_header.yml diff --git a/src/version-tag b/src/etc/version_tag similarity index 100% rename from src/version-tag rename to src/etc/version_tag diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py index ade6c87..ef2e6b5 100644 --- a/src/kiwi/_constants.py +++ b/src/kiwi/_constants.py @@ -15,9 +15,9 @@ KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") # FILE NAMES # text files inside kiwi-config "src" directory -HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_header.yml" -DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/kiwi_default.yml" -VERSION_TAG_NAME = f"{KIWI_ROOT}/version-tag" +HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_header.yml" +DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_default.yml" +VERSION_TAG_NAME = f"{KIWI_ROOT}/etc/version_tag" # special config directory in projects CONF_DIRECTORY_NAME = 'conf' From 6615982e95137239c8826db40977bbf2f3d8503f Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 13:50:21 +0200 Subject: [PATCH 075/118] FlexCommand class --- src/kiwi/subcommands/_subcommand.py | 39 +++++++++++++++++++++ src/kiwi/subcommands/down.py | 53 +++++++++++++---------------- src/kiwi/subcommands/up.py | 37 ++++++++------------ 3 files changed, 77 insertions(+), 52 deletions(-) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index e7c0274..2ff6d20 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -1,3 +1,6 @@ +# local +from .utils.project import get_project_name, list_projects + # parent from ..parser import Parser @@ -63,3 +66,39 @@ class ServiceCommand(ProjectCommand): 'services', metavar='service', nargs=num_services, type=str, help=f"select {services} in a project" ) + + +class FlexCommand(ServiceCommand): + def __init__(self, name, **kwargs): + super().__init__( + name, num_projects='?', num_services='*', + **kwargs + ) + + def _run_instance(self, runner, config, args): + result = True + + for project_name in list_projects(config): + args.projects = project_name + result &= runner.run(str(self)) + + return result + + def _run_project(self, runner, config, args): + pass + + def _run_services(self, runner, config, args): + pass + + def run(self, runner, config, args): + if 'projects' not in args or args.projects is None: + # command for entire instance + return self._run_instance(runner, config, args) + + elif 'services' not in args or not args.services: + # command for whole project + return self._run_project(runner, config, args) + + else: + # command for service(s) inside project + return self._run_services(runner, config, args) diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index 790f3bb..34c8681 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -2,48 +2,41 @@ import logging # local -from ._subcommand import ServiceCommand +from ._subcommand import FlexCommand from .utils.dockercommand import DockerCommand from .utils.project import get_project_name, list_projects from .utils.user_input import are_you_sure -class DownCommand(ServiceCommand): +class DownCommand(FlexCommand): """kiwi down""" def __init__(self): super().__init__( - 'down', num_projects='?', num_services='*', - description="Bring down the whole instance, a project or service(s) inside a project" + 'down', description="Bring down the whole instance, a project or service(s) inside a project" ) - def run(self, runner, config, args): - if 'projects' not in args or args.projects is None: - # "down" for all projects - if are_you_sure("This will bring down the entire instance."): - for project_name in list_projects(config): - args.projects = project_name - runner.run('down') - else: - return False + def _run_instance(self, runner, config, args): + if are_you_sure("This will bring down the entire instance."): + super()._run_instance(runner, config, args) - elif 'services' in args and args.services: - # "down" for service(s) inside project - logging.info(f"Bringing down services {args.services} in project '{get_project_name(args)}'") + return False - DockerCommand('docker-compose').run( - config, args, ['stop', *args.services] - ) - DockerCommand('docker-compose').run( - config, args, ['rm', '-f', *args.services] - ) - - else: - # "down" for project - logging.info(f"Bringing down project '{get_project_name(args)}'") - - DockerCommand('docker-compose').run( - config, args, ['down'] - ) + def _run_project(self, runner, config, args): + logging.info(f"Bringing down project '{get_project_name(args)}'") + DockerCommand('docker-compose').run( + config, args, ['down'] + ) + return True + + def _run_services(self, runner, config, args): + logging.info(f"Bringing down services {args.services} in project '{get_project_name(args)}'") + + DockerCommand('docker-compose').run( + config, args, ['stop', *args.services] + ) + DockerCommand('docker-compose').run( + config, args, ['rm', '-f', *args.services] + ) return True diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index 0681d83..1a9b61a 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -2,38 +2,31 @@ import logging # local -from ._subcommand import ServiceCommand +from ._subcommand import FlexCommand from .utils.dockercommand import DockerCommand from .utils.project import get_project_name, list_projects -class UpCommand(ServiceCommand): +class UpCommand(FlexCommand): """kiwi up""" def __init__(self): super().__init__( - 'up', num_projects='?', num_services='*', - description="Bring up the whole instance, a project or service(s) inside a project" + 'up', description="Bring up the whole instance, a project or service(s) inside a project" ) - def run(self, runner, config, args): - if 'projects' not in args or args.projects is None: - # "up" for all projects - for project_name in list_projects(config): - args.projects = project_name - runner.run('up') + def _run_project(self, runner, config, args): + logging.info(f"Bringing up project '{get_project_name(args)}'") - return + DockerCommand('docker-compose').run( + config, args, ['up', '-d'] + ) + return True - if 'services' in args and args.services: - logging.info(f"Bringing up services {args.services} in project '{get_project_name(args)}'") - else: - logging.info(f"Bringing up project '{get_project_name(args)}'") + def _run_services(self, runner, config, args): + logging.info(f"Bringing up services {args.services} in project '{get_project_name(args)}'") - if runner.run('net-up'): - DockerCommand('docker-compose').run( - config, args, ['up', '-d', *args.services] - ) - return True - - return False + DockerCommand('docker-compose').run( + config, args, ['up', '-d', *args.services] + ) + return True From c336de4d9c70a71b2a20a342993ff0e3927a250c Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 14:04:10 +0200 Subject: [PATCH 076/118] FlexCommand: project_name / services in params --- src/kiwi/subcommands/_subcommand.py | 26 +++++++------ src/kiwi/subcommands/conf.py | 2 +- src/kiwi/subcommands/down.py | 17 ++++---- src/kiwi/subcommands/net.py | 2 +- src/kiwi/subcommands/up.py | 11 +++--- .../utils/{project.py => _misc.py} | 39 +++++++++++++++++++ src/kiwi/subcommands/utils/dockercommand.py | 3 +- src/kiwi/subcommands/utils/user_input.py | 29 -------------- 8 files changed, 71 insertions(+), 58 deletions(-) rename src/kiwi/subcommands/utils/{project.py => _misc.py} (53%) delete mode 100644 src/kiwi/subcommands/utils/user_input.py diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index 2ff6d20..66687c6 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -1,5 +1,5 @@ # local -from .utils.project import get_project_name, list_projects +from .utils._misc import get_project_name, get_services, list_projects # parent from ..parser import Parser @@ -69,6 +69,8 @@ class ServiceCommand(ProjectCommand): class FlexCommand(ServiceCommand): + """this command concerns the entire instance, a whole project or just service(s) in a project""" + def __init__(self, name, **kwargs): super().__init__( name, num_projects='?', num_services='*', @@ -84,21 +86,23 @@ class FlexCommand(ServiceCommand): return result - def _run_project(self, runner, config, args): + def _run_project(self, runner, config, args, project_name): pass - def _run_services(self, runner, config, args): + def _run_services(self, runner, config, args, project_name, services): pass def run(self, runner, config, args): - if 'projects' not in args or args.projects is None: - # command for entire instance + project_name = get_project_name(args) + services = get_services(args) + + if project_name is None: + # no project given, run for entire instance return self._run_instance(runner, config, args) - elif 'services' not in args or not args.services: - # command for whole project - return self._run_project(runner, config, args) + if services is None: + # no services given, run for whole project + return self._run_project(runner, config, args, project_name) - else: - # command for service(s) inside project - return self._run_services(runner, config, args) + # run for service(s) inside project + return self._run_services(runner, config, args, project_name, services) diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index 9c0682d..ea2981d 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -5,7 +5,7 @@ import subprocess # local from ._subcommand import SubCommand -from .utils.project import list_projects, get_project_dir +from .utils._misc import list_projects, get_project_dir from .utils.rootkit import Rootkit, prefix_path_mnt # parent diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index 34c8681..6a9fc7d 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -4,8 +4,7 @@ import logging # local from ._subcommand import FlexCommand from .utils.dockercommand import DockerCommand -from .utils.project import get_project_name, list_projects -from .utils.user_input import are_you_sure +from .utils._misc import are_you_sure class DownCommand(FlexCommand): @@ -18,25 +17,25 @@ class DownCommand(FlexCommand): def _run_instance(self, runner, config, args): if are_you_sure("This will bring down the entire instance."): - super()._run_instance(runner, config, args) + return super()._run_instance(runner, config, args) return False - def _run_project(self, runner, config, args): - logging.info(f"Bringing down project '{get_project_name(args)}'") + def _run_project(self, runner, config, args, project_name): + logging.info(f"Bringing down project '{project_name}'") DockerCommand('docker-compose').run( config, args, ['down'] ) return True - def _run_services(self, runner, config, args): - logging.info(f"Bringing down services {args.services} in project '{get_project_name(args)}'") + def _run_services(self, runner, config, args, project_name, services): + logging.info(f"Bringing down services {services} in project '{project_name}'") DockerCommand('docker-compose').run( - config, args, ['stop', *args.services] + config, args, ['stop', *services] ) DockerCommand('docker-compose').run( - config, args, ['rm', '-f', *args.services] + config, args, ['rm', '-f', *services] ) return True diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 4a5e488..10bd92c 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -5,7 +5,7 @@ import subprocess # local from ._subcommand import SubCommand from .utils.dockercommand import DockerCommand -from .utils.user_input import are_you_sure +from .utils._misc import are_you_sure def _find_net(config, args): diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index 1a9b61a..a1dedbb 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -4,7 +4,6 @@ import logging # local from ._subcommand import FlexCommand from .utils.dockercommand import DockerCommand -from .utils.project import get_project_name, list_projects class UpCommand(FlexCommand): @@ -15,18 +14,18 @@ class UpCommand(FlexCommand): 'up', description="Bring up the whole instance, a project or service(s) inside a project" ) - def _run_project(self, runner, config, args): - logging.info(f"Bringing up project '{get_project_name(args)}'") + def _run_project(self, runner, config, args, project_name): + logging.info(f"Bringing up project '{project_name}'") DockerCommand('docker-compose').run( config, args, ['up', '-d'] ) return True - def _run_services(self, runner, config, args): - logging.info(f"Bringing up services {args.services} in project '{get_project_name(args)}'") + def _run_services(self, runner, config, args, project_name, services): + logging.info(f"Bringing up services {services} in project '{project_name}'") DockerCommand('docker-compose').run( - config, args, ['up', '-d', *args.services] + config, args, ['up', '-d', *services] ) return True diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/subcommands/utils/_misc.py similarity index 53% rename from src/kiwi/subcommands/utils/project.py rename to src/kiwi/subcommands/utils/_misc.py index 12f9a9a..c9aeee3 100644 --- a/src/kiwi/subcommands/utils/project.py +++ b/src/kiwi/subcommands/utils/_misc.py @@ -13,6 +13,15 @@ def get_project_name(args): return None +def get_services(args): + """get services list from CLI args""" + + if args is not None and 'services' in args: + return args.services + + return None + + def get_project_dir(config, project_name): """get project directory""" @@ -41,3 +50,33 @@ def list_projects(config): projects = [p[:-len(config['markers:project'])] for p in project_dirs] return projects + + +def _surround(string, bang): + midlane = f"{bang * 3} {string} {bang * 3}" + sidelane = bang*len(midlane) + + return f"{sidelane}\n{midlane}\n{sidelane}" + + +def are_you_sure(prompt, default="no"): + if default.lower() == 'yes': + suffix = "[YES|no]" + else: + suffix = "[yes|NO]" + + answer = input( + f"{_surround('CAREFULING IN PROGRESS', '!')}\n" + f"\n" + f">>> {prompt} <<<\n" + f"\n" + f"Are you sure you want to proceed? {suffix} " + ).strip().lower() + + if answer == '': + answer = default + + while answer not in ['yes', 'no']: + answer = input("Please type 'yes' or 'no' explicitly: ").strip().lower() + + return answer == 'yes' diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 6ef40b6..5b1bebe 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -1,10 +1,11 @@ # system import logging +import os import subprocess # local from .executable import Executable -from .project import * +from ._misc import get_project_dir, get_project_name # parent from ..._constants import CONF_DIRECTORY_NAME diff --git a/src/kiwi/subcommands/utils/user_input.py b/src/kiwi/subcommands/utils/user_input.py deleted file mode 100644 index 51b71e5..0000000 --- a/src/kiwi/subcommands/utils/user_input.py +++ /dev/null @@ -1,29 +0,0 @@ - -def _surround(string, bang): - midlane = f"{bang * 3} {string} {bang * 3}" - sidelane = bang*len(midlane) - - return f"{sidelane}\n{midlane}\n{sidelane}" - - -def are_you_sure(prompt, default="no"): - if default.lower() == 'yes': - suffix = "[YES|no]" - else: - suffix = "[yes|NO]" - - answer = input( - f"{_surround('CAREFULING IN PROGRESS', '!')}\n" - f"\n" - f">>> {prompt} <<<\n" - f"\n" - f"Are you sure you want to proceed? {suffix} " - ).strip().lower() - - if answer == '': - answer = default - - while answer not in ['yes', 'no']: - answer = input("Please type 'yes' or 'no' explicitly: ").strip().lower() - - return answer == 'yes' From 6976480dd7b221a576e3f4cfb5c0d25830b20679 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 14:18:54 +0200 Subject: [PATCH 077/118] FlexCommand automatic logging with action creation param --- src/kiwi/subcommands/_subcommand.py | 28 ++++++++++++++----- src/kiwi/subcommands/conf.py | 2 +- src/kiwi/subcommands/down.py | 13 ++++----- src/kiwi/subcommands/net.py | 2 +- src/kiwi/subcommands/up.py | 11 +++----- src/kiwi/subcommands/utils/dockercommand.py | 2 +- .../subcommands/utils/{_misc.py => misc.py} | 0 7 files changed, 33 insertions(+), 25 deletions(-) rename src/kiwi/subcommands/utils/{_misc.py => misc.py} (100%) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index 66687c6..61db871 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -1,5 +1,8 @@ +# system +import logging + # local -from .utils._misc import get_project_name, get_services, list_projects +from .utils.misc import get_project_name, get_services, list_projects # parent from ..parser import Parser @@ -71,12 +74,20 @@ class ServiceCommand(ProjectCommand): class FlexCommand(ServiceCommand): """this command concerns the entire instance, a whole project or just service(s) in a project""" - def __init__(self, name, **kwargs): + __action = None + + def __init__(self, name, action='', **kwargs): super().__init__( name, num_projects='?', num_services='*', **kwargs ) + if not action: + # default action string + self.__action = f"Running '{str(self)}' for" + else: + self.__action = action + def _run_instance(self, runner, config, args): result = True @@ -86,10 +97,10 @@ class FlexCommand(ServiceCommand): return result - def _run_project(self, runner, config, args, project_name): + def _run_project(self, runner, config, args): pass - def _run_services(self, runner, config, args, project_name, services): + def _run_services(self, runner, config, args, services): pass def run(self, runner, config, args): @@ -98,11 +109,14 @@ class FlexCommand(ServiceCommand): if project_name is None: # no project given, run for entire instance + logging.info(f"{self.__action} this instance") return self._run_instance(runner, config, args) - if services is None: + if not services: # no services given, run for whole project - return self._run_project(runner, config, args, project_name) + logging.info(f"{self.__action} project '{project_name}'") + return self._run_project(runner, config, args) # run for service(s) inside project - return self._run_services(runner, config, args, project_name, services) + logging.info(f"{self.__action} services {services} in project '{project_name}'") + return self._run_services(runner, config, args, services) diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index ea2981d..702261e 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -5,7 +5,7 @@ import subprocess # local from ._subcommand import SubCommand -from .utils._misc import list_projects, get_project_dir +from .utils.misc import list_projects, get_project_dir from .utils.rootkit import Rootkit, prefix_path_mnt # parent diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index 6a9fc7d..2950cdd 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -4,7 +4,7 @@ import logging # local from ._subcommand import FlexCommand from .utils.dockercommand import DockerCommand -from .utils._misc import are_you_sure +from .utils.misc import are_you_sure class DownCommand(FlexCommand): @@ -12,7 +12,8 @@ class DownCommand(FlexCommand): def __init__(self): super().__init__( - 'down', description="Bring down the whole instance, a project or service(s) inside a project" + 'down', "Bringing down", + description="Bring down the whole instance, a project or service(s) inside a project" ) def _run_instance(self, runner, config, args): @@ -21,17 +22,13 @@ class DownCommand(FlexCommand): return False - def _run_project(self, runner, config, args, project_name): - logging.info(f"Bringing down project '{project_name}'") - + def _run_project(self, runner, config, args): DockerCommand('docker-compose').run( config, args, ['down'] ) return True - def _run_services(self, runner, config, args, project_name, services): - logging.info(f"Bringing down services {services} in project '{project_name}'") - + def _run_services(self, runner, config, args, services): DockerCommand('docker-compose').run( config, args, ['stop', *services] ) diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 10bd92c..6c007f8 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -5,7 +5,7 @@ import subprocess # local from ._subcommand import SubCommand from .utils.dockercommand import DockerCommand -from .utils._misc import are_you_sure +from .utils.misc import are_you_sure def _find_net(config, args): diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index a1dedbb..1bf8c7c 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -11,20 +11,17 @@ class UpCommand(FlexCommand): def __init__(self): super().__init__( - 'up', description="Bring up the whole instance, a project or service(s) inside a project" + 'up', "Bringing up", + description="Bring up the whole instance, a project or service(s) inside a project" ) - def _run_project(self, runner, config, args, project_name): - logging.info(f"Bringing up project '{project_name}'") - + def _run_project(self, runner, config, args): DockerCommand('docker-compose').run( config, args, ['up', '-d'] ) return True - def _run_services(self, runner, config, args, project_name, services): - logging.info(f"Bringing up services {services} in project '{project_name}'") - + def _run_services(self, runner, config, args, services): DockerCommand('docker-compose').run( config, args, ['up', '-d', *services] ) diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 5b1bebe..6093dfd 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -5,7 +5,7 @@ import subprocess # local from .executable import Executable -from ._misc import get_project_dir, get_project_name +from .misc import get_project_dir, get_project_name # parent from ..._constants import CONF_DIRECTORY_NAME diff --git a/src/kiwi/subcommands/utils/_misc.py b/src/kiwi/subcommands/utils/misc.py similarity index 100% rename from src/kiwi/subcommands/utils/_misc.py rename to src/kiwi/subcommands/utils/misc.py From 6f99de30d0da3c3e7dbb4eacac8b02c6cf0eac06 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 14:23:56 +0200 Subject: [PATCH 078/118] Regression: Run net-up automatically --- src/kiwi/subcommands/up.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index 1bf8c7c..f8290a6 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -16,13 +16,19 @@ class UpCommand(FlexCommand): ) def _run_project(self, runner, config, args): - DockerCommand('docker-compose').run( - config, args, ['up', '-d'] - ) - return True + if runner.run('net-up'): + DockerCommand('docker-compose').run( + config, args, ['up', '-d'] + ) + return True + + return False def _run_services(self, runner, config, args, services): - DockerCommand('docker-compose').run( - config, args, ['up', '-d', *services] - ) - return True + if runner.run('net-up'): + DockerCommand('docker-compose').run( + config, args, ['up', '-d', *services] + ) + return True + + return False From c2d34aeaffab703549b536433efc4b3c2ba070d3 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 14:28:03 +0200 Subject: [PATCH 079/118] "up" for project same command --- src/kiwi/subcommands/_subcommand.py | 2 +- src/kiwi/subcommands/down.py | 3 --- src/kiwi/subcommands/up.py | 12 ------------ 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index 61db871..75d8d0c 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -98,7 +98,7 @@ class FlexCommand(ServiceCommand): return result def _run_project(self, runner, config, args): - pass + return self._run_services(runner, config, args, []) def _run_services(self, runner, config, args, services): pass diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index 2950cdd..013f6a4 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -1,6 +1,3 @@ -# system -import logging - # local from ._subcommand import FlexCommand from .utils.dockercommand import DockerCommand diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index f8290a6..0c16713 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -1,6 +1,3 @@ -# system -import logging - # local from ._subcommand import FlexCommand from .utils.dockercommand import DockerCommand @@ -15,15 +12,6 @@ class UpCommand(FlexCommand): description="Bring up the whole instance, a project or service(s) inside a project" ) - def _run_project(self, runner, config, args): - if runner.run('net-up'): - DockerCommand('docker-compose').run( - config, args, ['up', '-d'] - ) - return True - - return False - def _run_services(self, runner, config, args, services): if runner.run('net-up'): DockerCommand('docker-compose').run( From 679a47e4fa17a96d44bc7579cd82d449ab309d0e Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 14:37:07 +0200 Subject: [PATCH 080/118] subcommands "build", "pull", "update" --- src/kiwi/subcommands/__init__.py | 6 ++++++ src/kiwi/subcommands/build.py | 19 +++++++++++++++++++ src/kiwi/subcommands/pull.py | 19 +++++++++++++++++++ src/kiwi/subcommands/update.py | 21 +++++++++++++++++++++ 4 files changed, 65 insertions(+) create mode 100644 src/kiwi/subcommands/build.py create mode 100644 src/kiwi/subcommands/pull.py create mode 100644 src/kiwi/subcommands/update.py diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 6b37a7e..11e6dfc 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,15 +1,19 @@ # local +from .build import BuildCommand from .cmd import CmdCommand from .conf import ConfCopyCommand, ConfPurgeCommand from .down import DownCommand from .init import InitCommand from .logs import LogsCommand from .net import NetUpCommand, NetDownCommand +from .pull import PullCommand from .sh import ShCommand from .show import ShowCommand from .up import UpCommand +from .update import UpdateCommand __all__ = [ + 'BuildCommand', 'CmdCommand', 'ConfCopyCommand', 'ConfPurgeCommand', @@ -18,7 +22,9 @@ __all__ = [ 'LogsCommand', 'NetUpCommand', 'NetDownCommand', + 'PullCommand', 'ShCommand', 'ShowCommand', 'UpCommand', + 'UpdateCommand', ] diff --git a/src/kiwi/subcommands/build.py b/src/kiwi/subcommands/build.py new file mode 100644 index 0000000..bc13757 --- /dev/null +++ b/src/kiwi/subcommands/build.py @@ -0,0 +1,19 @@ +# local +from ._subcommand import FlexCommand +from .utils.dockercommand import DockerCommand + + +class BuildCommand(FlexCommand): + """kiwi build""" + + def __init__(self): + super().__init__( + 'build', "Building images", + description="Build images for the whole instance, a project or service(s) inside a project" + ) + + def _run_services(self, runner, config, args, services): + DockerCommand('docker-compose').run( + config, args, ['build', '--pull', *services] + ) + return True diff --git a/src/kiwi/subcommands/pull.py b/src/kiwi/subcommands/pull.py new file mode 100644 index 0000000..3497517 --- /dev/null +++ b/src/kiwi/subcommands/pull.py @@ -0,0 +1,19 @@ +# local +from ._subcommand import FlexCommand +from .utils.dockercommand import DockerCommand + + +class PullCommand(FlexCommand): + """kiwi pull""" + + def __init__(self): + super().__init__( + 'pull', "Pulling images", + description="Pull images for the whole instance, a project or service(s) inside a project" + ) + + def _run_services(self, runner, config, args, services): + DockerCommand('docker-compose').run( + config, args, ['pull', '--ignore-pull-failures', *services] + ) + return True diff --git a/src/kiwi/subcommands/update.py b/src/kiwi/subcommands/update.py new file mode 100644 index 0000000..045c192 --- /dev/null +++ b/src/kiwi/subcommands/update.py @@ -0,0 +1,21 @@ +# local +from ._subcommand import FlexCommand + + +class UpdateCommand(FlexCommand): + """kiwi update""" + + def __init__(self): + super().__init__( + 'update', "Updating", + description="Update the whole instance, a project or service(s) inside a project" + ) + + def _run_services(self, runner, config, args, services): + result = runner.run('build') + result &= runner.run('pull') + result &= runner.run('conf-copy') + result &= runner.run('down') + result &= runner.run('up') + + return result From 81b597cb586d78dfd06f4ddb39704139f5ae745f Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 15:05:19 +0200 Subject: [PATCH 081/118] neater prompts --- src/kiwi/subcommands/down.py | 7 ++++++- src/kiwi/subcommands/update.py | 13 +++++++++++++ src/kiwi/subcommands/utils/misc.py | 12 ++++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index 013f6a4..ebb4a4c 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -14,7 +14,12 @@ class DownCommand(FlexCommand): ) def _run_instance(self, runner, config, args): - if are_you_sure("This will bring down the entire instance."): + if are_you_sure([ + "This will bring down the entire instance.", + "", + "This may not be what you intended, because:", + " - Bringing down the instance stops ALL services in here", + ]): return super()._run_instance(runner, config, args) return False diff --git a/src/kiwi/subcommands/update.py b/src/kiwi/subcommands/update.py index 045c192..cfac6fe 100644 --- a/src/kiwi/subcommands/update.py +++ b/src/kiwi/subcommands/update.py @@ -1,5 +1,6 @@ # local from ._subcommand import FlexCommand +from .utils.misc import are_you_sure class UpdateCommand(FlexCommand): @@ -11,6 +12,18 @@ class UpdateCommand(FlexCommand): description="Update the whole instance, a project or service(s) inside a project" ) + def _run_instance(self, runner, config, args): + if are_you_sure([ + "This will update the entire instance at once.", + "", + "This is probably not what you intended, because:", + " - Updates may take a long time", + " - Updates may break beloved functionality", + ]): + return super()._run_instance(runner, config, args) + + return False + def _run_services(self, runner, config, args, services): result = runner.run('build') result &= runner.run('pull') diff --git a/src/kiwi/subcommands/utils/misc.py b/src/kiwi/subcommands/utils/misc.py index c9aeee3..93d5e9b 100644 --- a/src/kiwi/subcommands/utils/misc.py +++ b/src/kiwi/subcommands/utils/misc.py @@ -59,6 +59,14 @@ def _surround(string, bang): return f"{sidelane}\n{midlane}\n{sidelane}" +def _emphasize(lines): + if isinstance(lines, list): + return '\n'.join([_emphasize(line) for line in lines]) + elif lines: + return f">>> {lines} <<<" + else: + return lines + def are_you_sure(prompt, default="no"): if default.lower() == 'yes': suffix = "[YES|no]" @@ -66,9 +74,9 @@ def are_you_sure(prompt, default="no"): suffix = "[yes|NO]" answer = input( - f"{_surround('CAREFULING IN PROGRESS', '!')}\n" + f"{_surround('MUST HAVE CAREFULING IN PROGRESS', '!')}\n" f"\n" - f">>> {prompt} <<<\n" + f"{_emphasize(prompt)}\n" f"\n" f"Are you sure you want to proceed? {suffix} " ).strip().lower() From 9617597257f1a1f956f472e29b8fa4065a94eb87 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 15:12:08 +0200 Subject: [PATCH 082/118] subcommand "push" --- src/kiwi/subcommands/__init__.py | 2 ++ src/kiwi/subcommands/build.py | 2 +- src/kiwi/subcommands/pull.py | 2 +- src/kiwi/subcommands/push.py | 19 +++++++++++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 src/kiwi/subcommands/push.py diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 11e6dfc..864cea2 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -7,6 +7,7 @@ from .init import InitCommand from .logs import LogsCommand from .net import NetUpCommand, NetDownCommand from .pull import PullCommand +from .push import PushCommand from .sh import ShCommand from .show import ShowCommand from .up import UpCommand @@ -23,6 +24,7 @@ __all__ = [ 'NetUpCommand', 'NetDownCommand', 'PullCommand', + 'PushCommand', 'ShCommand', 'ShowCommand', 'UpCommand', diff --git a/src/kiwi/subcommands/build.py b/src/kiwi/subcommands/build.py index bc13757..d53c50d 100644 --- a/src/kiwi/subcommands/build.py +++ b/src/kiwi/subcommands/build.py @@ -8,7 +8,7 @@ class BuildCommand(FlexCommand): def __init__(self): super().__init__( - 'build', "Building images", + 'build', "Building images for", description="Build images for the whole instance, a project or service(s) inside a project" ) diff --git a/src/kiwi/subcommands/pull.py b/src/kiwi/subcommands/pull.py index 3497517..019c007 100644 --- a/src/kiwi/subcommands/pull.py +++ b/src/kiwi/subcommands/pull.py @@ -8,7 +8,7 @@ class PullCommand(FlexCommand): def __init__(self): super().__init__( - 'pull', "Pulling images", + 'pull', "Pulling images for", description="Pull images for the whole instance, a project or service(s) inside a project" ) diff --git a/src/kiwi/subcommands/push.py b/src/kiwi/subcommands/push.py new file mode 100644 index 0000000..f0b00ad --- /dev/null +++ b/src/kiwi/subcommands/push.py @@ -0,0 +1,19 @@ +# local +from ._subcommand import FlexCommand +from .utils.dockercommand import DockerCommand + + +class PushCommand(FlexCommand): + """kiwi push""" + + def __init__(self): + super().__init__( + 'push', "Pushing images for", + description="Push images for the whole instance, a project or service(s) inside a project" + ) + + def _run_services(self, runner, config, args, services): + DockerCommand('docker-compose').run( + config, args, ['push', *services] + ) + return True From 3014e7057cc9ac207657c8fbafc2534d89485188 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 15:56:22 +0200 Subject: [PATCH 083/118] readability --- src/kiwi/subcommands/_subcommand.py | 12 ++++---- src/kiwi/subcommands/utils/dockercommand.py | 4 +-- src/kiwi/subcommands/utils/misc.py | 33 ++++++++++++++++----- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index 75d8d0c..be50e67 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -2,7 +2,7 @@ import logging # local -from .utils.misc import get_project_name, get_services, list_projects +from .utils.misc import get_first_project_name, get_services, list_projects # parent from ..parser import Parser @@ -42,8 +42,8 @@ class ProjectCommand(SubCommand): projects = "a project" - if not str(num_projects) == '1': - projects = "projects" + if not num_projects == 1: + projects = "project(s)" self._sub_parser.add_argument( 'projects', metavar='project', nargs=num_projects, type=str, @@ -62,8 +62,8 @@ class ServiceCommand(ProjectCommand): services = "a service" - if not str(num_services) == '1': - services = "services" + if not num_services == 1: + services = "service(s)" self._sub_parser.add_argument( 'services', metavar='service', nargs=num_services, type=str, @@ -104,7 +104,7 @@ class FlexCommand(ServiceCommand): pass def run(self, runner, config, args): - project_name = get_project_name(args) + project_name = get_first_project_name(args) services = get_services(args) if project_name is None: diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 6093dfd..62ad1b0 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -5,7 +5,7 @@ import subprocess # local from .executable import Executable -from .misc import get_project_dir, get_project_name +from .misc import get_project_dir, get_first_project_name # parent from ..._constants import CONF_DIRECTORY_NAME @@ -13,7 +13,7 @@ from ..._constants import CONF_DIRECTORY_NAME def _update_kwargs(config, args, **kwargs): # project given in args: command affects a project in this instance - project_name = get_project_name(args) + project_name = get_first_project_name(args) if project_name is not None: # execute command in project directory kwargs['cwd'] = get_project_dir(config, project_name) diff --git a/src/kiwi/subcommands/utils/misc.py b/src/kiwi/subcommands/utils/misc.py index 93d5e9b..eb607fe 100644 --- a/src/kiwi/subcommands/utils/misc.py +++ b/src/kiwi/subcommands/utils/misc.py @@ -1,14 +1,27 @@ import os -def get_project_name(args): - """get project name from CLI args""" +def get_project_names(args): + """get project names from CLI args""" if args is not None and 'projects' in args: - if isinstance(args.projects, list) and len(args.projects) > 0: - return args.projects[0] - else: - return args.projects + if isinstance(args.projects, list): + if args.projects: + return args.projects + else: + return None + elif isinstance(args.projects, str): + return [args.projects] + + return None + + +def get_first_project_name(args): + """get first project name from CLI args""" + + names = get_project_names(args) + if names is not None: + return names[0] return None @@ -28,10 +41,16 @@ def get_project_dir(config, project_name): return f"{project_name}{config['markers:project']}" +def get_project_down_dir(config, project_name): + """get project directory""" + + return f"{get_project_dir(config, project_name)}{config['markers:down']}" + + def get_target_dir(config, project_name): """get project's target directory""" - return f"{config['runtime:storage']}{get_project_dir(config, project_name)}" + return os.path.join(config['runtime:storage'], get_project_dir(config, project_name)) def list_projects(config): From 1a98137c716d08d43d91cd322249e68ce606a875 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 15:56:58 +0200 Subject: [PATCH 084/118] subcommands "new", "enable", "disable" --- src/etc/docker-compose_default.yml | 19 +++++++++++++++ src/kiwi/_constants.py | 1 + src/kiwi/subcommands/__init__.py | 6 +++++ src/kiwi/subcommands/disable.py | 38 +++++++++++++++++++++++++++++ src/kiwi/subcommands/enable.py | 38 +++++++++++++++++++++++++++++ src/kiwi/subcommands/new.py | 39 ++++++++++++++++++++++++++++++ 6 files changed, 141 insertions(+) create mode 100644 src/etc/docker-compose_default.yml create mode 100644 src/kiwi/subcommands/disable.py create mode 100644 src/kiwi/subcommands/enable.py create mode 100644 src/kiwi/subcommands/new.py diff --git a/src/etc/docker-compose_default.yml b/src/etc/docker-compose_default.yml new file mode 100644 index 0000000..e6b96b6 --- /dev/null +++ b/src/etc/docker-compose_default.yml @@ -0,0 +1,19 @@ +version: "2" + +networks: + # reachable from outside + default: + driver: bridge + # interconnects projects + kiwi_hub: + external: + name: $KIWI_HUB_NAME + +services: + something: + image: maintainer/repo:tag + restart: unless-stopped + networks: + - default + - kiwi_hub + [...] diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py index ef2e6b5..715372e 100644 --- a/src/kiwi/_constants.py +++ b/src/kiwi/_constants.py @@ -18,6 +18,7 @@ KIWI_CONF_NAME = os.getenv('KIWI_CONF_NAME', "kiwi.yml") HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_header.yml" DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_default.yml" VERSION_TAG_NAME = f"{KIWI_ROOT}/etc/version_tag" +DEFAULT_DOCKER_COMPOSE_NAME = f"{KIWI_ROOT}/etc/docker-compose_default.yml" # special config directory in projects CONF_DIRECTORY_NAME = 'conf' diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 864cea2..ce70da2 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -2,10 +2,13 @@ from .build import BuildCommand from .cmd import CmdCommand from .conf import ConfCopyCommand, ConfPurgeCommand +from .disable import DisableCommand from .down import DownCommand +from .enable import EnableCommand from .init import InitCommand from .logs import LogsCommand from .net import NetUpCommand, NetDownCommand +from .new import NewCommand from .pull import PullCommand from .push import PushCommand from .sh import ShCommand @@ -18,11 +21,14 @@ __all__ = [ 'CmdCommand', 'ConfCopyCommand', 'ConfPurgeCommand', + 'DisableCommand', 'DownCommand', + 'EnableCommand', 'InitCommand', 'LogsCommand', 'NetUpCommand', 'NetDownCommand', + 'NewCommand', 'PullCommand', 'PushCommand', 'ShCommand', diff --git a/src/kiwi/subcommands/disable.py b/src/kiwi/subcommands/disable.py new file mode 100644 index 0000000..ccdd477 --- /dev/null +++ b/src/kiwi/subcommands/disable.py @@ -0,0 +1,38 @@ +# system +import logging +import os + +# local +from .utils.misc import get_project_names, get_project_dir, get_project_down_dir +from ._subcommand import ProjectCommand + + +class DisableCommand(ProjectCommand): + """kiwi disable""" + + def __init__(self): + super().__init__( + 'disable', num_projects='+', + description="Disable whole project(s) in this instance" + ) + + def run(self, runner, config, args): + result = True + + for project_name in get_project_names(args): + project_dir = get_project_dir(config, project_name) + project_down_dir = get_project_down_dir(config, project_name) + + if os.path.isdir(project_dir): + logging.info(f"Disabling project '{project_name}'") + os.rename(project_dir, project_down_dir) + + elif os.path.isdir(project_down_dir): + logging.warning(f"Project '{project_name}' is already disabled!") + result = False + + else: + logging.warning(f"Project '{project_name}' not found in instance!") + result = False + + return result diff --git a/src/kiwi/subcommands/enable.py b/src/kiwi/subcommands/enable.py new file mode 100644 index 0000000..1d67124 --- /dev/null +++ b/src/kiwi/subcommands/enable.py @@ -0,0 +1,38 @@ +# system +import logging +import os + +# local +from .utils.misc import get_project_names, get_project_dir, get_project_down_dir +from ._subcommand import ProjectCommand + + +class EnableCommand(ProjectCommand): + """kiwi enable""" + + def __init__(self): + super().__init__( + 'enable', num_projects='+', + description="Enable whole project(s) in this instance" + ) + + def run(self, runner, config, args): + result = True + + for project_name in get_project_names(args): + project_dir = get_project_dir(config, project_name) + project_down_dir = get_project_down_dir(config, project_name) + + if os.path.isdir(project_down_dir): + logging.info(f"Enabling project '{project_name}'") + os.rename(project_down_dir, project_dir) + + elif os.path.isdir(project_dir): + logging.warning(f"Project '{project_name}' is already enabled!") + result = False + + else: + logging.warning(f"Project '{project_name}' not found in instance!") + result = False + + return result diff --git a/src/kiwi/subcommands/new.py b/src/kiwi/subcommands/new.py new file mode 100644 index 0000000..e30a915 --- /dev/null +++ b/src/kiwi/subcommands/new.py @@ -0,0 +1,39 @@ +# system +import logging +import os +import shutil + +# local +from .utils.misc import get_project_names, get_project_dir, get_project_down_dir +from ._subcommand import ProjectCommand + +# parent +from .._constants import DEFAULT_DOCKER_COMPOSE_NAME + + +class NewCommand(ProjectCommand): + """kiwi new""" + + def __init__(self): + super().__init__( + 'new', num_projects='+', + description="Create new empty project(s) in this instance" + ) + + def run(self, runner, config, args): + result = True + + for project_name in get_project_names(args): + project_dir = get_project_dir(config, project_name) + project_down_dir = get_project_down_dir(config, project_name) + + if os.path.isdir(project_dir) or os.path.isdir(project_down_dir): + logging.error(f"Project '{project_name}' exists in this instance!") + result = False + + else: + logging.info(f"Creating project '{project_name}'") + os.mkdir(project_dir) + shutil.copy(DEFAULT_DOCKER_COMPOSE_NAME, os.path.join(project_dir, "docker-compose.yml")) + + return result From e45d3ae83f3700fe37ceb3accd69ead5f739543b Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 16:02:45 +0200 Subject: [PATCH 085/118] Run conf-copy before upping instance --- src/kiwi/subcommands/up.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index 0c16713..0f958cd 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -12,6 +12,12 @@ class UpCommand(FlexCommand): description="Bring up the whole instance, a project or service(s) inside a project" ) + def _run_instance(self, runner, config, args): + if runner.run('conf-copy'): + return super()._run_instance(runner, config, args) + + return False + def _run_services(self, runner, config, args, services): if runner.run('net-up'): DockerCommand('docker-compose').run( From 301b505efadefb6489d055ca354735c9df07e47b Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 16:36:29 +0200 Subject: [PATCH 086/118] yml whitespace fix --- example/kiwi.yml | 2 +- src/kiwi/config.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/example/kiwi.yml b/example/kiwi.yml index 323d023..5957503 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -7,7 +7,7 @@ version: '0.1' runtime: storage: /tmp/kiwi shells: - - /bin/bash + - /bin/bash env: null markers: diff --git a/src/kiwi/config.py b/src/kiwi/config.py index f557c10..84cc462 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -49,7 +49,10 @@ class Config: """dump into textual representation""" # dump yml content - yml_string = yaml.dump(self.__yml_content, default_flow_style=False, sort_keys=False) + yml_string = yaml.dump( + self.__yml_content, + default_flow_style=False, sort_keys=False + ).strip() # insert newline before every main key yml_string = re.sub(r'^(\S)', r'\n\1', yml_string, flags=re.MULTILINE) @@ -82,6 +85,7 @@ class Config: with open(KIWI_CONF_NAME, 'w') as stream: stream.write(str(self)) + stream.write('\n') class DefaultConfig(Config): From cd3d0518dff10c2e3eb30a0ee08a25826137a6d9 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 16:36:39 +0200 Subject: [PATCH 087/118] subcommand "list" --- src/kiwi/subcommands/__init__.py | 2 + src/kiwi/subcommands/list.py | 77 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 src/kiwi/subcommands/list.py diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index ce70da2..13e27a8 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -6,6 +6,7 @@ from .disable import DisableCommand from .down import DownCommand from .enable import EnableCommand from .init import InitCommand +from .list import ListCommand from .logs import LogsCommand from .net import NetUpCommand, NetDownCommand from .new import NewCommand @@ -25,6 +26,7 @@ __all__ = [ 'DownCommand', 'EnableCommand', 'InitCommand', + 'ListCommand', 'LogsCommand', 'NetUpCommand', 'NetDownCommand', diff --git a/src/kiwi/subcommands/list.py b/src/kiwi/subcommands/list.py new file mode 100644 index 0000000..0f8e3aa --- /dev/null +++ b/src/kiwi/subcommands/list.py @@ -0,0 +1,77 @@ +# system +import logging +import os +import subprocess + +# local +from ._subcommand import FlexCommand +from .utils.dockercommand import DockerCommand +from .utils.misc import list_projects, get_first_project_name, get_project_dir + + +def _print_list(strings): + if isinstance(strings, list): + for string in strings: + print(f" - {string}") + + elif isinstance(strings, str): + _print_list(strings.strip().split('\n')) + + elif isinstance(strings, bytes): + _print_list(str(strings, 'utf-8')) + + +class ListCommand(FlexCommand): + """kiwi list""" + + def __init__(self): + super().__init__( + 'list', "Listing", + description="List projects in this instance, services inside a project or service(s) inside a project" + ) + + def _run_instance(self, runner, config, args): + print(f"Projects in instance {os.getcwd()}:") + print("") + + _print_list(list_projects(config)) + + return True + + def _run_project(self, runner, config, args): + project_name = get_first_project_name(args) + print(f"Services in project '{project_name}':") + print("") + + ps = DockerCommand('docker-compose').run( + config, args, ['config', '--services'], + stdout=subprocess.PIPE + ) + _print_list(ps.stdout) + + return True + + def _run_services(self, runner, config, args, services): + import yaml + + project_name = get_first_project_name(args) + project_dir = get_project_dir(config, project_name) + print(f"Configuration of services {services} in project '{project_name}':") + print("") + + with open(os.path.join(project_dir, 'docker-compose.yml'), 'r') as stream: + try: + docker_compose_yml = yaml.safe_load(stream) + + for service_name in services: + print(yaml.dump( + {service_name: docker_compose_yml['services'][service_name]}, + default_flow_style=False, sort_keys=False + ).strip()) + + return True + + except yaml.YAMLError as exc: + logging.error(exc) + + return False From f9cb78476094d76107b537c6121ad745cd2b1a4c Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 16:40:38 +0200 Subject: [PATCH 088/118] default config fix --- src/etc/docker-compose_default.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/etc/docker-compose_default.yml b/src/etc/docker-compose_default.yml index e6b96b6..3baa9e3 100644 --- a/src/etc/docker-compose_default.yml +++ b/src/etc/docker-compose_default.yml @@ -10,10 +10,13 @@ networks: name: $KIWI_HUB_NAME services: + # an example service something: + # uses an image image: maintainer/repo:tag + # will get restarted restart: unless-stopped - networks: - - default - - kiwi_hub - [...] + # is also connected to the instance hub + networks: + - default + - kiwi_hub From b3456f8ec3194f3e46ec3e16aabda2fb2d2e9b73 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 17:49:49 +0200 Subject: [PATCH 089/118] subcommand "conf-clean" --- src/kiwi/subcommands/__init__.py | 3 ++- src/kiwi/subcommands/conf.py | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 13e27a8..637f9be 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,7 +1,7 @@ # local from .build import BuildCommand from .cmd import CmdCommand -from .conf import ConfCopyCommand, ConfPurgeCommand +from .conf import ConfCopyCommand, ConfPurgeCommand, ConfCleanCommand from .disable import DisableCommand from .down import DownCommand from .enable import EnableCommand @@ -22,6 +22,7 @@ __all__ = [ 'CmdCommand', 'ConfCopyCommand', 'ConfPurgeCommand', + 'ConfCleanCommand', 'DisableCommand', 'DownCommand', 'EnableCommand', diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index 702261e..d221943 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -62,3 +62,40 @@ class ConfPurgeCommand(SubCommand): ) return True + + +class ConfCleanCommand(SubCommand): + """kiwi conf-clean""" + + def __init__(self): + super().__init__( + 'conf-clean', + description="Cleanly sync all configs to target folder, relaunch affected projects" + ) + + def run(self, runner, config, args): + result = True + + # down all projects with config directories + affected_projects = [] + + for project_name in list_projects(config): + project_conf = f"{get_project_dir(config, project_name)}/{CONF_DIRECTORY_NAME}" + + if os.path.isdir(project_conf): + affected_projects.append(project_name) + + for project_name in affected_projects: + args.projects = project_name + result &= runner.run('down') + + # cleanly sync configs + result &= runner.run('conf-purge') + result &= runner.run('conf-purge') + + # bring projects back up + for project_name in affected_projects: + args.projects = project_name + result &= runner.run('up') + + return result \ No newline at end of file From a2c32b33149bd84d64eac3670e6cf644c86c9694 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 17:50:13 +0200 Subject: [PATCH 090/118] crude main help --- src/etc/command_help.txt | 25 +++++++++++++++++++++++++ src/kiwi/_constants.py | 1 + src/kiwi/parser.py | 14 ++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 src/etc/command_help.txt diff --git a/src/etc/command_help.txt b/src/etc/command_help.txt new file mode 100644 index 0000000..781e9ff --- /dev/null +++ b/src/etc/command_help.txt @@ -0,0 +1,25 @@ +COMMANDS +======== + +net-up +up +down +net-down +update +enable +disable + +show +list +logs +sh + +build +pull +push + +init +conf-copy +conf-purge + +cmd diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py index 715372e..e4bb229 100644 --- a/src/kiwi/_constants.py +++ b/src/kiwi/_constants.py @@ -19,6 +19,7 @@ HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_header.yml" DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_default.yml" VERSION_TAG_NAME = f"{KIWI_ROOT}/etc/version_tag" DEFAULT_DOCKER_COMPOSE_NAME = f"{KIWI_ROOT}/etc/docker-compose_default.yml" +COMMAND_HELP_TEXT_NAME = f"{KIWI_ROOT}/etc/command_help.txt" # special config directory in projects CONF_DIRECTORY_NAME = 'conf' diff --git a/src/kiwi/parser.py b/src/kiwi/parser.py index 2814c4c..01c869f 100644 --- a/src/kiwi/parser.py +++ b/src/kiwi/parser.py @@ -1,6 +1,9 @@ # system import argparse +# local +from ._constants import COMMAND_HELP_TEXT_NAME + class Parser: """Singleton: Main CLI arguments parser""" @@ -14,10 +17,17 @@ class Parser: __args = None def __init__(self): - # create main parsers + # add version data from separate file (keeps default config cleaner) + with open(COMMAND_HELP_TEXT_NAME, 'r') as stream: + command_help_text = stream.read().strip() + + # create main parser self.__parser = argparse.ArgumentParser( - description='kiwi-config' + description='kiwi-config', + usage='%(prog)s [command]', + epilog=command_help_text, ) + self.__parser.formatter_class = argparse.RawDescriptionHelpFormatter # main arguments self.__parser.add_argument( From a345baf1dad1c0825a9c59e54e3e654b02ccd157 Mon Sep 17 00:00:00 2001 From: ldericher Date: Tue, 18 Aug 2020 17:51:03 +0200 Subject: [PATCH 091/118] typo --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2480bcf..83fea49 100644 --- a/Makefile +++ b/Makefile @@ -249,8 +249,8 @@ services: something: image: maintainer/repo:tag restart: unless-stopped - networks: - - default - - kiwihub + networks: + - default + - kiwi_hub [...] endef From 0cf934965e01857344ab6caf22c51dcd19e64979 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 03:22:03 +0200 Subject: [PATCH 092/118] Project class --- src/kiwi/subcommands/disable.py | 23 +------ src/kiwi/subcommands/enable.py | 23 +------ src/kiwi/subcommands/utils/project.py | 99 +++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 40 deletions(-) create mode 100644 src/kiwi/subcommands/utils/project.py diff --git a/src/kiwi/subcommands/disable.py b/src/kiwi/subcommands/disable.py index ccdd477..c7a00ea 100644 --- a/src/kiwi/subcommands/disable.py +++ b/src/kiwi/subcommands/disable.py @@ -1,9 +1,5 @@ -# system -import logging -import os - # local -from .utils.misc import get_project_names, get_project_dir, get_project_down_dir +from .utils.project import Project from ._subcommand import ProjectCommand @@ -19,20 +15,7 @@ class DisableCommand(ProjectCommand): def run(self, runner, config, args): result = True - for project_name in get_project_names(args): - project_dir = get_project_dir(config, project_name) - project_down_dir = get_project_down_dir(config, project_name) - - if os.path.isdir(project_dir): - logging.info(f"Disabling project '{project_name}'") - os.rename(project_dir, project_down_dir) - - elif os.path.isdir(project_down_dir): - logging.warning(f"Project '{project_name}' is already disabled!") - result = False - - else: - logging.warning(f"Project '{project_name}' not found in instance!") - result = False + for project in Project.from_args(args): + result = project.disable() return result diff --git a/src/kiwi/subcommands/enable.py b/src/kiwi/subcommands/enable.py index 1d67124..6778c4d 100644 --- a/src/kiwi/subcommands/enable.py +++ b/src/kiwi/subcommands/enable.py @@ -1,9 +1,5 @@ -# system -import logging -import os - # local -from .utils.misc import get_project_names, get_project_dir, get_project_down_dir +from .utils.project import Project from ._subcommand import ProjectCommand @@ -19,20 +15,7 @@ class EnableCommand(ProjectCommand): def run(self, runner, config, args): result = True - for project_name in get_project_names(args): - project_dir = get_project_dir(config, project_name) - project_down_dir = get_project_down_dir(config, project_name) - - if os.path.isdir(project_down_dir): - logging.info(f"Enabling project '{project_name}'") - os.rename(project_down_dir, project_dir) - - elif os.path.isdir(project_dir): - logging.warning(f"Project '{project_name}' is already enabled!") - result = False - - else: - logging.warning(f"Project '{project_name}' not found in instance!") - result = False + for project in Project.from_args(args): + result = project.enable() return result diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/subcommands/utils/project.py new file mode 100644 index 0000000..15d5377 --- /dev/null +++ b/src/kiwi/subcommands/utils/project.py @@ -0,0 +1,99 @@ +import logging +import os + +from kiwi._constants import CONF_DIRECTORY_NAME +from kiwi.config import LoadedConfig + + +class Project: + __name = None + __config = None + + def __init__(self, name): + self.__name = name + self.__config = LoadedConfig.get() + + @classmethod + def from_names(cls, names): + return [cls(name) for name in names] + + @classmethod + def all(cls): + # current directory content + content = os.listdir() + + # filter subdirectories + dirs = [dir_name for dir_name in content if os.path.isdir(dir_name)] + + # filter by suffix + project_dirs = [dir_name for dir_name in dirs if dir_name.endswith(cls.__config['markers:project'])] + + # remove suffix + project_names = [project_name[:-len(cls.__config['markers:project'])] for project_name in project_dirs] + + return cls.from_names(project_names) + + @classmethod + def from_args(cls, args): + if args is not None and 'projects' in args: + if isinstance(args.projects, list) and args.projects: + return cls.from_names(args.projects) + elif isinstance(args.projects, str): + return cls.from_names([args.projects]) + + return [] + + def get_name(self): + return self.__name + + def dir_name(self): + return f"{self.__name}{self.__config['markers:project']}" + + def down_dir_name(self): + return f"{self.dir_name()}{self.__config['markers:down']}" + + def conf_dir_name(self): + return os.path.join(self.dir_name(), CONF_DIRECTORY_NAME) + + def target_dir_name(self): + return os.path.join(self.__config['runtime:storage'], self.dir_name()) + + def exists(self): + return os.path.isdir(self.dir_name()) or os.path.isdir(self.down_dir_name()) + + def is_enabled(self): + return os.path.isdir(self.dir_name()) + + def is_disabled(self): + return os.path.isdir(self.down_dir_name()) + + def has_configs(self): + return os.path.isdir(self.dir_name()) + + def enable(self): + if self.is_disabled(): + logging.info(f"Enabling project '{self.get_name()}'") + os.rename(self.down_dir_name(), self.dir_name()) + + elif self.is_enabled(): + logging.warning(f"Project '{self.get_name()}' is enabled!") + + else: + logging.warning(f"Project '{self.get_name()}' not found in instance!") + return False + + return True + + def disable(self): + if self.is_enabled(): + logging.info(f"Disabling project '{self.get_name()}'") + os.rename(self.dir_name(), self.down_dir_name()) + + elif self.is_disabled(): + logging.warning(f"Project '{self.get_name()}' is disabled!") + + else: + logging.warning(f"Project '{self.get_name()}' not found in instance!") + return False + + return True \ No newline at end of file From 267f85fa8daff785c4d10e905b49a35666293ed6 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 11:58:13 +0200 Subject: [PATCH 093/118] abandoned utils.misc for project handling --- src/kiwi/subcommands/_subcommand.py | 44 ++++---- src/kiwi/subcommands/conf.py | 29 ++--- src/kiwi/subcommands/disable.py | 12 +- src/kiwi/subcommands/enable.py | 12 +- src/kiwi/subcommands/list.py | 55 ++++++--- src/kiwi/subcommands/new.py | 18 ++- src/kiwi/subcommands/utils/dockercommand.py | 24 ++-- src/kiwi/subcommands/utils/misc.py | 74 +----------- src/kiwi/subcommands/utils/project.py | 118 ++++++++++++-------- 9 files changed, 182 insertions(+), 204 deletions(-) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index be50e67..34c6aae 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -2,7 +2,7 @@ import logging # local -from .utils.misc import get_first_project_name, get_services, list_projects +from .utils.project import Projects # parent from ..parser import Parser @@ -16,12 +16,13 @@ class SubCommand: # command parser _sub_parser = None - def __init__(self, name, **kwargs): + def __init__(self, name, add_parser=True, **kwargs): self.__name = name - self._sub_parser = Parser().get_subparsers().add_parser( - name, - **kwargs - ) + if add_parser: + self._sub_parser = Parser().get_subparsers().add_parser( + name, + **kwargs + ) def __str__(self): return self.__name @@ -34,9 +35,9 @@ class SubCommand: class ProjectCommand(SubCommand): """this command concerns a project in current instance""" - def __init__(self, name, num_projects, **kwargs): + def __init__(self, name, num_projects, add_parser=True, **kwargs): super().__init__( - name, + name, add_parser=add_parser, **kwargs ) @@ -54,9 +55,9 @@ class ProjectCommand(SubCommand): class ServiceCommand(ProjectCommand): """this command concerns service(s) in a project""" - def __init__(self, name, num_projects, num_services, **kwargs): + def __init__(self, name, num_projects, num_services, add_parser=True, **kwargs): super().__init__( - name, num_projects=num_projects, + name, num_projects=num_projects, add_parser=add_parser, **kwargs ) @@ -76,9 +77,9 @@ class FlexCommand(ServiceCommand): __action = None - def __init__(self, name, action='', **kwargs): + def __init__(self, name, action='', add_parser=True, **kwargs): super().__init__( - name, num_projects='?', num_services='*', + name, num_projects='?', num_services='*', add_parser=add_parser, **kwargs ) @@ -91,8 +92,8 @@ class FlexCommand(ServiceCommand): def _run_instance(self, runner, config, args): result = True - for project_name in list_projects(config): - args.projects = project_name + for project in Projects.from_args(args): + args.projects = project.get_name() result &= runner.run(str(self)) return result @@ -104,19 +105,18 @@ class FlexCommand(ServiceCommand): pass def run(self, runner, config, args): - project_name = get_first_project_name(args) - services = get_services(args) - - if project_name is None: + projects = Projects.from_args(args) + if not projects: # no project given, run for entire instance logging.info(f"{self.__action} this instance") return self._run_instance(runner, config, args) - if not services: + project = projects[0] + if args is None or 'services' not in args or not args.services: # no services given, run for whole project - logging.info(f"{self.__action} project '{project_name}'") + logging.info(f"{self.__action} project '{project.get_name()}'") return self._run_project(runner, config, args) # run for service(s) inside project - logging.info(f"{self.__action} services {services} in project '{project_name}'") - return self._run_services(runner, config, args, services) + logging.info(f"{self.__action} services {args.services} in project '{project.get_name()}'") + return self._run_services(runner, config, args, args.services) diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index d221943..7656f07 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -5,7 +5,7 @@ import subprocess # local from ._subcommand import SubCommand -from .utils.misc import list_projects, get_project_dir +from .utils.project import Projects from .utils.rootkit import Rootkit, prefix_path_mnt # parent @@ -22,13 +22,11 @@ class ConfCopyCommand(SubCommand): ) def run(self, runner, config, args): - conf_dirs = [] - - for project_name in list_projects(config): - project_conf = f"{get_project_dir(config, project_name)}/{CONF_DIRECTORY_NAME}" - - if os.path.isdir(project_conf): - conf_dirs.append(project_conf) + conf_dirs = [ + project.conf_dir_name() + for project in Projects.all() + if project.is_enabled() + ] if conf_dirs: # add target directory @@ -76,14 +74,11 @@ class ConfCleanCommand(SubCommand): def run(self, runner, config, args): result = True - # down all projects with config directories - affected_projects = [] - - for project_name in list_projects(config): - project_conf = f"{get_project_dir(config, project_name)}/{CONF_DIRECTORY_NAME}" - - if os.path.isdir(project_conf): - affected_projects.append(project_name) + affected_projects = [ + project.conf_dir_name() + for project in Projects.all() + if project.has_configs() + ] for project_name in affected_projects: args.projects = project_name @@ -91,7 +86,7 @@ class ConfCleanCommand(SubCommand): # cleanly sync configs result &= runner.run('conf-purge') - result &= runner.run('conf-purge') + result &= runner.run('conf-copy') # bring projects back up for project_name in affected_projects: diff --git a/src/kiwi/subcommands/disable.py b/src/kiwi/subcommands/disable.py index c7a00ea..29b4f88 100644 --- a/src/kiwi/subcommands/disable.py +++ b/src/kiwi/subcommands/disable.py @@ -1,5 +1,5 @@ # local -from .utils.project import Project +from .utils.project import Projects from ._subcommand import ProjectCommand @@ -13,9 +13,7 @@ class DisableCommand(ProjectCommand): ) def run(self, runner, config, args): - result = True - - for project in Project.from_args(args): - result = project.disable() - - return result + return all([ + project.disable() + for project in Projects.from_args(args) + ]) diff --git a/src/kiwi/subcommands/enable.py b/src/kiwi/subcommands/enable.py index 6778c4d..9392b95 100644 --- a/src/kiwi/subcommands/enable.py +++ b/src/kiwi/subcommands/enable.py @@ -1,5 +1,5 @@ # local -from .utils.project import Project +from .utils.project import Projects from ._subcommand import ProjectCommand @@ -13,9 +13,7 @@ class EnableCommand(ProjectCommand): ) def run(self, runner, config, args): - result = True - - for project in Project.from_args(args): - result = project.enable() - - return result + return all([ + project.enable() + for project in Projects.from_args(args) + ]) diff --git a/src/kiwi/subcommands/list.py b/src/kiwi/subcommands/list.py index 0f8e3aa..6e4cd9f 100644 --- a/src/kiwi/subcommands/list.py +++ b/src/kiwi/subcommands/list.py @@ -2,11 +2,12 @@ import logging import os import subprocess +import yaml # local from ._subcommand import FlexCommand from .utils.dockercommand import DockerCommand -from .utils.misc import list_projects, get_first_project_name, get_project_dir +from .utils.project import Projects def _print_list(strings): @@ -31,35 +32,61 @@ class ListCommand(FlexCommand): ) def _run_instance(self, runner, config, args): - print(f"Projects in instance {os.getcwd()}:") - print("") + print(f"kiwi-config instance at '{os.getcwd()}'") + print("#########") + projects = Projects.all() - _print_list(list_projects(config)) + enableds = [ + project.get_name() + for project in projects + if project.is_enabled() + ] + + if enableds: + print(f"Enabled projects:") + _print_list(enableds) + + disableds = [ + project.get_name() + for project in projects + if project.is_disabled() + ] + + if disableds: + print(f"Disabled projects:") + _print_list(disableds) return True def _run_project(self, runner, config, args): - project_name = get_first_project_name(args) - print(f"Services in project '{project_name}':") - print("") + project = Projects.from_args(args)[0] + + if not project.exists(): + logging.error(f"Project '{project.get_name()}' not found") + return False + + print(f"Services in project '{project.get_name()}':") + print("#########") ps = DockerCommand('docker-compose').run( config, args, ['config', '--services'], stdout=subprocess.PIPE ) - _print_list(ps.stdout) + _print_list(ps.stdout) return True def _run_services(self, runner, config, args, services): - import yaml + project = Projects.from_args(args)[0] - project_name = get_first_project_name(args) - project_dir = get_project_dir(config, project_name) - print(f"Configuration of services {services} in project '{project_name}':") - print("") + if not project.exists(): + logging.error(f"Project '{project.get_name()}' not found") + return False - with open(os.path.join(project_dir, 'docker-compose.yml'), 'r') as stream: + print(f"Configuration of services {services} in project '{project.get_name()}':") + print("#########") + + with open(project.compose_file_name(), 'r') as stream: try: docker_compose_yml = yaml.safe_load(stream) diff --git a/src/kiwi/subcommands/new.py b/src/kiwi/subcommands/new.py index e30a915..913f134 100644 --- a/src/kiwi/subcommands/new.py +++ b/src/kiwi/subcommands/new.py @@ -4,7 +4,7 @@ import os import shutil # local -from .utils.misc import get_project_names, get_project_dir, get_project_down_dir +from .utils.project import Projects from ._subcommand import ProjectCommand # parent @@ -22,18 +22,16 @@ class NewCommand(ProjectCommand): def run(self, runner, config, args): result = True + projects = Projects.from_args(args) - for project_name in get_project_names(args): - project_dir = get_project_dir(config, project_name) - project_down_dir = get_project_down_dir(config, project_name) - - if os.path.isdir(project_dir) or os.path.isdir(project_down_dir): - logging.error(f"Project '{project_name}' exists in this instance!") + for project in projects: + if project.exists(): + logging.error(f"Project '{project.get_name()}' exists in this instance!") result = False else: - logging.info(f"Creating project '{project_name}'") - os.mkdir(project_dir) - shutil.copy(DEFAULT_DOCKER_COMPOSE_NAME, os.path.join(project_dir, "docker-compose.yml")) + logging.info(f"Creating project '{project.get_name()}'") + os.mkdir(project.enabled_dir_name()) + shutil.copy(DEFAULT_DOCKER_COMPOSE_NAME, project.compose_file_name()) return result diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 62ad1b0..5863f1d 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -5,18 +5,24 @@ import subprocess # local from .executable import Executable -from .misc import get_project_dir, get_first_project_name +from .project import Projects # parent from ..._constants import CONF_DIRECTORY_NAME +from ...parser import Parser +from ...config import LoadedConfig -def _update_kwargs(config, args, **kwargs): +def _update_kwargs(**kwargs): + config = LoadedConfig.get() + projects = Projects.from_args(Parser().get_args()) + # project given in args: command affects a project in this instance - project_name = get_first_project_name(args) - if project_name is not None: + if projects: + project = projects[0] + # execute command in project directory - kwargs['cwd'] = get_project_dir(config, project_name) + kwargs['cwd'] = project.dir_name() # ensure there is an environment if 'env' not in kwargs: @@ -24,10 +30,10 @@ def _update_kwargs(config, args, **kwargs): # create environment variables for docker commands kwargs['env'].update({ - 'COMPOSE_PROJECT_NAME': project_name, + 'COMPOSE_PROJECT_NAME': project.get_name(), 'KIWI_HUB_NAME': config['network:name'], 'CONFDIR': os.path.join(config['runtime:storage'], CONF_DIRECTORY_NAME), - 'TARGETDIR': os.path.join(config['runtime:storage'], get_project_dir(config, project_name)) + 'TARGETDIR': project.target_dir_name() }) logging.debug(f"kwargs updated: {kwargs}") @@ -53,7 +59,7 @@ class DockerCommand(Executable): raise PermissionError("Cannot access docker, please get into the docker group or run as root!") def run(self, config, args, process_args, **kwargs): - kwargs = _update_kwargs(config, args, **kwargs) + kwargs = _update_kwargs(**kwargs) # equivalent to 'super().run' but agnostic of nested class construct return super().__getattr__("run")( @@ -62,7 +68,7 @@ class DockerCommand(Executable): ) def run_less(self, config, args, process_args, **kwargs): - kwargs = _update_kwargs(config, args, **kwargs) + kwargs = _update_kwargs(**kwargs) return super().__getattr__("run_less")( process_args, config, diff --git a/src/kiwi/subcommands/utils/misc.py b/src/kiwi/subcommands/utils/misc.py index eb607fe..90753ab 100644 --- a/src/kiwi/subcommands/utils/misc.py +++ b/src/kiwi/subcommands/utils/misc.py @@ -1,76 +1,3 @@ -import os - - -def get_project_names(args): - """get project names from CLI args""" - - if args is not None and 'projects' in args: - if isinstance(args.projects, list): - if args.projects: - return args.projects - else: - return None - elif isinstance(args.projects, str): - return [args.projects] - - return None - - -def get_first_project_name(args): - """get first project name from CLI args""" - - names = get_project_names(args) - if names is not None: - return names[0] - - return None - - -def get_services(args): - """get services list from CLI args""" - - if args is not None and 'services' in args: - return args.services - - return None - - -def get_project_dir(config, project_name): - """get project directory""" - - return f"{project_name}{config['markers:project']}" - - -def get_project_down_dir(config, project_name): - """get project directory""" - - return f"{get_project_dir(config, project_name)}{config['markers:down']}" - - -def get_target_dir(config, project_name): - """get project's target directory""" - - return os.path.join(config['runtime:storage'], get_project_dir(config, project_name)) - - -def list_projects(config): - """list projects in current instance""" - - # complete dir listing - content = os.listdir() - - # filter directories - dirs = [d for d in content if os.path.isdir(d)] - - # filter suffix - project_dirs = [p for p in dirs if p.endswith(config['markers:project'])] - - # remove suffix - projects = [p[:-len(config['markers:project'])] for p in project_dirs] - - return projects - - def _surround(string, bang): midlane = f"{bang * 3} {string} {bang * 3}" sidelane = bang*len(midlane) @@ -86,6 +13,7 @@ def _emphasize(lines): else: return lines + def are_you_sure(prompt, default="no"): if default.lower() == 'yes': suffix = "[YES|no]" diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/subcommands/utils/project.py index 15d5377..7551de2 100644 --- a/src/kiwi/subcommands/utils/project.py +++ b/src/kiwi/subcommands/utils/project.py @@ -1,79 +1,58 @@ import logging import os -from kiwi._constants import CONF_DIRECTORY_NAME -from kiwi.config import LoadedConfig +from ..._constants import CONF_DIRECTORY_NAME +from ...config import LoadedConfig class Project: __name = None - __config = None def __init__(self, name): self.__name = name - self.__config = LoadedConfig.get() - - @classmethod - def from_names(cls, names): - return [cls(name) for name in names] - - @classmethod - def all(cls): - # current directory content - content = os.listdir() - - # filter subdirectories - dirs = [dir_name for dir_name in content if os.path.isdir(dir_name)] - - # filter by suffix - project_dirs = [dir_name for dir_name in dirs if dir_name.endswith(cls.__config['markers:project'])] - - # remove suffix - project_names = [project_name[:-len(cls.__config['markers:project'])] for project_name in project_dirs] - - return cls.from_names(project_names) - - @classmethod - def from_args(cls, args): - if args is not None and 'projects' in args: - if isinstance(args.projects, list) and args.projects: - return cls.from_names(args.projects) - elif isinstance(args.projects, str): - return cls.from_names([args.projects]) - - return [] def get_name(self): return self.__name def dir_name(self): - return f"{self.__name}{self.__config['markers:project']}" + if self.is_enabled(): + return self.enabled_dir_name() + elif self.is_disabled(): + return self.disabled_dir_name() + else: + return None - def down_dir_name(self): - return f"{self.dir_name()}{self.__config['markers:down']}" + def enabled_dir_name(self): + return f"{self.__name}{LoadedConfig.get()['markers:project']}" + + def disabled_dir_name(self): + return f"{self.enabled_dir_name()}{LoadedConfig.get()['markers:down']}" def conf_dir_name(self): return os.path.join(self.dir_name(), CONF_DIRECTORY_NAME) + def compose_file_name(self): + return os.path.join(self.dir_name(), 'docker-compose.yml') + def target_dir_name(self): - return os.path.join(self.__config['runtime:storage'], self.dir_name()) + return os.path.join(LoadedConfig.get()['runtime:storage'], self.enabled_dir_name()) def exists(self): - return os.path.isdir(self.dir_name()) or os.path.isdir(self.down_dir_name()) + return os.path.isdir(self.enabled_dir_name()) or os.path.isdir(self.disabled_dir_name()) def is_enabled(self): - return os.path.isdir(self.dir_name()) + return os.path.isdir(self.enabled_dir_name()) def is_disabled(self): - return os.path.isdir(self.down_dir_name()) + return os.path.isdir(self.disabled_dir_name()) def has_configs(self): - return os.path.isdir(self.dir_name()) + return os.path.isdir(self.conf_dir_name()) def enable(self): if self.is_disabled(): logging.info(f"Enabling project '{self.get_name()}'") - os.rename(self.down_dir_name(), self.dir_name()) + os.rename(self.dir_name(), self.enabled_dir_name()) elif self.is_enabled(): logging.warning(f"Project '{self.get_name()}' is enabled!") @@ -87,7 +66,7 @@ class Project: def disable(self): if self.is_enabled(): logging.info(f"Disabling project '{self.get_name()}'") - os.rename(self.dir_name(), self.down_dir_name()) + os.rename(self.dir_name(), self.disabled_dir_name()) elif self.is_disabled(): logging.warning(f"Project '{self.get_name()}' is disabled!") @@ -96,4 +75,53 @@ class Project: logging.warning(f"Project '{self.get_name()}' not found in instance!") return False - return True \ No newline at end of file + return True + + +def _extract_project_name(file_name): + config = LoadedConfig.get() + enabled_suffix = config['markers:project'] + disabled_suffix = f"{enabled_suffix}{config['markers:down']}" + + if os.path.isdir(file_name): + # all subdirectories + if file_name.endswith(enabled_suffix): + # enabled projects + return file_name[:-len(enabled_suffix)] + + elif file_name.endswith(disabled_suffix): + # disabled projects + return file_name[:-len(disabled_suffix)] + + return None + + +class Projects: + __projects = None + + def __init__(self, names): + self.__projects = [ + Project(name) + for name in names if isinstance(name, str) + ] + + def __getitem__(self, item): + return self.__projects[item] + + @classmethod + def all(cls): + return cls([ + _extract_project_name(file_name) + for file_name in os.listdir() + ]) + + @classmethod + def from_args(cls, args): + if args is not None and 'projects' in args: + if isinstance(args.projects, list) and args.projects: + return cls(args.projects) + + elif isinstance(args.projects, str): + return cls([args.projects]) + + return [] From 0060bf8878eb41b87b5fdd52fd685766c12abafd Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 12:02:46 +0200 Subject: [PATCH 094/118] marker --- example/kiwi.yml | 2 +- src/etc/kiwi_default.yml | 2 +- src/kiwi/subcommands/init.py | 2 +- src/kiwi/subcommands/utils/project.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/example/kiwi.yml b/example/kiwi.yml index 5957503..d8dab05 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -12,7 +12,7 @@ runtime: markers: project: .project - down: .down + disabled: .disabled network: name: kiwi_hub diff --git a/src/etc/kiwi_default.yml b/src/etc/kiwi_default.yml index 8f477ee..1404379 100644 --- a/src/etc/kiwi_default.yml +++ b/src/etc/kiwi_default.yml @@ -6,7 +6,7 @@ runtime: env: null markers: project: .project - down: .down + disabled: .disabled network: name: kiwi_hub cidr: 10.22.46.0/24 diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 1f3714b..a637c79 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -58,7 +58,7 @@ class InitCommand(SubCommand): # markers user_input(config, 'markers:project', "Enter marker string for project directories") - user_input(config, 'markers:down', "Enter marker string for disabled projects") + user_input(config, 'markers:disabled', "Enter marker string for disabled projects") # network user_input(config, 'network:name', "Enter name for local docker network") diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/subcommands/utils/project.py index 7551de2..b248abe 100644 --- a/src/kiwi/subcommands/utils/project.py +++ b/src/kiwi/subcommands/utils/project.py @@ -26,7 +26,7 @@ class Project: return f"{self.__name}{LoadedConfig.get()['markers:project']}" def disabled_dir_name(self): - return f"{self.enabled_dir_name()}{LoadedConfig.get()['markers:down']}" + return f"{self.enabled_dir_name()}{LoadedConfig.get()['markers:disabled']}" def conf_dir_name(self): return os.path.join(self.dir_name(), CONF_DIRECTORY_NAME) @@ -81,7 +81,7 @@ class Project: def _extract_project_name(file_name): config = LoadedConfig.get() enabled_suffix = config['markers:project'] - disabled_suffix = f"{enabled_suffix}{config['markers:down']}" + disabled_suffix = f"{enabled_suffix}{config['markers:disabled']}" if os.path.isdir(file_name): # all subdirectories From 93d0b56eb7e0fd38c4ce0f8ffb815daf54fefdb1 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 16:10:56 +0200 Subject: [PATCH 095/118] Rework: FlexCommand defaults into lower hierarchy (override protected _run variants by default) --- src/kiwi/parser.py | 3 +- src/kiwi/runner.py | 9 +- src/kiwi/subcommands/_subcommand.py | 114 +++++++++++--------- src/kiwi/subcommands/build.py | 15 +-- src/kiwi/subcommands/cmd.py | 26 ++++- src/kiwi/subcommands/conf.py | 32 +++--- src/kiwi/subcommands/disable.py | 5 +- src/kiwi/subcommands/down.py | 34 +++--- src/kiwi/subcommands/enable.py | 5 +- src/kiwi/subcommands/init.py | 5 +- src/kiwi/subcommands/list.py | 88 +++++++-------- src/kiwi/subcommands/logs.py | 9 +- src/kiwi/subcommands/net.py | 68 ++++++------ src/kiwi/subcommands/new.py | 6 +- src/kiwi/subcommands/pull.py | 15 +-- src/kiwi/subcommands/push.py | 15 +-- src/kiwi/subcommands/sh.py | 33 +++--- src/kiwi/subcommands/show.py | 7 +- src/kiwi/subcommands/up.py | 19 ++-- src/kiwi/subcommands/update.py | 17 +-- src/kiwi/subcommands/utils/dockercommand.py | 29 ++--- src/kiwi/subcommands/utils/executable.py | 38 +++---- src/kiwi/subcommands/utils/project.py | 105 ++++++++++++------ src/kiwi/subcommands/utils/rootkit.py | 65 +++++------ 24 files changed, 414 insertions(+), 348 deletions(-) diff --git a/src/kiwi/parser.py b/src/kiwi/parser.py index 01c869f..a54bb05 100644 --- a/src/kiwi/parser.py +++ b/src/kiwi/parser.py @@ -46,7 +46,8 @@ class Parser: def get_args(self): if self.__args is None: # parse args if needed - self.__args = self.__parser.parse_args() + self.__args, unknowns = self.__parser.parse_known_args() + self.__args.unknowns = unknowns return self.__args diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index d84cca1..026f25b 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -3,7 +3,6 @@ import logging # local from . import subcommands -from .config import LoadedConfig from .parser import Parser @@ -13,7 +12,6 @@ class Runner: class __Runner: """Singleton type""" - __parser = None __commands = [] def __init__(self): @@ -22,10 +20,11 @@ class Runner: cmd = getattr(subcommands, className) self.__commands.append(cmd()) - def run(self, command=None): + def run(self, command=None, args=None): """run the desired subcommand""" - args = Parser().get_args() + if args is None: + args = Parser().get_args() if command is None: command = args.command @@ -36,7 +35,7 @@ class Runner: logging.debug(f"Running '{cmd}' with args: {args}") try: - result = cmd.run(self, LoadedConfig.get(), args) + result = cmd.run(self, args) except KeyboardInterrupt: print() diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index 34c6aae..9566443 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -1,5 +1,6 @@ # system import logging +import os # local from .utils.project import Projects @@ -16,7 +17,9 @@ class SubCommand: # command parser _sub_parser = None - def __init__(self, name, add_parser=True, **kwargs): + _action = None + + def __init__(self, name, action='', add_parser=True, **kwargs): self.__name = name if add_parser: self._sub_parser = Parser().get_subparsers().add_parser( @@ -24,26 +27,38 @@ class SubCommand: **kwargs ) + if not action: + # default action string + self._action = f"Running '{str(self)}' for" + else: + self._action = action + def __str__(self): return self.__name - def run(self, runner, config, args): - """actually run command with this dir's config and parsed CLI args""" + def _run_instance(self, runner, args): pass + def run(self, runner, args): + """actually run command with parsed CLI args""" + + # run for entire instance + logging.info(f"{self._action} kiwi-config instance at '{os.getcwd()}'") + return self._run_instance(runner, args) + class ProjectCommand(SubCommand): """this command concerns a project in current instance""" - def __init__(self, name, num_projects, add_parser=True, **kwargs): + def __init__(self, name, num_projects, action='', add_parser=True, **kwargs): super().__init__( - name, add_parser=add_parser, + name, action=action, add_parser=add_parser, **kwargs ) - projects = "a project" - - if not num_projects == 1: + if num_projects == 1: + projects = "a project" + else: projects = "project(s)" self._sub_parser.add_argument( @@ -51,19 +66,41 @@ class ProjectCommand(SubCommand): help=f"select {projects} in this instance" ) + def _run_instance(self, runner, args): + # default: run for all enabled projects + return self._run_projects(runner, args, Projects.from_dir().filter_enabled()) + + def _run_projects(self, runner, args, projects): + pass + + def run(self, runner, args): + projects = Projects.from_args(args) + + if not projects.empty(): + # project(s) given + logging.info(f"{self._action} projects {projects}") + return self._run_projects(runner, args, projects) + + else: + return super().run(runner, args) + class ServiceCommand(ProjectCommand): """this command concerns service(s) in a project""" - def __init__(self, name, num_projects, num_services, add_parser=True, **kwargs): + def __init__(self, name, num_projects, num_services, action='', add_parser=True, **kwargs): super().__init__( - name, num_projects=num_projects, add_parser=add_parser, + name, num_projects=num_projects, action=action, add_parser=add_parser, **kwargs ) - services = "a service" + if (isinstance(num_projects, str) and num_projects == '*') \ + or (isinstance(num_projects, int) and num_projects > 1): + logging.warning(f"Invalid choice for project count: {num_projects}") - if not num_services == 1: + if num_services == 1: + services = "a service" + else: services = "service(s)" self._sub_parser.add_argument( @@ -71,52 +108,25 @@ class ServiceCommand(ProjectCommand): help=f"select {services} in a project" ) - -class FlexCommand(ServiceCommand): - """this command concerns the entire instance, a whole project or just service(s) in a project""" - - __action = None - - def __init__(self, name, action='', add_parser=True, **kwargs): - super().__init__( - name, num_projects='?', num_services='*', add_parser=add_parser, - **kwargs - ) - - if not action: - # default action string - self.__action = f"Running '{str(self)}' for" - else: - self.__action = action - - def _run_instance(self, runner, config, args): + def _run_projects(self, runner, args, projects): result = True - for project in Projects.from_args(args): - args.projects = project.get_name() - result &= runner.run(str(self)) + # default: run without services for all given + for project in projects: + result &= self._run_services(runner, args, project, []) return result - def _run_project(self, runner, config, args): - return self._run_services(runner, config, args, []) - - def _run_services(self, runner, config, args, services): + def _run_services(self, runner, args, project, services): pass - def run(self, runner, config, args): - projects = Projects.from_args(args) - if not projects: - # no project given, run for entire instance - logging.info(f"{self.__action} this instance") - return self._run_instance(runner, config, args) + def run(self, runner, args): + if 'services' in args and args.services: + project = Projects.from_args(args)[0] - project = projects[0] - if args is None or 'services' not in args or not args.services: - # no services given, run for whole project - logging.info(f"{self.__action} project '{project.get_name()}'") - return self._run_project(runner, config, args) + # run for service(s) inside project + logging.info(f"{self._action} project '{project.get_name()}', services {args.services}") + return self._run_services(runner, args, project, args.services) - # run for service(s) inside project - logging.info(f"{self.__action} services {args.services} in project '{project.get_name()}'") - return self._run_services(runner, config, args, args.services) + else: + return super().run(runner, args) diff --git a/src/kiwi/subcommands/build.py b/src/kiwi/subcommands/build.py index d53c50d..5676c17 100644 --- a/src/kiwi/subcommands/build.py +++ b/src/kiwi/subcommands/build.py @@ -1,19 +1,20 @@ # local -from ._subcommand import FlexCommand +from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand -class BuildCommand(FlexCommand): +class BuildCommand(ServiceCommand): """kiwi build""" def __init__(self): super().__init__( - 'build', "Building images for", + 'build', num_projects='?', num_services='*', + action="Building images for", description="Build images for the whole instance, a project or service(s) inside a project" ) - def _run_services(self, runner, config, args, services): - DockerCommand('docker-compose').run( - config, args, ['build', '--pull', *services] - ) + def _run_services(self, runner, args, project, services): + DockerCommand('docker-compose').run(project, [ + 'build', '--pull', *services + ]) return True diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index 3182f11..f08c6dc 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -1,3 +1,6 @@ +# system +import logging + # local from ._subcommand import ProjectCommand from .utils.dockercommand import DockerCommand @@ -9,19 +12,32 @@ class CmdCommand(ProjectCommand): def __init__(self): super().__init__( 'cmd', num_projects=1, + action="Running docker-compose in", description="Run raw docker-compose command in a project" ) - # command string after docker-compose + # command for docker-compose self._sub_parser.add_argument( 'compose_cmd', metavar='cmd', type=str, - help="runs `docker-compose `" + help="command for 'docker-compose'" ) - def run(self, runner, config, args): - import shlex + # arguments for docker-compose command + self._sub_parser.add_argument( + 'compose_args', metavar='arg', nargs='*', type=str, + help="arguments for 'docker-compose' commands" + ) + + def _run_projects(self, runner, args, projects): + if args.unknowns: + args.compose_args = [*args.compose_args, *args.unknowns] + args.unknowns = [] + + logging.debug(f"Updated args: {args}") # run with split compose_cmd argument - DockerCommand('docker-compose').run(config, args, shlex.split(args.compose_cmd)) + DockerCommand('docker-compose').run(projects[0], [ + args.compose_cmd, *args.compose_args + ]) return True diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index 7656f07..b3d511f 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -10,6 +10,7 @@ from .utils.rootkit import Rootkit, prefix_path_mnt # parent from .._constants import CONF_DIRECTORY_NAME +from ..config import LoadedConfig class ConfCopyCommand(SubCommand): @@ -21,22 +22,20 @@ class ConfCopyCommand(SubCommand): description="Synchronize all config files to target directory" ) - def run(self, runner, config, args): + def _run_instance(self, runner, args): conf_dirs = [ project.conf_dir_name() - for project in Projects.all() - if project.is_enabled() + for project in Projects.from_dir().filter_enabled() ] if conf_dirs: # add target directory - conf_dirs.append(config['runtime:storage']) + conf_dirs.append(LoadedConfig.get()['runtime:storage']) logging.info(f"Sync directories: {conf_dirs}") - Rootkit('rsync').run( - config, args, ['rsync', '-r', *prefix_path_mnt(conf_dirs)], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) + Rootkit('rsync').run([ + 'rsync', '-r', *prefix_path_mnt(conf_dirs) + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) return True @@ -50,14 +49,13 @@ class ConfPurgeCommand(SubCommand): description="Remove all config files in target directory" ) - def run(self, runner, config, args): - conf_target = f"{config['runtime:storage']}/{CONF_DIRECTORY_NAME}" + def _run_instance(self, runner, args): + conf_target = f"{LoadedConfig.get()['runtime:storage']}/{CONF_DIRECTORY_NAME}" logging.info(f"Purging directories: {conf_target}") - Rootkit().run( - config, args, ['rm', '-rf', prefix_path_mnt(conf_target)], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) + Rootkit().run([ + 'rm', '-rf', prefix_path_mnt(conf_target) + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) return True @@ -71,12 +69,12 @@ class ConfCleanCommand(SubCommand): description="Cleanly sync all configs to target folder, relaunch affected projects" ) - def run(self, runner, config, args): + def _run_instance(self, runner, args): result = True affected_projects = [ project.conf_dir_name() - for project in Projects.all() + for project in Projects.from_dir() if project.has_configs() ] @@ -93,4 +91,4 @@ class ConfCleanCommand(SubCommand): args.projects = project_name result &= runner.run('up') - return result \ No newline at end of file + return result diff --git a/src/kiwi/subcommands/disable.py b/src/kiwi/subcommands/disable.py index 29b4f88..f2e9362 100644 --- a/src/kiwi/subcommands/disable.py +++ b/src/kiwi/subcommands/disable.py @@ -9,11 +9,12 @@ class DisableCommand(ProjectCommand): def __init__(self): super().__init__( 'disable', num_projects='+', + action="Disabling", description="Disable whole project(s) in this instance" ) - def run(self, runner, config, args): + def _run_projects(self, runner, args, projects): return all([ project.disable() - for project in Projects.from_args(args) + for project in projects ]) diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index ebb4a4c..286f65f 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -1,40 +1,42 @@ # local -from ._subcommand import FlexCommand +from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand from .utils.misc import are_you_sure -class DownCommand(FlexCommand): +class DownCommand(ServiceCommand): """kiwi down""" def __init__(self): super().__init__( - 'down', "Bringing down", + 'down', num_projects='?', num_services='*', + action="Bringing down", description="Bring down the whole instance, a project or service(s) inside a project" ) - def _run_instance(self, runner, config, args): + def _run_instance(self, runner, args): if are_you_sure([ "This will bring down the entire instance.", "", "This may not be what you intended, because:", " - Bringing down the instance stops ALL services in here", ]): - return super()._run_instance(runner, config, args) + return super()._run_instance(runner, args) return False - def _run_project(self, runner, config, args): - DockerCommand('docker-compose').run( - config, args, ['down'] - ) + def _run_projects(self, runner, args, projects): + for project in projects: + DockerCommand('docker-compose').run(project, [ + 'down' + ]) return True - def _run_services(self, runner, config, args, services): - DockerCommand('docker-compose').run( - config, args, ['stop', *services] - ) - DockerCommand('docker-compose').run( - config, args, ['rm', '-f', *services] - ) + def _run_services(self, runner, args, project, services): + DockerCommand('docker-compose').run(project, [ + 'stop', *services + ]) + DockerCommand('docker-compose').run(project, [ + 'rm', '-f', *services + ]) return True diff --git a/src/kiwi/subcommands/enable.py b/src/kiwi/subcommands/enable.py index 9392b95..9432bdf 100644 --- a/src/kiwi/subcommands/enable.py +++ b/src/kiwi/subcommands/enable.py @@ -9,11 +9,12 @@ class EnableCommand(ProjectCommand): def __init__(self): super().__init__( 'enable', num_projects='+', + action="Enabling", description="Enable whole project(s) in this instance" ) - def run(self, runner, config, args): + def _run_projects(self, runner, args, projects): return all([ project.enable() - for project in Projects.from_args(args) + for project in projects ]) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index a637c79..53ed9e3 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -7,6 +7,7 @@ from ._subcommand import SubCommand # parent (display purposes only) from .._constants import KIWI_CONF_NAME +from ..config import DefaultConfig, LoadedConfig def user_input(config, key, prompt): @@ -40,12 +41,12 @@ class InitCommand(SubCommand): help=f"use default values even if {KIWI_CONF_NAME} is present" ) - def run(self, runner, config, args): + def _run_instance(self, runner, args): logging.info(f"Initializing '{KIWI_CONF_NAME}' in '{os.getcwd()}'") + config = LoadedConfig.get() # check force switch if args.force and os.path.isfile(KIWI_CONF_NAME): - from ..config import DefaultConfig logging.warning(f"Overwriting existing '{KIWI_CONF_NAME}'!") config = DefaultConfig.get() diff --git a/src/kiwi/subcommands/list.py b/src/kiwi/subcommands/list.py index 6e4cd9f..df8068c 100644 --- a/src/kiwi/subcommands/list.py +++ b/src/kiwi/subcommands/list.py @@ -1,84 +1,75 @@ # system import logging import os -import subprocess import yaml # local -from ._subcommand import FlexCommand -from .utils.dockercommand import DockerCommand -from .utils.project import Projects +from ._subcommand import ServiceCommand +from .utils.project import Project, Projects def _print_list(strings): - if isinstance(strings, list): + if isinstance(strings, str): + print(f" - {strings}") + + elif isinstance(strings, Project): + _print_list(strings.get_name()) + + elif isinstance(strings, list): for string in strings: - print(f" - {string}") + _print_list(string) - elif isinstance(strings, str): - _print_list(strings.strip().split('\n')) - - elif isinstance(strings, bytes): - _print_list(str(strings, 'utf-8')) + else: + _print_list(list(strings)) -class ListCommand(FlexCommand): +class ListCommand(ServiceCommand): """kiwi list""" def __init__(self): super().__init__( - 'list', "Listing", + 'list', num_projects='?', num_services='*', + action="Listing", description="List projects in this instance, services inside a project or service(s) inside a project" ) - def _run_instance(self, runner, config, args): + def _run_instance(self, runner, args): print(f"kiwi-config instance at '{os.getcwd()}'") print("#########") - projects = Projects.all() + projects = Projects.from_dir() - enableds = [ - project.get_name() - for project in projects - if project.is_enabled() - ] - - if enableds: + enabled_projects = projects.filter_enabled() + if not enabled_projects.empty(): print(f"Enabled projects:") - _print_list(enableds) + _print_list(enabled_projects) - disableds = [ - project.get_name() - for project in projects - if project.is_disabled() - ] - - if disableds: + disabled_projects = projects.filter_disabled() + if not disabled_projects.empty(): print(f"Disabled projects:") - _print_list(disableds) + _print_list(disabled_projects) return True - def _run_project(self, runner, config, args): - project = Projects.from_args(args)[0] - + def _run_projects(self, runner, args, projects): + project = projects[0] if not project.exists(): - logging.error(f"Project '{project.get_name()}' not found") + logging.warning(f"Project '{project.get_name()}' not found") return False print(f"Services in project '{project.get_name()}':") print("#########") - ps = DockerCommand('docker-compose').run( - config, args, ['config', '--services'], - stdout=subprocess.PIPE - ) + with open(project.compose_file_name(), 'r') as stream: + try: + docker_compose_yml = yaml.safe_load(stream) + _print_list(docker_compose_yml['services'].keys()) + + except yaml.YAMLError as exc: + logging.error(exc) - _print_list(ps.stdout) return True - def _run_services(self, runner, config, args, services): - project = Projects.from_args(args)[0] - + def _run_services(self, runner, args, project, services): if not project.exists(): logging.error(f"Project '{project.get_name()}' not found") return False @@ -91,10 +82,13 @@ class ListCommand(FlexCommand): docker_compose_yml = yaml.safe_load(stream) for service_name in services: - print(yaml.dump( - {service_name: docker_compose_yml['services'][service_name]}, - default_flow_style=False, sort_keys=False - ).strip()) + try: + print(yaml.dump( + {service_name: docker_compose_yml['services'][service_name]}, + default_flow_style=False, sort_keys=False + ).strip()) + except KeyError: + logging.error(f"Service '{service_name}' not found") return True diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index a60122f..280a966 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -9,6 +9,7 @@ class LogsCommand(ServiceCommand): def __init__(self): super().__init__( 'logs', num_projects=1, num_services='*', + action="Showing logs of", description="Show logs of a project or service(s) of a project" ) @@ -18,7 +19,7 @@ class LogsCommand(ServiceCommand): help="output appended data as log grows" ) - def run(self, runner, config, args): + def _run_services(self, runner, args, project, services): # include timestamps compose_cmd = ['logs', '-t'] @@ -27,13 +28,13 @@ class LogsCommand(ServiceCommand): compose_cmd = [*compose_cmd, '-f', '--tail=10'] # append if one or more services are given - if args.services: + if services: compose_cmd = [*compose_cmd, *args.services] # use 'less' viewer if output will be static if args.follow: - DockerCommand('docker-compose').run(config, args, compose_cmd) + DockerCommand('docker-compose').run(project, compose_cmd) else: - DockerCommand('docker-compose').run_less(config, args, compose_cmd) + DockerCommand('docker-compose').run_less(project, compose_cmd) return True diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 6c007f8..e83f513 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -7,16 +7,18 @@ from ._subcommand import SubCommand from .utils.dockercommand import DockerCommand from .utils.misc import are_you_sure +# parent +from ..config import LoadedConfig -def _find_net(config, args): - ps = DockerCommand('docker').run( - config, args, ['network', 'ls', '--filter', f"name={config['network:name']}", '--format', '{{.Name}}'], - stdout=subprocess.PIPE - ) + +def _find_net(net_name): + ps = DockerCommand('docker').run(None, [ + 'network', 'ls', '--filter', f"name={net_name}", '--format', '{{.Name}}' + ], stdout=subprocess.PIPE) net_found = str(ps.stdout, 'utf-8').strip() - return net_found == config['network:name'] + return net_found == net_name class NetUpCommand(SubCommand): @@ -25,30 +27,31 @@ class NetUpCommand(SubCommand): def __init__(self): super().__init__( 'net-up', + action="Creating the local network hub", description="Create the local network hub for this instance" ) - def run(self, runner, config, args): - if _find_net(config, args): - logging.info(f"Network '{config['network:name']}' already exists") + def _run_instance(self, runner, args): + config = LoadedConfig.get() + net_name = config['network:name'] + net_cidr = config['network:cidr'] + + if _find_net(net_name): + logging.info(f"Network '{net_name}' already exists") return True try: - DockerCommand('docker').run( - config, args, - [ - 'network', 'create', - '--driver', 'bridge', - '--internal', - '--subnet', config['network:cidr'], - config['network:name'] - ], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - logging.info(f"Network '{config['network:name']}' created") + DockerCommand('docker').run(None, [ + 'network', 'create', + '--driver', 'bridge', + '--internal', + '--subnet', net_cidr, + net_name + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + logging.info(f"Network '{net_name}' created") except subprocess.CalledProcessError: - logging.error(f"Error creating network '{config['network:name']}'") + logging.error(f"Error creating network '{net_name}'") return False return True @@ -63,25 +66,26 @@ class NetDownCommand(SubCommand): description="Remove the local network hub for this instance" ) - def run(self, runner, config, args): - if not _find_net(config, args): - logging.info(f"Network '{config['network:name']}' does not exist") + def _run_instance(self, runner, args): + net_name = LoadedConfig.get()['network:name'] + + if not _find_net(net_name): + logging.info(f"Network '{net_name}' does not exist") return True try: if are_you_sure("This will bring down this instance's hub network!"): if runner.run('down'): - DockerCommand('docker').run( - config, args, - ['network', 'rm', config['network:name']], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - logging.info(f"Network '{config['network:name']}' removed") + DockerCommand('docker').run(None, [ + 'network', 'rm', net_name + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + logging.info(f"Network '{net_name}' removed") else: return False except subprocess.CalledProcessError: - logging.error(f"Error removing network '{config['network:name']}'") + logging.error(f"Error removing network '{net_name}'") return False return True diff --git a/src/kiwi/subcommands/new.py b/src/kiwi/subcommands/new.py index 913f134..e92142c 100644 --- a/src/kiwi/subcommands/new.py +++ b/src/kiwi/subcommands/new.py @@ -17,12 +17,12 @@ class NewCommand(ProjectCommand): def __init__(self): super().__init__( 'new', num_projects='+', + action="Creating", description="Create new empty project(s) in this instance" ) - def run(self, runner, config, args): + def _run_projects(self, runner, args, projects): result = True - projects = Projects.from_args(args) for project in projects: if project.exists(): @@ -31,7 +31,7 @@ class NewCommand(ProjectCommand): else: logging.info(f"Creating project '{project.get_name()}'") - os.mkdir(project.enabled_dir_name()) + os.mkdir(project.disabled_dir_name()) shutil.copy(DEFAULT_DOCKER_COMPOSE_NAME, project.compose_file_name()) return result diff --git a/src/kiwi/subcommands/pull.py b/src/kiwi/subcommands/pull.py index 019c007..debe467 100644 --- a/src/kiwi/subcommands/pull.py +++ b/src/kiwi/subcommands/pull.py @@ -1,19 +1,20 @@ # local -from ._subcommand import FlexCommand +from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand -class PullCommand(FlexCommand): +class PullCommand(ServiceCommand): """kiwi pull""" def __init__(self): super().__init__( - 'pull', "Pulling images for", + 'pull', num_projects='?', num_services='*', + action="Pulling images for", description="Pull images for the whole instance, a project or service(s) inside a project" ) - def _run_services(self, runner, config, args, services): - DockerCommand('docker-compose').run( - config, args, ['pull', '--ignore-pull-failures', *services] - ) + def _run_services(self, runner, args, project, services): + DockerCommand('docker-compose').run(project, [ + 'pull', '--ignore-pull-failures', *services + ]) return True diff --git a/src/kiwi/subcommands/push.py b/src/kiwi/subcommands/push.py index f0b00ad..b026b1b 100644 --- a/src/kiwi/subcommands/push.py +++ b/src/kiwi/subcommands/push.py @@ -1,19 +1,20 @@ # local -from ._subcommand import FlexCommand +from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand -class PushCommand(FlexCommand): +class PushCommand(ServiceCommand): """kiwi push""" def __init__(self): super().__init__( - 'push', "Pushing images for", + 'push', num_projects='?', num_services='*', + action="Pushing images for", description="Push images for the whole instance, a project or service(s) inside a project" ) - def _run_services(self, runner, config, args, services): - DockerCommand('docker-compose').run( - config, args, ['push', *services] - ) + def _run_services(self, runner, args, project, services): + DockerCommand('docker-compose').run(project, [ + 'push', *services + ]) return True diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index 7e3d31e..c945d42 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -6,19 +6,21 @@ import subprocess from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand +# parent +from ..config import LoadedConfig -def _service_has_executable(config, args, compose_cmd, exe_name): + +def _service_has_executable(project, service, exe_name): """ - Test if container (as of compose_cmd array) has an executable exe_name in its PATH. + Test if service in project has an executable exe_name in its PATH. Requires /bin/sh and which. """ try: # test if desired shell exists - DockerCommand('docker-compose').run( - config, args, [*compose_cmd, '/bin/sh', '-c', f"which {exe_name}"], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) + DockerCommand('docker-compose').run(project, [ + 'exec', service, '/bin/sh', '-c', f"which {exe_name}" + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) return True except subprocess.CalledProcessError as e: @@ -26,13 +28,14 @@ def _service_has_executable(config, args, compose_cmd, exe_name): return False -def _find_shell(config, args, compose_cmd): - """find first working shell (provided by config and args) in container (as of compose_cmd array)""" +def _find_shell(args, project, service): + """find first working shell (provided by config and args) in service in project""" # builtin shells: as a last resort, fallback to '/bin/sh' and 'sh' shells = ['/bin/sh', 'sh'] # load favorite shells from config + config = LoadedConfig.get() if config['runtime:shells']: shells = [*config['runtime:shells'], *shells] @@ -44,7 +47,7 @@ def _find_shell(config, args, compose_cmd): # actually try shells for i, shell in enumerate(shells): - if _service_has_executable(config, args, compose_cmd, shell): + if _service_has_executable(project, service, shell): # found working shell logging.debug(f"Using shell '{shell}'") return shell @@ -82,15 +85,15 @@ class ShCommand(ServiceCommand): help="shell to spawn" ) - def run(self, runner, config, args): - compose_cmd = ['exec', args.services[0]] - shell = _find_shell(config, args, compose_cmd) + def _run_services(self, runner, args, project, services): + service = services[0] + shell = _find_shell(args, project, service) if shell is not None: # spawn shell - DockerCommand('docker-compose').run( - config, args, [*compose_cmd, shell] - ) + DockerCommand('docker-compose').run(project, [ + 'exec', service, shell + ]) return True return False diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 4448734..99834df 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -1,6 +1,9 @@ # local from ._subcommand import SubCommand +# parent +from ..config import LoadedConfig + class ShowCommand(SubCommand): """kiwi show""" @@ -11,6 +14,6 @@ class ShowCommand(SubCommand): description="Show effective kiwi.yml" ) - def run(self, runner, config, args): - print(config) + def _run_instance(self, runner, args): + print(LoadedConfig.get()) return True diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index 0f958cd..ae4261d 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -1,28 +1,29 @@ # local -from ._subcommand import FlexCommand +from ._subcommand import ServiceCommand from .utils.dockercommand import DockerCommand -class UpCommand(FlexCommand): +class UpCommand(ServiceCommand): """kiwi up""" def __init__(self): super().__init__( - 'up', "Bringing up", + 'up', num_projects='?', num_services='*', + action="Bringing up", description="Bring up the whole instance, a project or service(s) inside a project" ) - def _run_instance(self, runner, config, args): + def _run_instance(self, runner, args): if runner.run('conf-copy'): - return super()._run_instance(runner, config, args) + return super()._run_instance(runner, args) return False - def _run_services(self, runner, config, args, services): + def _run_services(self, runner, args, project, services): if runner.run('net-up'): - DockerCommand('docker-compose').run( - config, args, ['up', '-d', *services] - ) + DockerCommand('docker-compose').run(project, [ + 'up', '-d', *services + ]) return True return False diff --git a/src/kiwi/subcommands/update.py b/src/kiwi/subcommands/update.py index cfac6fe..ccb58a3 100644 --- a/src/kiwi/subcommands/update.py +++ b/src/kiwi/subcommands/update.py @@ -1,18 +1,19 @@ # local -from ._subcommand import FlexCommand +from ._subcommand import ServiceCommand from .utils.misc import are_you_sure -class UpdateCommand(FlexCommand): +class UpdateCommand(ServiceCommand): """kiwi update""" def __init__(self): super().__init__( - 'update', "Updating", + 'update', num_projects='?', num_services='*', + action="Updating", description="Update the whole instance, a project or service(s) inside a project" ) - def _run_instance(self, runner, config, args): + def _run_instance(self, runner, args): if are_you_sure([ "This will update the entire instance at once.", "", @@ -20,12 +21,14 @@ class UpdateCommand(FlexCommand): " - Updates may take a long time", " - Updates may break beloved functionality", ]): - return super()._run_instance(runner, config, args) + return super()._run_instance(runner, args) return False - def _run_services(self, runner, config, args, services): - result = runner.run('build') + def _run_services(self, runner, args, project, services): + result = True + + result &= runner.run('build') result &= runner.run('pull') result &= runner.run('conf-copy') result &= runner.run('down') diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py index 5863f1d..ea2cc2f 100644 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ b/src/kiwi/subcommands/utils/dockercommand.py @@ -5,21 +5,16 @@ import subprocess # local from .executable import Executable -from .project import Projects # parent from ..._constants import CONF_DIRECTORY_NAME -from ...parser import Parser from ...config import LoadedConfig -def _update_kwargs(**kwargs): - config = LoadedConfig.get() - projects = Projects.from_args(Parser().get_args()) - - # project given in args: command affects a project in this instance - if projects: - project = projects[0] +def _update_kwargs(project, **kwargs): + # enabled project given: command affects a project in this instance + if project is not None and project.is_enabled(): + config = LoadedConfig.get() # execute command in project directory kwargs['cwd'] = project.dir_name() @@ -58,19 +53,15 @@ class DockerCommand(Executable): except subprocess.CalledProcessError: raise PermissionError("Cannot access docker, please get into the docker group or run as root!") - def run(self, config, args, process_args, **kwargs): - kwargs = _update_kwargs(**kwargs) - + def run(self, project, process_args, **kwargs): # equivalent to 'super().run' but agnostic of nested class construct return super().__getattr__("run")( - process_args, config, - **kwargs + process_args, + **_update_kwargs(project, **kwargs) ) - def run_less(self, config, args, process_args, **kwargs): - kwargs = _update_kwargs(**kwargs) - + def run_less(self, project, process_args, **kwargs): return super().__getattr__("run_less")( - process_args, config, - **kwargs + process_args, + **_update_kwargs(project, **kwargs) ) diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/subcommands/utils/executable.py index 4b7fecb..7f2a537 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/subcommands/utils/executable.py @@ -3,18 +3,22 @@ import logging import os import subprocess +# parent +from ...config import LoadedConfig -def _update_kwargs(config, **kwargs): - if config is not None: - # ensure there is an environment - if 'env' not in kwargs: - kwargs['env'] = {} - # add common environment from config - if config['runtime:env'] is not None: - kwargs['env'].update(config['runtime:env']) +def _update_kwargs(**kwargs): + config = LoadedConfig.get() - logging.debug(f"kwargs updated: {kwargs}") + # ensure there is an environment + if 'env' not in kwargs: + kwargs['env'] = {} + + # add common environment from config + if config['runtime:env'] is not None: + kwargs['env'].update(config['runtime:env']) + + logging.debug(f"kwargs updated: {kwargs}") return kwargs @@ -48,28 +52,24 @@ class Executable: logging.debug(f"Executable cmd{cmd}, kwargs{kwargs}") return cmd - def run(self, process_args, config=None, **kwargs): - kwargs = _update_kwargs(config, **kwargs) - + def run(self, process_args, **kwargs): return subprocess.run( - self.__build_cmd(process_args, **kwargs), + self.__build_cmd(process_args, **_update_kwargs(**kwargs)), **kwargs ) - def Popen(self, process_args, config=None, **kwargs): - kwargs = _update_kwargs(config, **kwargs) - + def Popen(self, process_args, **kwargs): return subprocess.Popen( - self.__build_cmd(process_args, **kwargs), + self.__build_cmd(process_args, **_update_kwargs(**kwargs)), **kwargs ) - def run_less(self, process_args, config=None, **kwargs): + def run_less(self, process_args, **kwargs): kwargs['stdout'] = subprocess.PIPE kwargs['stderr'] = subprocess.DEVNULL process = self.Popen( - process_args, config, + process_args, **kwargs ) diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/subcommands/utils/project.py index b248abe..d371d83 100644 --- a/src/kiwi/subcommands/utils/project.py +++ b/src/kiwi/subcommands/utils/project.py @@ -11,6 +11,20 @@ class Project: def __init__(self, name): self.__name = name + @classmethod + def from_file_name(cls, file_name): + if os.path.isdir(file_name): + config = LoadedConfig.get() + + if file_name.endswith(config['markers:disabled']): + file_name = file_name[:-len(config['markers:disabled'])] + + if file_name.endswith(config['markers:project']): + file_name = file_name[:-len(config['markers:project'])] + return cls(file_name) + + return None + def get_name(self): return self.__name @@ -78,40 +92,41 @@ class Project: return True -def _extract_project_name(file_name): - config = LoadedConfig.get() - enabled_suffix = config['markers:project'] - disabled_suffix = f"{enabled_suffix}{config['markers:disabled']}" - - if os.path.isdir(file_name): - # all subdirectories - if file_name.endswith(enabled_suffix): - # enabled projects - return file_name[:-len(enabled_suffix)] - - elif file_name.endswith(disabled_suffix): - # disabled projects - return file_name[:-len(disabled_suffix)] - - return None - - class Projects: __projects = None - def __init__(self, names): - self.__projects = [ - Project(name) - for name in names if isinstance(name, str) - ] - def __getitem__(self, item): return self.__projects[item] + def __str__(self): + return str([ + project.get_name() + for project + in self.__projects + ]) + @classmethod - def all(cls): - return cls([ - _extract_project_name(file_name) + def from_names(cls, project_names): + result = cls() + result.__projects = [ + Project(name) + for name in project_names if isinstance(name, str) + ] + return result + + @classmethod + def from_projects(cls, projects): + result = cls() + result.__projects = [ + project + for project in projects if isinstance(project, Project) + ] + return result + + @classmethod + def from_dir(cls): + return cls.from_projects([ + Project.from_file_name(file_name) for file_name in os.listdir() ]) @@ -119,9 +134,39 @@ class Projects: def from_args(cls, args): if args is not None and 'projects' in args: if isinstance(args.projects, list) and args.projects: - return cls(args.projects) + return cls.from_names(args.projects) elif isinstance(args.projects, str): - return cls([args.projects]) + return cls.from_names([args.projects]) - return [] + return cls() + + def empty(self): + return not self.__projects + + def filter_exists(self): + result = Projects() + result.__projects = [ + project + for project in self.__projects + if project.exists() + ] + return result + + def filter_enabled(self): + result = Projects() + result.__projects = [ + project + for project in self.__projects + if project.is_enabled() + ] + return result + + def filter_disabled(self): + result = Projects() + result.__projects = [ + project + for project in self.__projects + if project.is_disabled() + ] + return result diff --git a/src/kiwi/subcommands/utils/rootkit.py b/src/kiwi/subcommands/utils/rootkit.py index b3686c7..7f890eb 100644 --- a/src/kiwi/subcommands/utils/rootkit.py +++ b/src/kiwi/subcommands/utils/rootkit.py @@ -14,6 +14,7 @@ def _prefix_path(prefix, path): if isinstance(path, str): abs_path = os.path.abspath(path) return os.path.realpath(f"{prefix}/{abs_path}") + elif isinstance(path, list): return [_prefix_path(prefix, p) for p in path] @@ -36,55 +37,43 @@ class Rootkit: def __init__(self, image_tag=None): self.__image_tag = image_tag - def __exists(self, config, args): - ps = DockerCommand('docker').run( - config, args, [ - 'images', - '--filter', f"reference={_image_name(self.__image_tag)}", - '--format', '{{.Repository}}:{{.Tag}}' - ], - stdout=subprocess.PIPE - ) + def __exists(self): + ps = DockerCommand('docker').run(None, [ + 'images', + '--filter', f"reference={_image_name(self.__image_tag)}", + '--format', '{{.Repository}}:{{.Tag}}' + ], stdout=subprocess.PIPE) return str(ps.stdout, 'utf-8').strip() == _image_name(self.__image_tag) - def __build_image(self, config, args): - if self.__exists(config, args): + def __build_image(self): + if self.__exists(): logging.info(f"Using image {_image_name(self.__image_tag)}") else: if self.__image_tag is None: logging.info(f"Pulling image {_image_name(self.__image_tag)}") - DockerCommand('docker').run( - config, args, ['pull', _image_name(self.__image_tag)], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) + DockerCommand('docker').run(None, [ + 'pull', _image_name(self.__image_tag) + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: logging.info(f"Building image {_image_name(self.__image_tag)}") - DockerCommand('docker').run( - config, args, - [ - 'build', - '-t', _image_name(self.__image_tag), - '-f', f"{IMAGES_DIRECTORY_NAME}/{self.__image_tag}.Dockerfile", - f"{IMAGES_DIRECTORY_NAME}" - ], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) + DockerCommand('docker').run(None, [ + 'build', + '-t', _image_name(self.__image_tag), + '-f', f"{IMAGES_DIRECTORY_NAME}/{self.__image_tag}.Dockerfile", + f"{IMAGES_DIRECTORY_NAME}" + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - def run(self, config, args, process_args, **kwargs): - self.__build_image(config, args) - DockerCommand('docker').run( - config, args, - [ - 'run', '--rm', - '-v', '/:/mnt', - '-u', 'root', - _image_name(self.__image_tag), - *process_args - ], - **kwargs - ) + def run(self, process_args, **kwargs): + self.__build_image() + DockerCommand('docker').run(None, [ + 'run', '--rm', + '-v', '/:/mnt', + '-u', 'root', + _image_name(self.__image_tag), + *process_args + ], **kwargs) __image_tag = None __instances = {} From fd681418f59ee8af1f359c0cb48a7b554eccd997 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 16:23:52 +0200 Subject: [PATCH 096/118] action strings everywhere --- src/kiwi/subcommands/_subcommand.py | 14 ++++---------- src/kiwi/subcommands/conf.py | 3 +++ src/kiwi/subcommands/init.py | 1 + src/kiwi/subcommands/net.py | 3 ++- src/kiwi/subcommands/sh.py | 1 + src/kiwi/subcommands/show.py | 1 + 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index 9566443..dccb00d 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -19,7 +19,7 @@ class SubCommand: _action = None - def __init__(self, name, action='', add_parser=True, **kwargs): + def __init__(self, name, action, add_parser=True, **kwargs): self.__name = name if add_parser: self._sub_parser = Parser().get_subparsers().add_parser( @@ -27,12 +27,6 @@ class SubCommand: **kwargs ) - if not action: - # default action string - self._action = f"Running '{str(self)}' for" - else: - self._action = action - def __str__(self): return self.__name @@ -50,7 +44,7 @@ class SubCommand: class ProjectCommand(SubCommand): """this command concerns a project in current instance""" - def __init__(self, name, num_projects, action='', add_parser=True, **kwargs): + def __init__(self, name, num_projects, action, add_parser=True, **kwargs): super().__init__( name, action=action, add_parser=add_parser, **kwargs @@ -88,7 +82,7 @@ class ProjectCommand(SubCommand): class ServiceCommand(ProjectCommand): """this command concerns service(s) in a project""" - def __init__(self, name, num_projects, num_services, action='', add_parser=True, **kwargs): + def __init__(self, name, num_projects, num_services, action, add_parser=True, **kwargs): super().__init__( name, num_projects=num_projects, action=action, add_parser=add_parser, **kwargs @@ -96,7 +90,7 @@ class ServiceCommand(ProjectCommand): if (isinstance(num_projects, str) and num_projects == '*') \ or (isinstance(num_projects, int) and num_projects > 1): - logging.warning(f"Invalid choice for project count: {num_projects}") + raise ValueError(f"Invalid choice for project count: {num_projects}") if num_services == 1: services = "a service" diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index b3d511f..3030b98 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -19,6 +19,7 @@ class ConfCopyCommand(SubCommand): def __init__(self): super().__init__( 'conf-copy', + action="Syncing all configs for", description="Synchronize all config files to target directory" ) @@ -46,6 +47,7 @@ class ConfPurgeCommand(SubCommand): def __init__(self): super().__init__( 'conf-purge', + action="Removing all configs for", description="Remove all config files in target directory" ) @@ -66,6 +68,7 @@ class ConfCleanCommand(SubCommand): def __init__(self): super().__init__( 'conf-clean', + action="Cleaning all configs for", description="Cleanly sync all configs to target folder, relaunch affected projects" ) diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 53ed9e3..5f9298a 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -31,6 +31,7 @@ class InitCommand(SubCommand): def __init__(self): super().__init__( 'init', + action="Creating", description="Create a new kiwi-config instance" ) diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index e83f513..63236fc 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -27,7 +27,7 @@ class NetUpCommand(SubCommand): def __init__(self): super().__init__( 'net-up', - action="Creating the local network hub", + action="Creating the local network hub for", description="Create the local network hub for this instance" ) @@ -63,6 +63,7 @@ class NetDownCommand(SubCommand): def __init__(self): super().__init__( 'net-down', + action="Removing the local network hub for", description="Remove the local network hub for this instance" ) diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index c945d42..bf1d586 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -76,6 +76,7 @@ class ShCommand(ServiceCommand): def __init__(self): super().__init__( 'sh', num_projects=1, num_services=1, + action="Spawning shell in", description="Spawn shell inside a project's service" ) diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 99834df..1aed901 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -11,6 +11,7 @@ class ShowCommand(SubCommand): def __init__(self): super().__init__( 'show', + action="Printing", description="Show effective kiwi.yml" ) From 4290ed3743ddbd5d3b4d35dc384a9a7002269a46 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 16:25:57 +0200 Subject: [PATCH 097/118] command list (plans) --- src/etc/command_help.txt | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/etc/command_help.txt b/src/etc/command_help.txt index 781e9ff..07163bd 100644 --- a/src/etc/command_help.txt +++ b/src/etc/command_help.txt @@ -1,25 +1,27 @@ COMMANDS ======== -net-up +init -> config +show -> config --show + up down -net-down update +new enable disable -show -list +list -> show logs sh +cmd build pull push -init -conf-copy -conf-purge - -cmd +net-up +net-down +conf-copy (hide) +conf-purge (hide) +conf-clean -> clean From 1f1f3b27b2872326fdd3f635fcc49fed0badf037 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 17:09:43 +0200 Subject: [PATCH 098/118] Call compose from Project objects --- src/kiwi/runner.py | 12 ++++ src/kiwi/subcommands/build.py | 6 +- src/kiwi/subcommands/cmd.py | 5 +- src/kiwi/subcommands/down.py | 15 ++--- src/kiwi/subcommands/logs.py | 8 +-- src/kiwi/subcommands/net.py | 8 +-- src/kiwi/subcommands/pull.py | 6 +- src/kiwi/subcommands/push.py | 6 +- src/kiwi/subcommands/sh.py | 12 ++-- src/kiwi/subcommands/up.py | 5 +- src/kiwi/subcommands/utils/dockercommand.py | 67 --------------------- src/kiwi/subcommands/utils/project.py | 37 ++++++++++++ src/kiwi/subcommands/utils/rootkit.py | 10 +-- 13 files changed, 80 insertions(+), 117 deletions(-) delete mode 100644 src/kiwi/subcommands/utils/dockercommand.py diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 026f25b..3fba2c9 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -1,8 +1,10 @@ # system import logging +import subprocess # local from . import subcommands +from .subcommands.utils.executable import Executable from .parser import Parser @@ -15,6 +17,16 @@ class Runner: __commands = [] def __init__(self): + # probe for Docker access + try: + Executable('docker').run( + ['ps'], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + + except subprocess.CalledProcessError: + raise PermissionError("Cannot access docker, please get into the docker group or run as root!") + # setup all subcommands for className in subcommands.__all__: cmd = getattr(subcommands, className) diff --git a/src/kiwi/subcommands/build.py b/src/kiwi/subcommands/build.py index 5676c17..ed4b4a6 100644 --- a/src/kiwi/subcommands/build.py +++ b/src/kiwi/subcommands/build.py @@ -1,6 +1,5 @@ # local from ._subcommand import ServiceCommand -from .utils.dockercommand import DockerCommand class BuildCommand(ServiceCommand): @@ -14,7 +13,6 @@ class BuildCommand(ServiceCommand): ) def _run_services(self, runner, args, project, services): - DockerCommand('docker-compose').run(project, [ - 'build', '--pull', *services - ]) + project.compose_run(['build', '--pull', *services]) + return True diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index f08c6dc..0b12122 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -3,7 +3,6 @@ import logging # local from ._subcommand import ProjectCommand -from .utils.dockercommand import DockerCommand class CmdCommand(ProjectCommand): @@ -36,8 +35,6 @@ class CmdCommand(ProjectCommand): logging.debug(f"Updated args: {args}") # run with split compose_cmd argument - DockerCommand('docker-compose').run(projects[0], [ - args.compose_cmd, *args.compose_args - ]) + projects[0].compose_run([args.compose_cmd, *args.compose_args]) return True diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index 286f65f..6023919 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -1,6 +1,5 @@ # local from ._subcommand import ServiceCommand -from .utils.dockercommand import DockerCommand from .utils.misc import are_you_sure @@ -27,16 +26,12 @@ class DownCommand(ServiceCommand): def _run_projects(self, runner, args, projects): for project in projects: - DockerCommand('docker-compose').run(project, [ - 'down' - ]) + project.compose_run(['down']) + return True def _run_services(self, runner, args, project, services): - DockerCommand('docker-compose').run(project, [ - 'stop', *services - ]) - DockerCommand('docker-compose').run(project, [ - 'rm', '-f', *services - ]) + project.compose_run(['stop', *services]) + project.compose_run(['rm', '-f', *services]) + return True diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index 280a966..fcb1d3f 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,6 +1,5 @@ # local from ._subcommand import ServiceCommand -from .utils.dockercommand import DockerCommand class LogsCommand(ServiceCommand): @@ -31,10 +30,11 @@ class LogsCommand(ServiceCommand): if services: compose_cmd = [*compose_cmd, *args.services] - # use 'less' viewer if output will be static if args.follow: - DockerCommand('docker-compose').run(project, compose_cmd) + project.compose_run(compose_cmd) + else: - DockerCommand('docker-compose').run_less(project, compose_cmd) + # use 'less' viewer if output is static + project.compose_run_less(compose_cmd) return True diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 63236fc..8a630b1 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -4,7 +4,7 @@ import subprocess # local from ._subcommand import SubCommand -from .utils.dockercommand import DockerCommand +from .utils.executable import Executable from .utils.misc import are_you_sure # parent @@ -12,7 +12,7 @@ from ..config import LoadedConfig def _find_net(net_name): - ps = DockerCommand('docker').run(None, [ + ps = Executable('docker').run([ 'network', 'ls', '--filter', f"name={net_name}", '--format', '{{.Name}}' ], stdout=subprocess.PIPE) @@ -41,7 +41,7 @@ class NetUpCommand(SubCommand): return True try: - DockerCommand('docker').run(None, [ + Executable('docker').run([ 'network', 'create', '--driver', 'bridge', '--internal', @@ -77,7 +77,7 @@ class NetDownCommand(SubCommand): try: if are_you_sure("This will bring down this instance's hub network!"): if runner.run('down'): - DockerCommand('docker').run(None, [ + Executable('docker').run([ 'network', 'rm', net_name ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) diff --git a/src/kiwi/subcommands/pull.py b/src/kiwi/subcommands/pull.py index debe467..a54144f 100644 --- a/src/kiwi/subcommands/pull.py +++ b/src/kiwi/subcommands/pull.py @@ -1,6 +1,5 @@ # local from ._subcommand import ServiceCommand -from .utils.dockercommand import DockerCommand class PullCommand(ServiceCommand): @@ -14,7 +13,6 @@ class PullCommand(ServiceCommand): ) def _run_services(self, runner, args, project, services): - DockerCommand('docker-compose').run(project, [ - 'pull', '--ignore-pull-failures', *services - ]) + project.compose_run(['pull', '--ignore-pull-failures', *services]) + return True diff --git a/src/kiwi/subcommands/push.py b/src/kiwi/subcommands/push.py index b026b1b..48c3ab2 100644 --- a/src/kiwi/subcommands/push.py +++ b/src/kiwi/subcommands/push.py @@ -1,6 +1,5 @@ # local from ._subcommand import ServiceCommand -from .utils.dockercommand import DockerCommand class PushCommand(ServiceCommand): @@ -14,7 +13,6 @@ class PushCommand(ServiceCommand): ) def _run_services(self, runner, args, project, services): - DockerCommand('docker-compose').run(project, [ - 'push', *services - ]) + project.compose_run(['push', *services]) + return True diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index bf1d586..08a41b1 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -4,7 +4,6 @@ import subprocess # local from ._subcommand import ServiceCommand -from .utils.dockercommand import DockerCommand # parent from ..config import LoadedConfig @@ -18,9 +17,10 @@ def _service_has_executable(project, service, exe_name): try: # test if desired shell exists - DockerCommand('docker-compose').run(project, [ - 'exec', service, '/bin/sh', '-c', f"which {exe_name}" - ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + project.compose_run( + ['exec', service, '/bin/sh', '-c', f"which {exe_name}"], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) return True except subprocess.CalledProcessError as e: @@ -92,9 +92,7 @@ class ShCommand(ServiceCommand): if shell is not None: # spawn shell - DockerCommand('docker-compose').run(project, [ - 'exec', service, shell - ]) + project.compose_run(['exec', service, shell]) return True return False diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index ae4261d..f28e310 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -1,6 +1,5 @@ # local from ._subcommand import ServiceCommand -from .utils.dockercommand import DockerCommand class UpCommand(ServiceCommand): @@ -21,9 +20,7 @@ class UpCommand(ServiceCommand): def _run_services(self, runner, args, project, services): if runner.run('net-up'): - DockerCommand('docker-compose').run(project, [ - 'up', '-d', *services - ]) + project.compose_run(['up', '-d', *services]) return True return False diff --git a/src/kiwi/subcommands/utils/dockercommand.py b/src/kiwi/subcommands/utils/dockercommand.py deleted file mode 100644 index ea2cc2f..0000000 --- a/src/kiwi/subcommands/utils/dockercommand.py +++ /dev/null @@ -1,67 +0,0 @@ -# system -import logging -import os -import subprocess - -# local -from .executable import Executable - -# parent -from ..._constants import CONF_DIRECTORY_NAME -from ...config import LoadedConfig - - -def _update_kwargs(project, **kwargs): - # enabled project given: command affects a project in this instance - if project is not None and project.is_enabled(): - config = LoadedConfig.get() - - # execute command in project directory - kwargs['cwd'] = project.dir_name() - - # ensure there is an environment - if 'env' not in kwargs: - kwargs['env'] = {} - - # create environment variables for docker commands - kwargs['env'].update({ - 'COMPOSE_PROJECT_NAME': project.get_name(), - 'KIWI_HUB_NAME': config['network:name'], - 'CONFDIR': os.path.join(config['runtime:storage'], CONF_DIRECTORY_NAME), - 'TARGETDIR': project.target_dir_name() - }) - - logging.debug(f"kwargs updated: {kwargs}") - - return kwargs - - -class DockerCommand(Executable): - __has_tried = False - - def __init__(self, exe_name): - super().__init__(exe_name) - - if not DockerCommand.__has_tried: - try: - Executable('docker').run( - ['ps'], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - DockerCommand.__has_tried = True - - except subprocess.CalledProcessError: - raise PermissionError("Cannot access docker, please get into the docker group or run as root!") - - def run(self, project, process_args, **kwargs): - # equivalent to 'super().run' but agnostic of nested class construct - return super().__getattr__("run")( - process_args, - **_update_kwargs(project, **kwargs) - ) - - def run_less(self, project, process_args, **kwargs): - return super().__getattr__("run_less")( - process_args, - **_update_kwargs(project, **kwargs) - ) diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/subcommands/utils/project.py index d371d83..1037770 100644 --- a/src/kiwi/subcommands/utils/project.py +++ b/src/kiwi/subcommands/utils/project.py @@ -1,6 +1,8 @@ import logging import os +from .executable import Executable + from ..._constants import CONF_DIRECTORY_NAME from ...config import LoadedConfig @@ -63,6 +65,41 @@ class Project: def has_configs(self): return os.path.isdir(self.conf_dir_name()) + def __update_kwargs(self, kwargs): + if not self.is_enabled(): + # cannot compose in a disabled project + logging.warning(f"Project '{self.get_name()}' is not enabled!") + return False + + config = LoadedConfig.get() + + # execute command in project directory + kwargs['cwd'] = self.dir_name() + + # ensure there is an environment + if 'env' not in kwargs: + kwargs['env'] = {} + + # create environment variables for docker commands + kwargs['env'].update({ + 'COMPOSE_PROJECT_NAME': self.get_name(), + 'KIWI_HUB_NAME': config['network:name'], + 'CONFDIR': os.path.join(config['runtime:storage'], CONF_DIRECTORY_NAME), + 'TARGETDIR': self.target_dir_name() + }) + + logging.debug(f"kwargs updated: {kwargs}") + + return True + + def compose_run(self, compose_args, **kwargs): + if self.__update_kwargs(kwargs): + Executable('docker-compose').run(compose_args, **kwargs) + + def compose_run_less(self, compose_args, **kwargs): + if self.__update_kwargs(kwargs): + Executable('docker-compose').run_less(compose_args, **kwargs) + def enable(self): if self.is_disabled(): logging.info(f"Enabling project '{self.get_name()}'") diff --git a/src/kiwi/subcommands/utils/rootkit.py b/src/kiwi/subcommands/utils/rootkit.py index 7f890eb..c286030 100644 --- a/src/kiwi/subcommands/utils/rootkit.py +++ b/src/kiwi/subcommands/utils/rootkit.py @@ -4,7 +4,7 @@ import os import subprocess # local -from .dockercommand import DockerCommand +from .executable import Executable # parent from ..._constants import IMAGES_DIRECTORY_NAME, LOCAL_IMAGES_NAME, DEFAULT_IMAGE_NAME @@ -38,7 +38,7 @@ class Rootkit: self.__image_tag = image_tag def __exists(self): - ps = DockerCommand('docker').run(None, [ + ps = Executable('docker').run([ 'images', '--filter', f"reference={_image_name(self.__image_tag)}", '--format', '{{.Repository}}:{{.Tag}}' @@ -52,13 +52,13 @@ class Rootkit: else: if self.__image_tag is None: logging.info(f"Pulling image {_image_name(self.__image_tag)}") - DockerCommand('docker').run(None, [ + Executable('docker').run([ 'pull', _image_name(self.__image_tag) ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: logging.info(f"Building image {_image_name(self.__image_tag)}") - DockerCommand('docker').run(None, [ + Executable('docker').run([ 'build', '-t', _image_name(self.__image_tag), '-f', f"{IMAGES_DIRECTORY_NAME}/{self.__image_tag}.Dockerfile", @@ -67,7 +67,7 @@ class Rootkit: def run(self, process_args, **kwargs): self.__build_image() - DockerCommand('docker').run(None, [ + Executable('docker').run([ 'run', '--rm', '-v', '/:/mnt', '-u', 'root', From 717010d9b41f7b827ad463f6f593937064cc24c3 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 17:15:00 +0200 Subject: [PATCH 099/118] move utils package into top level --- src/kiwi/{subcommands/utils => }/executable.py | 2 +- src/kiwi/{subcommands/utils => }/misc.py | 0 src/kiwi/{subcommands/utils => }/project.py | 4 ++-- src/kiwi/{subcommands/utils => }/rootkit.py | 4 +--- src/kiwi/runner.py | 2 +- src/kiwi/subcommands/_subcommand.py | 4 +--- src/kiwi/subcommands/conf.py | 5 ++--- src/kiwi/subcommands/disable.py | 1 - src/kiwi/subcommands/down.py | 2 +- src/kiwi/subcommands/enable.py | 1 - src/kiwi/subcommands/list.py | 4 +++- src/kiwi/subcommands/net.py | 4 ++-- src/kiwi/subcommands/new.py | 1 - src/kiwi/subcommands/update.py | 4 +++- src/kiwi/subcommands/utils/__init__.py | 1 - 15 files changed, 17 insertions(+), 22 deletions(-) rename src/kiwi/{subcommands/utils => }/executable.py (98%) rename src/kiwi/{subcommands/utils => }/misc.py (100%) rename src/kiwi/{subcommands/utils => }/project.py (98%) rename src/kiwi/{subcommands/utils => }/rootkit.py (96%) delete mode 100644 src/kiwi/subcommands/utils/__init__.py diff --git a/src/kiwi/subcommands/utils/executable.py b/src/kiwi/executable.py similarity index 98% rename from src/kiwi/subcommands/utils/executable.py rename to src/kiwi/executable.py index 7f2a537..4681f92 100644 --- a/src/kiwi/subcommands/utils/executable.py +++ b/src/kiwi/executable.py @@ -4,7 +4,7 @@ import os import subprocess # parent -from ...config import LoadedConfig +from .config import LoadedConfig def _update_kwargs(**kwargs): diff --git a/src/kiwi/subcommands/utils/misc.py b/src/kiwi/misc.py similarity index 100% rename from src/kiwi/subcommands/utils/misc.py rename to src/kiwi/misc.py diff --git a/src/kiwi/subcommands/utils/project.py b/src/kiwi/project.py similarity index 98% rename from src/kiwi/subcommands/utils/project.py rename to src/kiwi/project.py index 1037770..60e1beb 100644 --- a/src/kiwi/subcommands/utils/project.py +++ b/src/kiwi/project.py @@ -3,8 +3,8 @@ import os from .executable import Executable -from ..._constants import CONF_DIRECTORY_NAME -from ...config import LoadedConfig +from ._constants import CONF_DIRECTORY_NAME +from .config import LoadedConfig class Project: diff --git a/src/kiwi/subcommands/utils/rootkit.py b/src/kiwi/rootkit.py similarity index 96% rename from src/kiwi/subcommands/utils/rootkit.py rename to src/kiwi/rootkit.py index c286030..a3c7937 100644 --- a/src/kiwi/subcommands/utils/rootkit.py +++ b/src/kiwi/rootkit.py @@ -4,11 +4,9 @@ import os import subprocess # local +from ._constants import IMAGES_DIRECTORY_NAME, LOCAL_IMAGES_NAME, DEFAULT_IMAGE_NAME from .executable import Executable -# parent -from ..._constants import IMAGES_DIRECTORY_NAME, LOCAL_IMAGES_NAME, DEFAULT_IMAGE_NAME - def _prefix_path(prefix, path): if isinstance(path, str): diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 3fba2c9..9f4058d 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -4,7 +4,7 @@ import subprocess # local from . import subcommands -from .subcommands.utils.executable import Executable +from .executable import Executable from .parser import Parser diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommands/_subcommand.py index dccb00d..1925bc7 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommands/_subcommand.py @@ -2,11 +2,9 @@ import logging import os -# local -from .utils.project import Projects - # parent from ..parser import Parser +from ..project import Projects class SubCommand: diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index 3030b98..0d590d1 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -1,16 +1,15 @@ # system import logging -import os import subprocess # local from ._subcommand import SubCommand -from .utils.project import Projects -from .utils.rootkit import Rootkit, prefix_path_mnt # parent from .._constants import CONF_DIRECTORY_NAME from ..config import LoadedConfig +from ..project import Projects +from ..rootkit import Rootkit, prefix_path_mnt class ConfCopyCommand(SubCommand): diff --git a/src/kiwi/subcommands/disable.py b/src/kiwi/subcommands/disable.py index f2e9362..82f8e5c 100644 --- a/src/kiwi/subcommands/disable.py +++ b/src/kiwi/subcommands/disable.py @@ -1,5 +1,4 @@ # local -from .utils.project import Projects from ._subcommand import ProjectCommand diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index 6023919..cd838fb 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -1,6 +1,6 @@ # local from ._subcommand import ServiceCommand -from .utils.misc import are_you_sure +from ..misc import are_you_sure class DownCommand(ServiceCommand): diff --git a/src/kiwi/subcommands/enable.py b/src/kiwi/subcommands/enable.py index 9432bdf..d0ea8a0 100644 --- a/src/kiwi/subcommands/enable.py +++ b/src/kiwi/subcommands/enable.py @@ -1,5 +1,4 @@ # local -from .utils.project import Projects from ._subcommand import ProjectCommand diff --git a/src/kiwi/subcommands/list.py b/src/kiwi/subcommands/list.py index df8068c..406ecb9 100644 --- a/src/kiwi/subcommands/list.py +++ b/src/kiwi/subcommands/list.py @@ -5,7 +5,9 @@ import yaml # local from ._subcommand import ServiceCommand -from .utils.project import Project, Projects + +# parent +from ..project import Project, Projects def _print_list(strings): diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 8a630b1..30a25ad 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -4,11 +4,11 @@ import subprocess # local from ._subcommand import SubCommand -from .utils.executable import Executable -from .utils.misc import are_you_sure # parent from ..config import LoadedConfig +from ..executable import Executable +from ..misc import are_you_sure def _find_net(net_name): diff --git a/src/kiwi/subcommands/new.py b/src/kiwi/subcommands/new.py index e92142c..9571ebe 100644 --- a/src/kiwi/subcommands/new.py +++ b/src/kiwi/subcommands/new.py @@ -4,7 +4,6 @@ import os import shutil # local -from .utils.project import Projects from ._subcommand import ProjectCommand # parent diff --git a/src/kiwi/subcommands/update.py b/src/kiwi/subcommands/update.py index ccb58a3..5335e37 100644 --- a/src/kiwi/subcommands/update.py +++ b/src/kiwi/subcommands/update.py @@ -1,6 +1,8 @@ # local from ._subcommand import ServiceCommand -from .utils.misc import are_you_sure + +# parent +from ..misc import are_you_sure class UpdateCommand(ServiceCommand): diff --git a/src/kiwi/subcommands/utils/__init__.py b/src/kiwi/subcommands/utils/__init__.py deleted file mode 100644 index 5a65bd6..0000000 --- a/src/kiwi/subcommands/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# git keep From bf5bed87347476072e542e00f1ba7eeddc4ad24e Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 17:21:38 +0200 Subject: [PATCH 100/118] move subcommand.py --- src/kiwi/executable.py | 2 +- src/kiwi/{subcommands/_subcommand.py => subcommand.py} | 6 +++--- src/kiwi/subcommands/build.py | 2 +- src/kiwi/subcommands/cmd.py | 2 +- src/kiwi/subcommands/conf.py | 4 +--- src/kiwi/subcommands/disable.py | 2 +- src/kiwi/subcommands/down.py | 2 +- src/kiwi/subcommands/enable.py | 2 +- src/kiwi/subcommands/init.py | 4 +--- src/kiwi/subcommands/list.py | 4 +--- src/kiwi/subcommands/logs.py | 2 +- src/kiwi/subcommands/net.py | 4 +--- src/kiwi/subcommands/new.py | 4 +--- src/kiwi/subcommands/pull.py | 2 +- src/kiwi/subcommands/push.py | 2 +- src/kiwi/subcommands/sh.py | 4 +--- src/kiwi/subcommands/show.py | 4 +--- src/kiwi/subcommands/up.py | 2 +- src/kiwi/subcommands/update.py | 4 +--- 19 files changed, 21 insertions(+), 37 deletions(-) rename src/kiwi/{subcommands/_subcommand.py => subcommand.py} (98%) diff --git a/src/kiwi/executable.py b/src/kiwi/executable.py index 4681f92..9f8aa8d 100644 --- a/src/kiwi/executable.py +++ b/src/kiwi/executable.py @@ -3,7 +3,7 @@ import logging import os import subprocess -# parent +# local from .config import LoadedConfig diff --git a/src/kiwi/subcommands/_subcommand.py b/src/kiwi/subcommand.py similarity index 98% rename from src/kiwi/subcommands/_subcommand.py rename to src/kiwi/subcommand.py index 1925bc7..18b518a 100644 --- a/src/kiwi/subcommands/_subcommand.py +++ b/src/kiwi/subcommand.py @@ -2,9 +2,9 @@ import logging import os -# parent -from ..parser import Parser -from ..project import Projects +# local +from .parser import Parser +from .project import Projects class SubCommand: diff --git a/src/kiwi/subcommands/build.py b/src/kiwi/subcommands/build.py index ed4b4a6..35cb4e5 100644 --- a/src/kiwi/subcommands/build.py +++ b/src/kiwi/subcommands/build.py @@ -1,5 +1,5 @@ # local -from ._subcommand import ServiceCommand +from ..subcommand import ServiceCommand class BuildCommand(ServiceCommand): diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index 0b12122..da90fc9 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -2,7 +2,7 @@ import logging # local -from ._subcommand import ProjectCommand +from ..subcommand import ProjectCommand class CmdCommand(ProjectCommand): diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index 0d590d1..77296fb 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -3,10 +3,8 @@ import logging import subprocess # local -from ._subcommand import SubCommand - -# parent from .._constants import CONF_DIRECTORY_NAME +from ..subcommand import SubCommand from ..config import LoadedConfig from ..project import Projects from ..rootkit import Rootkit, prefix_path_mnt diff --git a/src/kiwi/subcommands/disable.py b/src/kiwi/subcommands/disable.py index 82f8e5c..9715c27 100644 --- a/src/kiwi/subcommands/disable.py +++ b/src/kiwi/subcommands/disable.py @@ -1,5 +1,5 @@ # local -from ._subcommand import ProjectCommand +from ..subcommand import ProjectCommand class DisableCommand(ProjectCommand): diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index cd838fb..213eca9 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -1,5 +1,5 @@ # local -from ._subcommand import ServiceCommand +from ..subcommand import ServiceCommand from ..misc import are_you_sure diff --git a/src/kiwi/subcommands/enable.py b/src/kiwi/subcommands/enable.py index d0ea8a0..cf4079e 100644 --- a/src/kiwi/subcommands/enable.py +++ b/src/kiwi/subcommands/enable.py @@ -1,5 +1,5 @@ # local -from ._subcommand import ProjectCommand +from ..subcommand import ProjectCommand class EnableCommand(ProjectCommand): diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 5f9298a..0a2cc4e 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -3,10 +3,8 @@ import logging import os # local -from ._subcommand import SubCommand - -# parent (display purposes only) from .._constants import KIWI_CONF_NAME +from ..subcommand import SubCommand from ..config import DefaultConfig, LoadedConfig diff --git a/src/kiwi/subcommands/list.py b/src/kiwi/subcommands/list.py index 406ecb9..63e773d 100644 --- a/src/kiwi/subcommands/list.py +++ b/src/kiwi/subcommands/list.py @@ -4,9 +4,7 @@ import os import yaml # local -from ._subcommand import ServiceCommand - -# parent +from ..subcommand import ServiceCommand from ..project import Project, Projects diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index fcb1d3f..b2d49b7 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -1,5 +1,5 @@ # local -from ._subcommand import ServiceCommand +from ..subcommand import ServiceCommand class LogsCommand(ServiceCommand): diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py index 30a25ad..9db49a5 100644 --- a/src/kiwi/subcommands/net.py +++ b/src/kiwi/subcommands/net.py @@ -3,9 +3,7 @@ import logging import subprocess # local -from ._subcommand import SubCommand - -# parent +from ..subcommand import SubCommand from ..config import LoadedConfig from ..executable import Executable from ..misc import are_you_sure diff --git a/src/kiwi/subcommands/new.py b/src/kiwi/subcommands/new.py index 9571ebe..cfac9b1 100644 --- a/src/kiwi/subcommands/new.py +++ b/src/kiwi/subcommands/new.py @@ -4,10 +4,8 @@ import os import shutil # local -from ._subcommand import ProjectCommand - -# parent from .._constants import DEFAULT_DOCKER_COMPOSE_NAME +from ..subcommand import ProjectCommand class NewCommand(ProjectCommand): diff --git a/src/kiwi/subcommands/pull.py b/src/kiwi/subcommands/pull.py index a54144f..a1ed790 100644 --- a/src/kiwi/subcommands/pull.py +++ b/src/kiwi/subcommands/pull.py @@ -1,5 +1,5 @@ # local -from ._subcommand import ServiceCommand +from ..subcommand import ServiceCommand class PullCommand(ServiceCommand): diff --git a/src/kiwi/subcommands/push.py b/src/kiwi/subcommands/push.py index 48c3ab2..0f83c52 100644 --- a/src/kiwi/subcommands/push.py +++ b/src/kiwi/subcommands/push.py @@ -1,5 +1,5 @@ # local -from ._subcommand import ServiceCommand +from ..subcommand import ServiceCommand class PushCommand(ServiceCommand): diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/sh.py index 08a41b1..c8ef46e 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/sh.py @@ -3,9 +3,7 @@ import logging import subprocess # local -from ._subcommand import ServiceCommand - -# parent +from ..subcommand import ServiceCommand from ..config import LoadedConfig diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py index 1aed901..8b47e75 100644 --- a/src/kiwi/subcommands/show.py +++ b/src/kiwi/subcommands/show.py @@ -1,7 +1,5 @@ # local -from ._subcommand import SubCommand - -# parent +from ..subcommand import SubCommand from ..config import LoadedConfig diff --git a/src/kiwi/subcommands/up.py b/src/kiwi/subcommands/up.py index f28e310..b369545 100644 --- a/src/kiwi/subcommands/up.py +++ b/src/kiwi/subcommands/up.py @@ -1,5 +1,5 @@ # local -from ._subcommand import ServiceCommand +from ..subcommand import ServiceCommand class UpCommand(ServiceCommand): diff --git a/src/kiwi/subcommands/update.py b/src/kiwi/subcommands/update.py index 5335e37..0440cdc 100644 --- a/src/kiwi/subcommands/update.py +++ b/src/kiwi/subcommands/update.py @@ -1,7 +1,5 @@ # local -from ._subcommand import ServiceCommand - -# parent +from ..subcommand import ServiceCommand from ..misc import are_you_sure From e6f87277c6c31ad6ce0a09c75edf34f399325aeb Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 17:22:40 +0200 Subject: [PATCH 101/118] move Projects class --- src/kiwi/project.py | 80 ---------------------------------- src/kiwi/projects.py | 83 ++++++++++++++++++++++++++++++++++++ src/kiwi/subcommand.py | 2 +- src/kiwi/subcommands/conf.py | 2 +- src/kiwi/subcommands/list.py | 3 +- 5 files changed, 87 insertions(+), 83 deletions(-) create mode 100644 src/kiwi/projects.py diff --git a/src/kiwi/project.py b/src/kiwi/project.py index 60e1beb..8b59149 100644 --- a/src/kiwi/project.py +++ b/src/kiwi/project.py @@ -127,83 +127,3 @@ class Project: return False return True - - -class Projects: - __projects = None - - def __getitem__(self, item): - return self.__projects[item] - - def __str__(self): - return str([ - project.get_name() - for project - in self.__projects - ]) - - @classmethod - def from_names(cls, project_names): - result = cls() - result.__projects = [ - Project(name) - for name in project_names if isinstance(name, str) - ] - return result - - @classmethod - def from_projects(cls, projects): - result = cls() - result.__projects = [ - project - for project in projects if isinstance(project, Project) - ] - return result - - @classmethod - def from_dir(cls): - return cls.from_projects([ - Project.from_file_name(file_name) - for file_name in os.listdir() - ]) - - @classmethod - def from_args(cls, args): - if args is not None and 'projects' in args: - if isinstance(args.projects, list) and args.projects: - return cls.from_names(args.projects) - - elif isinstance(args.projects, str): - return cls.from_names([args.projects]) - - return cls() - - def empty(self): - return not self.__projects - - def filter_exists(self): - result = Projects() - result.__projects = [ - project - for project in self.__projects - if project.exists() - ] - return result - - def filter_enabled(self): - result = Projects() - result.__projects = [ - project - for project in self.__projects - if project.is_enabled() - ] - return result - - def filter_disabled(self): - result = Projects() - result.__projects = [ - project - for project in self.__projects - if project.is_disabled() - ] - return result diff --git a/src/kiwi/projects.py b/src/kiwi/projects.py new file mode 100644 index 0000000..4459363 --- /dev/null +++ b/src/kiwi/projects.py @@ -0,0 +1,83 @@ +import os + +from kiwi.project import Project + + +class Projects: + __projects = None + + def __getitem__(self, item): + return self.__projects[item] + + def __str__(self): + return str([ + project.get_name() + for project + in self.__projects + ]) + + @classmethod + def from_names(cls, project_names): + result = cls() + result.__projects = [ + Project(name) + for name in project_names if isinstance(name, str) + ] + return result + + @classmethod + def from_projects(cls, projects): + result = cls() + result.__projects = [ + project + for project in projects if isinstance(project, Project) + ] + return result + + @classmethod + def from_dir(cls): + return cls.from_projects([ + Project.from_file_name(file_name) + for file_name in os.listdir() + ]) + + @classmethod + def from_args(cls, args): + if args is not None and 'projects' in args: + if isinstance(args.projects, list) and args.projects: + return cls.from_names(args.projects) + + elif isinstance(args.projects, str): + return cls.from_names([args.projects]) + + return cls() + + def empty(self): + return not self.__projects + + def filter_exists(self): + result = Projects() + result.__projects = [ + project + for project in self.__projects + if project.exists() + ] + return result + + def filter_enabled(self): + result = Projects() + result.__projects = [ + project + for project in self.__projects + if project.is_enabled() + ] + return result + + def filter_disabled(self): + result = Projects() + result.__projects = [ + project + for project in self.__projects + if project.is_disabled() + ] + return result diff --git a/src/kiwi/subcommand.py b/src/kiwi/subcommand.py index 18b518a..59a64ee 100644 --- a/src/kiwi/subcommand.py +++ b/src/kiwi/subcommand.py @@ -4,7 +4,7 @@ import os # local from .parser import Parser -from .project import Projects +from .projects import Projects class SubCommand: diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index 77296fb..86a379e 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -6,7 +6,7 @@ import subprocess from .._constants import CONF_DIRECTORY_NAME from ..subcommand import SubCommand from ..config import LoadedConfig -from ..project import Projects +from ..projects import Projects from ..rootkit import Rootkit, prefix_path_mnt diff --git a/src/kiwi/subcommands/list.py b/src/kiwi/subcommands/list.py index 63e773d..cd75af8 100644 --- a/src/kiwi/subcommands/list.py +++ b/src/kiwi/subcommands/list.py @@ -5,7 +5,8 @@ import yaml # local from ..subcommand import ServiceCommand -from ..project import Project, Projects +from ..project import Project +from ..projects import Projects def _print_list(strings): From a085586baa94f17ba9fc5bc816503c6cf5280d44 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 17:25:39 +0200 Subject: [PATCH 102/118] use action strings --- src/kiwi/subcommand.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kiwi/subcommand.py b/src/kiwi/subcommand.py index 59a64ee..fb472ff 100644 --- a/src/kiwi/subcommand.py +++ b/src/kiwi/subcommand.py @@ -19,6 +19,8 @@ class SubCommand: def __init__(self, name, action, add_parser=True, **kwargs): self.__name = name + self._action = action + if add_parser: self._sub_parser = Parser().get_subparsers().add_parser( name, From d99f26e3cf84712098e176445d94014a55ee7d51 Mon Sep 17 00:00:00 2001 From: ldericher Date: Wed, 19 Aug 2020 17:42:09 +0200 Subject: [PATCH 103/118] exit status --- src/kiwi-config.py | 3 ++- src/kiwi/__init__.py | 2 +- src/kiwi/subcommands/init.py | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kiwi-config.py b/src/kiwi-config.py index 4c5acad..7435727 100755 --- a/src/kiwi-config.py +++ b/src/kiwi-config.py @@ -31,7 +31,8 @@ def main(): set_verbosity(logging.getLogger(), log_handler, kiwi.verbosity()) # run the app - kiwi.run() + if not kiwi.run(): + quit(1) if __name__ == "__main__": diff --git a/src/kiwi/__init__.py b/src/kiwi/__init__.py index 8a4a6db..ba8912f 100644 --- a/src/kiwi/__init__.py +++ b/src/kiwi/__init__.py @@ -11,7 +11,7 @@ def verbosity(): def run(): # pass down - Runner().run() + return Runner().run() __all__ = [ diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 0a2cc4e..7e0fa59 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -29,7 +29,7 @@ class InitCommand(SubCommand): def __init__(self): super().__init__( 'init', - action="Creating", + action=f"Initializing '{KIWI_CONF_NAME}' in", description="Create a new kiwi-config instance" ) @@ -41,7 +41,6 @@ class InitCommand(SubCommand): ) def _run_instance(self, runner, args): - logging.info(f"Initializing '{KIWI_CONF_NAME}' in '{os.getcwd()}'") config = LoadedConfig.get() # check force switch From 34685704033b66f0afc5908efb51eb575a4d14ff Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 12:30:06 +0200 Subject: [PATCH 104/118] load from directory --- src/kiwi/config.py | 16 +++++++--------- src/kiwi/projects.py | 4 ++-- src/kiwi/subcommands/conf.py | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/kiwi/config.py b/src/kiwi/config.py index 84cc462..ac56457 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -113,20 +113,18 @@ class LoadedConfig(Config): __instances = {} @classmethod - def get(cls): - cwd = os.getcwd() - - if cwd not in LoadedConfig.__instances: + def get(cls, directory='.'): + if directory not in LoadedConfig.__instances: # create singleton for new path result = DefaultConfig.get() - # update with current dir's kiwi.yml + # update with that dir's kiwi.yml try: - result = result._update_from_file(KIWI_CONF_NAME) + result = result._update_from_file(os.path.join(directory, KIWI_CONF_NAME)) except FileNotFoundError: - logging.info(f"No '{KIWI_CONF_NAME}' found at '{cwd}'. Using defaults.") + logging.info(f"No '{KIWI_CONF_NAME}' found at '{directory}'. Using defaults.") - LoadedConfig.__instances[cwd] = result + LoadedConfig.__instances[directory] = result # return singleton - return LoadedConfig.__instances[cwd] + return LoadedConfig.__instances[directory] diff --git a/src/kiwi/projects.py b/src/kiwi/projects.py index 4459363..65c75d0 100644 --- a/src/kiwi/projects.py +++ b/src/kiwi/projects.py @@ -35,10 +35,10 @@ class Projects: return result @classmethod - def from_dir(cls): + def from_dir(cls, directory='.'): return cls.from_projects([ Project.from_file_name(file_name) - for file_name in os.listdir() + for file_name in os.listdir(directory) ]) @classmethod diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/conf.py index 86a379e..bfbd933 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/conf.py @@ -73,7 +73,7 @@ class ConfCleanCommand(SubCommand): result = True affected_projects = [ - project.conf_dir_name() + project.get_name() for project in Projects.from_dir() if project.has_configs() ] From 3a28af7f7e64ef32d2fc42ae861449d74ebbbea9 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 13:29:08 +0200 Subject: [PATCH 105/118] user query into Config object --- src/kiwi/config.py | 27 +++++++++++++++++++++++++++ src/kiwi/subcommands/init.py | 27 ++++++--------------------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/kiwi/config.py b/src/kiwi/config.py index ac56457..d5de124 100644 --- a/src/kiwi/config.py +++ b/src/kiwi/config.py @@ -13,6 +13,19 @@ class Config: """represents a kiwi.yml""" __yml_content = {} + __keys = { + 'version': "kiwi-config version to use in this instance", + + 'runtime:storage': "local directory for service data", + 'runtime:shells': "shell preference for working in service containers", + 'runtime:env': "common environment for compose yml", + + 'markers:project': "marker string for project directories", + 'markers:disabled': "marker string for disabled projects", + + 'network:name': "name for local network hub", + 'network:cidr': "CIDR block for local network hub", + } def __key_resolve(self, key): """ @@ -80,6 +93,20 @@ class Config: except yaml.YAMLError as exc: logging.error(exc) + def user_query(self, key): + """query user for new config value""" + + # prompt user as per argument + try: + result = input(f"Enter {self.__keys[key]} [{self[key]}] ").strip() + except EOFError: + print() + result = None + + # store result if present + if result: + self[key] = result + def save(self): """save current yml representation in current directory's kiwi.yml""" diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/init.py index 7e0fa59..05012c0 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/init.py @@ -8,21 +8,6 @@ from ..subcommand import SubCommand from ..config import DefaultConfig, LoadedConfig -def user_input(config, key, prompt): - """query user for new config value""" - - # prompt user as per argument - try: - result = input(f"{prompt} [{config[key]}] ").strip() - except EOFError: - print() - result = None - - # store result if present - if result: - config[key] = result - - class InitCommand(SubCommand): """kiwi init""" @@ -50,18 +35,18 @@ class InitCommand(SubCommand): config = DefaultConfig.get() # version - user_input(config, 'version', "Enter kiwi-config version for this instance") + config.user_query('version') # runtime - user_input(config, 'runtime:storage', "Enter local directory for service data") + config.user_query('runtime:storage') # markers - user_input(config, 'markers:project', "Enter marker string for project directories") - user_input(config, 'markers:disabled', "Enter marker string for disabled projects") + config.user_query('markers:project') + config.user_query('markers:disabled') # network - user_input(config, 'network:name', "Enter name for local docker network") - user_input(config, 'network:cidr', "Enter CIDR block for local docker network") + config.user_query('network:name') + config.user_query('network:cidr') config.save() return True From 3ccdffe73790598b12d897fa58bc239ec513ef59 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 14:15:38 +0200 Subject: [PATCH 106/118] CLI commands cleanup --- src/etc/command_help.txt | 27 +++--- src/kiwi/subcommands/__init__.py | 31 +++---- src/kiwi/subcommands/{conf.py => _hidden.py} | 62 ++++++++------ src/kiwi/subcommands/clean.py | 37 ++++++++ src/kiwi/subcommands/{init.py => config.py} | 22 +++-- src/kiwi/subcommands/{list.py => inspect.py} | 10 +-- src/kiwi/subcommands/net.py | 90 -------------------- src/kiwi/subcommands/purge.py | 45 ++++++++++ src/kiwi/subcommands/{sh.py => shell.py} | 6 +- src/kiwi/subcommands/show.py | 18 ---- 10 files changed, 175 insertions(+), 173 deletions(-) rename src/kiwi/subcommands/{conf.py => _hidden.py} (54%) create mode 100644 src/kiwi/subcommands/clean.py rename src/kiwi/subcommands/{init.py => config.py} (68%) rename src/kiwi/subcommands/{list.py => inspect.py} (90%) delete mode 100644 src/kiwi/subcommands/net.py create mode 100644 src/kiwi/subcommands/purge.py rename src/kiwi/subcommands/{sh.py => shell.py} (96%) delete mode 100644 src/kiwi/subcommands/show.py diff --git a/src/etc/command_help.txt b/src/etc/command_help.txt index 07163bd..ea1ecf6 100644 --- a/src/etc/command_help.txt +++ b/src/etc/command_help.txt @@ -1,27 +1,30 @@ COMMANDS ======== -init -> config -show -> config --show +# operation +config up down update +clean + +# management + new enable disable - -list -> show -logs -sh cmd +purge + +# inspection + +inspect +logs +shell + +# imaging build pull push - -net-up -net-down -conf-copy (hide) -conf-purge (hide) -conf-clean -> clean diff --git a/src/kiwi/subcommands/__init__.py b/src/kiwi/subcommands/__init__.py index 637f9be..76ff7ab 100644 --- a/src/kiwi/subcommands/__init__.py +++ b/src/kiwi/subcommands/__init__.py @@ -1,41 +1,42 @@ # local +from ._hidden import ConfCopyCommand, ConfPurgeCommand, NetUpCommand + from .build import BuildCommand +from .clean import CleanCommand from .cmd import CmdCommand -from .conf import ConfCopyCommand, ConfPurgeCommand, ConfCleanCommand +from .config import ConfigCommand from .disable import DisableCommand from .down import DownCommand from .enable import EnableCommand -from .init import InitCommand -from .list import ListCommand +from .inspect import InspectCommand from .logs import LogsCommand -from .net import NetUpCommand, NetDownCommand from .new import NewCommand from .pull import PullCommand +from .purge import PurgeCommand from .push import PushCommand -from .sh import ShCommand -from .show import ShowCommand +from .shell import ShellCommand from .up import UpCommand from .update import UpdateCommand __all__ = [ - 'BuildCommand', - 'CmdCommand', 'ConfCopyCommand', 'ConfPurgeCommand', - 'ConfCleanCommand', + 'NetUpCommand', + + 'BuildCommand', + 'CleanCommand', + 'CmdCommand', + 'ConfigCommand', 'DisableCommand', 'DownCommand', 'EnableCommand', - 'InitCommand', - 'ListCommand', + 'InspectCommand', 'LogsCommand', - 'NetUpCommand', - 'NetDownCommand', 'NewCommand', 'PullCommand', + 'PurgeCommand', 'PushCommand', - 'ShCommand', - 'ShowCommand', + 'ShellCommand', 'UpCommand', 'UpdateCommand', ] diff --git a/src/kiwi/subcommands/conf.py b/src/kiwi/subcommands/_hidden.py similarity index 54% rename from src/kiwi/subcommands/conf.py rename to src/kiwi/subcommands/_hidden.py index bfbd933..bd7bbc8 100644 --- a/src/kiwi/subcommands/conf.py +++ b/src/kiwi/subcommands/_hidden.py @@ -4,6 +4,7 @@ import subprocess # local from .._constants import CONF_DIRECTORY_NAME +from ..executable import Executable from ..subcommand import SubCommand from ..config import LoadedConfig from ..projects import Projects @@ -16,7 +17,7 @@ class ConfCopyCommand(SubCommand): def __init__(self): super().__init__( 'conf-copy', - action="Syncing all configs for", + action="Syncing all configs for", add_parser=False, description="Synchronize all config files to target directory" ) @@ -44,7 +45,7 @@ class ConfPurgeCommand(SubCommand): def __init__(self): super().__init__( 'conf-purge', - action="Removing all configs for", + action="Removing all configs for", add_parser=False, description="Remove all config files in target directory" ) @@ -59,36 +60,47 @@ class ConfPurgeCommand(SubCommand): return True -class ConfCleanCommand(SubCommand): - """kiwi conf-clean""" +def _find_net(net_name): + ps = Executable('docker').run([ + 'network', 'ls', '--filter', f"name={net_name}", '--format', '{{.Name}}' + ], stdout=subprocess.PIPE) + + net_found = str(ps.stdout, 'utf-8').strip() + + return net_found == net_name + + +class NetUpCommand(SubCommand): + """kiwi net-up""" def __init__(self): super().__init__( - 'conf-clean', - action="Cleaning all configs for", - description="Cleanly sync all configs to target folder, relaunch affected projects" + 'net-up', + action="Creating the local network hub for", add_parser=False, + description="Create the local network hub for this instance" ) def _run_instance(self, runner, args): - result = True + config = LoadedConfig.get() + net_name = config['network:name'] + net_cidr = config['network:cidr'] - affected_projects = [ - project.get_name() - for project in Projects.from_dir() - if project.has_configs() - ] + if _find_net(net_name): + logging.info(f"Network '{net_name}' already exists") + return True - for project_name in affected_projects: - args.projects = project_name - result &= runner.run('down') + try: + Executable('docker').run([ + 'network', 'create', + '--driver', 'bridge', + '--internal', + '--subnet', net_cidr, + net_name + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + logging.info(f"Network '{net_name}' created") - # cleanly sync configs - result &= runner.run('conf-purge') - result &= runner.run('conf-copy') + except subprocess.CalledProcessError: + logging.error(f"Error creating network '{net_name}'") + return False - # bring projects back up - for project_name in affected_projects: - args.projects = project_name - result &= runner.run('up') - - return result + return True diff --git a/src/kiwi/subcommands/clean.py b/src/kiwi/subcommands/clean.py new file mode 100644 index 0000000..22422e5 --- /dev/null +++ b/src/kiwi/subcommands/clean.py @@ -0,0 +1,37 @@ +from ..projects import Projects +from ..subcommand import SubCommand + + +class CleanCommand(SubCommand): + """kiwi clean""" + + def __init__(self): + super().__init__( + 'clean', + action="Cleaning all configs for", + description="Cleanly sync all configs to target folder, then relaunch affected projects" + ) + + def _run_instance(self, runner, args): + result = True + + affected_projects = [ + project.get_name() + for project in Projects.from_dir() + if project.has_configs() + ] + + for project_name in affected_projects: + args.projects = project_name + result &= runner.run('down') + + # cleanly sync configs + result &= runner.run('conf-purge') + result &= runner.run('conf-copy') + + # bring projects back up + for project_name in affected_projects: + args.projects = project_name + result &= runner.run('up') + + return result diff --git a/src/kiwi/subcommands/init.py b/src/kiwi/subcommands/config.py similarity index 68% rename from src/kiwi/subcommands/init.py rename to src/kiwi/subcommands/config.py index 05012c0..1174a3b 100644 --- a/src/kiwi/subcommands/init.py +++ b/src/kiwi/subcommands/config.py @@ -8,14 +8,14 @@ from ..subcommand import SubCommand from ..config import DefaultConfig, LoadedConfig -class InitCommand(SubCommand): - """kiwi init""" +class ConfigCommand(SubCommand): + """kiwi config""" def __init__(self): super().__init__( - 'init', - action=f"Initializing '{KIWI_CONF_NAME}' in", - description="Create a new kiwi-config instance" + 'config', + action=f"Configuring '{KIWI_CONF_NAME}' in", + description="Configure kiwi-config instance" ) # -f switch: Initialize with default config @@ -25,9 +25,21 @@ class InitCommand(SubCommand): help=f"use default values even if {KIWI_CONF_NAME} is present" ) + # -s switch: Show current config instead + self._sub_parser.add_argument( + '-s', '--show', + action='store_true', + help=f"show effective {KIWI_CONF_NAME} contents instead" + ) + def _run_instance(self, runner, args): config = LoadedConfig.get() + # check show switch + if args.show: + print(config) + return True + # check force switch if args.force and os.path.isfile(KIWI_CONF_NAME): diff --git a/src/kiwi/subcommands/list.py b/src/kiwi/subcommands/inspect.py similarity index 90% rename from src/kiwi/subcommands/list.py rename to src/kiwi/subcommands/inspect.py index cd75af8..ae679f1 100644 --- a/src/kiwi/subcommands/list.py +++ b/src/kiwi/subcommands/inspect.py @@ -24,14 +24,14 @@ def _print_list(strings): _print_list(list(strings)) -class ListCommand(ServiceCommand): - """kiwi list""" +class InspectCommand(ServiceCommand): + """kiwi inspect""" def __init__(self): super().__init__( - 'list', num_projects='?', num_services='*', - action="Listing", - description="List projects in this instance, services inside a project or service(s) inside a project" + 'inspect', num_projects='?', num_services='*', + action="Inspecting", + description="Inspect projects in this instance, services inside a project or service(s) inside a project" ) def _run_instance(self, runner, args): diff --git a/src/kiwi/subcommands/net.py b/src/kiwi/subcommands/net.py deleted file mode 100644 index 9db49a5..0000000 --- a/src/kiwi/subcommands/net.py +++ /dev/null @@ -1,90 +0,0 @@ -# system -import logging -import subprocess - -# local -from ..subcommand import SubCommand -from ..config import LoadedConfig -from ..executable import Executable -from ..misc import are_you_sure - - -def _find_net(net_name): - ps = Executable('docker').run([ - 'network', 'ls', '--filter', f"name={net_name}", '--format', '{{.Name}}' - ], stdout=subprocess.PIPE) - - net_found = str(ps.stdout, 'utf-8').strip() - - return net_found == net_name - - -class NetUpCommand(SubCommand): - """kiwi net-up""" - - def __init__(self): - super().__init__( - 'net-up', - action="Creating the local network hub for", - description="Create the local network hub for this instance" - ) - - def _run_instance(self, runner, args): - config = LoadedConfig.get() - net_name = config['network:name'] - net_cidr = config['network:cidr'] - - if _find_net(net_name): - logging.info(f"Network '{net_name}' already exists") - return True - - try: - Executable('docker').run([ - 'network', 'create', - '--driver', 'bridge', - '--internal', - '--subnet', net_cidr, - net_name - ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - logging.info(f"Network '{net_name}' created") - - except subprocess.CalledProcessError: - logging.error(f"Error creating network '{net_name}'") - return False - - return True - - -class NetDownCommand(SubCommand): - """kiwi net-down""" - - def __init__(self): - super().__init__( - 'net-down', - action="Removing the local network hub for", - description="Remove the local network hub for this instance" - ) - - def _run_instance(self, runner, args): - net_name = LoadedConfig.get()['network:name'] - - if not _find_net(net_name): - logging.info(f"Network '{net_name}' does not exist") - return True - - try: - if are_you_sure("This will bring down this instance's hub network!"): - if runner.run('down'): - Executable('docker').run([ - 'network', 'rm', net_name - ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - logging.info(f"Network '{net_name}' removed") - else: - return False - - except subprocess.CalledProcessError: - logging.error(f"Error removing network '{net_name}'") - return False - - return True diff --git a/src/kiwi/subcommands/purge.py b/src/kiwi/subcommands/purge.py new file mode 100644 index 0000000..fbf771b --- /dev/null +++ b/src/kiwi/subcommands/purge.py @@ -0,0 +1,45 @@ +# system +import logging +import subprocess + +# local +from ._hidden import _find_net +from ..subcommand import SubCommand +from ..config import LoadedConfig +from ..executable import Executable +from ..misc import are_you_sure + + +class PurgeCommand(SubCommand): + """kiwi purge""" + + def __init__(self): + super().__init__( + 'purge', + action="Tearing down", + description="Remove all ephemeral artifacts of this instance" + ) + + def _run_instance(self, runner, args): + net_name = LoadedConfig.get()['network:name'] + + if not _find_net(net_name): + logging.info(f"Network '{net_name}' does not exist") + return True + + try: + if are_you_sure("This will bring down this instance's hub network!"): + if runner.run('down'): + Executable('docker').run([ + 'network', 'rm', net_name + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + logging.info(f"Network '{net_name}' removed") + else: + return False + + except subprocess.CalledProcessError: + logging.error(f"Error removing network '{net_name}'") + return False + + return True diff --git a/src/kiwi/subcommands/sh.py b/src/kiwi/subcommands/shell.py similarity index 96% rename from src/kiwi/subcommands/sh.py rename to src/kiwi/subcommands/shell.py index c8ef46e..3a3ed00 100644 --- a/src/kiwi/subcommands/sh.py +++ b/src/kiwi/subcommands/shell.py @@ -68,12 +68,12 @@ def _find_shell(args, project, service): return None -class ShCommand(ServiceCommand): - """kiwi sh""" +class ShellCommand(ServiceCommand): + """kiwi shell""" def __init__(self): super().__init__( - 'sh', num_projects=1, num_services=1, + 'shell', num_projects=1, num_services=1, action="Spawning shell in", description="Spawn shell inside a project's service" ) diff --git a/src/kiwi/subcommands/show.py b/src/kiwi/subcommands/show.py deleted file mode 100644 index 8b47e75..0000000 --- a/src/kiwi/subcommands/show.py +++ /dev/null @@ -1,18 +0,0 @@ -# local -from ..subcommand import SubCommand -from ..config import LoadedConfig - - -class ShowCommand(SubCommand): - """kiwi show""" - - def __init__(self): - super().__init__( - 'show', - action="Printing", - description="Show effective kiwi.yml" - ) - - def _run_instance(self, runner, args): - print(LoadedConfig.get()) - return True From 4bd602a245fa642b08074d7225f4d6b1fbaf4e96 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 14:40:49 +0200 Subject: [PATCH 107/118] help command --- src/etc/command_help.txt | 47 ++++++++++++++------------------- src/etc/usage.txt | 6 +++++ src/kiwi/_constants.py | 1 + src/kiwi/parser.py | 9 ++++--- src/kiwi/subcommands/config.py | 2 +- src/kiwi/subcommands/disable.py | 2 +- src/kiwi/subcommands/enable.py | 2 +- src/kiwi/subcommands/logs.py | 2 +- src/kiwi/subcommands/purge.py | 2 +- 9 files changed, 38 insertions(+), 35 deletions(-) create mode 100644 src/etc/usage.txt diff --git a/src/etc/command_help.txt b/src/etc/command_help.txt index ea1ecf6..fe27d16 100644 --- a/src/etc/command_help.txt +++ b/src/etc/command_help.txt @@ -1,30 +1,23 @@ -COMMANDS -======== +Commands for Operation: + up Bring up the whole instance, a project or service(s) inside a project + down Bring down the whole instance, a project or service(s) inside a project + update Update the whole instance, a project or service(s) inside a project + clean Cleanly sync all configs to target folder, then relaunch affected projects + purge Remove all running docker artifacts of this instance -# operation +Commands for Instance Management: + config Create or configure kiwi-config instance + inspect Inspect projects in this instance, services inside a project or service(s) inside a project + cmd Run raw docker-compose command in a project -config -up -down -update -clean +Commands for Project and Service Management: + new Create new empty project(s) in this instance + enable Enable project(s) in this instance + disable Disable project(s) in this instance + logs Show logs of a project or service(s) inside a project + shell Spawn shell inside a service inside a project -# management - -new -enable -disable -cmd -purge - -# inspection - -inspect -logs -shell - -# imaging - -build -pull -push +Commands for Image Handling: + build Build images for the whole instance, a project or service(s) inside a project + pull Pull images for the whole instance, a project or service(s) inside a project + push Push images for the whole instance, a project or service(s) inside a project \ No newline at end of file diff --git a/src/etc/usage.txt b/src/etc/usage.txt new file mode 100644 index 0000000..0e25174 --- /dev/null +++ b/src/etc/usage.txt @@ -0,0 +1,6 @@ + + %(prog)s {up,down,update,clean,purge} [ARGS...] + %(prog)s {config,inspect,cmd} [ARGS...] + %(prog)s {new,enable,disable,logs,shell} [ARGS...] + %(prog)s {build,pull,push} [ARGS...] + %(prog)s -h|--help diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py index e4bb229..bb6b4cb 100644 --- a/src/kiwi/_constants.py +++ b/src/kiwi/_constants.py @@ -20,6 +20,7 @@ DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_default.yml" VERSION_TAG_NAME = f"{KIWI_ROOT}/etc/version_tag" DEFAULT_DOCKER_COMPOSE_NAME = f"{KIWI_ROOT}/etc/docker-compose_default.yml" COMMAND_HELP_TEXT_NAME = f"{KIWI_ROOT}/etc/command_help.txt" +USAGE_TEXT_NAME = f"{KIWI_ROOT}/etc/usage.txt" # special config directory in projects CONF_DIRECTORY_NAME = 'conf' diff --git a/src/kiwi/parser.py b/src/kiwi/parser.py index a54bb05..88114fa 100644 --- a/src/kiwi/parser.py +++ b/src/kiwi/parser.py @@ -2,7 +2,7 @@ import argparse # local -from ._constants import COMMAND_HELP_TEXT_NAME +from ._constants import COMMAND_HELP_TEXT_NAME, USAGE_TEXT_NAME class Parser: @@ -19,12 +19,15 @@ class Parser: def __init__(self): # add version data from separate file (keeps default config cleaner) with open(COMMAND_HELP_TEXT_NAME, 'r') as stream: - command_help_text = stream.read().strip() + command_help_text = stream.read() + + with open(USAGE_TEXT_NAME, 'r') as stream: + usage_text = stream.read() # create main parser self.__parser = argparse.ArgumentParser( description='kiwi-config', - usage='%(prog)s [command]', + usage=usage_text, epilog=command_help_text, ) self.__parser.formatter_class = argparse.RawDescriptionHelpFormatter diff --git a/src/kiwi/subcommands/config.py b/src/kiwi/subcommands/config.py index 1174a3b..15f3c9c 100644 --- a/src/kiwi/subcommands/config.py +++ b/src/kiwi/subcommands/config.py @@ -15,7 +15,7 @@ class ConfigCommand(SubCommand): super().__init__( 'config', action=f"Configuring '{KIWI_CONF_NAME}' in", - description="Configure kiwi-config instance" + description="Create or configure kiwi-config instance" ) # -f switch: Initialize with default config diff --git a/src/kiwi/subcommands/disable.py b/src/kiwi/subcommands/disable.py index 9715c27..c10eec9 100644 --- a/src/kiwi/subcommands/disable.py +++ b/src/kiwi/subcommands/disable.py @@ -9,7 +9,7 @@ class DisableCommand(ProjectCommand): super().__init__( 'disable', num_projects='+', action="Disabling", - description="Disable whole project(s) in this instance" + description="Disable project(s) in this instance" ) def _run_projects(self, runner, args, projects): diff --git a/src/kiwi/subcommands/enable.py b/src/kiwi/subcommands/enable.py index cf4079e..730fb69 100644 --- a/src/kiwi/subcommands/enable.py +++ b/src/kiwi/subcommands/enable.py @@ -9,7 +9,7 @@ class EnableCommand(ProjectCommand): super().__init__( 'enable', num_projects='+', action="Enabling", - description="Enable whole project(s) in this instance" + description="Enable project(s) in this instance" ) def _run_projects(self, runner, args, projects): diff --git a/src/kiwi/subcommands/logs.py b/src/kiwi/subcommands/logs.py index b2d49b7..2407220 100644 --- a/src/kiwi/subcommands/logs.py +++ b/src/kiwi/subcommands/logs.py @@ -9,7 +9,7 @@ class LogsCommand(ServiceCommand): super().__init__( 'logs', num_projects=1, num_services='*', action="Showing logs of", - description="Show logs of a project or service(s) of a project" + description="Show logs of a project or service(s) inside a project" ) # -f switch: Follow logs diff --git a/src/kiwi/subcommands/purge.py b/src/kiwi/subcommands/purge.py index fbf771b..2200302 100644 --- a/src/kiwi/subcommands/purge.py +++ b/src/kiwi/subcommands/purge.py @@ -17,7 +17,7 @@ class PurgeCommand(SubCommand): super().__init__( 'purge', action="Tearing down", - description="Remove all ephemeral artifacts of this instance" + description="Remove all running docker artifacts of this instance" ) def _run_instance(self, runner, args): From 40e476ba7b92a1ca6f8e11189fc34005a6954b32 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 15:08:41 +0200 Subject: [PATCH 108/118] QoL: Projects default bool conversion (nonempty) --- src/kiwi/projects.py | 6 +++--- src/kiwi/subcommand.py | 2 +- src/kiwi/subcommands/inspect.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/kiwi/projects.py b/src/kiwi/projects.py index 65c75d0..620fc81 100644 --- a/src/kiwi/projects.py +++ b/src/kiwi/projects.py @@ -16,6 +16,9 @@ class Projects: in self.__projects ]) + def __bool__(self): + return bool(self.__projects) + @classmethod def from_names(cls, project_names): result = cls() @@ -52,9 +55,6 @@ class Projects: return cls() - def empty(self): - return not self.__projects - def filter_exists(self): result = Projects() result.__projects = [ diff --git a/src/kiwi/subcommand.py b/src/kiwi/subcommand.py index fb472ff..150a222 100644 --- a/src/kiwi/subcommand.py +++ b/src/kiwi/subcommand.py @@ -70,7 +70,7 @@ class ProjectCommand(SubCommand): def run(self, runner, args): projects = Projects.from_args(args) - if not projects.empty(): + if projects: # project(s) given logging.info(f"{self._action} projects {projects}") return self._run_projects(runner, args, projects) diff --git a/src/kiwi/subcommands/inspect.py b/src/kiwi/subcommands/inspect.py index ae679f1..1a36b03 100644 --- a/src/kiwi/subcommands/inspect.py +++ b/src/kiwi/subcommands/inspect.py @@ -40,12 +40,12 @@ class InspectCommand(ServiceCommand): projects = Projects.from_dir() enabled_projects = projects.filter_enabled() - if not enabled_projects.empty(): + if enabled_projects: print(f"Enabled projects:") _print_list(enabled_projects) disabled_projects = projects.filter_disabled() - if not disabled_projects.empty(): + if disabled_projects: print(f"Disabled projects:") _print_list(disabled_projects) From 6819eabb6739ace3afc4ecac18e2fe04badef781 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 15:09:15 +0200 Subject: [PATCH 109/118] QoL: Added _run_project overridable with automatic call for multiple projects --- src/kiwi/subcommand.py | 18 ++++++++++-------- src/kiwi/subcommands/cmd.py | 4 ++-- src/kiwi/subcommands/disable.py | 7 ++----- src/kiwi/subcommands/down.py | 6 ++---- src/kiwi/subcommands/enable.py | 7 ++----- src/kiwi/subcommands/inspect.py | 3 +-- src/kiwi/subcommands/new.py | 22 +++++++++------------- 7 files changed, 28 insertions(+), 39 deletions(-) diff --git a/src/kiwi/subcommand.py b/src/kiwi/subcommand.py index 150a222..072ce3f 100644 --- a/src/kiwi/subcommand.py +++ b/src/kiwi/subcommand.py @@ -65,6 +65,13 @@ class ProjectCommand(SubCommand): return self._run_projects(runner, args, Projects.from_dir().filter_enabled()) def _run_projects(self, runner, args, projects): + # default: run for all given projects + return all([ + self._run_project(runner, args, project) + for project in projects + ]) + + def _run_project(self, runner, args, project): pass def run(self, runner, args): @@ -102,14 +109,9 @@ class ServiceCommand(ProjectCommand): help=f"select {services} in a project" ) - def _run_projects(self, runner, args, projects): - result = True - - # default: run without services for all given - for project in projects: - result &= self._run_services(runner, args, project, []) - - return result + def _run_project(self, runner, args, project): + # default: run with empty service list + return self._run_services(runner, args, project, []) def _run_services(self, runner, args, project, services): pass diff --git a/src/kiwi/subcommands/cmd.py b/src/kiwi/subcommands/cmd.py index da90fc9..c890edf 100644 --- a/src/kiwi/subcommands/cmd.py +++ b/src/kiwi/subcommands/cmd.py @@ -27,7 +27,7 @@ class CmdCommand(ProjectCommand): help="arguments for 'docker-compose' commands" ) - def _run_projects(self, runner, args, projects): + def _run_project(self, runner, args, project): if args.unknowns: args.compose_args = [*args.compose_args, *args.unknowns] args.unknowns = [] @@ -35,6 +35,6 @@ class CmdCommand(ProjectCommand): logging.debug(f"Updated args: {args}") # run with split compose_cmd argument - projects[0].compose_run([args.compose_cmd, *args.compose_args]) + project.compose_run([args.compose_cmd, *args.compose_args]) return True diff --git a/src/kiwi/subcommands/disable.py b/src/kiwi/subcommands/disable.py index c10eec9..d4d762d 100644 --- a/src/kiwi/subcommands/disable.py +++ b/src/kiwi/subcommands/disable.py @@ -12,8 +12,5 @@ class DisableCommand(ProjectCommand): description="Disable project(s) in this instance" ) - def _run_projects(self, runner, args, projects): - return all([ - project.disable() - for project in projects - ]) + def _run_project(self, runner, args, project): + return project.disable() diff --git a/src/kiwi/subcommands/down.py b/src/kiwi/subcommands/down.py index 213eca9..cd53b3e 100644 --- a/src/kiwi/subcommands/down.py +++ b/src/kiwi/subcommands/down.py @@ -24,10 +24,8 @@ class DownCommand(ServiceCommand): return False - def _run_projects(self, runner, args, projects): - for project in projects: - project.compose_run(['down']) - + def _run_project(self, runner, args, project): + project.compose_run(['down']) return True def _run_services(self, runner, args, project, services): diff --git a/src/kiwi/subcommands/enable.py b/src/kiwi/subcommands/enable.py index 730fb69..1262a7f 100644 --- a/src/kiwi/subcommands/enable.py +++ b/src/kiwi/subcommands/enable.py @@ -12,8 +12,5 @@ class EnableCommand(ProjectCommand): description="Enable project(s) in this instance" ) - def _run_projects(self, runner, args, projects): - return all([ - project.enable() - for project in projects - ]) + def _run_project(self, runner, args, project): + return project.enable() diff --git a/src/kiwi/subcommands/inspect.py b/src/kiwi/subcommands/inspect.py index 1a36b03..a7c139e 100644 --- a/src/kiwi/subcommands/inspect.py +++ b/src/kiwi/subcommands/inspect.py @@ -51,8 +51,7 @@ class InspectCommand(ServiceCommand): return True - def _run_projects(self, runner, args, projects): - project = projects[0] + def _run_project(self, runner, args, project): if not project.exists(): logging.warning(f"Project '{project.get_name()}' not found") return False diff --git a/src/kiwi/subcommands/new.py b/src/kiwi/subcommands/new.py index cfac9b1..5fc5243 100644 --- a/src/kiwi/subcommands/new.py +++ b/src/kiwi/subcommands/new.py @@ -18,17 +18,13 @@ class NewCommand(ProjectCommand): description="Create new empty project(s) in this instance" ) - def _run_projects(self, runner, args, projects): - result = True + def _run_project(self, runner, args, project): + if project.exists(): + logging.error(f"Project '{project.get_name()}' exists in this instance!") + return False - for project in projects: - if project.exists(): - logging.error(f"Project '{project.get_name()}' exists in this instance!") - result = False - - else: - logging.info(f"Creating project '{project.get_name()}'") - os.mkdir(project.disabled_dir_name()) - shutil.copy(DEFAULT_DOCKER_COMPOSE_NAME, project.compose_file_name()) - - return result + else: + os.mkdir(project.disabled_dir_name()) + shutil.copy(DEFAULT_DOCKER_COMPOSE_NAME, project.compose_file_name()) + logging.debug(f"Project '{project.get_name()}' created") + return True From 934a1bd15e2f5c11ad759423f53c645e33ad5b02 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 15:29:07 +0200 Subject: [PATCH 110/118] kiwi help --- .../hello-world.project/docker-compose.yml | 6 +++- src/etc/command_help.txt | 32 +++++++++---------- src/etc/docker-compose_default.yml | 2 +- src/etc/kiwi_help.txt | 9 ++++++ src/etc/usage.txt | 6 ---- src/kiwi/_constants.py | 2 +- src/kiwi/parser.py | 11 +++---- 7 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 src/etc/kiwi_help.txt delete mode 100644 src/etc/usage.txt diff --git a/example/hello-world.project/docker-compose.yml b/example/hello-world.project/docker-compose.yml index 6d66369..35951b8 100644 --- a/example/hello-world.project/docker-compose.yml +++ b/example/hello-world.project/docker-compose.yml @@ -7,12 +7,16 @@ networks: # interconnects projects kiwi_hub: external: - name: $KIWI_HUB_NAME + name: ${KIWI_HUB_NAME} services: hello-world: image: alpine:latest command: sh -c 'LOOP=1; while :; do echo Hello World "$$LOOP"; LOOP=$$(($$LOOP + 1)); sleep 10; done' + networks: + - default + - kiwi_hub + foo-bar: image: alpine:latest command: sh -c 'LOOP=1; while :; do echo Foo Bar "$$LOOP"; LOOP=$$(($$LOOP + 1)); sleep 20; done' diff --git a/src/etc/command_help.txt b/src/etc/command_help.txt index fe27d16..51ae022 100644 --- a/src/etc/command_help.txt +++ b/src/etc/command_help.txt @@ -1,23 +1,23 @@ Commands for Operation: - up Bring up the whole instance, a project or service(s) inside a project - down Bring down the whole instance, a project or service(s) inside a project - update Update the whole instance, a project or service(s) inside a project - clean Cleanly sync all configs to target folder, then relaunch affected projects - purge Remove all running docker artifacts of this instance + up Bring up the whole instance, a project or service(s) inside a project + down Bring down the whole instance, a project or service(s) inside a project + update Update the whole instance, a project or service(s) inside a project + clean Cleanly sync all configs to target folder, then relaunch affected projects + purge Remove all running docker artifacts of this instance Commands for Instance Management: - config Create or configure kiwi-config instance - inspect Inspect projects in this instance, services inside a project or service(s) inside a project - cmd Run raw docker-compose command in a project + config Create or configure kiwi-config instance + inspect Inspect projects in this instance, services inside a project or service(s) inside a project + cmd Run raw docker-compose command in a project Commands for Project and Service Management: - new Create new empty project(s) in this instance - enable Enable project(s) in this instance - disable Disable project(s) in this instance - logs Show logs of a project or service(s) inside a project - shell Spawn shell inside a service inside a project + new Create new empty project(s) in this instance + enable Enable project(s) in this instance + disable Disable project(s) in this instance + logs Show logs of a project or service(s) inside a project + shell Spawn shell inside a service inside a project Commands for Image Handling: - build Build images for the whole instance, a project or service(s) inside a project - pull Pull images for the whole instance, a project or service(s) inside a project - push Push images for the whole instance, a project or service(s) inside a project \ No newline at end of file + build Build images for the whole instance, a project or service(s) inside a project + pull Pull images for the whole instance, a project or service(s) inside a project + push Push images for the whole instance, a project or service(s) inside a project \ No newline at end of file diff --git a/src/etc/docker-compose_default.yml b/src/etc/docker-compose_default.yml index 3baa9e3..47f85e4 100644 --- a/src/etc/docker-compose_default.yml +++ b/src/etc/docker-compose_default.yml @@ -7,7 +7,7 @@ networks: # interconnects projects kiwi_hub: external: - name: $KIWI_HUB_NAME + name: ${KIWI_HUB_NAME} services: # an example service diff --git a/src/etc/kiwi_help.txt b/src/etc/kiwi_help.txt new file mode 100644 index 0000000..e08f116 --- /dev/null +++ b/src/etc/kiwi_help.txt @@ -0,0 +1,9 @@ +kiwi-config is the tool for container server management. + +Features: + - Group your services into projects using their own docker-compose.yml + - Bind to file system by using ${TARGETDIR} as volume in docker-compose.yml + - Add instance-global config files by using ${CONFDIR} as volume in docker-compose.yml + - Add instance-global custom values inside docker-compose.yml using config:runtime:env + - Build service-specific, private docker images from Dockerfiles + - Check full instances into any version control system diff --git a/src/etc/usage.txt b/src/etc/usage.txt deleted file mode 100644 index 0e25174..0000000 --- a/src/etc/usage.txt +++ /dev/null @@ -1,6 +0,0 @@ - - %(prog)s {up,down,update,clean,purge} [ARGS...] - %(prog)s {config,inspect,cmd} [ARGS...] - %(prog)s {new,enable,disable,logs,shell} [ARGS...] - %(prog)s {build,pull,push} [ARGS...] - %(prog)s -h|--help diff --git a/src/kiwi/_constants.py b/src/kiwi/_constants.py index bb6b4cb..11726b0 100644 --- a/src/kiwi/_constants.py +++ b/src/kiwi/_constants.py @@ -19,8 +19,8 @@ HEADER_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_header.yml" DEFAULT_KIWI_CONF_NAME = f"{KIWI_ROOT}/etc/kiwi_default.yml" VERSION_TAG_NAME = f"{KIWI_ROOT}/etc/version_tag" DEFAULT_DOCKER_COMPOSE_NAME = f"{KIWI_ROOT}/etc/docker-compose_default.yml" +KIWI_HELP_TEXT_NAME = f"{KIWI_ROOT}/etc/kiwi_help.txt" COMMAND_HELP_TEXT_NAME = f"{KIWI_ROOT}/etc/command_help.txt" -USAGE_TEXT_NAME = f"{KIWI_ROOT}/etc/usage.txt" # special config directory in projects CONF_DIRECTORY_NAME = 'conf' diff --git a/src/kiwi/parser.py b/src/kiwi/parser.py index 88114fa..d8c1e71 100644 --- a/src/kiwi/parser.py +++ b/src/kiwi/parser.py @@ -2,7 +2,7 @@ import argparse # local -from ._constants import COMMAND_HELP_TEXT_NAME, USAGE_TEXT_NAME +from ._constants import COMMAND_HELP_TEXT_NAME, KIWI_HELP_TEXT_NAME class Parser: @@ -18,16 +18,15 @@ class Parser: def __init__(self): # add version data from separate file (keeps default config cleaner) + with open(KIWI_HELP_TEXT_NAME, 'r') as stream: + kiwi_help = stream.read() + with open(COMMAND_HELP_TEXT_NAME, 'r') as stream: command_help_text = stream.read() - with open(USAGE_TEXT_NAME, 'r') as stream: - usage_text = stream.read() - # create main parser self.__parser = argparse.ArgumentParser( - description='kiwi-config', - usage=usage_text, + description=kiwi_help, epilog=command_help_text, ) self.__parser.formatter_class = argparse.RawDescriptionHelpFormatter From 6118cb932a08c046c256bea97cf782c228ae3257 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 15:36:28 +0200 Subject: [PATCH 111/118] format; kwargs fix --- src/kiwi/executable.py | 29 ++++++----------------------- src/kiwi/project.py | 4 ++++ src/kiwi/runner.py | 7 +++---- 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/src/kiwi/executable.py b/src/kiwi/executable.py index 9f8aa8d..3d79983 100644 --- a/src/kiwi/executable.py +++ b/src/kiwi/executable.py @@ -7,22 +7,6 @@ import subprocess from .config import LoadedConfig -def _update_kwargs(**kwargs): - config = LoadedConfig.get() - - # ensure there is an environment - if 'env' not in kwargs: - kwargs['env'] = {} - - # add common environment from config - if config['runtime:env'] is not None: - kwargs['env'].update(config['runtime:env']) - - logging.debug(f"kwargs updated: {kwargs}") - - return kwargs - - def _is_executable(filename): if filename is None: return False @@ -46,7 +30,7 @@ class Executable: def __init__(self, exe_name): self.__exe_path = _find_exe_file(exe_name) - def __build_cmd(self, args, **kwargs): + def __build_cmd(self, args, kwargs): cmd = [self.__exe_path, *args] logging.debug(f"Executable cmd{cmd}, kwargs{kwargs}") @@ -54,13 +38,13 @@ class Executable: def run(self, process_args, **kwargs): return subprocess.run( - self.__build_cmd(process_args, **_update_kwargs(**kwargs)), + self.__build_cmd(process_args, kwargs), **kwargs ) def Popen(self, process_args, **kwargs): return subprocess.Popen( - self.__build_cmd(process_args, **_update_kwargs(**kwargs)), + self.__build_cmd(process_args, kwargs), **kwargs ) @@ -73,10 +57,9 @@ class Executable: **kwargs ) - less_process = Executable('less').run( - ['-R', '+G'], - stdin=process.stdout - ) + less_process = Executable('less').run([ + '-R', '+G' + ], stdin=process.stdout) process.communicate() return less_process diff --git a/src/kiwi/project.py b/src/kiwi/project.py index 8b59149..5072aa8 100644 --- a/src/kiwi/project.py +++ b/src/kiwi/project.py @@ -88,6 +88,10 @@ class Project: 'TARGETDIR': self.target_dir_name() }) + # add common environment from config + if config['runtime:env'] is not None: + kwargs['env'].update(config['runtime:env']) + logging.debug(f"kwargs updated: {kwargs}") return True diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index 9f4058d..affa16e 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -19,10 +19,9 @@ class Runner: def __init__(self): # probe for Docker access try: - Executable('docker').run( - ['ps'], - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) + Executable('docker').run([ + 'ps' + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except subprocess.CalledProcessError: raise PermissionError("Cannot access docker, please get into the docker group or run as root!") From 0aee2eadd583f3de90f84c63a39f0df78b0b7be1 Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 17:38:25 +0200 Subject: [PATCH 112/118] Distribution script, lesser dependencies --- example/kiwi | 1 - kiwi | 109 +++++++++++++++++++++++++++++----- src/kiwi/parser.py | 1 + src/kiwi/subcommands/shell.py | 4 +- 4 files changed, 96 insertions(+), 19 deletions(-) delete mode 120000 example/kiwi diff --git a/example/kiwi b/example/kiwi deleted file mode 120000 index d7c36e5..0000000 --- a/example/kiwi +++ /dev/null @@ -1 +0,0 @@ -../src/kiwi-config.py \ No newline at end of file diff --git a/kiwi b/kiwi index 0e66857..d7c927d 100755 --- a/kiwi +++ b/kiwi @@ -5,30 +5,81 @@ ############# # base config filename -export KIWI_CONF_NAME="kiwi.conf" +KIWI_CONF_NAME="kiwi.yml" +# version tag filename +KIWI_VERSION_TAG="etc/version_tag" + +# dependencies to run kiwi-config +KIWI_DEPS=(python3 pipenv less docker docker-compose) # base install dir KIWI_BASEDIR="${HOME}/.cache/kiwi-config" +# per-user env setup script +KIWI_ENVFILE="${HOME}/.kiwienv" + # repository uri KIWI_REPO="https://github.com/ldericher/kiwi-config" +# use latest version by default +KIWI_VERSION="master" + +################### +# DYNAMIC STRINGS # +################### + +# directory of correct installation +function kiwi_installdir() { + echo "${KIWI_BASEDIR}/${KIWI_VERSION}" +} + +# src directory in installed version +function kiwi_root() { + echo "$(kiwi_installdir)/src" +} + +# main script in installed version +function kiwi_executable() { + echo "$(kiwi_root)/kiwi-config.py" +} + +# cache current work directory +WORKDIR="$(pwd)" + +################## +# PER-USER SETUP # +################## + +# add in environment setup +if [ -f "${KIWI_ENVFILE}" ]; then + # shellcheck source=$HOME/.kiwienv + source "${KIWI_ENVFILE}" +fi + +########## +# CHECKS # +########## + +for dep in "${KIWI_DEPS[@]}"; do + if ! command -v "${dep}" &> /dev/null; then + echo "Dependency '${dep}' not found, please install!" + exit 1 + fi +done ######## # MAIN # ######## -# use latest version by default -export KIWI_VERSION="master" - # check if pwd is a kiwi folder if [ -f "./${KIWI_CONF_NAME}" ]; then # determine needed kiwi-config version - export KIWI_VERSION=$(source "./${KIWI_CONF_NAME}" && echo "${VERSION}") + re_version_line='version\s*:\s*' + eval "$(grep -E "${re_version_line}" "./${KIWI_CONF_NAME}" | sed -r 's/'"${re_version_line}"'/KIWI_VERSION=/')" fi # install if kiwi-config not found -if [ ! -x "${KIWI_BASEDIR}/${KIWI_VERSION}/bin/main.sh" ]; then +if [ ! -x "$(kiwi_executable)" ]; then echo -n "Installing kiwi-config v${KIWI_VERSION} into ${KIWI_BASEDIR} ... " - ### production version ### + ### TODO: post-release version ### # # switch to temp dir # workdir=$(pwd) @@ -51,19 +102,45 @@ if [ ! -x "${KIWI_BASEDIR}/${KIWI_VERSION}/bin/main.sh" ]; then # cd "${workdir}" # rm -rf "${tmpdir}" - ### development version ### + # echo "OK" + + ### pre-release version ### + + # use this directory as archive + cd "$(dirname "$(readlink -f "${0}")")" ||: # read this version tag - export KIWI_VERSION=$(cat ./version-tag) + KIWI_VERSION=$(cat "./src/${KIWI_VERSION_TAG}") - # install this - mkdir -p "${KIWI_BASEDIR}" - ln -s "$(readlink -f ./src)" "${KIWI_BASEDIR}/${KIWI_VERSION}" - - echo "OK" + if [ -x "$(kiwi_executable)" ]; then + # after version update: no need to install + echo "kiwi-config v${KIWI_VERSION} found!" + else + # install this + mkdir -p "${KIWI_BASEDIR}" + ln -s "$(readlink -f .)" "$(kiwi_installdir)" + # cd + echo "OK" + fi fi -export KIWI_ROOT="${KIWI_BASEDIR}/${KIWI_VERSION}" +# check virtualenv +cd "$(kiwi_installdir)" ||: +if ! pipenv --venv &> /dev/null; then + # install virtualenv + echo -n "Preparing virtualenv ... " + pipenv sync &> /dev/null + echo "OK" +fi + +# go back to the original work directory +cd "${WORKDIR}" ||: + +# setup main environment +KIWI_ROOT="$(kiwi_root)" + +export KIWI_ROOT +export KIWI_CONF_NAME # run main script -exec "${KIWI_ROOT}/bin/main.sh" "${@}" \ No newline at end of file +exec pipenv run "$(kiwi_executable)" "${@}" \ No newline at end of file diff --git a/src/kiwi/parser.py b/src/kiwi/parser.py index d8c1e71..5276152 100644 --- a/src/kiwi/parser.py +++ b/src/kiwi/parser.py @@ -26,6 +26,7 @@ class Parser: # create main parser self.__parser = argparse.ArgumentParser( + prog='kiwi', description=kiwi_help, epilog=command_help_text, ) diff --git a/src/kiwi/subcommands/shell.py b/src/kiwi/subcommands/shell.py index 3a3ed00..0a240de 100644 --- a/src/kiwi/subcommands/shell.py +++ b/src/kiwi/subcommands/shell.py @@ -10,13 +10,13 @@ from ..config import LoadedConfig def _service_has_executable(project, service, exe_name): """ Test if service in project has an executable exe_name in its PATH. - Requires /bin/sh and which. + Requires /bin/sh. """ try: # test if desired shell exists project.compose_run( - ['exec', service, '/bin/sh', '-c', f"which {exe_name}"], + ['exec', service, '/bin/sh', '-c', f"command -v {exe_name}"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) return True From 96d4253fb3bd9b0df2ef71f77124f591d59c310c Mon Sep 17 00:00:00 2001 From: ldericher Date: Thu, 20 Aug 2020 17:39:33 +0200 Subject: [PATCH 113/118] remove legacy version --- Makefile | 256 ---------------------------------------------- example/Makefile | 1 - example/base.conf | 7 -- 3 files changed, 264 deletions(-) delete mode 100644 Makefile delete mode 120000 example/Makefile delete mode 100644 example/base.conf diff --git a/Makefile b/Makefile deleted file mode 100644 index 83fea49..0000000 --- a/Makefile +++ /dev/null @@ -1,256 +0,0 @@ -########## -# COMMANDS - -DOCKER:=docker -DOCKER_COMPOSE:=docker-compose -docker_bash=$(1) - -# Check if needs root privileges? -PRIVGROUP:=docker -ifneq ($(findstring $(PRIVGROUP),$(shell groups)),$(PRIVGROUP)) -DOCKER:=sudo $(DOCKER) -docker_bash=sudo bash -c '$(1)' -endif - -######### -# CONFIGS - -CONF_WILDC:=$(wildcard $(PWD)/*.conf) -# apply source to *all* configs! -CONF_SOURCE:=$(patsubst %,. %;,$(CONF_WILDC)) - -# extraction of env variables from *.conf files -confvalue=$(shell $(CONF_SOURCE) echo -n $${$(1)}) - -# docker network name -CONF_KIWI_HUB_NAME:=$(call confvalue,KIWI_HUB_NAME) -ifeq ($(CONF_KIWI_HUB_NAME),) -$(error KIWI_HUB_NAME not set in $(CONF_WILDC)) -endif - -# docker network CIDR -CONF_KIWI_HUB_CIDR:=$(call confvalue,KIWI_HUB_CIDR) -ifeq ($(CONF_KIWI_HUB_CIDR),) -$(error KIWI_HUB_CIDR not set in $(CONF_WILDC)) -endif - -# persistent data directory -CONF_TARGET_ROOT:=$(call confvalue,TARGET_ROOT) -ifeq ($(CONF_TARGET_ROOT),) -$(error TARGET_ROOT not set in $(CONF_WILDC)) -endif - -# suffix for project directories -PROJ_SUFFX:=$(call confvalue,SUFFIX_PROJECT) -ifeq ($(PROJ_SUFFX),) -$(error SUFFIX_PROJECT not set in $(CONF_WILDC)) -endif - -# suffix for disabled project directories -DOWN_SUFFX:=$(call confvalue,SUFFIX_DOWN) -ifeq ($(DOWN_SUFFX),) -$(error SUFFIX_DOWN not set in $(CONF_WILDC)) -endif - -######### -# CONSTANTS - -# file to store docker network cidr -KIWI_HUB_FILE:=$(CONF_TARGET_ROOT)/up-$(CONF_KIWI_HUB_NAME) - -# remove any suffix $2 from $1 -rmsuffix=$(patsubst %$2,%,$1) -# remove project suffix from $1 -projname=$(call rmsuffix,$1,$(PROJ_SUFFX)) - -# project directory handling -PROJ_WILDC:=$(wildcard *$(PROJ_SUFFX)) -PROJ_NAMES:=$(call projname,$(PROJ_WILDC)) - -######### -# FUNCTIONS - -# run DOCKER_COMPOSE: -# - in project directory -# - with sourced *.conf files -# - with COMPOSE_PROJECT_NAME, CONFDIR and TARGETDIR set -kiwicompose=$(call docker_bash,\ - cd "$<"; \ - $(CONF_SOURCE) \ - COMPOSE_PROJECT_NAME="$(call projname,$<)" \ - CONFDIR="$(CONF_TARGET_ROOT)/conf" \ - TARGETDIR="$(CONF_TARGET_ROOT)/$<" \ - $(DOCKER_COMPOSE) $(1)) - -######### -# TARGETS - -# default target -.PHONY: all -all: purge-conf up - -######### -# manage the docker network (container name local DNS) -$(KIWI_HUB_FILE): - -$(DOCKER) network create \ - --driver bridge \ - --internal \ - --subnet "$(CONF_KIWI_HUB_CIDR)" \ - "$(CONF_KIWI_HUB_NAME)" - @echo "Creating canary $(KIWI_HUB_FILE) ..." - @$(DOCKER) run --rm \ - -v "/:/mnt" -u root alpine:latest \ - ash -c '\ - mkdir -p "$(addprefix /mnt, $(CONF_TARGET_ROOT))"; \ - echo "$(CONF_KIWI_HUB_CIDR)" > "$(addprefix /mnt, $(KIWI_HUB_FILE))"; \ - ' - -.PHONY: net-up -net-up: $(KIWI_HUB_FILE) - -.PHONY: net-down -net-down: down - $(DOCKER) network rm "$(CONF_KIWI_HUB_NAME)" - @echo "Removing canary $(KIWI_HUB_FILE) ..." - @$(DOCKER) run --rm \ - -v "/:/mnt" -u root alpine:latest \ - ash -c '\ - rm -f "$(addprefix /mnt, $(KIWI_HUB_FILE))"; \ - ' - -######### -# sync project config directory to variable folder - -# Dockerfile for running rsync as root -define DOCKERFILE_RSYNC -FROM alpine:latest -RUN apk --no-cache add rsync -endef - -.PHONY: copy-conf -copy-conf: -ifneq ($(wildcard *${PROJ_SUFFX}/conf),) - $(eval export DOCKERFILE_RSYNC) - @echo "Building auxiliary image ldericher/kiwi-config:rsync ..." - @echo -e "$${DOCKERFILE_RSYNC}" | \ - $(DOCKER) build -t ldericher/kiwi-config:rsync . -f- &> /dev/null - - $(eval sources:=$(wildcard *${PROJ_SUFFX}/conf)) - @echo "Syncing $(sources) to $(CONF_TARGET_ROOT) ..." - - $(eval sources:=$(realpath $(sources))) - $(eval sources:=$(addprefix /mnt, $(sources))) - $(eval sources:=$(patsubst %,'%',$(sources))) - $(eval dest:='$(addprefix /mnt, $(CONF_TARGET_ROOT))') - - @$(DOCKER) run --rm \ - -v "/:/mnt" -u root ldericher/kiwi-config:rsync \ - ash -c '\ - rsync -r $(sources) $(dest); \ - ' -endif - -.PHONY: purge-conf -purge-conf: - @echo "Emptying $(CONF_TARGET_ROOT)/conf ..." - @$(DOCKER) run --rm \ - -v "/:/mnt" -u root alpine:latest \ - ash -c '\ - rm -rf "$(addprefix /mnt, $(CONF_TARGET_ROOT)/conf)"; \ - ' - -######### -# manage all projects -.PHONY: up down update -up: net-up copy-conf $(patsubst %,%-up,$(PROJ_NAMES)) -down: $(patsubst %,%-down,$(PROJ_NAMES)) -update: $(patsubst %,%-update,$(PROJ_NAMES)) - -######### -# manage single project -.PHONY: %-up -%-up: %$(PROJ_SUFFX) net-up - $(call kiwicompose,up -d $(x)) - -.PHONY: %-down -ifeq ($(x),) -%-down: %$(PROJ_SUFFX) - $(call kiwicompose,down) -else -%-down: %$(PROJ_SUFFX) - $(call kiwicompose,stop $(x)) - $(call kiwicompose,rm -f $(x)) -endif - -.PHONY: %-pull -%-pull: %$(PROJ_SUFFX) - $(call kiwicompose,pull --ignore-pull-failures $(x)) - -.PHONY: %-build -%-build: %$(PROJ_SUFFX) - $(call kiwicompose,build --pull $(x)) - -.PHONY: %-logs -%-logs: %$(PROJ_SUFFX) - $(call kiwicompose,logs -t $(x)) 2>/dev/null | less -R +G - -.PHONY: %-logf -%-logf: %$(PROJ_SUFFX) - $(call kiwicompose,logs -tf --tail=10 $(x)) ||: - -ifneq ($(x),) -s?=bash -.PHONY: %-sh -%-sh: %$(PROJ_SUFFX) - $(call kiwicompose,exec $(x) /bin/sh -c "[ -e /bin/$(s) ] && /bin/$(s) || /bin/sh") -endif - -# enabling and disabling -.PHONY: %-enable %-disable -%-enable: %$(PROJ_SUFFX)$(DOWN_SUFFX) - mv "$<" "$(call projname,$(call rmsuffix,$<,$(DOWN_SUFFX)))$(PROJ_SUFFX)" -%-disable: %$(PROJ_SUFFX) - mv "$<" "$<$(DOWN_SUFFX)" - -# Combinations -.PHONY: %-update -%-update: %$(PROJ_SUFFX) %-build %-pull copy-conf - $(MAKE) $(call projname,$<)-down - $(MAKE) $(call projname,$<)-up - -# Arbitrary compose command -.PHONY: %-cmd -%-cmd: %$(PROJ_SUFFX) - $(call kiwicompose,$(x)) - -######### -# project creation -.PHONY: %-new -%-new: - $(eval proj_dir:=$(patsubst %-new,%$(PROJ_SUFFX)$(DOWN_SUFFX),$@)) - mkdir $(proj_dir) - $(eval export COMPOSEFILE) - echo -e "$${COMPOSEFILE}" > $(proj_dir)/docker-compose.yml - -# default compose file -define COMPOSEFILE -version: "2" - -networks: - # reachable from outside - default: - driver: bridge - # interconnects projects - kiwi_hub: - external: - name: $$KIWI_HUB_NAME - -services: - something: - image: maintainer/repo:tag - restart: unless-stopped - networks: - - default - - kiwi_hub - [...] -endef diff --git a/example/Makefile b/example/Makefile deleted file mode 120000 index d0b0e8e..0000000 --- a/example/Makefile +++ /dev/null @@ -1 +0,0 @@ -../Makefile \ No newline at end of file diff --git a/example/base.conf b/example/base.conf deleted file mode 100644 index 3b2f803..0000000 --- a/example/base.conf +++ /dev/null @@ -1,7 +0,0 @@ -export SUFFIX_PROJECT=.project -export SUFFIX_DOWN=.down - -export KIWI_HUB_NAME=kiwi_hub -export KIWI_HUB_CIDR=10.22.46.0/24 - -export TARGET_ROOT=/tmp/kiwi From 4407899fbb62e758d70c22f47d7235e3009e741a Mon Sep 17 00:00:00 2001 From: ldericher Date: Fri, 21 Aug 2020 00:04:26 +0200 Subject: [PATCH 114/118] Format, license, distribution script --- LICENSE | 21 ++++++++++ kiwi | 94 +++++++++++++++++++++---------------------- src/etc/kiwi_help.txt | 4 +- 3 files changed, 70 insertions(+), 49 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d3884f0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jörn-Michael Miehe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/kiwi b/kiwi index d7c927d..6f09e4a 100755 --- a/kiwi +++ b/kiwi @@ -2,7 +2,7 @@ ############# # CONSTANTS # -############# +############# # base config filename KIWI_CONF_NAME="kiwi.yml" @@ -11,7 +11,7 @@ KIWI_VERSION_TAG="etc/version_tag" # dependencies to run kiwi-config KIWI_DEPS=(python3 pipenv less docker docker-compose) -# base install dir +# base install dir KIWI_BASEDIR="${HOME}/.cache/kiwi-config" # per-user env setup script KIWI_ENVFILE="${HOME}/.kiwienv" @@ -49,8 +49,8 @@ WORKDIR="$(pwd)" # add in environment setup if [ -f "${KIWI_ENVFILE}" ]; then - # shellcheck source=$HOME/.kiwienv - source "${KIWI_ENVFILE}" + # shellcheck source=$HOME/.kiwienv + source "${KIWI_ENVFILE}" fi ########## @@ -58,7 +58,7 @@ fi ########## for dep in "${KIWI_DEPS[@]}"; do - if ! command -v "${dep}" &> /dev/null; then + if ! command -v "${dep}" &>/dev/null; then echo "Dependency '${dep}' not found, please install!" exit 1 fi @@ -70,77 +70,77 @@ done # check if pwd is a kiwi folder if [ -f "./${KIWI_CONF_NAME}" ]; then - # determine needed kiwi-config version - re_version_line='version\s*:\s*' - eval "$(grep -E "${re_version_line}" "./${KIWI_CONF_NAME}" | sed -r 's/'"${re_version_line}"'/KIWI_VERSION=/')" + # determine needed kiwi-config version + re_version_line='version\s*:\s*' + eval "$(grep -E "${re_version_line}" "./${KIWI_CONF_NAME}" | sed -r 's/'"${re_version_line}"'/KIWI_VERSION=/')" fi # install if kiwi-config not found if [ ! -x "$(kiwi_executable)" ]; then - echo -n "Installing kiwi-config v${KIWI_VERSION} into ${KIWI_BASEDIR} ... " + echo -n "Installing kiwi-config v${KIWI_VERSION} into ${KIWI_BASEDIR} ... " - ### TODO: post-release version ### + ### TODO: post-release version ### - # # switch to temp dir - # workdir=$(pwd) - # tmpdir=$(mktemp -d) - # cd "${tmpdir}" + # # switch to temp dir + # tmpdir=$(mktemp -d) + # cd "${tmpdir}" || : - # # download archive - # wget "${KIWI_REPO}/archive/${KIWI_VERSION}.zip" - # unzip "${KIWI_VERSION}.zip" + # # download archive + # wget "${KIWI_REPO}/archive/${KIWI_VERSION}.zip" + # unzip "${KIWI_VERSION}.zip" - # # read archive version tag - # cd "kiwi-config-${KIWI_VERSION}" - # export KIWI_VERSION=$(cat ./version-tag) + # # read archive version tag + # cd "kiwi-config-${KIWI_VERSION}" || : + # KIWI_VERSION=$(cat "./src/${KIWI_VERSION_TAG}") - # # install archive - # mkdir -p "${KIWI_BASEDIR}" - # mv ./src "${KIWI_BASEDIR}/${KIWI_VERSION}" + # # install archive + # mkdir -p "$(kiwi_installdir)" + # mv "src" "Pipfile" "Pipfile.lock" "$(kiwi_installdir)/" - # # discard temp dir - # cd "${workdir}" - # rm -rf "${tmpdir}" + # # discard temp dir + # cd "${WORKDIR}" || : + # rm -rf "${tmpdir}" - # echo "OK" + # echo "OK" - ### pre-release version ### + ### pre-release version ### - # use this directory as archive - cd "$(dirname "$(readlink -f "${0}")")" ||: + # use this directory as archive + cd "$(dirname "$(readlink -f "${0}")")" || : - # read this version tag - KIWI_VERSION=$(cat "./src/${KIWI_VERSION_TAG}") + # read this version tag + KIWI_VERSION=$(cat "./src/${KIWI_VERSION_TAG}") - if [ -x "$(kiwi_executable)" ]; then - # after version update: no need to install - echo "kiwi-config v${KIWI_VERSION} found!" - else - # install this - mkdir -p "${KIWI_BASEDIR}" - ln -s "$(readlink -f .)" "$(kiwi_installdir)" - # cd - echo "OK" - fi + if [ -x "$(kiwi_executable)" ]; then + # after version update: no need to install + echo "kiwi-config v${KIWI_VERSION} found!" + else + # install this + mkdir -p "${KIWI_BASEDIR}" + ln -s "$(readlink -f .)" "$(kiwi_installdir)" + # cd + echo "OK" + fi fi # check virtualenv -cd "$(kiwi_installdir)" ||: -if ! pipenv --venv &> /dev/null; then +cd "$(kiwi_installdir)" || : +if ! pipenv --venv &>/dev/null; then # install virtualenv echo -n "Preparing virtualenv ... " - pipenv sync &> /dev/null + pipenv sync &>/dev/null echo "OK" fi # go back to the original work directory -cd "${WORKDIR}" ||: +cd "${WORKDIR}" || : # setup main environment KIWI_ROOT="$(kiwi_root)" export KIWI_ROOT export KIWI_CONF_NAME +export PIPENV_VERBOSITY=-1 # run main script -exec pipenv run "$(kiwi_executable)" "${@}" \ No newline at end of file +exec pipenv run "$(kiwi_executable)" "${@}" diff --git a/src/etc/kiwi_help.txt b/src/etc/kiwi_help.txt index e08f116..60f6ba2 100644 --- a/src/etc/kiwi_help.txt +++ b/src/etc/kiwi_help.txt @@ -1,8 +1,8 @@ kiwi-config is the tool for container server management. Features: - - Group your services into projects using their own docker-compose.yml - - Bind to file system by using ${TARGETDIR} as volume in docker-compose.yml + - Group services into projects using their own docker-compose.yml + - Bind to the local file system by using ${TARGETDIR} as volume in docker-compose.yml - Add instance-global config files by using ${CONFDIR} as volume in docker-compose.yml - Add instance-global custom values inside docker-compose.yml using config:runtime:env - Build service-specific, private docker images from Dockerfiles From 9c198622ca5480ee47b8d114db1258f76692fb7c Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 24 Aug 2020 14:48:50 +0200 Subject: [PATCH 115/118] Format --- kiwi | 2 +- src/kiwi/runner.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kiwi b/kiwi index 6f09e4a..71f2382 100755 --- a/kiwi +++ b/kiwi @@ -72,7 +72,7 @@ done if [ -f "./${KIWI_CONF_NAME}" ]; then # determine needed kiwi-config version re_version_line='version\s*:\s*' - eval "$(grep -E "${re_version_line}" "./${KIWI_CONF_NAME}" | sed -r 's/'"${re_version_line}"'/KIWI_VERSION=/')" + eval "$(grep -E "${re_version_line}" "./${KIWI_CONF_NAME}" | sed -r "s/${re_version_line}/KIWI_VERSION=/")" fi # install if kiwi-config not found diff --git a/src/kiwi/runner.py b/src/kiwi/runner.py index affa16e..a2b33dd 100644 --- a/src/kiwi/runner.py +++ b/src/kiwi/runner.py @@ -24,7 +24,8 @@ class Runner: ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except subprocess.CalledProcessError: - raise PermissionError("Cannot access docker, please get into the docker group or run as root!") + logging.critical("Cannot access docker, please get into the docker group or run as root!") + quit(1) # setup all subcommands for className in subcommands.__all__: From bf0fb911fded085009a5bd21922b2238b894dc18 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 24 Aug 2020 14:49:17 +0200 Subject: [PATCH 116/118] shell -u argument --- src/kiwi/subcommands/shell.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/kiwi/subcommands/shell.py b/src/kiwi/subcommands/shell.py index 0a240de..6ae4a0f 100644 --- a/src/kiwi/subcommands/shell.py +++ b/src/kiwi/subcommands/shell.py @@ -84,13 +84,21 @@ class ShellCommand(ServiceCommand): help="shell to spawn" ) + # -u argument: Run as user + self._sub_parser.add_argument( + '-u', '--user', type=str, + help="container user to run shell" + ) + def _run_services(self, runner, args, project, services): service = services[0] shell = _find_shell(args, project, service) + user_args = ['-u', args.user] if args.user else [] + if shell is not None: # spawn shell - project.compose_run(['exec', service, shell]) + project.compose_run(['exec', *user_args, service, shell]) return True return False From 772c22a49a993dfd1d859c123652c0232702f36e Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 24 Aug 2020 14:49:39 +0200 Subject: [PATCH 117/118] better example project --- .../hello-world.project/conf/html/index.html | 24 +++++++++++ example/hello-world.project/conf/test.txt | 1 - .../hello-world.project/docker-compose.yml | 41 +++++++++++++++++-- example/hello-world.project/web/Dockerfile | 2 + example/hello-world.project/web/index.html | 31 ++++++++++++++ 5 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 example/hello-world.project/conf/html/index.html delete mode 100644 example/hello-world.project/conf/test.txt create mode 100644 example/hello-world.project/web/Dockerfile create mode 100644 example/hello-world.project/web/index.html diff --git a/example/hello-world.project/conf/html/index.html b/example/hello-world.project/conf/html/index.html new file mode 100644 index 0000000..869330c --- /dev/null +++ b/example/hello-world.project/conf/html/index.html @@ -0,0 +1,24 @@ + + + + + + Hello kiwi #2! + + + + +

Another Hello World!

+ +

This service uses an off-the-shelf image and the conf-directory feature of kiwi-config.

+ +

Greetings, nginx.

+ + + \ No newline at end of file diff --git a/example/hello-world.project/conf/test.txt b/example/hello-world.project/conf/test.txt deleted file mode 100644 index 484ba93..0000000 --- a/example/hello-world.project/conf/test.txt +++ /dev/null @@ -1 +0,0 @@ -This is a test. diff --git a/example/hello-world.project/docker-compose.yml b/example/hello-world.project/docker-compose.yml index 35951b8..9a7ea1e 100644 --- a/example/hello-world.project/docker-compose.yml +++ b/example/hello-world.project/docker-compose.yml @@ -10,13 +10,46 @@ networks: name: ${KIWI_HUB_NAME} services: - hello-world: + # simple loop producing (rather boring) logs + greeter: image: alpine:latest command: sh -c 'LOOP=1; while :; do echo Hello World "$$LOOP"; LOOP=$$(($$LOOP + 1)); sleep 10; done' + + # basic webserver listening on localhost:8080 + web: + build: web + restart: unless-stopped + ports: + - "8080:80" + + # internal mariadb (mysql) instance with persistent storage + db: + image: mariadb:10 + restart: unless-stopped + networks: + - kiwi_hub + environment: + MYSQL_ROOT_PASSWORD: changeme + volumes: + - "${TARGETDIR}/db:/var/lib/mysql" + + # admin interface for databases + adminer: + image: adminer:standalone + restart: unless-stopped networks: - default - kiwi_hub + depends_on: + - db + ports: + - "8081:8080" - foo-bar: - image: alpine:latest - command: sh -c 'LOOP=1; while :; do echo Foo Bar "$$LOOP"; LOOP=$$(($$LOOP + 1)); sleep 20; done' + # Another webserver just to show off the ${CONFDIR} variable + another-web: + image: nginx:stable-alpine + restart: unless-stopped + ports: + - "8082:80" + volumes: + - "${CONFDIR}/html/index.html:/usr/share/nginx/html/index.html:ro" diff --git a/example/hello-world.project/web/Dockerfile b/example/hello-world.project/web/Dockerfile new file mode 100644 index 0000000..d479ce9 --- /dev/null +++ b/example/hello-world.project/web/Dockerfile @@ -0,0 +1,2 @@ +FROM nginx:stable-alpine +COPY index.html /usr/share/nginx/html \ No newline at end of file diff --git a/example/hello-world.project/web/index.html b/example/hello-world.project/web/index.html new file mode 100644 index 0000000..407e6bd --- /dev/null +++ b/example/hello-world.project/web/index.html @@ -0,0 +1,31 @@ + + + + + + Hello kiwi! + + + + +

Hello World!

+

kiwi-config works.

+ +

This is an example service on a custom image assembled by kiwi-config.
But wait, there's more!

+ +

Adminer

 +

There's a mySQL database included in this project. Login with user root and password changeme.

+ +

Another hello world

 +

Another web server, built in a different way.

+ +

Greetings, nginx.

+ + + \ No newline at end of file From eafc4ae55c374c9ff8f3ba8063db9987233096a2 Mon Sep 17 00:00:00 2001 From: ldericher Date: Mon, 24 Aug 2020 17:01:16 +0200 Subject: [PATCH 118/118] v0.0.1: First release, not expected to be fully functional --- README.md | 23 ++++++++++++ example/kiwi.yml | 2 +- install.sh | 85 +++++++++++++++++++++++++++++++++++++++++++++ kiwi | 61 +++++++++++--------------------- src/etc/version_tag | 2 +- 5 files changed, 130 insertions(+), 43 deletions(-) create mode 100644 README.md create mode 100755 install.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..b63427a --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# kiwi-config + +The simple tool for managing container servers + +## Quick start + +- Learn to use `docker` with `docker-compose` +- Install kiwi-config +- Look at [the example instance](../example) +- Look at the output of `kiwi --help` +- Start building your own instances + +## Installation + +```shell script +curl 'https://raw.githubusercontent.com/ldericher/kiwi-config/master/install.sh' | sh +``` + +That script checks for the basic dependencies of the `kiwi` command, then downloads the main script and installs it to a location of your choice. Please consider installing `kiwi` into a directory inside your $PATH. + +## Get started + +TODO diff --git a/example/kiwi.yml b/example/kiwi.yml index d8dab05..9965083 100644 --- a/example/kiwi.yml +++ b/example/kiwi.yml @@ -2,7 +2,7 @@ # kiwi-config instance configuration # ###################################### -version: '0.1' +version: '0.0.1' runtime: storage: /tmp/kiwi diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..3ee19c3 --- /dev/null +++ b/install.sh @@ -0,0 +1,85 @@ +#!/bin/sh + +############# +# CONSTANTS # +############# + +# dependencies to run kiwi-config +KIWI_DEPS="bash python3 pipenv less" + +########## +# CHECKS # +########## + +printf "checking dependencies ... " + +for dep in ${KIWI_DEPS}; do + printf "%s, " "${dep}" + + if ! command -v "${dep}" >/dev/null 2>/dev/null; then + echo + echo "Dependency '${dep}' not found, please install!" >/dev/stderr + exit 1 + fi +done + +echo "OK" + +######## +# MAIN # +######## + +# prompt for installation directory +install_dir_default="/usr/local/bin" +valid="no" + +while [ "${valid}" = "no" ]; do + printf "Select installation directory [Default: '%s']: " "${install_dir_default}" + read install_dir + install_dir="${install_dir:-${install_dir_default}}" + + # check + if [ -d "${install_dir}" ]; then + valid="yes" + + else + printf "Install directory doesn't exist. Try creating? [Y|n] " + read yesno + if [ ! "${yesno}" = "N" ] || [ ! "${yesno}" = "n" ]; then + + # check creation failure + if mkdir -p "${install_dir}"; then + valid="yes" + + else + echo "Invalid install directory." >/dev/stderr + exit 1 + fi + fi + fi +done + +# start actual installation +printf "Installing into '%s' ... " "${install_dir}" + +# install "kiwi" script +uri="https://raw.githubusercontent.com/ldericher/kiwi-config/master/kiwi" +tmp_file="$(mktemp)" + +if ! curl -o "${tmp_file}" "${uri}" >/dev/null 2>/dev/null; then + rm "${tmp_file}" + echo "Download 'kiwi' failed!" >/dev/stderr + exit 1 +fi + +if ! install -m 0755 "${tmp_file}" "${install_dir}/kiwi"; then + rm "${tmp_file}" + echo "Install 'kiwi' failed!" >/dev/stderr + exit 1 +fi + +rm "${tmp_file}" + +# finalization +echo "OK" +exit 0 \ No newline at end of file diff --git a/kiwi b/kiwi index 71f2382..e4cb044 100755 --- a/kiwi +++ b/kiwi @@ -10,9 +10,9 @@ KIWI_CONF_NAME="kiwi.yml" KIWI_VERSION_TAG="etc/version_tag" # dependencies to run kiwi-config -KIWI_DEPS=(python3 pipenv less docker docker-compose) +KIWI_DEPS=(docker docker-compose) # base install dir -KIWI_BASEDIR="${HOME}/.cache/kiwi-config" +KIWI_BASEDIR="${HOME}/.local/lib/kiwi-config" # per-user env setup script KIWI_ENVFILE="${HOME}/.kiwienv" @@ -59,7 +59,7 @@ fi for dep in "${KIWI_DEPS[@]}"; do if ! command -v "${dep}" &>/dev/null; then - echo "Dependency '${dep}' not found, please install!" + echo "Dependency '${dep}' not found, please install!" >/dev/stderr exit 1 fi done @@ -79,48 +79,27 @@ fi if [ ! -x "$(kiwi_executable)" ]; then echo -n "Installing kiwi-config v${KIWI_VERSION} into ${KIWI_BASEDIR} ... " - ### TODO: post-release version ### + # switch to temp dir + tmpdir=$(mktemp -d) + cd "${tmpdir}" || : - # # switch to temp dir - # tmpdir=$(mktemp -d) - # cd "${tmpdir}" || : + # download archive + curl -o "kiwi-config.zip" "${KIWI_REPO}/archive/${KIWI_VERSION}.zip" + unzip "kiwi-config.zip" - # # download archive - # wget "${KIWI_REPO}/archive/${KIWI_VERSION}.zip" - # unzip "${KIWI_VERSION}.zip" - - # # read archive version tag - # cd "kiwi-config-${KIWI_VERSION}" || : - # KIWI_VERSION=$(cat "./src/${KIWI_VERSION_TAG}") - - # # install archive - # mkdir -p "$(kiwi_installdir)" - # mv "src" "Pipfile" "Pipfile.lock" "$(kiwi_installdir)/" - - # # discard temp dir - # cd "${WORKDIR}" || : - # rm -rf "${tmpdir}" - - # echo "OK" - - ### pre-release version ### - - # use this directory as archive - cd "$(dirname "$(readlink -f "${0}")")" || : - - # read this version tag + # read archive version tag + cd "kiwi-config-${KIWI_VERSION}" || : KIWI_VERSION=$(cat "./src/${KIWI_VERSION_TAG}") - if [ -x "$(kiwi_executable)" ]; then - # after version update: no need to install - echo "kiwi-config v${KIWI_VERSION} found!" - else - # install this - mkdir -p "${KIWI_BASEDIR}" - ln -s "$(readlink -f .)" "$(kiwi_installdir)" - # cd - echo "OK" - fi + # install archive + mkdir -p "$(kiwi_installdir)" + mv "src" "Pipfile" "Pipfile.lock" "$(kiwi_installdir)/" + + # discard temp dir + cd "${WORKDIR}" || : + rm -rf "${tmpdir}" + + echo "OK" fi # check virtualenv diff --git a/src/etc/version_tag b/src/etc/version_tag index ceab6e1..8a9ecc2 100644 --- a/src/etc/version_tag +++ b/src/etc/version_tag @@ -1 +1 @@ -0.1 \ No newline at end of file +0.0.1 \ No newline at end of file