From a5b0cbe3969b8a6cf339a8ac277db6aecd80a023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 15:02:17 +0000 Subject: [PATCH 01/10] preparation for release 0.1.0 --- .vscode/launch.json | 2 +- Dockerfile | 20 ++++++++++++++++++++ dummy-metric | 8 -------- example/dummy-metric | 8 ++++++++ min_fail_example.py | 25 ------------------------- 5 files changed, 29 insertions(+), 34 deletions(-) create mode 100644 Dockerfile delete mode 100755 dummy-metric create mode 100755 example/dummy-metric delete mode 100644 min_fail_example.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 3049ac8..0bb267d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,7 +13,7 @@ "METRIC__INTERVAL": "5", "METRIC__LOG__ENABLED": "True", "METRIC__DISK__PATHS": "[\"/var\", \"/\", \"/dev\"]", - "METRIC__EXTERNAL__EXECUTABLES": "[\"${workspaceFolder}/dummy-metric\"]", + "METRIC__EXTERNAL__EXECUTABLES": "[\"${workspaceFolder}/example/dummy-metric\"]", }, "justMyCode": true } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..40d7e0d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11-alpine + +ENV \ + PYTHONUNBUFFERED=1 + +COPY . /usr/src/app + +RUN set -ex; \ + # buildtime deps + apk add --no-cache \ + --virtual .build-deps \ + build-base \ + gcc \ + linux-headers \ + ; + +RUN set -ex; \ + pip3 --no-cache-dir install /usr/src/app + +ENTRYPOINT ["kiwi-simple-metrics"] \ No newline at end of file diff --git a/dummy-metric b/dummy-metric deleted file mode 100755 index 284bcdb..0000000 --- a/dummy-metric +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -echo "Dummy" -echo "95" -echo "normal" -awk "BEGIN{srand(); r=rand(); print r * 100}" - -exit 0 \ No newline at end of file diff --git a/example/dummy-metric b/example/dummy-metric new file mode 100755 index 0000000..d869c76 --- /dev/null +++ b/example/dummy-metric @@ -0,0 +1,8 @@ +#!/bin/sh + +echo "Dummy" # name +echo "95" # threshold +echo "normal" # inversion +awk "BEGIN{srand(); r=rand(); print r * 100}" # value + +exit 0 \ No newline at end of file diff --git a/min_fail_example.py b/min_fail_example.py deleted file mode 100644 index c812183..0000000 --- a/min_fail_example.py +++ /dev/null @@ -1,25 +0,0 @@ -from pydantic import BaseModel - - -class SubModel(BaseModel): - optional: int = 42 - required: str - - -class Settings(BaseModel): - sub: SubModel = SubModel(required="foo") - sub2: SubModel = SubModel(required="bar") - - -def main() -> None: - settings = Settings.model_validate({ - "sub": {"optional": "69"}, - }) - - # settings = Settings() - - print(settings.model_dump()) - - -if __name__ == "__main__": - main() From 61ede82f3655ad5d257634fbb69bfb9aaea077d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 15:13:49 +0000 Subject: [PATCH 02/10] default values for lists in SETTINGS --- kiwi_simple_metrics/settings/metric.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kiwi_simple_metrics/settings/metric.py b/kiwi_simple_metrics/settings/metric.py index d1b5b68..9b3f71b 100644 --- a/kiwi_simple_metrics/settings/metric.py +++ b/kiwi_simple_metrics/settings/metric.py @@ -1,8 +1,8 @@ import math from typing import Any, Literal -from pydantic import (BaseModel, DirectoryPath, Field, FieldValidationInfo, - FilePath, field_validator) +from pydantic import (BaseModel, DirectoryPath, FieldValidationInfo, FilePath, + field_validator) class MetricSettings(BaseModel): @@ -81,7 +81,7 @@ class DiskMS(MetricSettings): count: int | None = 1 # paths to check for disk space - paths: list[DirectoryPath] = Field(default_factory=list) + paths: list[DirectoryPath] = [DirectoryPath("/")] class ExternalMS(MetricSettings): @@ -89,7 +89,7 @@ class ExternalMS(MetricSettings): threshold: float = 0 # path to executable files - executables: list[FilePath] = Field(default_factory=list) + executables: list[FilePath] = [] # wait at most this many seconds for each executable timeout: int = 60 From ec322276ddf84d15e8941134f5c7eb66486d4ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 15:16:42 +0000 Subject: [PATCH 03/10] handle empty aggregate reports correctly --- .vscode/launch.json | 1 - kiwi_simple_metrics/metrics/_report.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3049ac8..f018c2c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,7 +12,6 @@ "env": { "METRIC__INTERVAL": "5", "METRIC__LOG__ENABLED": "True", - "METRIC__DISK__PATHS": "[\"/var\", \"/\", \"/dev\"]", "METRIC__EXTERNAL__EXECUTABLES": "[\"${workspaceFolder}/dummy-metric\"]", }, "justMyCode": true diff --git a/kiwi_simple_metrics/metrics/_report.py b/kiwi_simple_metrics/metrics/_report.py index e974ab1..5cf4044 100644 --- a/kiwi_simple_metrics/metrics/_report.py +++ b/kiwi_simple_metrics/metrics/_report.py @@ -102,6 +102,9 @@ class Report: reports = [data.report for data in get_data()] + if not reports: + return None + return cls( result=settings.report_outer.format( name=settings.name, From 71b8fe745412c5668bbdf643c095866b60a3dd85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 15:17:23 +0000 Subject: [PATCH 04/10] disable external metric by default --- .vscode/launch.json | 1 + kiwi_simple_metrics/settings/metric.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index f018c2c..e1aebf7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,6 +12,7 @@ "env": { "METRIC__INTERVAL": "5", "METRIC__LOG__ENABLED": "True", + "METRIC__EXTERNAL__ENABLED": "True", "METRIC__EXTERNAL__EXECUTABLES": "[\"${workspaceFolder}/dummy-metric\"]", }, "justMyCode": true diff --git a/kiwi_simple_metrics/settings/metric.py b/kiwi_simple_metrics/settings/metric.py index 9b3f71b..dc597b8 100644 --- a/kiwi_simple_metrics/settings/metric.py +++ b/kiwi_simple_metrics/settings/metric.py @@ -86,6 +86,7 @@ class DiskMS(MetricSettings): class ExternalMS(MetricSettings): name: str = "External Metric" + enabled: bool = False threshold: float = 0 # path to executable files From ef6d13a7044ae4e8cb1cffcefb4d0813b8320992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 15:26:33 +0000 Subject: [PATCH 05/10] env_prefix --- .vscode/launch.json | 8 ++++---- kiwi_simple_metrics/settings/__init__.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index c3af61b..901abe3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,10 +10,10 @@ "request": "launch", "module": "kiwi_simple_metrics.main", "env": { - "METRIC__INTERVAL": "5", - "METRIC__LOG__ENABLED": "True", - "METRIC__EXTERNAL__ENABLED": "True", - "METRIC__EXTERNAL__EXECUTABLES": "[\"${workspaceFolder}/example/dummy-metric\"]", + "METRICS__INTERVAL": "5", + "METRICS__LOG__ENABLED": "True", + "METRICS__EXTERNAL__ENABLED": "True", + "METRICS__EXTERNAL__EXECUTABLES": "[\"${workspaceFolder}/example/dummy-metric\"]", }, "justMyCode": true } diff --git a/kiwi_simple_metrics/settings/__init__.py b/kiwi_simple_metrics/settings/__init__.py index 0d98373..7c42a83 100644 --- a/kiwi_simple_metrics/settings/__init__.py +++ b/kiwi_simple_metrics/settings/__init__.py @@ -6,7 +6,7 @@ from .metric import MetricSettings class Settings(BaseSettings): model_config = SettingsConfigDict( - env_prefix="METRIC__", + env_prefix="METRICS__", env_nested_delimiter="__", ) From e68b3f7957a7d5208eff095d975b3a417cd48d62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:20:59 +0000 Subject: [PATCH 06/10] README --- README.md | 54 +++++++++++++++++++++++++- kiwi_simple_metrics/settings/metric.py | 2 +- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a3b3729..be7747b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,53 @@ -# vscode-python3 +# kiwi-simple-metrics -Use this template to jumpstart python development using Visual Studio Code! \ No newline at end of file +[![Build Status](https://github.drone.yavook.de/api/badges/yavook/kiwi-simple-metrics/status.svg)](https://github.drone.yavook.de/yavook/kiwi-simple-metrics) + +> `kiwi` - simple, consistent, powerful + +A lightweight monitoring solution for [`kiwi-scp`](https://github.com/yavook/kiwi-scp), created with [`uptime-kuma`](https://github.com/louislam/uptime-kuma) in mind. Also [on Docker Hub](https://hub.docker.com/r/yavook/kiwi-simple-metrics). + +## Quick start + +The minimal config to add to one of your `docker-compose.yml` is this: + +```yaml +metrics: + image: yavook/kiwi-simple-metrics:0.1 +``` + +- admittedly not useful, but it *does* run monitoring +- every 600 seconds (10 minutes), metrics are evaluated +- measures cpu, memory and swap, and disk usage at "/" +- does not log to stdout +- does not trigger any webhooks + +Every aspect of kiwi-simple-metrics can be tweaked by environment variables, so this is a more reasonable configuration example: + +```yaml +metrics: + image: yavook/kiwi-simple-metrics:0.1 + environment: + METRICS__LOG__ENABLED: "True" + METRICS__WEBHOOK__URL: "https://my.webhook.host/success?report={}" + METRICS__WEBHOOK__FAIL: "https://my.webhook.host/failure?report={}" +``` + +- same metrics as above +- logs reports to stdout +- triggers webhooks (`{}` is the placeholder for the result string) + +## Configuration + +These are the environment variables you most likely need: + +- `METRICS__INTERVAL`: Time in seconds between metrics evaluation (default: `600`, i.e. 10 minutes) +- `METRICS__LOG__ENABLED`: If truthy, logs reports to stdout (default: `False`) +- `METRICS__[M]__ENABLED`, `METRICS__[M]__THRESHOLD` (with `[M]` from `CPU`, `MEMORY`, `DISK`): Enable or disable each metric, and set its failure threshold +- `METRICS__MEMORY__SWAP`: How swap space is handled in the "memory" report (default: `include`) +- `METRICS__DISK__PATHS`: At which paths the disk usage is measured (default: `["/"]`) +- `METRICS__EXTERNAL__ENABLED`, `METRICS__EXTERNAL__EXECUTABLES`: Setup for `external values`, as further defined in [`metrics/external.py`](./tree/master/kiwi_simple_metrics/metrics/external.py) (default: `False`, `[]`) +- `METRICS__WEBHOOK__URL`, `METRICS__WEBHOOK__FAIL`: Which webhooks to push the reports to (default: `None`, `None`) + +All settings can be found in the `SETTINGS` variable defined by module [`kiwi_simple_metrics.settings` in `settings/__init__.py`](./tree/master/kiwi_simple_metrics/settings/__init__.py). For default values, refer to the respective python files. + +Example: The above `METRICS__LOG__ENABLED` refers to `SETTINGS.log.enabled`, defined by model [`LogSettings` in `settings/misc.py`](./tree/master/kiwi_simple_metrics/settings/misc.py). diff --git a/kiwi_simple_metrics/settings/metric.py b/kiwi_simple_metrics/settings/metric.py index dc597b8..99d2988 100644 --- a/kiwi_simple_metrics/settings/metric.py +++ b/kiwi_simple_metrics/settings/metric.py @@ -54,7 +54,7 @@ class CpuMS(MetricSettings): name: str = "CPU" threshold: float = math.inf - # timespan to analyze average CPU usage + # timespan in seconds to measure average CPU usage interval: float = 1 From e1797f61eca031c8761652f00c712562efd06f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:22:58 +0000 Subject: [PATCH 07/10] more pydantic way of forcing `SETTINGS.external.count` to `None` --- kiwi_simple_metrics/settings/metric.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/kiwi_simple_metrics/settings/metric.py b/kiwi_simple_metrics/settings/metric.py index dc597b8..48b3201 100644 --- a/kiwi_simple_metrics/settings/metric.py +++ b/kiwi_simple_metrics/settings/metric.py @@ -89,14 +89,11 @@ class ExternalMS(MetricSettings): enabled: bool = False threshold: float = 0 + # always include all defined external values! + count: None = None + # path to executable files executables: list[FilePath] = [] # wait at most this many seconds for each executable timeout: int = 60 - - @field_validator("count", mode="after") - @classmethod - def force_none(cls, _) -> int | None: - """Don't accept a `count` value for the external metric!""" - return None From 41a9446f55e5f47070181b3da58bbcd54a5a9121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:28:10 +0000 Subject: [PATCH 08/10] package description --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 657e38b..5e5ff75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] authors = ["Jörn-Michael Miehe <40151420+ldericher@users.noreply.github.com>"] -description = "" +description = "A lightweight monitoring solution for kiwi-scp" license = "MIT" name = "kiwi-simple-metrics" packages = [{include = "kiwi_simple_metrics"}] @@ -9,8 +9,8 @@ version = "0.1.0" [tool.poetry.dependencies] psutil = "^5.9.5" -python = "^3.11" pydantic-settings = "^2.0.3" +python = "^3.11" requests = "^2.31.0" [tool.poetry.group.dev.dependencies] From 2572afac22e5364be0d8192ea7786d69295b5c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:30:54 +0000 Subject: [PATCH 09/10] "result" param for webhooks --- README.md | 6 +++--- kiwi_simple_metrics/metrics/_report.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index be7747b..4199891 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,13 @@ metrics: image: yavook/kiwi-simple-metrics:0.1 environment: METRICS__LOG__ENABLED: "True" - METRICS__WEBHOOK__URL: "https://my.webhook.host/success?report={}" - METRICS__WEBHOOK__FAIL: "https://my.webhook.host/failure?report={}" + METRICS__WEBHOOK__URL: "https://my.webhook.host/success?report={result}" + METRICS__WEBHOOK__FAIL: "https://my.webhook.host/failure?report={result}" ``` - same metrics as above - logs reports to stdout -- triggers webhooks (`{}` is the placeholder for the result string) +- triggers webhooks (`{result}` is the placeholder for the result string) ## Configuration diff --git a/kiwi_simple_metrics/metrics/_report.py b/kiwi_simple_metrics/metrics/_report.py index 5cf4044..5d8bac4 100644 --- a/kiwi_simple_metrics/metrics/_report.py +++ b/kiwi_simple_metrics/metrics/_report.py @@ -126,7 +126,7 @@ class Report: requests.get( url=str(url).format( - urllib.parse.quote_plus(self.result) + result=urllib.parse.quote_plus(self.result) ), verify=not SETTINGS.webhook.insecure, ) From 553c6f91d48128872732a63e4f093875766c6351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:33:46 +0000 Subject: [PATCH 10/10] drone config --- .drone.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..2c68ba6 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,15 @@ +--- +kind: pipeline +name: default +type: docker + +steps: +- name: docker + image: plugins/docker + settings: + repo: yavook/kiwi-simple-metrics + auto_tag: true + username: + from_secret: DOCKER_USERNAME + password: + from_secret: DOCKER_PASSWORD \ No newline at end of file