From 4633a4923f6e61e678dc81954d6086cf57364f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 01:11:13 +0100 Subject: [PATCH 01/13] kiwi-cron as base image; separate requirements.txt --- Dockerfile | 78 +++++++++++++++++++++++------------------------- requirements.txt | 42 ++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 40 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index bfe26fc..07ba3e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-alpine3.13 +FROM yavook/kiwi-cron:0.1 AS deps LABEL maintainer="jmm@yavook.de" # Previous work: https://github.com/wernight/docker-duplicity @@ -19,13 +19,19 @@ RUN set -ex; \ libxslt \ openssh-client \ openssl \ + python3 \ rsync \ ; \ - update-ca-certificates; \ + update-ca-certificates; + +COPY requirements.txt /tmp/ + +RUN set -ex; \ \ - # dependencies to build python packages + # python packages buildtime dependencies apk --no-cache add --virtual .build-deps \ gcc \ + git \ libffi-dev \ librsync-dev \ libxml2-dev \ @@ -33,47 +39,41 @@ RUN set -ex; \ make \ musl-dev \ openssl-dev \ + python3-dev \ + py3-pip \ + cargo \ + ; \ + # make use of prebuilt wheels where possible + python3 -m pip --no-cache-dir \ + install wheel \ ; \ \ - # make use of "wheel" python packages - pip3 --no-cache-dir install wheel ; \ - \ - pip3 --no-cache-dir install \ - # main app - duplicity \ - \ - # general duplicity requirements, based on - # http://duplicity.nongnu.org/vers8/README - # https://git.launchpad.net/duplicity/tree/requirements.txt - fasteners \ - future \ - mock \ - paramiko \ - python-gettext \ - requests \ - urllib3 \ - \ - # backend requirements - azure-mgmt-storage \ - b2sdk \ - boto \ - boto3 \ - dropbox \ - gdata \ - jottalib \ - mediafire \ - mega.py \ - pydrive \ - pyrax \ - python-swiftclient \ - requests_oauthlib \ + python3 -m pip --no-cache-dir \ + install -r /tmp/requirements.txt \ ; \ \ # remove buildtime dependencies - pip3 --no-cache-dir uninstall -y wheel; \ - apk del --purge .build-deps + python3 -m pip --no-cache-dir \ + uninstall -y wheel \ + ; \ + apk del --purge .build-deps; -VOLUME ["/root/.cache/duplicity", "/backup/target"] +RUN set -ex; \ + \ + # create non-root user + adduser -D -u 1368 duplicity; \ + mkdir -p /home/duplicity/.cache/duplicity; \ + mkdir -p /home/duplicity/.gnupg; \ + chmod -R go+rwx /home/duplicity/; + +USER duplicity + +VOLUME [ "/home/duplicity/.cache/duplicity" ] + +# confirm this is working +RUN set -ex; \ + \ + duplicity --version ENV \ ################# @@ -104,6 +104,4 @@ ENV \ GPG_KEY_ID="" \ GPG_PASSPHRASE="" -COPY do-plicity /usr/local/bin/ - CMD ["do-plicity"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..365df1a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,42 @@ +##### main app ##### + +duplicity==0.8.21 + +# dependencies as per +# https://gitlab.com/duplicity/duplicity/-/blob/rel.0.8.21/requirements.txt + +##### basic requirements ##### + +fasteners +future +python-gettext +setuptools>=44.1.1 +setuptools-scm>=5.0.2 + + +##### backend libraries ##### + +azure-storage-blob ; python_version >= '3.6' +b2sdk ; python_version >= '3.6' +boto +boto3 +boxsdk[jwt] ; python_version >= '3.6' +dropbox +gdata ; python_version == '2.7' +gdata-python3 ; python_version >= '3.6' +google-api-python-client +google-auth-oauthlib +jottalib +keyring +mediafire +megatools ; python_version >= '3.6' +paramiko +pexpect +psutil +pydrive ; python_version >= '3.6' +pydrive2 ; python_version >= '3.6' +pyrax ; python_version >= '3.6' +python-swiftclient +python-keystoneclient +requests +requests-oauthlib From b78a20d91211c1dc9aa21ba645bcdc59c30bfd7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 02:26:13 +0100 Subject: [PATCH 02/13] method to install duplicity itself; purge cargo artifacts --- Dockerfile | 9 ++++++--- requirements.txt | 4 ---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07ba3e7..93e1af8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,6 @@ FROM yavook/kiwi-cron:0.1 AS deps LABEL maintainer="jmm@yavook.de" -# Previous work: https://github.com/wernight/docker-duplicity - RUN set -ex; \ \ # create backup source @@ -48,15 +46,20 @@ RUN set -ex; \ install wheel \ ; \ \ + # install duplicity python3 -m pip --no-cache-dir \ install -r /tmp/requirements.txt \ ; \ + python3 -m pip --no-cache-dir \ + install duplicity \ + ; \ \ # remove buildtime dependencies python3 -m pip --no-cache-dir \ uninstall -y wheel \ ; \ - apk del --purge .build-deps; + apk del --purge .build-deps; \ + rm -rf "${USER}/.cargo"; RUN set -ex; \ \ diff --git a/requirements.txt b/requirements.txt index 365df1a..3bb3b2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,3 @@ -##### main app ##### - -duplicity==0.8.21 - # dependencies as per # https://gitlab.com/duplicity/duplicity/-/blob/rel.0.8.21/requirements.txt From 52befda4d8833d4e482588f4bf94b95c9a4c4ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 02:39:19 +0100 Subject: [PATCH 03/13] user creation --- .drone.yml | 1 + Dockerfile | 20 +++++++++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.drone.yml b/.drone.yml index 80e60ab..1ca1490 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,6 +1,7 @@ --- kind: pipeline name: default +type: docker steps: - name: docker diff --git a/Dockerfile b/Dockerfile index 93e1af8..ec58d2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,22 +61,20 @@ RUN set -ex; \ apk del --purge .build-deps; \ rm -rf "${USER}/.cargo"; -RUN set -ex; \ - \ - # create non-root user - adduser -D -u 1368 duplicity; \ - mkdir -p /home/duplicity/.cache/duplicity; \ - mkdir -p /home/duplicity/.gnupg; \ - chmod -R go+rwx /home/duplicity/; +# create non-root user +RUN adduser -D -u 1368 duplicity; USER duplicity -VOLUME [ "/home/duplicity/.cache/duplicity" ] - -# confirm this is working RUN set -ex; \ \ - duplicity --version + mkdir -p "${HOME}/.cache/duplicity"; \ + mkdir -pm 600 "${HOME}/.gnupg"; + +VOLUME [ "/home/duplicity/.cache/duplicity" ] + +# confirm duplicity is working +RUN duplicity --version ENV \ ################# From b4c95c1be17af7fad74376eb27a5ff1d61ad2d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 02:45:09 +0100 Subject: [PATCH 04/13] typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ec58d2a..f5eb4e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,7 @@ RUN set -ex; \ uninstall -y wheel \ ; \ apk del --purge .build-deps; \ - rm -rf "${USER}/.cargo"; + rm -rf "${HOME}/.cargo"; # create non-root user RUN adduser -D -u 1368 duplicity; From d5c7f3e684168fd8a69b13135d7a6ae77b02eb98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 02:45:54 +0100 Subject: [PATCH 05/13] unneeded target --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f5eb4e0..c09f8d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM yavook/kiwi-cron:0.1 AS deps +FROM yavook/kiwi-cron:0.1 LABEL maintainer="jmm@yavook.de" RUN set -ex; \ From b1cd4ef52e6c9746e9df279795da6dde293edd79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 02:56:44 +0100 Subject: [PATCH 06/13] purge requirements.txt --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c09f8d9..5ed3bce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,6 +59,7 @@ RUN set -ex; \ uninstall -y wheel \ ; \ apk del --purge .build-deps; \ + rm -f "/tmp/requirements.txt"; \ rm -rf "${HOME}/.cargo"; # create non-root user From 8ebd564926136bcb6ffd8b3103a000f1d6bd3666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 17:30:22 +0100 Subject: [PATCH 07/13] modularity as in kiwi-cron --- Dockerfile | 26 ++--- bin/kiwi-backup | 5 + do-plicity | 177 ------------------------------ libexec/kiwi-backup/build_command | 81 ++++++++++++++ libexec/kiwi-backup/scheduler | 9 ++ 5 files changed, 108 insertions(+), 190 deletions(-) create mode 100755 bin/kiwi-backup delete mode 100755 do-plicity create mode 100755 libexec/kiwi-backup/build_command create mode 100755 libexec/kiwi-backup/scheduler diff --git a/Dockerfile b/Dockerfile index 5ed3bce..cc5d085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,14 @@ FROM yavook/kiwi-cron:0.1 LABEL maintainer="jmm@yavook.de" +COPY requirements.txt /tmp/ + RUN set -ex; \ \ # create backup source mkdir -p /backup/source; \ \ + # duplicity software dependencies apk --no-cache add \ ca-certificates \ gettext \ @@ -20,11 +23,7 @@ RUN set -ex; \ python3 \ rsync \ ; \ - update-ca-certificates; - -COPY requirements.txt /tmp/ - -RUN set -ex; \ + update-ca-certificates; \ \ # python packages buildtime dependencies apk --no-cache add --virtual .build-deps \ @@ -60,23 +59,23 @@ RUN set -ex; \ ; \ apk del --purge .build-deps; \ rm -f "/tmp/requirements.txt"; \ - rm -rf "${HOME}/.cargo"; - -# create non-root user -RUN adduser -D -u 1368 duplicity; + rm -rf "${HOME}/.cargo"; \ + \ + # create a non-root user + adduser -D -u 1368 duplicity; USER duplicity RUN set -ex; \ + \ + # confirm duplicity is working + duplicity --version; \ \ mkdir -p "${HOME}/.cache/duplicity"; \ mkdir -pm 600 "${HOME}/.gnupg"; VOLUME [ "/home/duplicity/.cache/duplicity" ] -# confirm duplicity is working -RUN duplicity --version - ENV \ ################# # BACKUP POLICY # @@ -106,4 +105,5 @@ ENV \ GPG_KEY_ID="" \ GPG_PASSPHRASE="" -CMD ["do-plicity"] +COPY bin /usr/local/bin/ +COPY libexec /usr/local/libexec/ diff --git a/bin/kiwi-backup b/bin/kiwi-backup new file mode 100755 index 0000000..0208e18 --- /dev/null +++ b/bin/kiwi-backup @@ -0,0 +1,5 @@ +#!/bin/sh + +exec /usr/local/libexec/kiwi-cron/run_cron \ + "${@}" \ + /usr/local/libexec/kiwi-backup/scheduler diff --git a/do-plicity b/do-plicity deleted file mode 100755 index 6d310cf..0000000 --- a/do-plicity +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/sh - -############# -# CONSTANTS # -############# - -# commands -this_exe="$(command -v "${0}")" -ionice_exe="$(command -v ionice)" -duplicity_exe="$(command -v duplicity)" - -# files -duplicity_secrets_file='/root/duplicity_secrets' - -############# -# FUNCTIONS # -############# - -append_options() { - ao_cmdline="${1}" - ao_options="${2}" - shift 1 - - if [ -n "${ao_cmdline}" ] && [ -n "${ao_options}" ]; then - # if both are given, stitch together with a space - echo "${ao_cmdline} ${ao_options}" - elif [ -n "${ao_options}" ]; then - # if only options are given, output them - echo "${ao_options}" - else - # if at most a cmdline is given, output that - echo "${ao_cmdline}" - fi -} - -print_command() { - pc_task="${1}" - shift 1 - - pc_cmdline="${ionice_exe} -c 3 ${duplicity_exe}" - pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_ALL}" )" - - case "${pc_task}" in - backup) - pc_cmdline="$( append_options "${pc_cmdline}" "--allow-source-mismatch" )" - pc_cmdline="$( append_options "${pc_cmdline}" "--volsize ${BACKUP_VOLSIZE}" )" - pc_cmdline="$( append_options "${pc_cmdline}" "--full-if-older-than ${FULL_BACKUP_FREQUENCY}" )" - pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_BACKUP}" )" - pc_cmdline="$( append_options "${pc_cmdline}" "/backup/source" )" - ;; - - cleanup) - pc_cmdline="$( append_options "${pc_cmdline}" "cleanup --force" )" - pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_CLEAN}" )" - ;; - - rmfull) - pc_cmdline="$( append_options "${pc_cmdline}" "remove-older-than ${BACKUP_RETENTION_TIME} --force" )" - pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_RMFULL}" )" - ;; - - rmincr) - pc_cmdline="$( append_options "${pc_cmdline}" "remove-all-inc-of-but-n-full ${KEEP_NUM_FULL_CHAINS} --force" )" - pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_RMINCR}" )" - ;; - esac - - pc_cmdline="${pc_cmdline} ${BACKUP_TARGET}" - echo "${pc_cmdline}" -} - -print_cron_schedule() { - pcs_min="$( echo "${1}" | awk '{print $1}' )" - pcs_hour="$( echo "${1}" | awk '{print $2}' )" - pcs_day="$( echo "${1}" | awk '{print $3}' )" - pcs_month="$( echo "${1}" | awk '{print $4}' )" - pcs_weekday="$( echo "${1}" | awk '{print $5}' )" - pcs_command="${2}" - shift 2 - - printf '%-8s%-8s%-8s%-8s%-8s%s\n' "${pcs_min}" "${pcs_hour}" "${pcs_day}" "${pcs_month}" "${pcs_weekday}" "${pcs_command}" -} - -print_crontab() { - echo '# crontab generated for kiwi-backup' - printf '# generation time: '; date - echo '#' - - # don't split the '#' from 'min' - print_cron_schedule '#_min hour day month weekday' 'command' | tr '_' ' ' - - print_cron_schedule "${SCHEDULE_BACKUP}" "${this_exe} backup" - print_cron_schedule "${SCHEDULE_CLEANUP}" "${this_exe} cleanup" - print_cron_schedule "${SCHEDULE_RMFULL}" "${this_exe} rmfull" - print_cron_schedule "${SCHEDULE_RMINCR}" "${this_exe} rmincr" -} - -############### -# ENVIRONMENT # -############### - -# load secrets file -if [ -f "${duplicity_secrets_file}" ]; then - # shellcheck disable=SC1090 - . "${duplicity_secrets_file}" -fi - -# check if uses encryption -if [ -n "${GPG_KEY_ID}" ]; then - # gpg key given - OPTIONS_ALL="$( append_options "${OPTIONS_ALL}" "--encrypt-key=${GPG_KEY_ID}" )" - - # handle more verbose "GPG_PASSPHRASE" env_var - PASSPHRASE="${GPG_PASSPHRASE:-${PASSPHRASE}}" - export PASSPHRASE - unset GPG_PASSPHRASE -else - # no key given - OPTIONS_ALL="$( append_options "${OPTIONS_ALL}" "--no-encryption" )" -fi - -######## -# MAIN # -######## - -if [ "${#}" -gt 0 ]; then - - # CLI - task="${1}" - shift 1 - - # run task - case "${task}" in - print-*) - task="${task##*-}" - - case "${task}" in - # print out the crontab - crontab) - print_crontab - ;; - - # print out a task - backup|cleanup|rmfull|rmincr) - print_command "${task}" - ;; - - # unknown task - *) - >&2 echo "Cannot print '${task}'." - >&2 echo "Choose from: crontab, backup, cleanup, rmfull, rmincr" - exit 1 - ;; - esac - ;; - - # execute single task - backup|cleanup|rmfull|rmincr) - # shellcheck disable=SC2091 - $(print_command "${task}") - ;; - - # unknown task - *) - >&2 echo "Unknown task '${task}'." - >&2 echo "Choose from: backup, cleanup, rmfull, rmincr" - exit 1 - ;; - esac -else - - # default run: replace crontab, then start crond - print_crontab | crontab - - crond -fl 8 -fi - -exit 0 \ No newline at end of file diff --git a/libexec/kiwi-backup/build_command b/libexec/kiwi-backup/build_command new file mode 100755 index 0000000..76a8525 --- /dev/null +++ b/libexec/kiwi-backup/build_command @@ -0,0 +1,81 @@ +#!/bin/sh + +############# +# CONSTANTS # +############# + +# commands +ionice_exe="$(command -v ionice)" +duplicity_exe="$(command -v duplicity)" + +# files +duplicity_secrets_file="${HOME}/duplicity_secrets" + +######## +# MAIN # +######## + +# load secrets file +if [ -f "${duplicity_secrets_file}" ]; then + # shellcheck disable=SC1090 + . "${duplicity_secrets_file}" +fi + +# check if uses encryption +if [ -n "${GPG_KEY_ID}" ]; then + # gpg key given + options_encryption="--encrypt-key=${GPG_KEY_ID}" + + # handle more verbose "GPG_PASSPHRASE" env_var + PASSPHRASE="${GPG_PASSPHRASE:-${PASSPHRASE}}" + export PASSPHRASE + unset GPG_PASSPHRASE +else + # no key given + options_encryption="--no-encryption" +fi + +task="${1}" +shift 1 + +add_space () { + if [ -n "${1}" ]; then + echo " ${1}" + fi +} + +printf "%s -c 3 %s%s %s" \ + "${ionice_exe}" \ + "${duplicity_exe}" \ + "$( add_space "${OPTIONS_ALL}" )" \ + "${options_encryption}" + +case "${task}" in + backup) + printf ' --allow-source-mismatch' + printf ' --volsize' + printf ' --full-if-older-than' + printf '%s' "$( add_space "${OPTIONS_BACKUP}" )" + printf ' /backup/source' + ;; + + cleanup) + printf ' cleanup' + printf ' --force' + printf '%s' "$( add_space "${OPTIONS_CLEAN}" )" + ;; + + rmfull) + printf ' remove-older-than %s' "${BACKUP_RETENTION_TIME}" + printf ' --force' + printf '%s' "$( add_space "${OPTIONS_RMFULL}" )" + ;; + + rmincr) + printf ' remove-all-inc-of-but-n-full %s' "${KEEP_NUM_FULL_CHAINS}" + printf ' --force' + printf '%s' "$( add_space "${OPTIONS_RMINCR}" )" + ;; +esac + +echo " ${BACKUP_TARGET}" diff --git a/libexec/kiwi-backup/scheduler b/libexec/kiwi-backup/scheduler new file mode 100755 index 0000000..52f4b42 --- /dev/null +++ b/libexec/kiwi-backup/scheduler @@ -0,0 +1,9 @@ +#!/bin/sh + +this_script="$( readlink -f "${0}" )" +this_dir="${this_script%/*}" + +echo "${SCHEDULE_BACKUP}" "$( "${this_dir}/build_command" backup )" +echo "${SCHEDULE_CLEANUP}" "$( "${this_dir}/build_command" cleanup )" +echo "${SCHEDULE_RMFULL}" "$( "${this_dir}/build_command" rmfull )" +echo "${SCHEDULE_RMINCR}" "$( "${this_dir}/build_command" rmincr )" From 5a4a6e56b0c394ac9bddc5d34cb8ed874185bc10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 18:41:15 +0100 Subject: [PATCH 08/13] missing option values --- libexec/kiwi-backup/build_command | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libexec/kiwi-backup/build_command b/libexec/kiwi-backup/build_command index 76a8525..b5dc61a 100755 --- a/libexec/kiwi-backup/build_command +++ b/libexec/kiwi-backup/build_command @@ -53,8 +53,8 @@ printf "%s -c 3 %s%s %s" \ case "${task}" in backup) printf ' --allow-source-mismatch' - printf ' --volsize' - printf ' --full-if-older-than' + printf ' --volsize %s' "${BACKUP_VOLSIZE}" + printf ' --full-if-older-than %s' "${FULL_BACKUP_FREQUENCY}" printf '%s' "$( add_space "${OPTIONS_BACKUP}" )" printf ' /backup/source' ;; From 8e149bea0645ff60f0517c0668c32d1e4c46f69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Wed, 2 Mar 2022 18:41:47 +0100 Subject: [PATCH 09/13] bunch of renamings --- Dockerfile | 35 ++++++++++++++++++------------- libexec/kiwi-backup/build_command | 2 +- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index cc5d085..088f3ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,6 @@ LABEL maintainer="jmm@yavook.de" COPY requirements.txt /tmp/ RUN set -ex; \ - \ - # create backup source - mkdir -p /backup/source; \ \ # duplicity software dependencies apk --no-cache add \ @@ -59,22 +56,29 @@ RUN set -ex; \ ; \ apk del --purge .build-deps; \ rm -f "/tmp/requirements.txt"; \ - rm -rf "${HOME}/.cargo"; \ - \ - # create a non-root user - adduser -D -u 1368 duplicity; - -USER duplicity + rm -rf "${HOME}/.cargo"; RUN set -ex; \ \ - # confirm duplicity is working - duplicity --version; \ + # create /kiwi-backup directories tree + mkdir -m 777 /kiwi-backup; \ + mkdir -m 777 /kiwi-backup/source; \ + mkdir -m 777 /kiwi-backup/target; \ + \ + # create a non-root user + adduser -D -u 1368 kiwi-backup; + +USER kiwi-backup + +RUN set -ex; \ \ mkdir -p "${HOME}/.cache/duplicity"; \ - mkdir -pm 600 "${HOME}/.gnupg"; + mkdir -pm 700 "${HOME}/.gnupg"; \ + \ + # confirm duplicity is working + duplicity --version; -VOLUME [ "/home/duplicity/.cache/duplicity" ] +VOLUME [ "/home/kiwi-backup/.cache/duplicity" ] ENV \ ################# @@ -92,7 +96,8 @@ ENV \ SCHEDULE_RMFULL="36 05 * * SAT" \ SCHEDULE_RMINCR="36 05 * * SUN" \ BACKUP_VOLSIZE=1024 \ - BACKUP_TARGET="file:///backup/target" \ + BACKUP_SOURCE="/kiwi-backup/source" \ + BACKUP_TARGET="file:///kiwi-backup/target" \ OPTIONS_ALL="" \ OPTIONS_BACKUP="" \ OPTIONS_CLEANUP="" \ @@ -107,3 +112,5 @@ ENV \ COPY bin /usr/local/bin/ COPY libexec /usr/local/libexec/ + +CMD [ "kiwi-backup" ] diff --git a/libexec/kiwi-backup/build_command b/libexec/kiwi-backup/build_command index b5dc61a..7845d06 100755 --- a/libexec/kiwi-backup/build_command +++ b/libexec/kiwi-backup/build_command @@ -56,7 +56,7 @@ case "${task}" in printf ' --volsize %s' "${BACKUP_VOLSIZE}" printf ' --full-if-older-than %s' "${FULL_BACKUP_FREQUENCY}" printf '%s' "$( add_space "${OPTIONS_BACKUP}" )" - printf ' /backup/source' + printf ' %s' "${BACKUP_SOURCE}" ;; cleanup) From 48d02483fbf4512bc002031e2dcf96f66a70a06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Thu, 3 Mar 2022 00:42:11 +0100 Subject: [PATCH 10/13] kiwi-cron:0.2 base; don't drop privilege level --- Dockerfile | 31 ++++++++++++++----------------- libexec/kiwi-backup/build_command | 2 +- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 088f3ae..e1a8ccd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ -FROM yavook/kiwi-cron:0.1 +FROM yavook/kiwi-cron:0.2 LABEL maintainer="jmm@yavook.de" COPY requirements.txt /tmp/ +# full install of duplicity distribution RUN set -ex; \ \ # duplicity software dependencies @@ -50,7 +51,7 @@ RUN set -ex; \ install duplicity \ ; \ \ - # remove buildtime dependencies + # cleanup python3 -m pip --no-cache-dir \ uninstall -y wheel \ ; \ @@ -58,34 +59,30 @@ RUN set -ex; \ rm -f "/tmp/requirements.txt"; \ rm -rf "${HOME}/.cargo"; +# start of kiwi additions here RUN set -ex; \ \ - # create /kiwi-backup directories tree + # create /kiwi-backup directory structure mkdir -m 777 /kiwi-backup; \ mkdir -m 777 /kiwi-backup/source; \ mkdir -m 777 /kiwi-backup/target; \ \ - # create a non-root user - adduser -D -u 1368 kiwi-backup; - -USER kiwi-backup - -RUN set -ex; \ - \ - mkdir -p "${HOME}/.cache/duplicity"; \ - mkdir -pm 700 "${HOME}/.gnupg"; \ + # we need to run as root in container. + # otherwise, we might miss directories in backup source! + mkdir -p "/root/.cache/duplicity"; \ + mkdir -pm 700 "/root/.gnupg"; \ \ # confirm duplicity is working duplicity --version; -VOLUME [ "/home/kiwi-backup/.cache/duplicity" ] +VOLUME [ "/root/.cache/duplicity" ] ENV \ ################# # BACKUP POLICY # ################# - SCHEDULE_BACKUP="36 02 * * *" \ - SCHEDULE_CLEANUP="36 04 * * *" \ + SCHEDULE_BACKUP="R 02 * * *" \ + SCHEDULE_CLEANUP="R 04 * * *" \ FULL_BACKUP_FREQUENCY=3M \ BACKUP_RETENTION_TIME=6M \ KEEP_NUM_FULL_CHAINS=2 \ @@ -93,8 +90,8 @@ ENV \ ###################### # ADDITIONAL OPTIONS # ###################### - SCHEDULE_RMFULL="36 05 * * SAT" \ - SCHEDULE_RMINCR="36 05 * * SUN" \ + SCHEDULE_RMFULL="R 05 * * SAT" \ + SCHEDULE_RMINCR="R 05 * * SUN" \ BACKUP_VOLSIZE=1024 \ BACKUP_SOURCE="/kiwi-backup/source" \ BACKUP_TARGET="file:///kiwi-backup/target" \ diff --git a/libexec/kiwi-backup/build_command b/libexec/kiwi-backup/build_command index 7845d06..6232c11 100755 --- a/libexec/kiwi-backup/build_command +++ b/libexec/kiwi-backup/build_command @@ -9,7 +9,7 @@ ionice_exe="$(command -v ionice)" duplicity_exe="$(command -v duplicity)" # files -duplicity_secrets_file="${HOME}/duplicity_secrets" +duplicity_secrets_file="/root/duplicity_secrets" ######## # MAIN # From af2d16f64b44284edbbd8c8bddaf2333207042b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Thu, 3 Mar 2022 01:02:38 +0100 Subject: [PATCH 11/13] randomized schedules --- Dockerfile | 8 ++++---- libexec/kiwi-backup/scheduler | 15 +++++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1a8ccd..7e94534 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,8 +81,8 @@ ENV \ ################# # BACKUP POLICY # ################# - SCHEDULE_BACKUP="R 02 * * *" \ - SCHEDULE_CLEANUP="R 04 * * *" \ + SCHEDULE_BACKUP="R 2 * * *" \ + SCHEDULE_CLEANUP="R 4 * * *" \ FULL_BACKUP_FREQUENCY=3M \ BACKUP_RETENTION_TIME=6M \ KEEP_NUM_FULL_CHAINS=2 \ @@ -90,8 +90,8 @@ ENV \ ###################### # ADDITIONAL OPTIONS # ###################### - SCHEDULE_RMFULL="R 05 * * SAT" \ - SCHEDULE_RMINCR="R 05 * * SUN" \ + SCHEDULE_RMFULL="R 5 * * SAT" \ + SCHEDULE_RMINCR="R 5 * * SUN" \ BACKUP_VOLSIZE=1024 \ BACKUP_SOURCE="/kiwi-backup/source" \ BACKUP_TARGET="file:///kiwi-backup/target" \ diff --git a/libexec/kiwi-backup/scheduler b/libexec/kiwi-backup/scheduler index 52f4b42..9226a14 100755 --- a/libexec/kiwi-backup/scheduler +++ b/libexec/kiwi-backup/scheduler @@ -3,7 +3,14 @@ this_script="$( readlink -f "${0}" )" this_dir="${this_script%/*}" -echo "${SCHEDULE_BACKUP}" "$( "${this_dir}/build_command" backup )" -echo "${SCHEDULE_CLEANUP}" "$( "${this_dir}/build_command" cleanup )" -echo "${SCHEDULE_RMFULL}" "$( "${this_dir}/build_command" rmfull )" -echo "${SCHEDULE_RMINCR}" "$( "${this_dir}/build_command" rmincr )" +echo "${SCHEDULE_BACKUP}" "$( "${this_dir}/build_command" backup )" \ + | /usr/local/libexec/kiwi-cron/randomize_schedule + +echo "${SCHEDULE_CLEANUP}" "$( "${this_dir}/build_command" cleanup )" \ + | /usr/local/libexec/kiwi-cron/randomize_schedule + +echo "${SCHEDULE_RMFULL}" "$( "${this_dir}/build_command" rmfull )" \ + | /usr/local/libexec/kiwi-cron/randomize_schedule + +echo "${SCHEDULE_RMINCR}" "$( "${this_dir}/build_command" rmincr )" \ + | /usr/local/libexec/kiwi-cron/randomize_schedule From 1b3dc526f77b179b7b359e5161b0f3502e50469b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Thu, 3 Mar 2022 01:05:33 +0100 Subject: [PATCH 12/13] README --- README.md | 67 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 7a8cf25..a659235 100644 --- a/README.md +++ b/README.md @@ -4,20 +4,20 @@ > `kiwi` - simple, consistent, powerful -The backup solution for [`kiwi-scp`](https://github.com/yavook/kiwi-scp) +The backup solution for [`kiwi-scp`](https://github.com/yavook/kiwi-scp). Also [on Docker Hub](https://hub.docker.com/r/yavook/kiwi-backup). ## Quick start -kiwi-backup is an image with [duplicity](http://duplicity.nongnu.org/), tailored to backup service data of `kiwi-scp` instances. +kiwi-backup is an image with [duplicity](https://duplicity.gitlab.io/duplicity-web/), tailored to backup service data of `kiwi-scp` instances. If you want backups in the host directory `/var/local/kiwi.backup`, just add this to one of your projects' `docker-compose.yml` to use the default configuration. ```yaml backup: - image: yavook/kiwi-backup + image: yavook/kiwi-backup:0.10 volumes: - - "${KIWI_INSTANCE}:/backup/source:ro" - - "/var/local/kiwi.backup:/backup/target" + - "${KIWI_INSTANCE}:/kiwi-backup/source:ro" + - "/var/local/kiwi.backup:/kiwi-backup/target" ``` - backups the entire service data directory @@ -26,7 +26,7 @@ backup: - a new full backup once every 3 months - keeps backups up to 6 months old - keeps daily backups for two recent sets (3-6 months) -- backup jobs run at 02:36 am UTC (time chosen by fair dice roll) +- backup jobs run at a random minute past 2 am Be aware though -- backups will use a fair bit of storage space! @@ -34,21 +34,27 @@ Be aware though -- backups will use a fair bit of storage space! The kiwi-backup image allows for extensive customization even without creating a local image variant. -Schedules in environment variables are to be provided [in cron notation](https://crontab.guru/). +Schedules in environment variables are to be provided [in cron notation](https://crontab.guru/). Additionally, the special value "R" is supported and will be replaced by a random value. + +### Time Zones + +Being based on [`kiwi-cron`](https://github.com/yavook/kiwi-cron), `kiwi-backup` makes changing time zones easy. Just change the container environment variable `TZ` to your liking, e.g. "Europe/Berlin". ### Backup Scope -kiwi-backup will backup everything in its `/backup/source` directory -- change the backup scope by adjusting what's mounted into that container directory. +kiwi-backup will backup everything in its `/kiwi-backup/source` directory -- change the backup scope by adjusting what's mounted into that container directory. ```yaml backup: # ... volumes: # change scope here! - - "${KIWI_INSTANCE}:/backup/source:ro" + - "${KIWI_INSTANCE}:/kiwi-backup/source:ro" ``` -You may of course create additional sources below the `/backup/source` directory to limit the backup to specific projects or services. For added safety, mount your backup sources read-only by appending `:ro`. +You may of course create additional sources below the `/kiwi-backup/source` directory to limit the backup to specific projects or services. For added safety, mount your backup source(s) read-only by appending `:ro`. + +You may also change the container environment variable `BACKUP_SOURCE`, though this is discouraged. ### Backup policy @@ -61,12 +67,12 @@ backup: # ... # when to run backups - # default: daily at 02:36 am UTC - SCHEDULE_BACKUP: "36 02 * * *" + # default: daily at a random minute past 2 am + SCHEDULE_BACKUP: "R 2 * * *" # when to remove leftovers from failed transactions - # default: daily at 04:36 am UTC - SCHEDULE_CLEANUP: "36 04 * * *" + # default: daily at a random minute past 4 am + SCHEDULE_CLEANUP: "R 4 * * *" # how often to opt for a full backup # default: every 3 months @@ -89,14 +95,14 @@ There are three major ways to for inject secrets into `kiwi-backup` environments #### Container environment -Just fire up your container using `docker run -e "FTP_PASSWORD=my_secret_here" yavook/kiwi-backup` +Just fire up your container using `docker run -e "FTP_PASSWORD=my_secret_here" yavook/kiwi-backup:0.10` #### Image environment Create a simple `Dockerfile` from following template. ```Dockerfile -FROM yavook/kiwi-backup +FROM yavook/kiwi-backup:0.10 ENV FTP_PASSWORD="my_secret_here" ``` @@ -123,20 +129,24 @@ backup: # ... # when to remove old full backup chains - # default: every saturday at 05:36 am UTC - SCHEDULE_RMFULL: "36 05 * * SAT" + # default: every Saturday at a random minute past 5 am + SCHEDULE_RMFULL: "R 5 * * SAT" # when to remove old incremental backups - # default: every sunday at 05:36 am UTC - SCHEDULE_RMINCR: "36 05 * * SUN" + # default: every Sunday at a random minute past 5 am + SCHEDULE_RMINCR: "R 5 * * SUN" # size of individual duplicity data volumes # default: 1GiB BACKUP_VOLSIZE: "1024" + # what to base backups on + # default: container directory "/kiwi-backup/source", usually mounted volume(s) + BACKUP_SOURCE: "/kiwi-backup/source" + # where to put backups - # default: some docker volume - BACKUP_TARGET: "file:///backup/target" + # default: container directory "/kiwi-backup/target", usually a mounted volume + BACKUP_TARGET: "file:///kiwi-backup/target" # Additional options for all "duplicity" commands OPTIONS_ALL: "" @@ -175,7 +185,7 @@ Reasonable defaults for a backup encryption key are: To quickly generate a key, use the following command, then enter a passphrase: ```sh -docker run --rm -it -v "gnupg.tmp:/root/.gnupg" yavook/kiwi-backup gpg --quick-gen-key --yes "Administrator " rsa4096 encr never +docker run --rm -it -v "gnupg.tmp:/root/.gnupg" yavook/kiwi-backup:0.10 gpg --quick-gen-key --yes "Administrator " rsa4096 encr never ``` To get a more in-depth generation wizard instead, use `gpg --full-gen-key` command without any more args and follow through. @@ -185,13 +195,13 @@ To get a more in-depth generation wizard instead, use `gpg --full-gen-key` comma This one-liner exports your generated key into a new subdirectory "backup": ```sh -docker run --rm -it -v "gnupg.tmp:/root/.gnupg" -v "$(pwd)/backup:/root/backup" -e "CURRENT_USER=$(id -u):$(id -g)" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --export-secret-keys --armor > secret.asc && gpg --export-ownertrust > ownertrust.txt && chown -R "${CURRENT_USER}" .' +docker run --rm -it -v "gnupg.tmp:/root/.gnupg" -v "$(pwd)/backup:/root/backup" -e "CURRENT_USER=$(id -u):$(id -g)" yavook/kiwi-backup:0.10 sh -c 'cd /root/backup && gpg --export-secret-keys --armor > secret.asc && gpg --export-ownertrust > ownertrust.txt && chown -R "${CURRENT_USER}" .' ``` You'll now find the "backup" subdirectory with files "secret.asc" and "ownertrust.txt" in it. Check your exported files: ```sh -docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --import --batch secret.asc 2>/dev/null && gpg --import-ownertrust ownertrust.txt 2>/dev/null && gpg -k 2>/dev/null | grep -A1 "^pub" | xargs | tail -c17' +docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/backup && gpg --import --batch secret.asc 2>/dev/null && gpg --import-ownertrust ownertrust.txt 2>/dev/null && gpg -k 2>/dev/null | grep -A1 "^pub" | xargs | tail -c17' ``` This should output your 16-digit Key-ID, so take note of it if you haven't already! Afterwards, run `docker volume rm gnupg.tmp` to get rid of the key generation volume. @@ -208,7 +218,7 @@ gpg --export-ownertrust > backup/ownertrust.txt You can still check your exported files :) ```sh -docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --import --batch secret.asc && gpg --import-ownertrust ownertrust.txt && gpg -k' +docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/backup && gpg --import --batch secret.asc && gpg --import-ownertrust ownertrust.txt && gpg -k' ``` ### Describe local kiwi-backup image @@ -216,7 +226,7 @@ docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd Now create a simple `Dockerfile` inside the "backup" directory from following template. ```Dockerfile -FROM yavook/kiwi-backup +FROM yavook/kiwi-backup:0.10 COPY secret.asc ownertrust.txt /root/ @@ -237,7 +247,7 @@ All that's left is to come back to your project's `docker-compose.yml`, where yo ```yaml backup: - image: yavook/kiwi-backup + image: yavook/kiwi-backup:0.10 # ... ``` @@ -253,3 +263,4 @@ That's it -- from now on, all new backups will be encrypted! ## Offsite Backups +TODO From fefad19993ad70e01ce6d1f5f77b3c481cdfedf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Thu, 3 Mar 2022 01:16:58 +0100 Subject: [PATCH 13/13] readme names --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a659235..db8015f 100644 --- a/README.md +++ b/README.md @@ -185,26 +185,26 @@ Reasonable defaults for a backup encryption key are: To quickly generate a key, use the following command, then enter a passphrase: ```sh -docker run --rm -it -v "gnupg.tmp:/root/.gnupg" yavook/kiwi-backup:0.10 gpg --quick-gen-key --yes "Administrator " rsa4096 encr never +docker run --rm -it -v "kiwi-backup.gnupg.tmp:/root/.gnupg" yavook/kiwi-backup:0.10 gpg --quick-gen-key --yes "Administrator " rsa4096 encr never ``` To get a more in-depth generation wizard instead, use `gpg --full-gen-key` command without any more args and follow through. ### Export the generated key -This one-liner exports your generated key into a new subdirectory "backup": +This one-liner exports your generated key into a new subdirectory "kiwi-backup.gnupg": ```sh -docker run --rm -it -v "gnupg.tmp:/root/.gnupg" -v "$(pwd)/backup:/root/backup" -e "CURRENT_USER=$(id -u):$(id -g)" yavook/kiwi-backup:0.10 sh -c 'cd /root/backup && gpg --export-secret-keys --armor > secret.asc && gpg --export-ownertrust > ownertrust.txt && chown -R "${CURRENT_USER}" .' +docker run --rm -it -v "kiwi-backup.gnupg.tmp:/root/.gnupg" -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg" -e "CURRENT_USER=$(id -u):$(id -g)" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --export-secret-keys --armor > secret.asc && gpg --export-ownertrust > ownertrust.txt && chown -R "${CURRENT_USER}" .' ``` -You'll now find the "backup" subdirectory with files "secret.asc" and "ownertrust.txt" in it. Check your exported files: +You'll now find the "kiwi-backup.gnupg" subdirectory with files "secret.asc" and "ownertrust.txt" in it. Check your exported files: ```sh -docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/backup && gpg --import --batch secret.asc 2>/dev/null && gpg --import-ownertrust ownertrust.txt 2>/dev/null && gpg -k 2>/dev/null | grep -A1 "^pub" | xargs | tail -c17' +docker run --rm -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --import --batch secret.asc 2>/dev/null && gpg --import-ownertrust ownertrust.txt 2>/dev/null && gpg -k 2>/dev/null | grep -A1 "^pub" | xargs | tail -c17' ``` -This should output your 16-digit Key-ID, so take note of it if you haven't already! Afterwards, run `docker volume rm gnupg.tmp` to get rid of the key generation volume. +This should output your 16-digit Key-ID, so take note of it if you haven't already! Afterwards, run `docker volume rm kiwi-backup.gnupg.tmp` to get rid of the key generation volume. ### Using a pre-generated key @@ -218,7 +218,7 @@ gpg --export-ownertrust > backup/ownertrust.txt You can still check your exported files :) ```sh -docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/backup && gpg --import --batch secret.asc && gpg --import-ownertrust ownertrust.txt && gpg -k' +docker run --rm -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --import --batch secret.asc && gpg --import-ownertrust ownertrust.txt && gpg -k' ``` ### Describe local kiwi-backup image