Yavook.de/kiwi-backup
1
0
Fork 0
mirror of https://github.com/yavook/kiwi-backup.git synced 2024-11-22 06:53:00 +00:00
Code Packages Activity

Merge branch 'release/0.10.0'

Browse source
This commit is contained in:
Jörn-Michael Miehe 2022-03-03 01:18:05 +01:00
commit 224f2552fd
8 changed files with 236 additions and 257 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.drone.yml
View file

@ -1,6 +1,7 @@
---
kind: pipeline
name: default
type: docker
steps:
- name: docker

102
Dockerfile
View file

@ -1,13 +1,12 @@
FROM python:3.9-alpine3.13
FROM yavook/kiwi-cron:0.2
LABEL maintainer="jmm@yavook.de"
# Previous work: https://github.com/wernight/docker-duplicity
COPY requirements.txt /tmp/
# full install of duplicity distribution
RUN set -ex; \
\
# create backup source
mkdir -p /backup/source; \
\
# duplicity software dependencies
apk --no-cache add \
ca-certificates \
gettext \
@ -19,13 +18,15 @@ RUN set -ex; \
libxslt \
openssh-client \
openssl \
python3 \
rsync \
; \
update-ca-certificates; \
\
# dependencies to build python packages
# python packages buildtime dependencies
apk --no-cache add --virtual .build-deps \
gcc \
git \
libffi-dev \
librsync-dev \
libxml2-dev \
@ -33,54 +34,55 @@ RUN set -ex; \
make \
musl-dev \
openssl-dev \
python3-dev \
py3-pip \
cargo \
; \
# make use of prebuilt wheels where possible
python3 -m pip --no-cache-dir \
install wheel \
; \
\
# make use of "wheel" python packages
pip3 --no-cache-dir install wheel ; \
\
pip3 --no-cache-dir install \
# main app
duplicity \
\
# general duplicity requirements, based on
# http://duplicity.nongnu.org/vers8/README
# https://git.launchpad.net/duplicity/tree/requirements.txt
fasteners \
future \
mock \
paramiko \
python-gettext \
requests \
urllib3 \
\
# backend requirements
azure-mgmt-storage \
b2sdk \
boto \
boto3 \
dropbox \
gdata \
jottalib \
mediafire \
mega.py \
pydrive \
pyrax \
python-swiftclient \
requests_oauthlib \
# install duplicity
python3 -m pip --no-cache-dir \
install -r /tmp/requirements.txt \
; \
python3 -m pip --no-cache-dir \
install duplicity \
; \
\
# remove buildtime dependencies
pip3 --no-cache-dir uninstall -y wheel; \
apk del --purge .build-deps
# cleanup
python3 -m pip --no-cache-dir \
uninstall -y wheel \
; \
apk del --purge .build-deps; \
rm -f "/tmp/requirements.txt"; \
rm -rf "${HOME}/.cargo";
VOLUME ["/root/.cache/duplicity", "/backup/target"]
# start of kiwi additions here
RUN set -ex; \
\
# create /kiwi-backup directory structure
mkdir -m 777 /kiwi-backup; \
mkdir -m 777 /kiwi-backup/source; \
mkdir -m 777 /kiwi-backup/target; \
\
# we need to run as root in container.
# otherwise, we might miss directories in backup source!
mkdir -p "/root/.cache/duplicity"; \
mkdir -pm 700 "/root/.gnupg"; \
\
# confirm duplicity is working
duplicity --version;
VOLUME [ "/root/.cache/duplicity" ]
ENV \
#################
# BACKUP POLICY #
#################
SCHEDULE_BACKUP="36 02 * * *" \
SCHEDULE_CLEANUP="36 04 * * *" \
SCHEDULE_BACKUP="R 2 * * *" \
SCHEDULE_CLEANUP="R 4 * * *" \
FULL_BACKUP_FREQUENCY=3M \
BACKUP_RETENTION_TIME=6M \
KEEP_NUM_FULL_CHAINS=2 \
@ -88,10 +90,11 @@ ENV \
######################
# ADDITIONAL OPTIONS #
######################
SCHEDULE_RMFULL="36 05 * * SAT" \
SCHEDULE_RMINCR="36 05 * * SUN" \
SCHEDULE_RMFULL="R 5 * * SAT" \
SCHEDULE_RMINCR="R 5 * * SUN" \
BACKUP_VOLSIZE=1024 \
BACKUP_TARGET="file:///backup/target" \
BACKUP_SOURCE="/kiwi-backup/source" \
BACKUP_TARGET="file:///kiwi-backup/target" \
OPTIONS_ALL="" \
OPTIONS_BACKUP="" \
OPTIONS_CLEANUP="" \
@ -104,6 +107,7 @@ ENV \
GPG_KEY_ID="" \
GPG_PASSPHRASE=""
COPY do-plicity /usr/local/bin/
COPY bin /usr/local/bin/
COPY libexec /usr/local/libexec/
CMD ["do-plicity"]
CMD [ "kiwi-backup" ]

73
README.md
View file

@ -4,20 +4,20 @@
> `kiwi` - simple, consistent, powerful
The backup solution for [`kiwi-scp`](https://github.com/yavook/kiwi-scp)
The backup solution for [`kiwi-scp`](https://github.com/yavook/kiwi-scp). Also [on Docker Hub](https://hub.docker.com/r/yavook/kiwi-backup).
## Quick start
kiwi-backup is an image with [duplicity](http://duplicity.nongnu.org/), tailored to backup service data of `kiwi-scp` instances.
kiwi-backup is an image with [duplicity](https://duplicity.gitlab.io/duplicity-web/), tailored to backup service data of `kiwi-scp` instances.
If you want backups in the host directory `/var/local/kiwi.backup`, just add this to one of your projects' `docker-compose.yml` to use the default configuration.
```yaml
backup:
image: yavook/kiwi-backup
image: yavook/kiwi-backup:0.10
volumes:
- "${KIWI_INSTANCE}:/backup/source:ro"
- "/var/local/kiwi.backup:/backup/target"
- "${KIWI_INSTANCE}:/kiwi-backup/source:ro"
- "/var/local/kiwi.backup:/kiwi-backup/target"
```
- backups the entire service data directory
@ -26,7 +26,7 @@ backup:
- a new full backup once every 3 months
- keeps backups up to 6 months old
- keeps daily backups for two recent sets (3-6 months)
- backup jobs run at 02:36 am UTC (time chosen by fair dice roll)
- backup jobs run at a random minute past 2 am
Be aware though -- backups will use a fair bit of storage space!
@ -34,21 +34,27 @@ Be aware though -- backups will use a fair bit of storage space!
The kiwi-backup image allows for extensive customization even without creating a local image variant.
Schedules in environment variables are to be provided [in cron notation](https://crontab.guru/).
Schedules in environment variables are to be provided [in cron notation](https://crontab.guru/). Additionally, the special value "R" is supported and will be replaced by a random value.
### Time Zones
Being based on [`kiwi-cron`](https://github.com/yavook/kiwi-cron), `kiwi-backup` makes changing time zones easy. Just change the container environment variable `TZ` to your liking, e.g. "Europe/Berlin".
### Backup Scope
kiwi-backup will backup everything in its `/backup/source` directory -- change the backup scope by adjusting what's mounted into that container directory.
kiwi-backup will backup everything in its `/kiwi-backup/source` directory -- change the backup scope by adjusting what's mounted into that container directory.
```yaml
backup:
# ...
volumes:
# change scope here!
- "${KIWI_INSTANCE}:/backup/source:ro"
- "${KIWI_INSTANCE}:/kiwi-backup/source:ro"
```
You may of course create additional sources below the `/backup/source` directory to limit the backup to specific projects or services. For added safety, mount your backup sources read-only by appending `:ro`.
You may of course create additional sources below the `/kiwi-backup/source` directory to limit the backup to specific projects or services. For added safety, mount your backup source(s) read-only by appending `:ro`.
You may also change the container environment variable `BACKUP_SOURCE`, though this is discouraged.
### Backup policy
@ -61,12 +67,12 @@ backup:
# ...
# when to run backups
# default: daily at 02:36 am UTC
SCHEDULE_BACKUP: "36 02 * * *"
# default: daily at a random minute past 2 am
SCHEDULE_BACKUP: "R 2 * * *"
# when to remove leftovers from failed transactions
# default: daily at 04:36 am UTC
SCHEDULE_CLEANUP: "36 04 * * *"
# default: daily at a random minute past 4 am
SCHEDULE_CLEANUP: "R 4 * * *"
# how often to opt for a full backup
# default: every 3 months
@ -89,14 +95,14 @@ There are three major ways to for inject secrets into `kiwi-backup` environments
#### Container environment
Just fire up your container using `docker run -e "FTP_PASSWORD=my_secret_here" yavook/kiwi-backup`
Just fire up your container using `docker run -e "FTP_PASSWORD=my_secret_here" yavook/kiwi-backup:0.10`
#### Image environment
Create a simple `Dockerfile` from following template.
```Dockerfile
FROM yavook/kiwi-backup
FROM yavook/kiwi-backup:0.10
ENV FTP_PASSWORD="my_secret_here"
```
@ -123,20 +129,24 @@ backup:
# ...
# when to remove old full backup chains
# default: every saturday at 05:36 am UTC
SCHEDULE_RMFULL: "36 05 * * SAT"
# default: every Saturday at a random minute past 5 am
SCHEDULE_RMFULL: "R 5 * * SAT"
# when to remove old incremental backups
# default: every sunday at 05:36 am UTC
SCHEDULE_RMINCR: "36 05 * * SUN"
# default: every Sunday at a random minute past 5 am
SCHEDULE_RMINCR: "R 5 * * SUN"
# size of individual duplicity data volumes
# default: 1GiB
BACKUP_VOLSIZE: "1024"
# what to base backups on
# default: container directory "/kiwi-backup/source", usually mounted volume(s)
BACKUP_SOURCE: "/kiwi-backup/source"
# where to put backups
# default: some docker volume
BACKUP_TARGET: "file:///backup/target"
# default: container directory "/kiwi-backup/target", usually a mounted volume
BACKUP_TARGET: "file:///kiwi-backup/target"
# Additional options for all "duplicity" commands
OPTIONS_ALL: ""
@ -175,26 +185,26 @@ Reasonable defaults for a backup encryption key are:
To quickly generate a key, use the following command, then enter a passphrase:
```sh
docker run --rm -it -v "gnupg.tmp:/root/.gnupg" yavook/kiwi-backup gpg --quick-gen-key --yes "Administrator <root@my-hostname.com>" rsa4096 encr never
docker run --rm -it -v "kiwi-backup.gnupg.tmp:/root/.gnupg" yavook/kiwi-backup:0.10 gpg --quick-gen-key --yes "Administrator <root@my-hostname.com>" rsa4096 encr never
```
To get a more in-depth generation wizard instead, use `gpg --full-gen-key` command without any more args and follow through.
### Export the generated key
This one-liner exports your generated key into a new subdirectory "backup":
This one-liner exports your generated key into a new subdirectory "kiwi-backup.gnupg":
```sh
docker run --rm -it -v "gnupg.tmp:/root/.gnupg" -v "$(pwd)/backup:/root/backup" -e "CURRENT_USER=$(id -u):$(id -g)" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --export-secret-keys --armor > secret.asc && gpg --export-ownertrust > ownertrust.txt && chown -R "${CURRENT_USER}" .'
docker run --rm -it -v "kiwi-backup.gnupg.tmp:/root/.gnupg" -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg" -e "CURRENT_USER=$(id -u):$(id -g)" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --export-secret-keys --armor > secret.asc && gpg --export-ownertrust > ownertrust.txt && chown -R "${CURRENT_USER}" .'
```
You'll now find the "backup" subdirectory with files "secret.asc" and "ownertrust.txt" in it. Check your exported files:
You'll now find the "kiwi-backup.gnupg" subdirectory with files "secret.asc" and "ownertrust.txt" in it. Check your exported files:
```sh
docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --import --batch secret.asc 2>/dev/null && gpg --import-ownertrust ownertrust.txt 2>/dev/null && gpg -k 2>/dev/null | grep -A1 "^pub" | xargs | tail -c17'
docker run --rm -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --import --batch secret.asc 2>/dev/null && gpg --import-ownertrust ownertrust.txt 2>/dev/null && gpg -k 2>/dev/null | grep -A1 "^pub" | xargs | tail -c17'
```
This should output your 16-digit Key-ID, so take note of it if you haven't already! Afterwards, run `docker volume rm gnupg.tmp` to get rid of the key generation volume.
This should output your 16-digit Key-ID, so take note of it if you haven't already! Afterwards, run `docker volume rm kiwi-backup.gnupg.tmp` to get rid of the key generation volume.
### Using a pre-generated key
@ -208,7 +218,7 @@ gpg --export-ownertrust > backup/ownertrust.txt
You can still check your exported files :)
```sh
docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --import --batch secret.asc && gpg --import-ownertrust ownertrust.txt && gpg -k'
docker run --rm -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --import --batch secret.asc && gpg --import-ownertrust ownertrust.txt && gpg -k'
```
### Describe local kiwi-backup image
@ -216,7 +226,7 @@ docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd
Now create a simple `Dockerfile` inside the "backup" directory from following template.
```Dockerfile
FROM yavook/kiwi-backup
FROM yavook/kiwi-backup:0.10
COPY secret.asc ownertrust.txt /root/
@ -237,7 +247,7 @@ All that's left is to come back to your project's `docker-compose.yml`, where yo
```yaml
backup:
image: yavook/kiwi-backup
image: yavook/kiwi-backup:0.10
# ...
```
@ -253,3 +263,4 @@ That's it -- from now on, all new backups will be encrypted!
## Offsite Backups
TODO

5
bin/kiwi-backup Executable file
View file

@ -0,0 +1,5 @@
#!/bin/sh
exec /usr/local/libexec/kiwi-cron/run_cron \
"${@}" \
/usr/local/libexec/kiwi-backup/scheduler

177
do-plicity
View file

@ -1,177 +0,0 @@
#!/bin/sh
#############
# CONSTANTS #
#############
# commands
this_exe="$(command -v "${0}")"
ionice_exe="$(command -v ionice)"
duplicity_exe="$(command -v duplicity)"
# files
duplicity_secrets_file='/root/duplicity_secrets'
#############
# FUNCTIONS #
#############
append_options() {
ao_cmdline="${1}"
ao_options="${2}"
shift 1
if [ -n "${ao_cmdline}" ] && [ -n "${ao_options}" ]; then
# if both are given, stitch together with a space
echo "${ao_cmdline} ${ao_options}"
elif [ -n "${ao_options}" ]; then
# if only options are given, output them
echo "${ao_options}"
else
# if at most a cmdline is given, output that
echo "${ao_cmdline}"
fi
}
print_command() {
pc_task="${1}"
shift 1
pc_cmdline="${ionice_exe} -c 3 ${duplicity_exe}"
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_ALL}" )"
case "${pc_task}" in
backup)
pc_cmdline="$( append_options "${pc_cmdline}" "--allow-source-mismatch" )"
pc_cmdline="$( append_options "${pc_cmdline}" "--volsize ${BACKUP_VOLSIZE}" )"
pc_cmdline="$( append_options "${pc_cmdline}" "--full-if-older-than ${FULL_BACKUP_FREQUENCY}" )"
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_BACKUP}" )"
pc_cmdline="$( append_options "${pc_cmdline}" "/backup/source" )"
;;
cleanup)
pc_cmdline="$( append_options "${pc_cmdline}" "cleanup --force" )"
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_CLEAN}" )"
;;
rmfull)
pc_cmdline="$( append_options "${pc_cmdline}" "remove-older-than ${BACKUP_RETENTION_TIME} --force" )"
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_RMFULL}" )"
;;
rmincr)
pc_cmdline="$( append_options "${pc_cmdline}" "remove-all-inc-of-but-n-full ${KEEP_NUM_FULL_CHAINS} --force" )"
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_RMINCR}" )"
;;
esac
pc_cmdline="${pc_cmdline} ${BACKUP_TARGET}"
echo "${pc_cmdline}"
}
print_cron_schedule() {
pcs_min="$( echo "${1}" | awk '{print $1}' )"
pcs_hour="$( echo "${1}" | awk '{print $2}' )"
pcs_day="$( echo "${1}" | awk '{print $3}' )"
pcs_month="$( echo "${1}" | awk '{print $4}' )"
pcs_weekday="$( echo "${1}" | awk '{print $5}' )"
pcs_command="${2}"
shift 2
printf '%-8s%-8s%-8s%-8s%-8s%s\n' "${pcs_min}" "${pcs_hour}" "${pcs_day}" "${pcs_month}" "${pcs_weekday}" "${pcs_command}"
}
print_crontab() {
echo '# crontab generated for kiwi-backup'
printf '# generation time: '; date
echo '#'
# don't split the '#' from 'min'
print_cron_schedule '#_min hour day month weekday' 'command' | tr '_' ' '
print_cron_schedule "${SCHEDULE_BACKUP}" "${this_exe} backup"
print_cron_schedule "${SCHEDULE_CLEANUP}" "${this_exe} cleanup"
print_cron_schedule "${SCHEDULE_RMFULL}" "${this_exe} rmfull"
print_cron_schedule "${SCHEDULE_RMINCR}" "${this_exe} rmincr"
}
###############
# ENVIRONMENT #
###############
# load secrets file
if [ -f "${duplicity_secrets_file}" ]; then
# shellcheck disable=SC1090
. "${duplicity_secrets_file}"
fi
# check if uses encryption
if [ -n "${GPG_KEY_ID}" ]; then
# gpg key given
OPTIONS_ALL="$( append_options "${OPTIONS_ALL}" "--encrypt-key=${GPG_KEY_ID}" )"
# handle more verbose "GPG_PASSPHRASE" env_var
PASSPHRASE="${GPG_PASSPHRASE:-${PASSPHRASE}}"
export PASSPHRASE
unset GPG_PASSPHRASE
else
# no key given
OPTIONS_ALL="$( append_options "${OPTIONS_ALL}" "--no-encryption" )"
fi
########
# MAIN #
########
if [ "${#}" -gt 0 ]; then
# CLI
task="${1}"
shift 1
# run task
case "${task}" in
print-*)
task="${task##*-}"
case "${task}" in
# print out the crontab
crontab)
print_crontab
;;
# print out a task
backup|cleanup|rmfull|rmincr)
print_command "${task}"
;;
# unknown task
*)
>&2 echo "Cannot print '${task}'."
>&2 echo "Choose from: crontab, backup, cleanup, rmfull, rmincr"
exit 1
;;
esac
;;
# execute single task
backup|cleanup|rmfull|rmincr)
# shellcheck disable=SC2091
$(print_command "${task}")
;;
# unknown task
*)
>&2 echo "Unknown task '${task}'."
>&2 echo "Choose from: backup, cleanup, rmfull, rmincr"
exit 1
;;
esac
else
# default run: replace crontab, then start crond
print_crontab | crontab -
crond -fl 8
fi
exit 0

81
libexec/kiwi-backup/build_command Executable file
View file

@ -0,0 +1,81 @@
#!/bin/sh
#############
# CONSTANTS #
#############
# commands
ionice_exe="$(command -v ionice)"
duplicity_exe="$(command -v duplicity)"
# files
duplicity_secrets_file="/root/duplicity_secrets"
########
# MAIN #
########
# load secrets file
if [ -f "${duplicity_secrets_file}" ]; then
# shellcheck disable=SC1090
. "${duplicity_secrets_file}"
fi
# check if uses encryption
if [ -n "${GPG_KEY_ID}" ]; then
# gpg key given
options_encryption="--encrypt-key=${GPG_KEY_ID}"
# handle more verbose "GPG_PASSPHRASE" env_var
PASSPHRASE="${GPG_PASSPHRASE:-${PASSPHRASE}}"
export PASSPHRASE
unset GPG_PASSPHRASE
else
# no key given
options_encryption="--no-encryption"
fi
task="${1}"
shift 1
add_space () {
if [ -n "${1}" ]; then
echo " ${1}"
fi
}
printf "%s -c 3 %s%s %s" \
"${ionice_exe}" \
"${duplicity_exe}" \
"$( add_space "${OPTIONS_ALL}" )" \
"${options_encryption}"
case "${task}" in
backup)
printf ' --allow-source-mismatch'
printf ' --volsize %s' "${BACKUP_VOLSIZE}"
printf ' --full-if-older-than %s' "${FULL_BACKUP_FREQUENCY}"
printf '%s' "$( add_space "${OPTIONS_BACKUP}" )"
printf ' %s' "${BACKUP_SOURCE}"
;;
cleanup)
printf ' cleanup'
printf ' --force'
printf '%s' "$( add_space "${OPTIONS_CLEAN}" )"
;;
rmfull)
printf ' remove-older-than %s' "${BACKUP_RETENTION_TIME}"
printf ' --force'
printf '%s' "$( add_space "${OPTIONS_RMFULL}" )"
;;
rmincr)
printf ' remove-all-inc-of-but-n-full %s' "${KEEP_NUM_FULL_CHAINS}"
printf ' --force'
printf '%s' "$( add_space "${OPTIONS_RMINCR}" )"
;;
esac
echo " ${BACKUP_TARGET}"

16
libexec/kiwi-backup/scheduler Executable file
View file

@ -0,0 +1,16 @@
#!/bin/sh
this_script="$( readlink -f "${0}" )"
this_dir="${this_script%/*}"
echo "${SCHEDULE_BACKUP}" "$( "${this_dir}/build_command" backup )" \
| /usr/local/libexec/kiwi-cron/randomize_schedule
echo "${SCHEDULE_CLEANUP}" "$( "${this_dir}/build_command" cleanup )" \
| /usr/local/libexec/kiwi-cron/randomize_schedule
echo "${SCHEDULE_RMFULL}" "$( "${this_dir}/build_command" rmfull )" \
| /usr/local/libexec/kiwi-cron/randomize_schedule
echo "${SCHEDULE_RMINCR}" "$( "${this_dir}/build_command" rmincr )" \
| /usr/local/libexec/kiwi-cron/randomize_schedule

38
requirements.txt Normal file
View file

@ -0,0 +1,38 @@
# dependencies as per
# https://gitlab.com/duplicity/duplicity/-/blob/rel.0.8.21/requirements.txt
##### basic requirements #####
fasteners
future
python-gettext
setuptools>=44.1.1
setuptools-scm>=5.0.2
##### backend libraries #####
azure-storage-blob ; python_version >= '3.6'
b2sdk ; python_version >= '3.6'
boto
boto3
boxsdk[jwt] ; python_version >= '3.6'
dropbox
gdata ; python_version == '2.7'
gdata-python3 ; python_version >= '3.6'
google-api-python-client
google-auth-oauthlib
jottalib
keyring
mediafire
megatools ; python_version >= '3.6'
paramiko
pexpect
psutil
pydrive ; python_version >= '3.6'
pydrive2 ; python_version >= '3.6'
pyrax ; python_version >= '3.6'
python-swiftclient
python-keystoneclient
requests
requests-oauthlib