mirror of
https://github.com/yavook/kiwi-backup.git
synced 2024-11-22 15:03:01 +00:00
Merge branch 'release/0.10.0'
This commit is contained in:
commit
224f2552fd
8 changed files with 236 additions and 257 deletions
|
@ -1,6 +1,7 @@
|
||||||
---
|
---
|
||||||
kind: pipeline
|
kind: pipeline
|
||||||
name: default
|
name: default
|
||||||
|
type: docker
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: docker
|
- name: docker
|
||||||
|
|
102
Dockerfile
102
Dockerfile
|
@ -1,13 +1,12 @@
|
||||||
FROM python:3.9-alpine3.13
|
FROM yavook/kiwi-cron:0.2
|
||||||
LABEL maintainer="jmm@yavook.de"
|
LABEL maintainer="jmm@yavook.de"
|
||||||
|
|
||||||
# Previous work: https://github.com/wernight/docker-duplicity
|
COPY requirements.txt /tmp/
|
||||||
|
|
||||||
|
# full install of duplicity distribution
|
||||||
RUN set -ex; \
|
RUN set -ex; \
|
||||||
\
|
\
|
||||||
# create backup source
|
# duplicity software dependencies
|
||||||
mkdir -p /backup/source; \
|
|
||||||
\
|
|
||||||
apk --no-cache add \
|
apk --no-cache add \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
gettext \
|
gettext \
|
||||||
|
@ -19,13 +18,15 @@ RUN set -ex; \
|
||||||
libxslt \
|
libxslt \
|
||||||
openssh-client \
|
openssh-client \
|
||||||
openssl \
|
openssl \
|
||||||
|
python3 \
|
||||||
rsync \
|
rsync \
|
||||||
; \
|
; \
|
||||||
update-ca-certificates; \
|
update-ca-certificates; \
|
||||||
\
|
\
|
||||||
# dependencies to build python packages
|
# python packages buildtime dependencies
|
||||||
apk --no-cache add --virtual .build-deps \
|
apk --no-cache add --virtual .build-deps \
|
||||||
gcc \
|
gcc \
|
||||||
|
git \
|
||||||
libffi-dev \
|
libffi-dev \
|
||||||
librsync-dev \
|
librsync-dev \
|
||||||
libxml2-dev \
|
libxml2-dev \
|
||||||
|
@ -33,54 +34,55 @@ RUN set -ex; \
|
||||||
make \
|
make \
|
||||||
musl-dev \
|
musl-dev \
|
||||||
openssl-dev \
|
openssl-dev \
|
||||||
|
python3-dev \
|
||||||
|
py3-pip \
|
||||||
|
cargo \
|
||||||
|
; \
|
||||||
|
# make use of prebuilt wheels where possible
|
||||||
|
python3 -m pip --no-cache-dir \
|
||||||
|
install wheel \
|
||||||
; \
|
; \
|
||||||
\
|
\
|
||||||
# make use of "wheel" python packages
|
# install duplicity
|
||||||
pip3 --no-cache-dir install wheel ; \
|
python3 -m pip --no-cache-dir \
|
||||||
\
|
install -r /tmp/requirements.txt \
|
||||||
pip3 --no-cache-dir install \
|
; \
|
||||||
# main app
|
python3 -m pip --no-cache-dir \
|
||||||
duplicity \
|
install duplicity \
|
||||||
\
|
|
||||||
# general duplicity requirements, based on
|
|
||||||
# http://duplicity.nongnu.org/vers8/README
|
|
||||||
# https://git.launchpad.net/duplicity/tree/requirements.txt
|
|
||||||
fasteners \
|
|
||||||
future \
|
|
||||||
mock \
|
|
||||||
paramiko \
|
|
||||||
python-gettext \
|
|
||||||
requests \
|
|
||||||
urllib3 \
|
|
||||||
\
|
|
||||||
# backend requirements
|
|
||||||
azure-mgmt-storage \
|
|
||||||
b2sdk \
|
|
||||||
boto \
|
|
||||||
boto3 \
|
|
||||||
dropbox \
|
|
||||||
gdata \
|
|
||||||
jottalib \
|
|
||||||
mediafire \
|
|
||||||
mega.py \
|
|
||||||
pydrive \
|
|
||||||
pyrax \
|
|
||||||
python-swiftclient \
|
|
||||||
requests_oauthlib \
|
|
||||||
; \
|
; \
|
||||||
\
|
\
|
||||||
# remove buildtime dependencies
|
# cleanup
|
||||||
pip3 --no-cache-dir uninstall -y wheel; \
|
python3 -m pip --no-cache-dir \
|
||||||
apk del --purge .build-deps
|
uninstall -y wheel \
|
||||||
|
; \
|
||||||
|
apk del --purge .build-deps; \
|
||||||
|
rm -f "/tmp/requirements.txt"; \
|
||||||
|
rm -rf "${HOME}/.cargo";
|
||||||
|
|
||||||
VOLUME ["/root/.cache/duplicity", "/backup/target"]
|
# start of kiwi additions here
|
||||||
|
RUN set -ex; \
|
||||||
|
\
|
||||||
|
# create /kiwi-backup directory structure
|
||||||
|
mkdir -m 777 /kiwi-backup; \
|
||||||
|
mkdir -m 777 /kiwi-backup/source; \
|
||||||
|
mkdir -m 777 /kiwi-backup/target; \
|
||||||
|
\
|
||||||
|
# we need to run as root in container.
|
||||||
|
# otherwise, we might miss directories in backup source!
|
||||||
|
mkdir -p "/root/.cache/duplicity"; \
|
||||||
|
mkdir -pm 700 "/root/.gnupg"; \
|
||||||
|
\
|
||||||
|
# confirm duplicity is working
|
||||||
|
duplicity --version;
|
||||||
|
|
||||||
|
VOLUME [ "/root/.cache/duplicity" ]
|
||||||
|
|
||||||
ENV \
|
ENV \
|
||||||
#################
|
#################
|
||||||
# BACKUP POLICY #
|
# BACKUP POLICY #
|
||||||
#################
|
#################
|
||||||
SCHEDULE_BACKUP="36 02 * * *" \
|
SCHEDULE_BACKUP="R 2 * * *" \
|
||||||
SCHEDULE_CLEANUP="36 04 * * *" \
|
SCHEDULE_CLEANUP="R 4 * * *" \
|
||||||
FULL_BACKUP_FREQUENCY=3M \
|
FULL_BACKUP_FREQUENCY=3M \
|
||||||
BACKUP_RETENTION_TIME=6M \
|
BACKUP_RETENTION_TIME=6M \
|
||||||
KEEP_NUM_FULL_CHAINS=2 \
|
KEEP_NUM_FULL_CHAINS=2 \
|
||||||
|
@ -88,10 +90,11 @@ ENV \
|
||||||
######################
|
######################
|
||||||
# ADDITIONAL OPTIONS #
|
# ADDITIONAL OPTIONS #
|
||||||
######################
|
######################
|
||||||
SCHEDULE_RMFULL="36 05 * * SAT" \
|
SCHEDULE_RMFULL="R 5 * * SAT" \
|
||||||
SCHEDULE_RMINCR="36 05 * * SUN" \
|
SCHEDULE_RMINCR="R 5 * * SUN" \
|
||||||
BACKUP_VOLSIZE=1024 \
|
BACKUP_VOLSIZE=1024 \
|
||||||
BACKUP_TARGET="file:///backup/target" \
|
BACKUP_SOURCE="/kiwi-backup/source" \
|
||||||
|
BACKUP_TARGET="file:///kiwi-backup/target" \
|
||||||
OPTIONS_ALL="" \
|
OPTIONS_ALL="" \
|
||||||
OPTIONS_BACKUP="" \
|
OPTIONS_BACKUP="" \
|
||||||
OPTIONS_CLEANUP="" \
|
OPTIONS_CLEANUP="" \
|
||||||
|
@ -104,6 +107,7 @@ ENV \
|
||||||
GPG_KEY_ID="" \
|
GPG_KEY_ID="" \
|
||||||
GPG_PASSPHRASE=""
|
GPG_PASSPHRASE=""
|
||||||
|
|
||||||
COPY do-plicity /usr/local/bin/
|
COPY bin /usr/local/bin/
|
||||||
|
COPY libexec /usr/local/libexec/
|
||||||
|
|
||||||
CMD ["do-plicity"]
|
CMD [ "kiwi-backup" ]
|
||||||
|
|
73
README.md
73
README.md
|
@ -4,20 +4,20 @@
|
||||||
|
|
||||||
> `kiwi` - simple, consistent, powerful
|
> `kiwi` - simple, consistent, powerful
|
||||||
|
|
||||||
The backup solution for [`kiwi-scp`](https://github.com/yavook/kiwi-scp)
|
The backup solution for [`kiwi-scp`](https://github.com/yavook/kiwi-scp). Also [on Docker Hub](https://hub.docker.com/r/yavook/kiwi-backup).
|
||||||
|
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
kiwi-backup is an image with [duplicity](http://duplicity.nongnu.org/), tailored to backup service data of `kiwi-scp` instances.
|
kiwi-backup is an image with [duplicity](https://duplicity.gitlab.io/duplicity-web/), tailored to backup service data of `kiwi-scp` instances.
|
||||||
|
|
||||||
If you want backups in the host directory `/var/local/kiwi.backup`, just add this to one of your projects' `docker-compose.yml` to use the default configuration.
|
If you want backups in the host directory `/var/local/kiwi.backup`, just add this to one of your projects' `docker-compose.yml` to use the default configuration.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backup:
|
backup:
|
||||||
image: yavook/kiwi-backup
|
image: yavook/kiwi-backup:0.10
|
||||||
volumes:
|
volumes:
|
||||||
- "${KIWI_INSTANCE}:/backup/source:ro"
|
- "${KIWI_INSTANCE}:/kiwi-backup/source:ro"
|
||||||
- "/var/local/kiwi.backup:/backup/target"
|
- "/var/local/kiwi.backup:/kiwi-backup/target"
|
||||||
```
|
```
|
||||||
|
|
||||||
- backups the entire service data directory
|
- backups the entire service data directory
|
||||||
|
@ -26,7 +26,7 @@ backup:
|
||||||
- a new full backup once every 3 months
|
- a new full backup once every 3 months
|
||||||
- keeps backups up to 6 months old
|
- keeps backups up to 6 months old
|
||||||
- keeps daily backups for two recent sets (3-6 months)
|
- keeps daily backups for two recent sets (3-6 months)
|
||||||
- backup jobs run at 02:36 am UTC (time chosen by fair dice roll)
|
- backup jobs run at a random minute past 2 am
|
||||||
|
|
||||||
Be aware though -- backups will use a fair bit of storage space!
|
Be aware though -- backups will use a fair bit of storage space!
|
||||||
|
|
||||||
|
@ -34,21 +34,27 @@ Be aware though -- backups will use a fair bit of storage space!
|
||||||
|
|
||||||
The kiwi-backup image allows for extensive customization even without creating a local image variant.
|
The kiwi-backup image allows for extensive customization even without creating a local image variant.
|
||||||
|
|
||||||
Schedules in environment variables are to be provided [in cron notation](https://crontab.guru/).
|
Schedules in environment variables are to be provided [in cron notation](https://crontab.guru/). Additionally, the special value "R" is supported and will be replaced by a random value.
|
||||||
|
|
||||||
|
### Time Zones
|
||||||
|
|
||||||
|
Being based on [`kiwi-cron`](https://github.com/yavook/kiwi-cron), `kiwi-backup` makes changing time zones easy. Just change the container environment variable `TZ` to your liking, e.g. "Europe/Berlin".
|
||||||
|
|
||||||
### Backup Scope
|
### Backup Scope
|
||||||
|
|
||||||
kiwi-backup will backup everything in its `/backup/source` directory -- change the backup scope by adjusting what's mounted into that container directory.
|
kiwi-backup will backup everything in its `/kiwi-backup/source` directory -- change the backup scope by adjusting what's mounted into that container directory.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backup:
|
backup:
|
||||||
# ...
|
# ...
|
||||||
volumes:
|
volumes:
|
||||||
# change scope here!
|
# change scope here!
|
||||||
- "${KIWI_INSTANCE}:/backup/source:ro"
|
- "${KIWI_INSTANCE}:/kiwi-backup/source:ro"
|
||||||
```
|
```
|
||||||
|
|
||||||
You may of course create additional sources below the `/backup/source` directory to limit the backup to specific projects or services. For added safety, mount your backup sources read-only by appending `:ro`.
|
You may of course create additional sources below the `/kiwi-backup/source` directory to limit the backup to specific projects or services. For added safety, mount your backup source(s) read-only by appending `:ro`.
|
||||||
|
|
||||||
|
You may also change the container environment variable `BACKUP_SOURCE`, though this is discouraged.
|
||||||
|
|
||||||
### Backup policy
|
### Backup policy
|
||||||
|
|
||||||
|
@ -61,12 +67,12 @@ backup:
|
||||||
# ...
|
# ...
|
||||||
|
|
||||||
# when to run backups
|
# when to run backups
|
||||||
# default: daily at 02:36 am UTC
|
# default: daily at a random minute past 2 am
|
||||||
SCHEDULE_BACKUP: "36 02 * * *"
|
SCHEDULE_BACKUP: "R 2 * * *"
|
||||||
|
|
||||||
# when to remove leftovers from failed transactions
|
# when to remove leftovers from failed transactions
|
||||||
# default: daily at 04:36 am UTC
|
# default: daily at a random minute past 4 am
|
||||||
SCHEDULE_CLEANUP: "36 04 * * *"
|
SCHEDULE_CLEANUP: "R 4 * * *"
|
||||||
|
|
||||||
# how often to opt for a full backup
|
# how often to opt for a full backup
|
||||||
# default: every 3 months
|
# default: every 3 months
|
||||||
|
@ -89,14 +95,14 @@ There are three major ways to for inject secrets into `kiwi-backup` environments
|
||||||
|
|
||||||
#### Container environment
|
#### Container environment
|
||||||
|
|
||||||
Just fire up your container using `docker run -e "FTP_PASSWORD=my_secret_here" yavook/kiwi-backup`
|
Just fire up your container using `docker run -e "FTP_PASSWORD=my_secret_here" yavook/kiwi-backup:0.10`
|
||||||
|
|
||||||
#### Image environment
|
#### Image environment
|
||||||
|
|
||||||
Create a simple `Dockerfile` from following template.
|
Create a simple `Dockerfile` from following template.
|
||||||
|
|
||||||
```Dockerfile
|
```Dockerfile
|
||||||
FROM yavook/kiwi-backup
|
FROM yavook/kiwi-backup:0.10
|
||||||
ENV FTP_PASSWORD="my_secret_here"
|
ENV FTP_PASSWORD="my_secret_here"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -123,20 +129,24 @@ backup:
|
||||||
# ...
|
# ...
|
||||||
|
|
||||||
# when to remove old full backup chains
|
# when to remove old full backup chains
|
||||||
# default: every saturday at 05:36 am UTC
|
# default: every Saturday at a random minute past 5 am
|
||||||
SCHEDULE_RMFULL: "36 05 * * SAT"
|
SCHEDULE_RMFULL: "R 5 * * SAT"
|
||||||
|
|
||||||
# when to remove old incremental backups
|
# when to remove old incremental backups
|
||||||
# default: every sunday at 05:36 am UTC
|
# default: every Sunday at a random minute past 5 am
|
||||||
SCHEDULE_RMINCR: "36 05 * * SUN"
|
SCHEDULE_RMINCR: "R 5 * * SUN"
|
||||||
|
|
||||||
# size of individual duplicity data volumes
|
# size of individual duplicity data volumes
|
||||||
# default: 1GiB
|
# default: 1GiB
|
||||||
BACKUP_VOLSIZE: "1024"
|
BACKUP_VOLSIZE: "1024"
|
||||||
|
|
||||||
|
# what to base backups on
|
||||||
|
# default: container directory "/kiwi-backup/source", usually mounted volume(s)
|
||||||
|
BACKUP_SOURCE: "/kiwi-backup/source"
|
||||||
|
|
||||||
# where to put backups
|
# where to put backups
|
||||||
# default: some docker volume
|
# default: container directory "/kiwi-backup/target", usually a mounted volume
|
||||||
BACKUP_TARGET: "file:///backup/target"
|
BACKUP_TARGET: "file:///kiwi-backup/target"
|
||||||
|
|
||||||
# Additional options for all "duplicity" commands
|
# Additional options for all "duplicity" commands
|
||||||
OPTIONS_ALL: ""
|
OPTIONS_ALL: ""
|
||||||
|
@ -175,26 +185,26 @@ Reasonable defaults for a backup encryption key are:
|
||||||
To quickly generate a key, use the following command, then enter a passphrase:
|
To quickly generate a key, use the following command, then enter a passphrase:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker run --rm -it -v "gnupg.tmp:/root/.gnupg" yavook/kiwi-backup gpg --quick-gen-key --yes "Administrator <root@my-hostname.com>" rsa4096 encr never
|
docker run --rm -it -v "kiwi-backup.gnupg.tmp:/root/.gnupg" yavook/kiwi-backup:0.10 gpg --quick-gen-key --yes "Administrator <root@my-hostname.com>" rsa4096 encr never
|
||||||
```
|
```
|
||||||
|
|
||||||
To get a more in-depth generation wizard instead, use `gpg --full-gen-key` command without any more args and follow through.
|
To get a more in-depth generation wizard instead, use `gpg --full-gen-key` command without any more args and follow through.
|
||||||
|
|
||||||
### Export the generated key
|
### Export the generated key
|
||||||
|
|
||||||
This one-liner exports your generated key into a new subdirectory "backup":
|
This one-liner exports your generated key into a new subdirectory "kiwi-backup.gnupg":
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker run --rm -it -v "gnupg.tmp:/root/.gnupg" -v "$(pwd)/backup:/root/backup" -e "CURRENT_USER=$(id -u):$(id -g)" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --export-secret-keys --armor > secret.asc && gpg --export-ownertrust > ownertrust.txt && chown -R "${CURRENT_USER}" .'
|
docker run --rm -it -v "kiwi-backup.gnupg.tmp:/root/.gnupg" -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg" -e "CURRENT_USER=$(id -u):$(id -g)" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --export-secret-keys --armor > secret.asc && gpg --export-ownertrust > ownertrust.txt && chown -R "${CURRENT_USER}" .'
|
||||||
```
|
```
|
||||||
|
|
||||||
You'll now find the "backup" subdirectory with files "secret.asc" and "ownertrust.txt" in it. Check your exported files:
|
You'll now find the "kiwi-backup.gnupg" subdirectory with files "secret.asc" and "ownertrust.txt" in it. Check your exported files:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --import --batch secret.asc 2>/dev/null && gpg --import-ownertrust ownertrust.txt 2>/dev/null && gpg -k 2>/dev/null | grep -A1 "^pub" | xargs | tail -c17'
|
docker run --rm -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --import --batch secret.asc 2>/dev/null && gpg --import-ownertrust ownertrust.txt 2>/dev/null && gpg -k 2>/dev/null | grep -A1 "^pub" | xargs | tail -c17'
|
||||||
```
|
```
|
||||||
|
|
||||||
This should output your 16-digit Key-ID, so take note of it if you haven't already! Afterwards, run `docker volume rm gnupg.tmp` to get rid of the key generation volume.
|
This should output your 16-digit Key-ID, so take note of it if you haven't already! Afterwards, run `docker volume rm kiwi-backup.gnupg.tmp` to get rid of the key generation volume.
|
||||||
|
|
||||||
### Using a pre-generated key
|
### Using a pre-generated key
|
||||||
|
|
||||||
|
@ -208,7 +218,7 @@ gpg --export-ownertrust > backup/ownertrust.txt
|
||||||
You can still check your exported files :)
|
You can still check your exported files :)
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd /root/backup && gpg --import --batch secret.asc && gpg --import-ownertrust ownertrust.txt && gpg -k'
|
docker run --rm -v "$(pwd)/kiwi-backup.gnupg:/root/kiwi-backup.gnupg:ro" yavook/kiwi-backup:0.10 sh -c 'cd /root/kiwi-backup.gnupg && gpg --import --batch secret.asc && gpg --import-ownertrust ownertrust.txt && gpg -k'
|
||||||
```
|
```
|
||||||
|
|
||||||
### Describe local kiwi-backup image
|
### Describe local kiwi-backup image
|
||||||
|
@ -216,7 +226,7 @@ docker run --rm -v "$(pwd)/backup:/root/backup:ro" yavook/kiwi-backup sh -c 'cd
|
||||||
Now create a simple `Dockerfile` inside the "backup" directory from following template.
|
Now create a simple `Dockerfile` inside the "backup" directory from following template.
|
||||||
|
|
||||||
```Dockerfile
|
```Dockerfile
|
||||||
FROM yavook/kiwi-backup
|
FROM yavook/kiwi-backup:0.10
|
||||||
|
|
||||||
COPY secret.asc ownertrust.txt /root/
|
COPY secret.asc ownertrust.txt /root/
|
||||||
|
|
||||||
|
@ -237,7 +247,7 @@ All that's left is to come back to your project's `docker-compose.yml`, where yo
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backup:
|
backup:
|
||||||
image: yavook/kiwi-backup
|
image: yavook/kiwi-backup:0.10
|
||||||
# ...
|
# ...
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -253,3 +263,4 @@ That's it -- from now on, all new backups will be encrypted!
|
||||||
|
|
||||||
## Offsite Backups
|
## Offsite Backups
|
||||||
|
|
||||||
|
TODO
|
||||||
|
|
5
bin/kiwi-backup
Executable file
5
bin/kiwi-backup
Executable file
|
@ -0,0 +1,5 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
exec /usr/local/libexec/kiwi-cron/run_cron \
|
||||||
|
"${@}" \
|
||||||
|
/usr/local/libexec/kiwi-backup/scheduler
|
177
do-plicity
177
do-plicity
|
@ -1,177 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
#############
|
|
||||||
# CONSTANTS #
|
|
||||||
#############
|
|
||||||
|
|
||||||
# commands
|
|
||||||
this_exe="$(command -v "${0}")"
|
|
||||||
ionice_exe="$(command -v ionice)"
|
|
||||||
duplicity_exe="$(command -v duplicity)"
|
|
||||||
|
|
||||||
# files
|
|
||||||
duplicity_secrets_file='/root/duplicity_secrets'
|
|
||||||
|
|
||||||
#############
|
|
||||||
# FUNCTIONS #
|
|
||||||
#############
|
|
||||||
|
|
||||||
append_options() {
|
|
||||||
ao_cmdline="${1}"
|
|
||||||
ao_options="${2}"
|
|
||||||
shift 1
|
|
||||||
|
|
||||||
if [ -n "${ao_cmdline}" ] && [ -n "${ao_options}" ]; then
|
|
||||||
# if both are given, stitch together with a space
|
|
||||||
echo "${ao_cmdline} ${ao_options}"
|
|
||||||
elif [ -n "${ao_options}" ]; then
|
|
||||||
# if only options are given, output them
|
|
||||||
echo "${ao_options}"
|
|
||||||
else
|
|
||||||
# if at most a cmdline is given, output that
|
|
||||||
echo "${ao_cmdline}"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
print_command() {
|
|
||||||
pc_task="${1}"
|
|
||||||
shift 1
|
|
||||||
|
|
||||||
pc_cmdline="${ionice_exe} -c 3 ${duplicity_exe}"
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_ALL}" )"
|
|
||||||
|
|
||||||
case "${pc_task}" in
|
|
||||||
backup)
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "--allow-source-mismatch" )"
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "--volsize ${BACKUP_VOLSIZE}" )"
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "--full-if-older-than ${FULL_BACKUP_FREQUENCY}" )"
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_BACKUP}" )"
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "/backup/source" )"
|
|
||||||
;;
|
|
||||||
|
|
||||||
cleanup)
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "cleanup --force" )"
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_CLEAN}" )"
|
|
||||||
;;
|
|
||||||
|
|
||||||
rmfull)
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "remove-older-than ${BACKUP_RETENTION_TIME} --force" )"
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_RMFULL}" )"
|
|
||||||
;;
|
|
||||||
|
|
||||||
rmincr)
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "remove-all-inc-of-but-n-full ${KEEP_NUM_FULL_CHAINS} --force" )"
|
|
||||||
pc_cmdline="$( append_options "${pc_cmdline}" "${OPTIONS_RMINCR}" )"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
pc_cmdline="${pc_cmdline} ${BACKUP_TARGET}"
|
|
||||||
echo "${pc_cmdline}"
|
|
||||||
}
|
|
||||||
|
|
||||||
print_cron_schedule() {
|
|
||||||
pcs_min="$( echo "${1}" | awk '{print $1}' )"
|
|
||||||
pcs_hour="$( echo "${1}" | awk '{print $2}' )"
|
|
||||||
pcs_day="$( echo "${1}" | awk '{print $3}' )"
|
|
||||||
pcs_month="$( echo "${1}" | awk '{print $4}' )"
|
|
||||||
pcs_weekday="$( echo "${1}" | awk '{print $5}' )"
|
|
||||||
pcs_command="${2}"
|
|
||||||
shift 2
|
|
||||||
|
|
||||||
printf '%-8s%-8s%-8s%-8s%-8s%s\n' "${pcs_min}" "${pcs_hour}" "${pcs_day}" "${pcs_month}" "${pcs_weekday}" "${pcs_command}"
|
|
||||||
}
|
|
||||||
|
|
||||||
print_crontab() {
|
|
||||||
echo '# crontab generated for kiwi-backup'
|
|
||||||
printf '# generation time: '; date
|
|
||||||
echo '#'
|
|
||||||
|
|
||||||
# don't split the '#' from 'min'
|
|
||||||
print_cron_schedule '#_min hour day month weekday' 'command' | tr '_' ' '
|
|
||||||
|
|
||||||
print_cron_schedule "${SCHEDULE_BACKUP}" "${this_exe} backup"
|
|
||||||
print_cron_schedule "${SCHEDULE_CLEANUP}" "${this_exe} cleanup"
|
|
||||||
print_cron_schedule "${SCHEDULE_RMFULL}" "${this_exe} rmfull"
|
|
||||||
print_cron_schedule "${SCHEDULE_RMINCR}" "${this_exe} rmincr"
|
|
||||||
}
|
|
||||||
|
|
||||||
###############
|
|
||||||
# ENVIRONMENT #
|
|
||||||
###############
|
|
||||||
|
|
||||||
# load secrets file
|
|
||||||
if [ -f "${duplicity_secrets_file}" ]; then
|
|
||||||
# shellcheck disable=SC1090
|
|
||||||
. "${duplicity_secrets_file}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# check if uses encryption
|
|
||||||
if [ -n "${GPG_KEY_ID}" ]; then
|
|
||||||
# gpg key given
|
|
||||||
OPTIONS_ALL="$( append_options "${OPTIONS_ALL}" "--encrypt-key=${GPG_KEY_ID}" )"
|
|
||||||
|
|
||||||
# handle more verbose "GPG_PASSPHRASE" env_var
|
|
||||||
PASSPHRASE="${GPG_PASSPHRASE:-${PASSPHRASE}}"
|
|
||||||
export PASSPHRASE
|
|
||||||
unset GPG_PASSPHRASE
|
|
||||||
else
|
|
||||||
# no key given
|
|
||||||
OPTIONS_ALL="$( append_options "${OPTIONS_ALL}" "--no-encryption" )"
|
|
||||||
fi
|
|
||||||
|
|
||||||
########
|
|
||||||
# MAIN #
|
|
||||||
########
|
|
||||||
|
|
||||||
if [ "${#}" -gt 0 ]; then
|
|
||||||
|
|
||||||
# CLI
|
|
||||||
task="${1}"
|
|
||||||
shift 1
|
|
||||||
|
|
||||||
# run task
|
|
||||||
case "${task}" in
|
|
||||||
print-*)
|
|
||||||
task="${task##*-}"
|
|
||||||
|
|
||||||
case "${task}" in
|
|
||||||
# print out the crontab
|
|
||||||
crontab)
|
|
||||||
print_crontab
|
|
||||||
;;
|
|
||||||
|
|
||||||
# print out a task
|
|
||||||
backup|cleanup|rmfull|rmincr)
|
|
||||||
print_command "${task}"
|
|
||||||
;;
|
|
||||||
|
|
||||||
# unknown task
|
|
||||||
*)
|
|
||||||
>&2 echo "Cannot print '${task}'."
|
|
||||||
>&2 echo "Choose from: crontab, backup, cleanup, rmfull, rmincr"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
;;
|
|
||||||
|
|
||||||
# execute single task
|
|
||||||
backup|cleanup|rmfull|rmincr)
|
|
||||||
# shellcheck disable=SC2091
|
|
||||||
$(print_command "${task}")
|
|
||||||
;;
|
|
||||||
|
|
||||||
# unknown task
|
|
||||||
*)
|
|
||||||
>&2 echo "Unknown task '${task}'."
|
|
||||||
>&2 echo "Choose from: backup, cleanup, rmfull, rmincr"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
else
|
|
||||||
|
|
||||||
# default run: replace crontab, then start crond
|
|
||||||
print_crontab | crontab -
|
|
||||||
crond -fl 8
|
|
||||||
fi
|
|
||||||
|
|
||||||
exit 0
|
|
81
libexec/kiwi-backup/build_command
Executable file
81
libexec/kiwi-backup/build_command
Executable file
|
@ -0,0 +1,81 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
#############
|
||||||
|
# CONSTANTS #
|
||||||
|
#############
|
||||||
|
|
||||||
|
# commands
|
||||||
|
ionice_exe="$(command -v ionice)"
|
||||||
|
duplicity_exe="$(command -v duplicity)"
|
||||||
|
|
||||||
|
# files
|
||||||
|
duplicity_secrets_file="/root/duplicity_secrets"
|
||||||
|
|
||||||
|
########
|
||||||
|
# MAIN #
|
||||||
|
########
|
||||||
|
|
||||||
|
# load secrets file
|
||||||
|
if [ -f "${duplicity_secrets_file}" ]; then
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
. "${duplicity_secrets_file}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# check if uses encryption
|
||||||
|
if [ -n "${GPG_KEY_ID}" ]; then
|
||||||
|
# gpg key given
|
||||||
|
options_encryption="--encrypt-key=${GPG_KEY_ID}"
|
||||||
|
|
||||||
|
# handle more verbose "GPG_PASSPHRASE" env_var
|
||||||
|
PASSPHRASE="${GPG_PASSPHRASE:-${PASSPHRASE}}"
|
||||||
|
export PASSPHRASE
|
||||||
|
unset GPG_PASSPHRASE
|
||||||
|
else
|
||||||
|
# no key given
|
||||||
|
options_encryption="--no-encryption"
|
||||||
|
fi
|
||||||
|
|
||||||
|
task="${1}"
|
||||||
|
shift 1
|
||||||
|
|
||||||
|
add_space () {
|
||||||
|
if [ -n "${1}" ]; then
|
||||||
|
echo " ${1}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
printf "%s -c 3 %s%s %s" \
|
||||||
|
"${ionice_exe}" \
|
||||||
|
"${duplicity_exe}" \
|
||||||
|
"$( add_space "${OPTIONS_ALL}" )" \
|
||||||
|
"${options_encryption}"
|
||||||
|
|
||||||
|
case "${task}" in
|
||||||
|
backup)
|
||||||
|
printf ' --allow-source-mismatch'
|
||||||
|
printf ' --volsize %s' "${BACKUP_VOLSIZE}"
|
||||||
|
printf ' --full-if-older-than %s' "${FULL_BACKUP_FREQUENCY}"
|
||||||
|
printf '%s' "$( add_space "${OPTIONS_BACKUP}" )"
|
||||||
|
printf ' %s' "${BACKUP_SOURCE}"
|
||||||
|
;;
|
||||||
|
|
||||||
|
cleanup)
|
||||||
|
printf ' cleanup'
|
||||||
|
printf ' --force'
|
||||||
|
printf '%s' "$( add_space "${OPTIONS_CLEAN}" )"
|
||||||
|
;;
|
||||||
|
|
||||||
|
rmfull)
|
||||||
|
printf ' remove-older-than %s' "${BACKUP_RETENTION_TIME}"
|
||||||
|
printf ' --force'
|
||||||
|
printf '%s' "$( add_space "${OPTIONS_RMFULL}" )"
|
||||||
|
;;
|
||||||
|
|
||||||
|
rmincr)
|
||||||
|
printf ' remove-all-inc-of-but-n-full %s' "${KEEP_NUM_FULL_CHAINS}"
|
||||||
|
printf ' --force'
|
||||||
|
printf '%s' "$( add_space "${OPTIONS_RMINCR}" )"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo " ${BACKUP_TARGET}"
|
16
libexec/kiwi-backup/scheduler
Executable file
16
libexec/kiwi-backup/scheduler
Executable file
|
@ -0,0 +1,16 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
this_script="$( readlink -f "${0}" )"
|
||||||
|
this_dir="${this_script%/*}"
|
||||||
|
|
||||||
|
echo "${SCHEDULE_BACKUP}" "$( "${this_dir}/build_command" backup )" \
|
||||||
|
| /usr/local/libexec/kiwi-cron/randomize_schedule
|
||||||
|
|
||||||
|
echo "${SCHEDULE_CLEANUP}" "$( "${this_dir}/build_command" cleanup )" \
|
||||||
|
| /usr/local/libexec/kiwi-cron/randomize_schedule
|
||||||
|
|
||||||
|
echo "${SCHEDULE_RMFULL}" "$( "${this_dir}/build_command" rmfull )" \
|
||||||
|
| /usr/local/libexec/kiwi-cron/randomize_schedule
|
||||||
|
|
||||||
|
echo "${SCHEDULE_RMINCR}" "$( "${this_dir}/build_command" rmincr )" \
|
||||||
|
| /usr/local/libexec/kiwi-cron/randomize_schedule
|
38
requirements.txt
Normal file
38
requirements.txt
Normal file
|
@ -0,0 +1,38 @@
|
||||||
|
# dependencies as per
|
||||||
|
# https://gitlab.com/duplicity/duplicity/-/blob/rel.0.8.21/requirements.txt
|
||||||
|
|
||||||
|
##### basic requirements #####
|
||||||
|
|
||||||
|
fasteners
|
||||||
|
future
|
||||||
|
python-gettext
|
||||||
|
setuptools>=44.1.1
|
||||||
|
setuptools-scm>=5.0.2
|
||||||
|
|
||||||
|
|
||||||
|
##### backend libraries #####
|
||||||
|
|
||||||
|
azure-storage-blob ; python_version >= '3.6'
|
||||||
|
b2sdk ; python_version >= '3.6'
|
||||||
|
boto
|
||||||
|
boto3
|
||||||
|
boxsdk[jwt] ; python_version >= '3.6'
|
||||||
|
dropbox
|
||||||
|
gdata ; python_version == '2.7'
|
||||||
|
gdata-python3 ; python_version >= '3.6'
|
||||||
|
google-api-python-client
|
||||||
|
google-auth-oauthlib
|
||||||
|
jottalib
|
||||||
|
keyring
|
||||||
|
mediafire
|
||||||
|
megatools ; python_version >= '3.6'
|
||||||
|
paramiko
|
||||||
|
pexpect
|
||||||
|
psutil
|
||||||
|
pydrive ; python_version >= '3.6'
|
||||||
|
pydrive2 ; python_version >= '3.6'
|
||||||
|
pyrax ; python_version >= '3.6'
|
||||||
|
python-swiftclient
|
||||||
|
python-keystoneclient
|
||||||
|
requests
|
||||||
|
requests-oauthlib
|
Loading…
Reference in a new issue