From 641e3c6551630673c812d15398d58f0e3ebb94ea Mon Sep 17 00:00:00 2001 From: andryyy Date: Mon, 18 Oct 2021 16:19:49 +0200 Subject: [PATCH] Add cold standby docs --- docs/b_n_r-accidental_deletion.md | 40 +++++++++++++ docs/b_n_r-backup.md | 97 +++++++++++++++++++++++++++++++ docs/b_n_r-coldstandby.md | 71 ++++++++++++++++++++++ docs/b_n_r-restore.md | 14 +++++ mkdocs.yml | 13 +++-- 5 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 docs/b_n_r-accidental_deletion.md create mode 100644 docs/b_n_r-backup.md create mode 100644 docs/b_n_r-coldstandby.md create mode 100644 docs/b_n_r-restore.md diff --git a/docs/b_n_r-accidental_deletion.md b/docs/b_n_r-accidental_deletion.md new file mode 100644 index 000000000..8f5597fd0 --- /dev/null +++ b/docs/b_n_r-accidental_deletion.md @@ -0,0 +1,40 @@ +So you deleted a mailbox and have no backups, he? + +If you noticed your mistake within a few hours, you can probably recover the users data. + +### SOGo + +We automatically create daily backups (24h interval starting from running up -d) in `/var/lib/docker/volumes/mailcowdockerized_sogo-userdata-backup-vol-1/_data/`. + +**Make sure the user you want to restore exists in your mailcow**. Re-create them if they are missing. + +Copy the file named after the user you want to restore to `__MAILCOW_DIRECTORY__/data/conf/sogo`. + +1\. Copy the backup: `cp /var/lib/docker/volumes/mailcowdockerized_sogo-userdata-backup-vol-1/_data/restoreme@example.org __MAILCOW_DIRECTORY__/data/conf/sogo` + +2\. Run `docker-compose exec -u sogo sogo-mailcow sogo-tool restore -F ALL /etc/sogo restoreme@example.org` + +Run `sogo-tool` without parameters to check for possible restore options. + +3\. Delete the copied backup by running `rm __MAILCOW_DIRECTORY__/data/conf/sogo` + +4\. Restart SOGo and Memcached: `docker-compose restart sogo-mailcow memcached-mailcow` + +### Mail + +In case of an accidental deletion of a mailbox, you will be able to recover for (by default) 5 days. This depends on the `MAILDIR_GC_TIME` parameter in `mailcow.conf`. + +A deleted mailbox is copied in its encrypted form to `/var/lib/docker/volumes/mailcowdockerized_vmail-vol-1/_data/_garbage`. + +The folder inside `_garbage` follows the structure `[timestamp]_[domain_sanitized][user_sanitized]`, for example `1629109708_exampleorgtest` in case of test@example.org deleted on 1629109708. + +To restore make sure you are actually restoring to the same mailcow it was deleted from or you use the same encryption keys in `crypt-vol-1`. + +**Make sure the user you want to restore exists in your mailcow**. Re-create them if they are missing. + +Copy the folders from `/var/lib/docker/volumes/mailcowdockerized_vmail-vol-1/_data/_garbage/[timestamp]_[domain_sanitized][user_sanitized]` back to `/var/lib/docker/volumes/mailcowdockerized_vmail-vol-1/_data/[domain]/[user]` and resync the folder and recalc the quota: + +``` +docker-compose exec dovecot-mailcow doveadm force-resync -u restoreme@example.net '*' +docker-compose exec dovecot-mailcow doveadm quota recalc -u restoreme@example.net +``` diff --git a/docs/b_n_r-backup.md b/docs/b_n_r-backup.md new file mode 100644 index 000000000..60e6c301e --- /dev/null +++ b/docs/b_n_r-backup.md @@ -0,0 +1,97 @@ +### Backup + +#### Manual + +You can use the provided script `helper-scripts/backup_and_restore.sh` to backup mailcow automatically. + +Please do not copy this script to another location. + +To run a backup, write "backup" as first parameter and either one or more components to backup as following parameters. +You can also use "all" as second parameter to backup all components. Append `--delete-days n` to delete backups older than n days. + +``` +# Syntax: +# ./helper-scripts/backup_and_restore.sh backup (vmail|crypt|redis|rspamd|postfix|mysql|all|--delete-days) + +# Backup all, delete backups older than 3 days +./helper-scripts/backup_and_restore.sh backup all --delete-days 3 + +# Backup vmail, crypt and mysql data, delete backups older than 30 days +./helper-scripts/backup_and_restore.sh backup vmail crypt mysql --delete-days 30 + +# Backup vmail +./helper-scripts/backup_and_restore.sh backup vmail + +``` + +The script will ask you for a backup location. Inside of this location it will create folders in the format "mailcow_DATE". +You should not rename those folders to not break the restore process. + +To run a backup unattended, define MAILCOW_BACKUP_LOCATION as environment variable before starting the script: + +``` +MAILCOW_BACKUP_LOCATION=/opt/backup /opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh backup all +``` + +#### Cronjob + +You can run the backup script regularly via cronjob. Make sure `BACKUP_LOCATION` exists: + +``` +5 4 * * * cd /opt/mailcow-dockerized/; MAILCOW_BACKUP_LOCATION=/mnt/mailcow_backups /opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh backup mysql crypt redis --delete-days 3 +``` + +Per default cron sends the full result of each backup operation by email. If you want cron to only mail on error (non-zero exit code) you may want to use the following snippet. Pathes need to be modified according to your setup (this script is a user contribution). + +This following script may be placed in `/etc/cron.daily/mailcow-backup` - do not forget to mark it as executable via `chmod +x`: + +``` +#!/bin/sh + +# Backup mailcow data +# https://mailcow.github.io/mailcow-dockerized-docs/b_n_r_backup/ + +set -e + +OUT="$(mktemp)" +export MAILCOW_BACKUP_LOCATION="/opt/backup" +SCRIPT="/opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh" +PARAMETERS="backup all" +OPTIONS="--delete-days 30" + +# run command +set +e +"${SCRIPT}" ${PARAMETERS} ${OPTIONS} 2>&1 > "$OUT" +RESULT=$? + +if [ $RESULT -ne 0 ] + then + echo "${SCRIPT} ${PARAMETERS} ${OPTIONS} encounters an error:" + echo "RESULT=$RESULT" + echo "STDOUT / STDERR:" + cat "$OUT" +fi +``` + +# Backup strategy with rsync and mailcow backup script + +Create the destination directory for mailcows helper script: +``` +mkdir -p /external_share/backups/backup_script +``` + +Create cronjobs: +``` +25 1 * * * rsync -aH --delete /opt/mailcow-dockerized /external_share/backups/mailcow-dockerized +40 2 * * * rsync -aH --delete /var/lib/docker/volumes /external_share/backups/var_lib_docker_volumes +5 4 * * * cd /opt/mailcow-dockerized/; BACKUP_LOCATION=/external_share/backups/backup_script /opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh backup mysql crypt redis --delete-days 3 +# If you want to, use the acl util to backup permissions of some/all folders/files: getfacl -Rn /path +``` + +On the destination (in this case `/external_share/backups`) you may want to have snapshot capabilities (ZFS, Btrfs etc.). Snapshot daily and keep for n days for a consistent backup. +Do **not** rsync to a Samba share, you need to keep the correct permissions! + +To restore you'd simply need to run rsync the other way round and restart Docker to re-read the volumes. Run `docker-compose pull` and `docker-compose up -d`. + +If you are lucky Redis and MariaDB can automatically fix the inconsistent databases (if they _are_ inconsistent). +In case of a corrupted database you'd need to use the helper script to restore the inconsistent elements. If a restore fails, try to extract the backups and copy the files back manually. Keep the file permissions! diff --git a/docs/b_n_r-coldstandby.md b/docs/b_n_r-coldstandby.md new file mode 100644 index 000000000..2baba6e6d --- /dev/null +++ b/docs/b_n_r-coldstandby.md @@ -0,0 +1,71 @@ +# Cold-standby backup + +mailcow offers an easy way to create a consistent copy of itself to be rsync'ed to a remote location without downtime. + +This may also be used to transfer your mailcow to a new server. + +## You should know + +The provided script will work on default installations. + +It may break when you use unsupported volume overrides. We don't support that and we will not include hacks to support that. Please run and maintain a fork if you plan to keep your changes. + +The script will use **the same pathes** as your default mailcow installation. That is the mailcow base directory - for most users `/opt/mailcow-dockerized` - as well as the mountpoints. + +To find the pathes of your source volumes we use `docker inspect` and read the destination directory of every volume related to your mailcow compose project. This means we will also transfer volumes you may have added in a override file. Local bind mounts may or may not work. + +The use rsync with the `--delete` flag. The destination will be an exact copy of the source. + +`mariabackup` is used to create a consistent copy of the SQL data directory. + +After rsync'ing the data we will run `docker-compose pull` and remove old image tags from the destination. + +Your source will not be changed at any time. + +**You may want to make sure to use the same `/etc/docker/daemon.json` on the remote target.** + +You should not run disk snapshots (e.g. via ZFS, LVM etc.) on the target at the very same time as this script is run. + +Versioning is not part of this script, we rely on the destination (snapshots or backups). You may also want to use any other tool for that. + +## Prepare + +You will need a SSH-enabled destination and a keyfile to connect to said destination. The key should not be protected by a password for the script to work unattended. + +In your mailcow base directory, e.g. `/opt/mailcow-dockerized` you will find a file `create_cold_standby.sh`. + +Edit this file and change the exported variables: + +``` +export REMOTE_SSH_KEY=/path/to/keyfile +export REMOTE_SSH_PORT=22 +export REMOTE_SSH_HOST=mailcow-backup.host.name +``` + +The key must be owned and readable by root only. + +Both the source and destination require `rsync` >= v3.1.0. +The destination must have Docker and docker-compose **v1** available. + +The script will detect errors automatically and exit. + +You may want to test the connection by running `ssh mailcow-backup.host.name -p22 -i/path/to/keyfile`. + +## Backup and refresh the cold-standby + +Run the first backup, this may take a while depending on the connection: + +``` +bash /opt/mailcow-dockerized/create_cold_standby.sh +``` + +That was easy, wasn't it? + +Updating your cold-standby is just as easy: + +``` +bash /opt/mailcow-dockerized/create_cold_standby.sh +``` + +It's the same command. + diff --git a/docs/b_n_r-restore.md b/docs/b_n_r-restore.md new file mode 100644 index 000000000..e8603d86c --- /dev/null +++ b/docs/b_n_r-restore.md @@ -0,0 +1,14 @@ +### Restore + +Please do not copy this script to another location. + +To run a restore, **start mailcow**, use the script with "restore" as first parameter. + +``` +# Syntax: +# ./helper-scripts/backup_and_restore.sh restore + +``` + +The script will ask you for a backup location containing the mailcow_DATE folders. + diff --git a/mkdocs.yml b/mkdocs.yml index be439bd91..600b6f073 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -62,13 +62,14 @@ nav: - 'Reset Passwords (incl. SQL)': 'debug-reset_pw.md' - 'Reset TLS certificates': 'debug-reset_tls.md' - 'Backup & Restore': - - 'Helper script': - - 'Backup': 'b_n_r_backup.md' - - 'Restore': 'b_n_r_restore.md' - - 'Manually': + - 'Component backup': + - 'Backup': 'b_n_r-backup.md' + - 'Restore': 'b_n_r-restore.md' + - 'Cold-standby (rolling)': 'b_n_r-coldstandby.md' + - 'Manual backups': - 'Maildir': 'u_e-backup_restore-maildir.md' - - 'MySQL': 'u_e-backup_restore-mysql.md' - - 'Automatic backups': + - 'MySQL (mysqldump)': 'u_e-backup_restore-mysql.md' + - 'mailcow-internal backups': - 'Recover accidentally deleted data': 'b_n_r_accidental_deletion.md' - 'Manual/Guides/Examples': - 'mailcow UI':