now with dnsec
This commit is contained in:
rnsrk
parent
b006c8f809
commit
fb22e9cab4
118 changed files with 8306 additions and 2337 deletions
|
|
@ -117,7 +117,7 @@ fi
|
|||
SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
|
||||
source "${SCRIPT_DIR}/../mailcow.conf"
|
||||
COMPOSE_FILE="${SCRIPT_DIR}/../docker-compose.yml"
|
||||
CMPS_PRJ=$(echo ${COMPOSE_PROJECT_NAME} | tr -cd 'A-Za-z-_')
|
||||
CMPS_PRJ=$(echo ${COMPOSE_PROJECT_NAME} | tr -cd '0-9A-Za-z-_')
|
||||
SQLIMAGE=$(grep -iEo '(mysql|mariadb)\:.+' "${COMPOSE_FILE}")
|
||||
|
||||
preflight_local_checks
|
||||
|
|
@ -169,7 +169,7 @@ if ! ssh -o StrictHostKeyChecking=no \
|
|||
-i "${REMOTE_SSH_KEY}" \
|
||||
${REMOTE_SSH_HOST} \
|
||||
-p ${REMOTE_SSH_PORT} \
|
||||
${COMPOSE_COMMAND} -f "${SCRIPT_DIR}/../docker-compose.yml" create 2>&1 ; then
|
||||
"cd \"${SCRIPT_DIR}/../\" && ${COMPOSE_COMMAND} create 2>&1" ; then
|
||||
>&2 echo -e "\e[31m[ERR]\e[0m - Could not create networks, volumes and containers on remote"
|
||||
fi
|
||||
|
||||
|
|
@ -284,7 +284,7 @@ echo "OK"
|
|||
-i "${REMOTE_SSH_KEY}" \
|
||||
${REMOTE_SSH_HOST} \
|
||||
-p ${REMOTE_SSH_PORT} \
|
||||
${COMPOSE_COMMAND} -f "${SCRIPT_DIR}/../docker-compose.yml" pull --quiet 2>&1 ; then
|
||||
"cd \"${SCRIPT_DIR}/../\" && ${COMPOSE_COMMAND} pull --quiet 2>&1" ; then
|
||||
>&2 echo -e "\e[31m[ERR]\e[0m - Could not pull images on remote"
|
||||
fi
|
||||
|
||||
|
|
@ -293,7 +293,7 @@ if ! ssh -o StrictHostKeyChecking=no \
|
|||
-i "${REMOTE_SSH_KEY}" \
|
||||
${REMOTE_SSH_HOST} \
|
||||
-p ${REMOTE_SSH_PORT} \
|
||||
${SCRIPT_DIR}/../update.sh -f --gc ; then
|
||||
"cd \"${SCRIPT_DIR}/../\" && ./update.sh -f --gc" ; then
|
||||
>&2 echo -e "\e[31m[ERR]\e[0m - Could not cleanup old images on remote"
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -91,6 +91,44 @@ if grep --help 2>&1 | head -n 1 | grep -q -i "busybox"; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Add image prefetch function
|
||||
function prefetch_image() {
|
||||
echo "Checking Docker image: ${DEBIAN_DOCKER_IMAGE}"
|
||||
|
||||
# Get local image digest if it exists
|
||||
local local_digest=$(docker image inspect ${DEBIAN_DOCKER_IMAGE} --format='{{index .RepoDigests 0}}' 2>/dev/null | cut -d'@' -f2)
|
||||
|
||||
# Get remote image digest without pulling
|
||||
local remote_digest=$(docker manifest inspect ${DEBIAN_DOCKER_IMAGE} 2>/dev/null | grep -oP '"digest":\s*"\K[^"]+' | head -1)
|
||||
|
||||
if [[ -z "${remote_digest}" ]]; then
|
||||
echo "Warning: Unable to check remote image"
|
||||
if [[ -n "${local_digest}" ]]; then
|
||||
echo "Using cached version"
|
||||
echo
|
||||
return 0
|
||||
else
|
||||
echo "Error: Image ${DEBIAN_DOCKER_IMAGE} not found locally or remotely"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${local_digest}" != "${remote_digest}" ]]; then
|
||||
echo "Image update available, pulling ${DEBIAN_DOCKER_IMAGE}"
|
||||
if docker pull ${DEBIAN_DOCKER_IMAGE} 2>/dev/null; then
|
||||
echo "Successfully pulled ${DEBIAN_DOCKER_IMAGE}"
|
||||
else
|
||||
echo "Error: Failed to pull ${DEBIAN_DOCKER_IMAGE}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Image is up to date (${remote_digest:0:12}...)"
|
||||
fi
|
||||
echo
|
||||
}
|
||||
|
||||
# Prefetch the image early in the script
|
||||
prefetch_image
|
||||
|
||||
function backup() {
|
||||
DATE=$(date +"%Y-%m-%d-%H-%M-%S")
|
||||
|
|
@ -110,32 +148,32 @@ function backup() {
|
|||
docker run --name mailcow-backup --rm \
|
||||
-v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_vmail-vol-1$):/vmail:ro,z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_vmail.tar.gz /vmail
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="zstd --rsyncable -T${THREADS}" -Pcvpf /backup/backup_vmail.tar.zst /vmail
|
||||
;;&
|
||||
crypt|all)
|
||||
docker run --name mailcow-backup --rm \
|
||||
-v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_crypt-vol-1$):/crypt:ro,z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_crypt.tar.gz /crypt
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="zstd --rsyncable -T${THREADS}" -Pcvpf /backup/backup_crypt.tar.zst /crypt
|
||||
;;&
|
||||
redis|all)
|
||||
docker exec $(docker ps -qf name=redis-mailcow) redis-cli -a ${REDISPASS} --no-auth-warning save
|
||||
docker run --name mailcow-backup --rm \
|
||||
-v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_redis-vol-1$):/redis:ro,z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_redis.tar.gz /redis
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="zstd --rsyncable -T${THREADS}" -Pcvpf /backup/backup_redis.tar.zst /redis
|
||||
;;&
|
||||
rspamd|all)
|
||||
docker run --name mailcow-backup --rm \
|
||||
-v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:ro,z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_rspamd.tar.gz /rspamd
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="zstd --rsyncable -T${THREADS}" -Pcvpf /backup/backup_rspamd.tar.zst /rspamd
|
||||
;;&
|
||||
postfix|all)
|
||||
docker run --name mailcow-backup --rm \
|
||||
-v ${BACKUP_LOCATION}/mailcow-${DATE}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_postfix-vol-1$):/postfix:ro,z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="pigz --rsyncable -p ${THREADS}" -Pcvpf /backup/backup_postfix.tar.gz /postfix
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --warning='no-file-ignored' --use-compress-program="zstd --rsyncable -T${THREADS}" -Pcvpf /backup/backup_postfix.tar.zst /postfix
|
||||
;;&
|
||||
mysql|all)
|
||||
SQLIMAGE=$(grep -iEo '(mysql|mariadb)\:.+' ${COMPOSE_FILE})
|
||||
|
|
@ -154,7 +192,7 @@ function backup() {
|
|||
${SQLIMAGE} /bin/sh -c "mariabackup --host mysql --user root --password ${DBROOT} --backup --rsync --target-dir=/backup_mariadb ; \
|
||||
mariabackup --prepare --target-dir=/backup_mariadb ; \
|
||||
chown -R 999:999 /backup_mariadb ; \
|
||||
/bin/tar --warning='no-file-ignored' --use-compress-program='gzip --rsyncable' -Pcvpf /backup/backup_mariadb.tar.gz /backup_mariadb ;"
|
||||
/bin/tar --warning='no-file-ignored' --use-compress-program='zstd --rsyncable' -Pcvpf /backup/backup_mariadb.tar.zst /backup_mariadb ;"
|
||||
fi
|
||||
;;&
|
||||
--delete-days)
|
||||
|
|
@ -170,6 +208,19 @@ function backup() {
|
|||
done
|
||||
}
|
||||
|
||||
function get_archive_info() {
|
||||
local backup_name="$1"
|
||||
local location="$2"
|
||||
|
||||
if [[ -f "${location}/${backup_name}.tar.zst" ]]; then
|
||||
echo "${backup_name}.tar.zst|zstd -d -T${THREADS}"
|
||||
elif [[ -f "${location}/${backup_name}.tar.gz" ]]; then
|
||||
echo "${backup_name}.tar.gz|pigz -d -p ${THREADS}"
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
function restore() {
|
||||
for bin in docker; do
|
||||
if [[ -z $(which ${bin}) ]]; then
|
||||
|
|
@ -199,10 +250,17 @@ function restore() {
|
|||
case "$1" in
|
||||
vmail)
|
||||
docker stop $(docker ps -qf name=dovecot-mailcow)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_vmail-vol-1$):/vmail:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_vmail.tar.gz
|
||||
ARCHIVE_INFO=$(get_archive_info "backup_vmail" "${RESTORE_LOCATION}")
|
||||
if [[ -z "${ARCHIVE_INFO}" ]]; then
|
||||
echo -e "\e[31mError: No backup file found for vmail (searched for .tar.zst and .tar.gz)\e[0m"
|
||||
else
|
||||
ARCHIVE_FILE=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f1)
|
||||
DECOMPRESS_PROG=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f2)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_vmail-vol-1$):/vmail:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="${DECOMPRESS_PROG}" -Pxvf /backup/${ARCHIVE_FILE}
|
||||
fi
|
||||
docker start $(docker ps -aqf name=dovecot-mailcow)
|
||||
echo
|
||||
echo "In most cases it is not required to run a full resync, you can run the command printed below at any time after testing wether the restore process broke a mailbox:"
|
||||
|
|
@ -218,31 +276,50 @@ function restore() {
|
|||
;;
|
||||
redis)
|
||||
docker stop $(docker ps -qf name=redis-mailcow)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_redis-vol-1$):/redis:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_redis.tar.gz
|
||||
ARCHIVE_INFO=$(get_archive_info "backup_redis" "${RESTORE_LOCATION}")
|
||||
if [[ -z "${ARCHIVE_INFO}" ]]; then
|
||||
echo -e "\e[31mError: No backup file found for redis (searched for .tar.zst and .tar.gz)\e[0m"
|
||||
else
|
||||
ARCHIVE_FILE=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f1)
|
||||
DECOMPRESS_PROG=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f2)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_redis-vol-1$):/redis:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="${DECOMPRESS_PROG}" -Pxvf /backup/${ARCHIVE_FILE}
|
||||
fi
|
||||
docker start $(docker ps -aqf name=redis-mailcow)
|
||||
;;
|
||||
crypt)
|
||||
docker stop $(docker ps -qf name=dovecot-mailcow)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_crypt-vol-1$):/crypt:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_crypt.tar.gz
|
||||
ARCHIVE_INFO=$(get_archive_info "backup_crypt" "${RESTORE_LOCATION}")
|
||||
if [[ -z "${ARCHIVE_INFO}" ]]; then
|
||||
echo -e "\e[31mError: No backup file found for crypt (searched for .tar.zst and .tar.gz)\e[0m"
|
||||
else
|
||||
ARCHIVE_FILE=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f1)
|
||||
DECOMPRESS_PROG=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f2)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_crypt-vol-1$):/crypt:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="${DECOMPRESS_PROG}" -Pxvf /backup/${ARCHIVE_FILE}
|
||||
fi
|
||||
docker start $(docker ps -aqf name=dovecot-mailcow)
|
||||
;;
|
||||
rspamd)
|
||||
if [[ $(find "${RESTORE_LOCATION}" \( -name '*x86*' -o -name '*aarch*' \) -exec basename {} \; | sed 's/^\.//' | sed 's/^\.//') == "" ]]; then
|
||||
ARCHIVE_INFO=$(get_archive_info "backup_rspamd" "${RESTORE_LOCATION}")
|
||||
if [[ -z "${ARCHIVE_INFO}" ]]; then
|
||||
echo -e "\e[31mError: No backup file found for rspamd (searched for .tar.zst and .tar.gz)\e[0m"
|
||||
elif [[ $(find "${RESTORE_LOCATION}" \( -name '*x86*' -o -name '*aarch*' \) -exec basename {} \; | sed 's/^\.//' | sed 's/^\.//') == "" ]]; then
|
||||
echo -e "\e[33mCould not find a architecture signature of the loaded backup... Maybe the backup was done before the multiarch update?"
|
||||
sleep 2
|
||||
echo -e "Continuing anyhow. If rspamd is crashing upon boot try remove the rspamd volume with docker volume rm ${CMPS_PRJ}_rspamd-vol-1 after you've stopped the stack.\e[0m"
|
||||
sleep 2
|
||||
docker stop $(docker ps -qf name=rspamd-mailcow)
|
||||
ARCHIVE_FILE=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f1)
|
||||
DECOMPRESS_PROG=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f2)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_rspamd.tar.gz
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="${DECOMPRESS_PROG}" -Pxvf /backup/${ARCHIVE_FILE}
|
||||
docker start $(docker ps -aqf name=rspamd-mailcow)
|
||||
elif [[ $ARCH != $(find "${RESTORE_LOCATION}" \( -name '*x86*' -o -name '*aarch*' \) -exec basename {} \; | sed 's/^\.//' | sed 's/^\.//') ]]; then
|
||||
echo -e "\e[31mThe Architecture of the backed up mailcow OS is different then your restoring mailcow OS..."
|
||||
|
|
@ -250,19 +327,28 @@ function restore() {
|
|||
echo -e "Skipping rspamd due to compatibility issues!\e[0m"
|
||||
else
|
||||
docker stop $(docker ps -qf name=rspamd-mailcow)
|
||||
ARCHIVE_FILE=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f1)
|
||||
DECOMPRESS_PROG=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f2)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_rspamd-vol-1$):/rspamd:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_rspamd.tar.gz
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="${DECOMPRESS_PROG}" -Pxvf /backup/${ARCHIVE_FILE}
|
||||
docker start $(docker ps -aqf name=rspamd-mailcow)
|
||||
fi
|
||||
;;
|
||||
postfix)
|
||||
docker stop $(docker ps -qf name=postfix-mailcow)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_postfix-vol-1$):/postfix:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="pigz -d -p ${THREADS}" -Pxvf /backup/backup_postfix.tar.gz
|
||||
ARCHIVE_INFO=$(get_archive_info "backup_postfix" "${RESTORE_LOCATION}")
|
||||
if [[ -z "${ARCHIVE_INFO}" ]]; then
|
||||
echo -e "\e[31mError: No backup file found for postfix (searched for .tar.zst and .tar.gz)\e[0m"
|
||||
else
|
||||
ARCHIVE_FILE=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f1)
|
||||
DECOMPRESS_PROG=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f2)
|
||||
docker run -i --name mailcow-backup --rm \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_postfix-vol-1$):/postfix:z \
|
||||
${DEBIAN_DOCKER_IMAGE} /bin/tar --use-compress-program="${DECOMPRESS_PROG}" -Pxvf /backup/${ARCHIVE_FILE}
|
||||
fi
|
||||
docker start $(docker ps -aqf name=postfix-mailcow)
|
||||
;;
|
||||
mysql|mariadb)
|
||||
|
|
@ -305,14 +391,19 @@ function restore() {
|
|||
echo Restoring... && \
|
||||
gunzip < backup/backup_mysql.gz | mysql -uroot && \
|
||||
mysql -uroot -e SHUTDOWN;"
|
||||
elif [[ -f "${RESTORE_LOCATION}/backup_mariadb.tar.gz" ]]; then
|
||||
docker run --name mailcow-backup --rm \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_mysql-vol-1$):/backup_mariadb/:rw,z \
|
||||
--entrypoint= \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
${SQLIMAGE} /bin/bash -c "shopt -s dotglob ; \
|
||||
/bin/rm -rf /backup_mariadb/* ; \
|
||||
/bin/tar -Pxvzf /backup/backup_mariadb.tar.gz"
|
||||
else
|
||||
ARCHIVE_INFO=$(get_archive_info "backup_mariadb" "${RESTORE_LOCATION}")
|
||||
if [[ -n "${ARCHIVE_INFO}" ]]; then
|
||||
ARCHIVE_FILE=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f1)
|
||||
DECOMPRESS_PROG=$(echo "${ARCHIVE_INFO}" | cut -d'|' -f2)
|
||||
docker run --name mailcow-backup --rm \
|
||||
-v $(docker volume ls -qf name=^${CMPS_PRJ}_mysql-vol-1$):/backup_mariadb/:rw,z \
|
||||
--entrypoint= \
|
||||
-v ${RESTORE_LOCATION}:/backup:z \
|
||||
${SQLIMAGE} /bin/bash -c "shopt -s dotglob ; \
|
||||
/bin/rm -rf /backup_mariadb/* ; \
|
||||
/bin/tar --use-compress-program='${DECOMPRESS_PROG}' -Pxvf /backup/${ARCHIVE_FILE}"
|
||||
fi
|
||||
fi
|
||||
echo "Modifying mailcow.conf..."
|
||||
source ${RESTORE_LOCATION}/mailcow.conf
|
||||
|
|
@ -363,8 +454,8 @@ elif [[ ${1} == "restore" ]]; then
|
|||
fi
|
||||
|
||||
echo "[ 0 ] - all"
|
||||
# find all files in folder with *.gz extension, print their base names, remove backup_, remove .tar (if present), remove .gz
|
||||
FILE_SELECTION[0]=$(find "${FOLDER_SELECTION[${input_sel}]}" -maxdepth 1 \( -type d -o -type f \) \( -name '*.gz' -o -name 'mysql' \) -printf '%f\n' | sed 's/backup_*//' | sed 's/\.[^.]*$//' | sed 's/\.[^.]*$//')
|
||||
# find all files in folder with *.zst or *.gz extension, print their base names, remove backup_, remove .tar (if present), remove .zst/.gz
|
||||
FILE_SELECTION[0]=$(find "${FOLDER_SELECTION[${input_sel}]}" -maxdepth 1 \( -type d -o -type f \) \( -name '*.zst' -o -name '*.gz' -o -name 'mysql' \) -printf '%f\n' | sed 's/backup_*//' | sed 's/\.[^.]*$//' | sed 's/\.[^.]*$//' | sort -u)
|
||||
for file in $(ls -f "${FOLDER_SELECTION[${input_sel}]}"); do
|
||||
if [[ ${file} =~ vmail ]]; then
|
||||
echo "[ ${i} ] - Mail directory (/var/vmail)"
|
||||
|
|
|
|||
301
mailcow/helper-scripts/dev_tests/test_backup_and_restore.sh
Executable file
301
mailcow/helper-scripts/dev_tests/test_backup_and_restore.sh
Executable file
|
|
@ -0,0 +1,301 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Test script for backup_and_restore.sh
|
||||
# Tests backward compatibility with .tar.gz and new .tar.zst format
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
BACKUP_IMAGE="${BACKUP_IMAGE:-ghcr.io/mailcow/backup:latest}"
|
||||
TEST_DIR="/tmp/mailcow_backup_test_$$"
|
||||
THREADS=2
|
||||
|
||||
echo "=== Mailcow Backup & Restore Test Suite ==="
|
||||
echo "Test directory: ${TEST_DIR}"
|
||||
echo "Backup image: ${BACKUP_IMAGE}"
|
||||
echo ""
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
echo "Cleaning up test files..."
|
||||
rm -rf "${TEST_DIR}"
|
||||
docker rmi mailcow-backup-test 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Create test directory structure
|
||||
mkdir -p "${TEST_DIR}"/{test_data,backup_zst,backup_gz,restore_zst,restore_gz,backup_large_zst,backup_large_gz}
|
||||
echo "Test data for mailcow backup compatibility test" > "${TEST_DIR}/test_data/test.txt"
|
||||
echo "Additional file to verify complete restore" > "${TEST_DIR}/test_data/test2.txt"
|
||||
|
||||
# Build test backup image with zstd support
|
||||
echo "=== Building backup image with zstd support ==="
|
||||
docker build -t mailcow-backup-test "${SCRIPT_DIR}/../data/Dockerfiles/backup/" || {
|
||||
echo "ERROR: Failed to build backup image"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Test 1: Create .tar.zst backup
|
||||
echo ""
|
||||
echo "=== Test 1: Creating .tar.zst backup ==="
|
||||
docker run --rm \
|
||||
-w /data \
|
||||
-v "${TEST_DIR}/test_data:/data:ro" \
|
||||
-v "${TEST_DIR}/backup_zst:/backup" \
|
||||
mailcow-backup-test \
|
||||
/bin/tar --use-compress-program="zstd --rsyncable -T${THREADS}" \
|
||||
-cvpf /backup/backup_test.tar.zst . \
|
||||
> /dev/null
|
||||
echo "✓ .tar.zst backup created: $(ls -lh ${TEST_DIR}/backup_zst/backup_test.tar.zst | awk '{print $5}')"
|
||||
|
||||
# Test 2: Create .tar.gz backup
|
||||
echo ""
|
||||
echo "=== Test 2: Creating .tar.gz backup (legacy) ==="
|
||||
docker run --rm \
|
||||
-w /data \
|
||||
-v "${TEST_DIR}/test_data:/data:ro" \
|
||||
-v "${TEST_DIR}/backup_gz:/backup" \
|
||||
mailcow-backup-test \
|
||||
/bin/tar --use-compress-program="pigz --rsyncable -p ${THREADS}" \
|
||||
-cvpf /backup/backup_test.tar.gz . \
|
||||
> /dev/null
|
||||
echo "✓ .tar.gz backup created: $(ls -lh ${TEST_DIR}/backup_gz/backup_test.tar.gz | awk '{print $5}')"
|
||||
|
||||
# Test 3: Test get_archive_info function
|
||||
echo ""
|
||||
echo "=== Test 3: Testing get_archive_info function ==="
|
||||
|
||||
# Extract and test the function directly
|
||||
get_archive_info() {
|
||||
local backup_name="$1"
|
||||
local location="$2"
|
||||
|
||||
if [[ -f "${location}/${backup_name}.tar.zst" ]]; then
|
||||
echo "${backup_name}.tar.zst|zstd -d -T${THREADS}"
|
||||
elif [[ -f "${location}/${backup_name}.tar.gz" ]]; then
|
||||
echo "${backup_name}.tar.gz|pigz -d -p ${THREADS}"
|
||||
else
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
# Test with .tar.zst
|
||||
result=$(get_archive_info "backup_test" "${TEST_DIR}/backup_zst")
|
||||
if [[ "${result}" =~ "zstd" ]]; then
|
||||
echo "✓ Correctly detects .tar.zst and returns zstd decompressor"
|
||||
else
|
||||
echo "✗ Failed to detect .tar.zst"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test with .tar.gz
|
||||
result=$(get_archive_info "backup_test" "${TEST_DIR}/backup_gz")
|
||||
if [[ "${result}" =~ "pigz" ]]; then
|
||||
echo "✓ Correctly detects .tar.gz and returns pigz decompressor"
|
||||
else
|
||||
echo "✗ Failed to detect .tar.gz"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test with no file
|
||||
result=$(get_archive_info "backup_test" "${TEST_DIR}")
|
||||
if [[ -z "${result}" ]]; then
|
||||
echo "✓ Correctly returns empty when no backup file found"
|
||||
else
|
||||
echo "✗ Should return empty but got: ${result}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 4: Restore from .tar.zst
|
||||
echo ""
|
||||
echo "=== Test 4: Restoring from .tar.zst ==="
|
||||
docker run --rm \
|
||||
-w /restore \
|
||||
-v "${TEST_DIR}/backup_zst:/backup:ro" \
|
||||
-v "${TEST_DIR}/restore_zst:/restore" \
|
||||
mailcow-backup-test \
|
||||
/bin/tar --use-compress-program="zstd -d -T${THREADS}" -xvpf /backup/backup_test.tar.zst \
|
||||
> /dev/null 2>&1
|
||||
|
||||
if [[ -f "${TEST_DIR}/restore_zst/test.txt" ]] && \
|
||||
[[ -f "${TEST_DIR}/restore_zst/test2.txt" ]]; then
|
||||
echo "✓ Successfully restored from .tar.zst"
|
||||
else
|
||||
echo "✗ Failed to restore from .tar.zst"
|
||||
ls -la "${TEST_DIR}/restore_zst/" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 5: Restore from .tar.gz
|
||||
echo ""
|
||||
echo "=== Test 5: Restoring from .tar.gz (backward compatibility) ==="
|
||||
docker run --rm \
|
||||
-w /restore \
|
||||
-v "${TEST_DIR}/backup_gz:/backup:ro" \
|
||||
-v "${TEST_DIR}/restore_gz:/restore" \
|
||||
mailcow-backup-test \
|
||||
/bin/tar --use-compress-program="pigz -d -p ${THREADS}" -xvpf /backup/backup_test.tar.gz \
|
||||
> /dev/null 2>&1
|
||||
|
||||
if [[ -f "${TEST_DIR}/restore_gz/test.txt" ]] && \
|
||||
[[ -f "${TEST_DIR}/restore_gz/test2.txt" ]]; then
|
||||
echo "✓ Successfully restored from .tar.gz (backward compatible)"
|
||||
else
|
||||
echo "✗ Failed to restore from .tar.gz"
|
||||
ls -la "${TEST_DIR}/restore_gz/" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 6: Verify content integrity
|
||||
echo ""
|
||||
echo "=== Test 6: Verifying content integrity ==="
|
||||
original_content=$(cat "${TEST_DIR}/test_data/test.txt")
|
||||
zst_content=$(cat "${TEST_DIR}/restore_zst/test.txt")
|
||||
gz_content=$(cat "${TEST_DIR}/restore_gz/test.txt")
|
||||
|
||||
if [[ "${original_content}" == "${zst_content}" ]] && \
|
||||
[[ "${original_content}" == "${gz_content}" ]]; then
|
||||
echo "✓ Content integrity verified for both formats"
|
||||
else
|
||||
echo "✗ Content mismatch detected"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 7: Compare compression ratios
|
||||
echo ""
|
||||
echo "=== Test 7: Compression comparison ==="
|
||||
zst_size=$(stat -f%z "${TEST_DIR}/backup_zst/backup_test.tar.zst" 2>/dev/null || stat -c%s "${TEST_DIR}/backup_zst/backup_test.tar.zst")
|
||||
gz_size=$(stat -f%z "${TEST_DIR}/backup_gz/backup_test.tar.gz" 2>/dev/null || stat -c%s "${TEST_DIR}/backup_gz/backup_test.tar.gz")
|
||||
improvement=$(echo "scale=2; (${gz_size} - ${zst_size}) * 100 / ${gz_size}" | bc)
|
||||
|
||||
echo " Small files - .tar.gz size: ${gz_size} bytes"
|
||||
echo " Small files - .tar.zst size: ${zst_size} bytes"
|
||||
echo " Small files - Improvement: ${improvement}% smaller with zstd"
|
||||
|
||||
# Test 8: Error handling - missing backup file
|
||||
echo ""
|
||||
echo "=== Test 8: Error handling - Missing backup file ==="
|
||||
result=$(get_archive_info "nonexistent_backup" "${TEST_DIR}/backup_zst")
|
||||
if [[ -z "${result}" ]]; then
|
||||
echo "✓ Correctly handles missing backup files"
|
||||
else
|
||||
echo "✗ Should return empty for missing files"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 9: Error handling - Empty directory
|
||||
echo ""
|
||||
echo "=== Test 9: Error handling - Empty directory ==="
|
||||
mkdir -p "${TEST_DIR}/empty_dir"
|
||||
result=$(get_archive_info "backup_test" "${TEST_DIR}/empty_dir")
|
||||
if [[ -z "${result}" ]]; then
|
||||
echo "✓ Correctly handles empty directories"
|
||||
else
|
||||
echo "✗ Should return empty for empty directories"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 10: Priority test - .tar.zst preferred over .tar.gz
|
||||
echo ""
|
||||
echo "=== Test 10: Format priority - .tar.zst preferred ==="
|
||||
mkdir -p "${TEST_DIR}/both_formats"
|
||||
touch "${TEST_DIR}/both_formats/backup_test.tar.gz"
|
||||
touch "${TEST_DIR}/both_formats/backup_test.tar.zst"
|
||||
result=$(get_archive_info "backup_test" "${TEST_DIR}/both_formats")
|
||||
if [[ "${result}" =~ "zstd" ]]; then
|
||||
echo "✓ Correctly prefers .tar.zst when both formats exist"
|
||||
else
|
||||
echo "✗ Should prefer .tar.zst over .tar.gz"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test 11: Large file compression test
|
||||
echo ""
|
||||
echo "=== Test 11: Large file compression test ==="
|
||||
mkdir -p "${TEST_DIR}/large_data"
|
||||
# Create ~10MB of compressible data (log-like content)
|
||||
for i in {1..50000}; do
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: Processing email message $i from user@example.com to recipient@domain.com" >> "${TEST_DIR}/large_data/maillog.txt"
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] DEBUG: SMTP connection established from 192.168.1.$((i % 255))" >> "${TEST_DIR}/large_data/maillog.txt"
|
||||
done 2>/dev/null
|
||||
|
||||
# Get size (portable: works on Linux and macOS)
|
||||
if du --version 2>/dev/null | grep -q GNU; then
|
||||
original_size=$(du -sb "${TEST_DIR}/large_data" | cut -f1)
|
||||
else
|
||||
# macOS
|
||||
original_size=$(find "${TEST_DIR}/large_data" -type f -exec stat -f%z {} \; | awk '{sum+=$1} END {print sum}')
|
||||
fi
|
||||
echo " Original data size: $(echo "scale=2; ${original_size} / 1024 / 1024" | bc) MB"
|
||||
|
||||
# Backup with zstd
|
||||
docker run --rm \
|
||||
-w /data \
|
||||
-v "${TEST_DIR}/large_data:/data:ro" \
|
||||
-v "${TEST_DIR}/backup_large_zst:/backup" \
|
||||
mailcow-backup-test \
|
||||
/bin/tar --use-compress-program="zstd --rsyncable -T${THREADS}" \
|
||||
-cvpf /backup/backup_large.tar.zst . \
|
||||
> /dev/null 2>&1
|
||||
|
||||
# Backup with pigz
|
||||
docker run --rm \
|
||||
-w /data \
|
||||
-v "${TEST_DIR}/large_data:/data:ro" \
|
||||
-v "${TEST_DIR}/backup_large_gz:/backup" \
|
||||
mailcow-backup-test \
|
||||
/bin/tar --use-compress-program="pigz --rsyncable -p ${THREADS}" \
|
||||
-cvpf /backup/backup_large.tar.gz . \
|
||||
> /dev/null 2>&1
|
||||
|
||||
zst_large_size=$(stat -f%z "${TEST_DIR}/backup_large_zst/backup_large.tar.zst" 2>/dev/null || stat -c%s "${TEST_DIR}/backup_large_zst/backup_large.tar.zst" 2>/dev/null || echo "0")
|
||||
gz_large_size=$(stat -f%z "${TEST_DIR}/backup_large_gz/backup_large.tar.gz" 2>/dev/null || stat -c%s "${TEST_DIR}/backup_large_gz/backup_large.tar.gz" 2>/dev/null || echo "0")
|
||||
|
||||
if [[ ${zst_large_size} -gt 0 ]] && [[ ${gz_large_size} -gt 0 ]]; then
|
||||
large_improvement=$(echo "scale=2; (${gz_large_size} - ${zst_large_size}) * 100 / ${gz_large_size}" | bc)
|
||||
|
||||
echo " .tar.gz compressed: $(echo "scale=2; ${gz_large_size} / 1024 / 1024" | bc) MB"
|
||||
echo " .tar.zst compressed: $(echo "scale=2; ${zst_large_size} / 1024 / 1024" | bc) MB"
|
||||
echo " Improvement: ${large_improvement}% smaller with zstd"
|
||||
else
|
||||
echo " ✗ Failed to get file sizes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $(echo "${large_improvement} > 0" | bc) -eq 1 ]]; then
|
||||
echo "✓ zstd provides better compression on realistic data"
|
||||
else
|
||||
echo "⚠ zstd compression similar or worse than gzip (unusual but not critical)"
|
||||
fi
|
||||
|
||||
# Test 12: Thread scaling test
|
||||
echo ""
|
||||
echo "=== Test 12: Multi-threading verification ==="
|
||||
# This test verifies that different thread counts work (not measuring speed difference)
|
||||
for thread_count in 1 4; do
|
||||
THREADS=${thread_count}
|
||||
result=$(get_archive_info "backup_test" "${TEST_DIR}/backup_zst")
|
||||
if [[ "${result}" =~ "-T${thread_count}" ]]; then
|
||||
echo "✓ Thread count ${thread_count} correctly configured"
|
||||
else
|
||||
echo "✗ Thread count not properly applied"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== All tests passed! ==="
|
||||
echo ""
|
||||
echo "Summary:"
|
||||
echo " ✓ zstd compression working"
|
||||
echo " ✓ pigz compression working (legacy)"
|
||||
echo " ✓ zstd decompression working"
|
||||
echo " ✓ pigz decompression working (backward compatible)"
|
||||
echo " ✓ Archive detection working"
|
||||
echo " ✓ Content integrity verified"
|
||||
echo " ✓ Format priority correct (.tar.zst preferred)"
|
||||
echo " ✓ Error handling for missing files"
|
||||
echo " ✓ Error handling for empty directories"
|
||||
echo " ✓ Multi-threading configuration verified"
|
||||
echo " ✓ Large file compression: ${large_improvement}% improvement"
|
||||
echo " ✓ Small file compression: ${improvement}% improvement"
|
||||
|
|
@ -6,9 +6,10 @@ SPFTOOLS_DIR=${WORKING_DIR}/spf-tools
|
|||
POSTWHITE_DIR=${WORKING_DIR}/postwhite
|
||||
POSTWHITE_CONF=${POSTWHITE_DIR}/postwhite.conf
|
||||
|
||||
CUSTOM_HOSTS='"web.de gmx.net mail.de freenet.de arcor.de unity-mail.de"'
|
||||
CUSTOM_HOSTS='"web.de gmx.net mail.de freenet.de arcor.de unity-mail.de protonmail.ch ionos.com strato.com t-online.de"'
|
||||
STATIC_HOSTS=(
|
||||
"194.25.134.0/24 permit # t-online.de"
|
||||
"49.12.4.251 permit # checks.mailcow.email"
|
||||
"2a01:4f8:c17:7906::10 permit # checks.mailcow.email"
|
||||
)
|
||||
|
||||
mkdir ${SCRIPT_DIR}/postwhite_tmp
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue