v4 Work In progress

This commit is contained in:
Geoffrey Frogeye 2019-08-02 08:15:01 +02:00
parent ad22ef40cf
commit ed2afd876c
3 changed files with 222 additions and 153 deletions

View file

@ -1,3 +1,12 @@
# ckupeye # ckupeye
Small bash script for entire server backups Small bash script for entire server backups.
It backups filesystems, sure, but also creates workable version of
PostgreSQL / MariaDB backups, Pacman / Apt software list, ACL.
It then uses [BorgBackup](https://www.borgbackup.org/) to create, compress, encrypt and send the backups.
## Notation
The machine is backed up is called the client,
the machine that stores the backup is called the server.

299
backup.sh
View file

@ -1,105 +1,230 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Backup script v3 ################################
# Uses restic # ckupeye executable #
# (Frogeye's backup script v4) #
################################
cd `dirname ${BASH_SOURCE-$0}` #
# Load config file
#
RUNFILE=/var/run/bkp CLIENT_CONFIG_FILE="${1:-/etc/ckupeye/config.sh}"
if [ -e $RUNFILE ] && ps -p $(cat $RUNFILE) > /dev/null
if [ -f "$CLIENT_CONFIG_FILE" ]
then then
>&2 echo "Another backup instance is running. Skipping" . "$CLIENT_CONFIG_FILE"
else
echo "Configuration file not found on $CLIENT_CONFIG_FILE."
exit 1 exit 1
fi fi
echo $$ > $RUNFILE #
echo → Backup started # Required configuration
#
source config.sh if [ -z ${BORG_REPO+x} ]
if [ ! -z ${ETCKEEPER_COMMIT_MESSAGE+x} ]
then then
etckeeper commit "$ETCKEEPER_COMMIT_MESSAGE" &> /dev/null & echo "Please specify BORG_REPO in the configuration file."
echo "See https://borgbackup.readthedocs.io/en/stable/usage/general.html?highlight=borg_repo#repository-urls"
exit 1
fi fi
# Dumping acl if [[ -z ${BORG_PASSPHRASE+x} && -z ${BORG_PASSCOMMAND+x} && -z ${BORG_PASSPHRASE_FD+x} ]]
ACLFILE=~/bkp/acl.gz
function bkpACL() {
touch "$ACLFILE"
chmod 700 "$ACLFILE"
sudo getfacl -R "$SRC" 2> /dev/null | gzip > "$ACLFILE"
}
# Dumping database
DBFILE=/var/bkp/psql.sql.gz
function bkpPgsql() {
touch "$DBFILE"
chmod 700 "$DBFILE"
pg_dumpall | gzip > "$DBFILE"
}
# Dumping database
DBFILE=./mysql.sql.gz
function bkpMysql() {
touch "$DBFILE"
chmod 700 "$DBFILE"
mysqldump --defaults-extra-file=mysql.cnf --all-databases --flush-privileges | gzip > "$DBFILE"
}
# Dumping package list
PACFILE=/var/bkp/pacman.list.gz
function bkpPacman() {
touch "$PACFILE"
chmod 700 "$PACFILE"
comm -23 <(pacman -Qeq | sort) <(pacman -Qgq base base-devel | sort) | gzip > "$PACFILE"
}
# Dumping package list
APTFILE=./apt.list.gz
function bkpApt() {
touch "$APTFILE"
chmod 700 "$APTFILE"
apt list --installed 2> /dev/null | gzip > "$APTFILE"
}
echo → Creating snapshots of databases
bkpPgsql &
bkpPacman &
wait
# echo "→ Restic: Transfering files"
#
# $RESTIC_CMD $RESTIC_OPTIONS backup $BACKUP_OPTIONS --cleanup-cache
#
# if [ ! -z ${FORGET_OPTIONS+x} ]
# then
#
# echo "→ Restic: Cleaning up"
#
# $RESTIC_CMD $RESTIC_OPTIONS forget $FORGET_OPTIONS
#
# fi
echo "→ Borg: Transfering files"
TAG="$(date -Isec)"
$BORG_CMD $BORG_OPTIONS create --stats $CREATE_OPTIONS "${BORG_REPO}::${TAG}" $FILES 2>&1
if [ ! -z ${PRUNE_OPTIONS+x} ]
then then
echo "Please specify one of BORG_PASSPHRASE, BORG_PASSCOMMAND or BORG_PASSPHRASE_FD in the configuration file."
echo "→ Borg: Cleaning up" echo "See https://borgbackup.readthedocs.io/en/stable/man_intro.html?highlight=borg_passphrase#environment-variables"
exit 1
$BORG_CMD $BORG_OPTIONS prune --stats $PRUNE_OPTIONS $BORG_REPO 2>&1
fi fi
echo → Remove temporary files #
rm "$PACFILE" "$DBFILE" # Default configuration
#
echo → Backup ended : "${BASE_DIR:="/var/ckupeye"}"
rm $RUNFILE
# Directories to backup
[ -z ${BACKUP_FILES+x} ] && BACKUP_FILES=("/") # What to include in the backup
[ -z ${BACKUP_EXCLUDES+x} ] && BACKUP_EXCLUDES=("/var/cache" "**.lock" "**/.cache/") # What to exclue from the backup
: "${BACKUP_TAG:="$(date -Isec)"}" # “Name” of the backup
: "${BACKUP_OPTIONS:="--one-file-system --compression auto,zstd"}" # Configuration of the backup
# Default folder where the database / package managers / ACL exported data will be stored.
: "${EXPORT_DIR:="$BASE_DIR"}"
# Things to export the data of
: "${EXPORT_MARIADB:="false"}"
: "${EXPORT_POSTGRESQL:="false"}"
: "${EXPORT_APT:="false"}"
: "${EXPORT_PACMAN:="false"}"
: "${EXPORT_ACL:="false"}"
: "${EXPORT_ETCKEEPER:="false"}"
# Where to put the exported data
: "${EXPORT_MARIADB_FILE:="$EXPORT_DIR/mariadb.sql"}"
: "${EXPORT_POSTGRESQL_FILE:="$EXPORT_DIR/postgresql.sql"}"
: "${EXPORT_APT_FILE:="$EXPORT_DIR/apt.list"}"
: "${EXPORT_PACMAN_FILE:="$EXPORT_DIR/pacman.list"}"
: "${EXPORT_ACL_FILE:="$EXPORT_DIR/acl.list"}"
# Options for exporting data
[ -z ${EXPORT_ACL_DIRECTORIES+x} ] && EXPORT_ACL_DIRECTORIES=("${BACKUP_FILES[@]}")
: "${EXPORT_ETCKEEPER_MESSAGE:="ckupeye backup for $(date -Isec)"}"
: "${EXPORT_POSTGRESQL_OPTIONS:=""}"
: "${EXPORT_MARIADB_OPTIONS:="--all-databases --flush-privileges"}"
: "${EXPORT_MARIADB_CREDENTIALS:="$BASE_DIR/mariadb.cnf"}"
# Pruning the backup
: "${PRUNE:="true"}"
: "${PRUNE_OPTIONS:="--keep-hourly 24 --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --keep-yearly 10"}"
# Borg specific
: "${BORG_BASE_DIR:="$BASE_DIR"}"
: "${BORG_EXECUTABLE:="$(command -v borg)"}" # Path to BorgBackup executable
: "${BORG_SSH_IDENTITY_FILE:="/etc/ckupeye/id_ed25519"}" # Identity file to use to connect to remote repo (will overwrite $BORG_RSH)
: "${BORG_RSH_EXTRA:=""}" # Options to add to $BORG_RSH (even after overriden by BORG_SSH_IDENTITY_FILE)
: "${BORG_OPTIONS:=""}" # Additional options to pass to BorgBackup
: "${BORG_ENCRYPTION:="repokey-blake2"}" # Passed `borg init`
# Answer borg questions
: "${BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK:=no}"
: "${BORG_RELOCATED_REPO_ACCESS_IS_OK:=no}"
: "${BORG_CHECK_I_KNOW_WHAT_I_AM_DOING:=NO}"
: "${BORG_DELETE_I_KNOW_WHAT_I_AM_DOING:=NO}"
: "${BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING:=NO}"
# Ckupeye specific
: "${CLIENT_RUNFILE:="$BASE_DIR/ckupeye.pid"}"
# Export borg environment variables
export BORG_REPO
export BORG_PASSPHRASE
export BORG_PASSCOMMAND
export BORG_PASSPHRASE_FD
export BORG_BASE_DIR
# this should be done in the configuration file too,
# for consistency
#
# Preparation
#
if [ -e $CLIENT_RUNFILE ] && ps -p $(cat $CLIENT_RUNFILE) > /dev/null
then
echo "Another backup instance is running. Skipping"
exit 1
fi
if ! touch $CLIENT_RUNFILE
then
echo "Could not create a runfile on $CLIENT_RUNFILE. Please change CLIENT_RUNFILE according to this user's permissions."
exit 1
fi
echo $$ > $CLIENT_RUNFILE
#
# Test repository, create it if not present
#
if [ ! -z "$BORG_SSH_IDENTITY_FILE" ]
then
export BORG_RSH="ssh -oBatchMode=yes -i $BORG_SSH_IDENTITY_FILE $BORG_RSH_EXTRA"
fi
echo "→ Opening repository"
$BORG_EXECUTABLE $BORG_OPTIONS info
if [ $? != 0 ]
then
echo "→ Could not open repository, trying to create it"
$BORG_EXECUTABLE $BORG_OPTIONS init --encryption "$BORG_ENCRYPTION"
if [ $? != 0 ]
then
echo "Could not create repository."
exit 1
fi
fi
#
# Exporting databases
#
if [ ! -z ${EXPORT_DIR+x} ]
then
mkdir -p "$EXPORT_DIR"
fi
if $EXPORT_ETCKEEPR
then
echo → Creating etckeeper commit
etckeeper commit "$EXPORT_ETCKEEPER_MESSAGE" &> /dev/null &
fi
export EXPORT_FILEs=""
function _feedExport() { # export_file
touch "$1"
chmod 700 "$1"
cat - > "$1"
export EXPORT_FILES="$1 $EXPORT_FILES"
}
if $EXPORT_ACL
then
echo → Exporting ACL
sudo getfacl -R "$EXPORT_ACL_DESTINATION[@]" 2> /dev/null | _feedExport "$EXPORT_ACL_FILE"
fi
if $EXPORT_POSTGRESQL
then
echo → Exporting PostgreSQL
pg_dumpall $EXPORT_POSTGRESQL_OPTIONS | _feedExport "$EXPORT_POSTGRESQL_FILE"
fi
if $EXPORT_MARIADB
then
echo → Exporting MariaDB
if [ ! -z $EXPORT_MARIADB_CREDENTIALS ]
then
EXPORT_MARIADB_OPTIONS="--defaults-extra-file=$EXPORT_MARIADB_CREDENTIALS $EXPORT_MARIADB_OPTIONS"
fi
mysqldump $EXPORT_MARIADB_OPTIONS | _feedExport "$EXPORT_MARIADB_FILE"
fi
if $EXPORT_PACMAN
then
echo → Exporting Pacman package list
pacman -Qeq | _feedExport "$EXPORT_PACMAN_FILE"
fi
if $EXPORT_APT
then
echo → Exporting Apt package list
apt list --installed 2> /dev/null | _feedExport "$EXPORT_APT_FILE"
fi
echo "→ Transfering files"
for exclude in "${BACKUP_EXCLUDES[@]}"
do
BACKUP_OPTIONS="$BORG_CREATE_OPTIONS --exclude=$exclude"
done
$BORG_EXECUTABLE $BORG_OPTIONS create --stats $BACKUP_OPTIONS "${BORG_REPO}::${BACKUP_TAG}" ${BACKUP_FILES[@]} 2>&1
echo "→ Remove temporary files"
rm -f $EXPORT_FILES
if $PRUNE
then
echo "→ Pruning old backups"
$BORG_EXECUTABLE $BORG_OPTIONS prune --stats $PRUNE_OPTIONS $BORG_REPO 2>&1
fi
echo "→ Done!"
rm "$CLIENT_RUNFILE"

View file

@ -1,65 +0,0 @@
#!/usr/bin/env bash
# Configuration options for the backup.sh script v4
# TODO Do the v4
cd `dirname ${BASH_SOURCE-$0}`
# Connection options
# Type of connection to the backup server
TYPE="sftp"
# Hostname/IP of the backup server
HOST="bkpuser@backup.example.com"
# Location of the backup directory on the backup server
DEST="/srv/bkp/host"
BDEST="/srv/bkp/host"
# You will need to create a `resticpass` file in this directory
# Exported variable for extra restic manipulation if needed
export RESTIC_REPOSITORY="$TYPE:$HOST:$DEST"
export BORG_REPO="ssh://$HOST$BDEST"
export RESTIC_PASSWORD_FILE="$PWD/resticpass"
export BORG_PASSPHRASE="$(cat $PWD/borgpass)"
# Export options
# Where the various exports will be saved
# The file will be deleted afterwards, so it is only to locate it in the backup
WORKING_DIR="/var/bkp"
# Create a etckeeper commit (make sure /etc is in $FILES for it to be saved) (unset if uneeded)
ETCKEEPER_COMMIT_MESSAGE="Sauvegarde automatique du $(date)"
# Backup options
# Directories to backup
FILES="/"
# Pattern to exclude
EXCLUDE=$(echo --exclude={"/var/cache","**.lock","**/.cache/"})
# Tags to add to the backups
TAGS=$(echo --tag={"auto","v3"})
# Options for the backup command
export BACKUP_OPTIONS="--one-file-system --cleanup-cache $FILES $EXCLUDE $TAGS"
export CREATE_OPTIONS="--one-file-system --compression auto,zstd $EXCLUDE"
# Forget options
# Which snapshots to keep (inclusive)
KEEP="--keep-hourly 24 --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --keep-yearly 10"
# Options for the restic forget command (unset if uneeded)
export FORGET_OPTIONS="$KEEP --prune"
export PRUNE_OPTIONS="$KEEP"
# Executable options
# Location of the restic command (you might want to add sudo in there)
# RESTIC_CMD="$(which restic)"
RESTIC_OPTIONS="--cache-dir /var/cache/restic"
BORG_CMD="$(which borg)"
BORG_OPTIONS=""