backup/backup-all.sh

#!/usr/bin/env bash
# backup/backup-all.sh
# Orchestrator. Runs every stage in sequence; stages are independently
# runnable and keep their own per-stage lock so one slow dump does not
# block the next one on a later run.
#
# Layout of artifacts on disk:
#   /srv/homelab/backup/
#     snapshots/<stage>/<ts>/
#     latest/           # symlinks to freshest <ts>
# After all stages pass, rclone-sync.sh ships latest/ off-site to B2.
#
# Docs: mercemay.top/src/homelab-compose/

set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
STAMP="/var/lib/homelab/backup.stamp"
LOCK="/var/lock/homelab-backup.lock"

# shellcheck source=/dev/null
. "${HERE}/../scripts/lib/log.sh"

exec 9>"${LOCK}"
if ! flock -n 9; then
    log_err "another backup run holds ${LOCK}, aborting"
    exit 1
fi

STAGES=(
    snapshot-btrfs
    pg-dump
    sqlite-backup
    docker-volumes
)

run_stage() {
    local stage="$1"
    local script="${HERE}/stages/${stage}.sh"
    if [[ ! -x "${script}" ]]; then
        log_err "missing stage script ${script}"
        return 2
    fi
    log_info "stage start ${stage}"
    local t0 t1 rc
    t0=$(date +%s)
    if "${script}"; then
        rc=0
    else
        rc=$?
    fi
    t1=$(date +%s)
    log_info "stage done  ${stage} rc=${rc} elapsed=$((t1 - t0))s"
    return "${rc}"
}

failed=0
for stage in "${STAGES[@]}"; do
    if ! run_stage "${stage}"; then
        log_err "stage ${stage} failed"
        failed=$((failed + 1))
    fi
done

if (( failed > 0 )); then
    log_err "${failed} stage(s) failed, skipping off-site sync"
    exit 1
fi

log_info "all local stages ok, starting off-site sync"
if "${HERE}/stages/rclone-sync.sh"; then
    install -d /var/lib/homelab
    date -u +%FT%TZ > "${STAMP}"
    log_info "backup complete, stamp=$(cat "${STAMP}")"
else
    log_err "rclone-sync failed"
    exit 1
fi