Backup#

┌──────────────────────┐  ┌──────────────────────┐ ┌── more backup CronJobs
│ backup-mariadb       │  │ backup-emby          │ │
│   (databases ns)     │  │   (media ns)         │ │
│ initContainer dumps  │  │ podAffinity:emby     │ │
│ → restic backup ────┐│  │ → restic backup ─────┼─┤
└─────────────────────┼┘  └──────────────────────┘ │
                      │                            │
                      │   ┌─────────────────────┐  │
                      │   │ backup.sh (shared)  │  │
                      │   │ ConfigMap mirrored  │  │
                      │   │ via Reflector       │  │
                      │   └─────────────────────┘  │
                      │                            │
                      ▼                            ▼
      ┌──────────────────────────────────────────────────┐
      │ rest-server.backup.svc.cluster.local:8000        │
      │   one repository, multiple tags                  │
      │   data on node04: /vol_raidz1/restic-local       │
      └────────────┬─────────────────────────────────────┘
                   │ daily 05:00, rclone copy
                   ▼
      ┌──────────────────────────────────────────────────┐
      │ Wasabi S3: k3s-at-home-01/restic-backup          │
      │   off-site mirror (read for restore only)        │
      └──────────────────────────────────────────────────┘
┌───────────────────────────────────────────────────────┐
│ restic-retentionpolicies (longhorn ns)                │
│   daily 03:05 forget + prune + check on local repo    │
└───────────────────────────────────────────────────────┘
┌───────────────────────────────────────────────────────┐
│ backup-restore-test (longhorn ns)                     │
│   weekly full-fleet restore from local repo           │
└───────────────────────────────────────────────────────┘
 metrics ──────────────── ntfy
      ▼                    ▼
Pushgateway           ntfy.geekbundle

apiVersion: batch/v1
kind: CronJob
metadata:
  name: backup-grafana
  namespace: monitoring
spec:
  schedule: "20 * * * *"
  concurrencyPolicy: Forbid
  failedJobsHistoryLimit: 1
  successfulJobsHistoryLimit: 1
  startingDeadlineSeconds: 3600
  jobTemplate:
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 540
      template:
        metadata:
          labels:
            app.kubernetes.io/name: backup-grafana
            app.kubernetes.io/component: backup
        spec:
          automountServiceAccountToken: false
          restartPolicy: Never
          securityContext:
            runAsUser: 472
            runAsGroup: 472
            fsGroup: 472
            runAsNonRoot: true
          affinity:
            podAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchLabels:
                      app.kubernetes.io/name: grafana
                  namespaces: [monitoring]
                  topologyKey: kubernetes.io/hostname
          containers:
            - name: restic
              image: ghcr.io/madic-creates/container-restic:latest
              command: ["/bin/sh", "/usr/local/bin/backup.sh"]
              env:
                - name: HOME
                  value: /home/restic
              envFrom:
                - secretRef: { name: backup-env-configuration-grafana }
              volumeMounts:
                - name: backup-script
                  mountPath: /usr/local/bin/backup.sh
                  subPath: backup.sh
                - name: grafana
                  mountPath: /backup/grafana
                  readOnly: true
                - name: home
                  mountPath: /home/restic
          volumes:
            - name: grafana
              persistentVolumeClaim: { claimName: longhorn-pvc-grafana }
            - name: backup-script
              configMap: { name: crontab-backup-script }
            - name: home
              emptyDir: { medium: Memory, sizeLimit: 200Mi }

kubectl run restic-shell -n monitoring --rm -it --restart=Never \
  --image=ghcr.io/madic-creates/container-restic:latest \
  --overrides='{
    "spec": {
      "containers": [{
        "name": "restic-shell",
        "image": "ghcr.io/madic-creates/container-restic:latest",
        "command": ["sh"],
        "stdin": true, "tty": true,
        "envFrom": [{"secretRef": {"name": "backup-env-configuration-grafana"}}]
      }]
    }
  }' -- sh

# Inside the pod
restic snapshots --tag grafana
restic restore latest --tag grafana --target /tmp/restore
ls /tmp/restore/backup/grafana/

# Inside a restic-shell pod, before running restic:
export RESTIC_REPOSITORY='s3:s3.eu-central-2.wasabisys.com/k3s-at-home-01/restic-backup'
export AWS_ACCESS_KEY_ID='<wasabi key>'
export AWS_SECRET_ACCESS_KEY='<wasabi secret>'
restic snapshots --tag grafana

# In a debug pod with restic and mariadb-backup tools
restic restore latest --tag mariadb --target /tmp/restore
mkdir /tmp/extracted
mbstream -x -C /tmp/extracted < /tmp/restore/backup/mariadb.xb
mariabackup --prepare --target-dir=/tmp/extracted

# Verify table counts (example)
mysql -e "SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_ROWS \
          FROM information_schema.TABLES \
          WHERE TABLE_SCHEMA NOT IN ('information_schema','performance_schema','mysql');"

restic restore latest --tag emby --target /tmp/emby-restore
ls /tmp/emby-restore/backup/emby-config/
# Look for system.xml, config files, and the metadata directory.

restic restore latest --tag nextpvr --target /tmp/nextpvr-restore
ls /tmp/nextpvr-restore/backup/nextpvr-config/

restic restore latest --tag downloader --target /tmp/restore-test

# SQLite integrity check (example for Sonarr)
sqlite3 /tmp/restore-test/backup/sonarr/sonarr.db "PRAGMA integrity_check;"
sqlite3 /tmp/restore-test/backup/radarr/radarr.db "PRAGMA integrity_check;"
sqlite3 /tmp/restore-test/backup/lidarr/lidarr.db "PRAGMA integrity_check;"

restic restore latest --tag paperless --target /tmp/paperless-restore
ls /tmp/paperless-restore/backup/paperless-data/
ls /tmp/paperless-restore/backup/paperless-media/documents/originals/
sqlite3 /tmp/paperless-restore/backup/paperless-data/db.sqlite3 "PRAGMA integrity_check;"

restic restore latest --tag grafana --target /tmp/grafana-restore
sqlite3 /tmp/grafana-restore/backup/grafana/grafana.db "PRAGMA integrity_check;"

apiVersion: v1
kind: Secret
metadata:
  name: backup-env-configuration-grafana
  namespace: monitoring
  annotations:
    kustomize.config.k8s.io/needs-hash: "true"
stringData:
  RESTIC_SOURCE: /backup/grafana
  RESTIC_REPOSITORY: rest:http://backup:<rest-server password>@rest-server.backup.svc.cluster.local:8000/
  RESTIC_PASSWORD: <shared restic password>
  RESTIC_HOSTNAME: grafana
  RESTIC_ADDITIONAL_BACKUP_PARAMETERS: --tag grafana --exclude lost+found
  RESTIC_RETENTION_POLICIES_ENABLED: "false"
  NTFY_ENABLED: "true"
  NTFY_CREDS: -u user:token
  NTFY_SERVER: https://ntfy.geekbundle.org
  NTFY_TOPIC: kubernetes-at-home
  PUSHGATEWAY_ENABLED: "true"
  PUSHGATEWAY_URL: http://prometheus-pushgateway.monitoring.svc.cluster.local:9091

#!/bin/sh

# Create run.log only if it does not exist
if [ ! -f /data/run.log ]; then
  touch /data/run.log
fi

# Echo current date to run.log for logging purposes
echo "$(date) Backup process started" >> /data/run.log

# Sync source to destination with a backup directory using 'year-month-day' format
backup_dir="$BUP_DST/$(hostname | cut -d '-' -f1)-$(date +%Y-%m-%d)"
rclone --config /config/rclone/rclone.conf sync "$BUP_SRC" "$BUP_DST/$(hostname | cut -d '-' -f 1)" -v --backup-dir="$backup_dir"

# Calculate the date for retention time
RETENTION_DATE=$(date -d '7 days ago' +%Y-%m-%d)

# List directories and delete those older than retention time
rclone --config /config/rclone/rclone.conf lsf "$BUP_DST/" --dirs-only | while read dir; do
  # Extract the date from the directory name
  dir_date=$(echo "$dir" | awk -F'-' '{print $NF}' | sed 's#/##g')
  # Compare directory date with RETENTION_DATE
  if [ "$dir_date" \< "$RETENTION_DATE" ]; then
    echo "Deleting old backup directory: $BUP_DST/$dir" >> /data/run.log
    rclone --config /config/rclone/rclone.conf purge "$BUP_DST/$dir"
  fi
done

# Log the completion of the backup process
echo "$(date) Backup process completed" >> /data/run.log

---
apiVersion: v1
kind: Secret
metadata:
  name: rclone-config
  namespace: whoami
stringData:
  rclone.conf: |
    [WasabiFrankfurt]
    type = s3
    provider = Wasabi
    env_auth = false
    access_key_id = <SuperSecretImportantPrivy>
    secret_access_key = <VeryVerySuperSecretImportantPrivy>
    region = eu-central-2
    endpoint = s3.eu-central-2.wasabisys.com
    acl =