[pve] Weekly backup 2026-07-05 — 20 files changed, 228 insertions(+), 71 deletions(-)

2026-07-28 13:22:48 -05:00 · 2026-07-05 03:00:03 -05:00
parent fd7b7bc615
commit 0bf6bda39e
20 changed files with 228 additions and 71 deletions
@@ -0,0 +1,119 @@
+#!/bin/bash
+# Collects MSP360 backup plan status from all known hosts and writes a JSON
+# status file, then pushes it to web.orbishosting.com (VM110).
+set -u
+CBB="/opt/local/MSP360 Backup/bin/cbb"
+SSHPW="Joker1974!!!"
+OUT=/tmp/backup-status.json
+NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+BACKUP_ROOT=/mnt/nas-backups/MSPBackups
+
+get_plan_status() {
+  # $1=host $2=user $3=planname $4=mode(local|hop)
+  local host="$1" user="$2" plan="$3" mode="$4" out
+  if [ "$mode" = "local" ]; then
+    out="$("$CBB" plan -l 2>&1)"
+  elif [ "$mode" = "hop" ]; then
+    out="$(sshpass -p "$SSHPW" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=8 "$user@$host" "'$CBB' plan -l 2>&1")"
+  fi
+  local state result id
+  state=$(echo "$out" | grep 'State:' | head -1 | sed 's/.*State: *//')
+  result=$(echo "$out" | grep 'Last result:' | head -1 | sed 's/.*Last result: *//')
+  id=$(echo "$out" | grep 'Id:' | head -1 | sed 's/.*Id: *//')
+  echo "${state:-unreachable}|${result:-unreachable}|${id:-}"
+}
+
+# Reads the destination generation folders on the NAS (mounted locally on
+# PVE1) to find the last backup that actually completed and wrote data.
+# More trustworthy than the plan's self-reported "Last result", since a
+# plan can report Warning/Fail on a run after previously succeeding, and
+# we want to know how stale the actual backed-up data is.
+get_last_success() {
+  local nas_folder="$1" plan_id="$2" latest
+  [ -z "$plan_id" ] && { echo "null"; return; }
+  latest=$(ls "$BACKUP_ROOT/$nas_folder/CBB_Configuration/${plan_id}.cbb\$/" 2>/dev/null | sort | tail -1)
+  if [ -z "$latest" ]; then
+    echo "null"
+  else
+    date -u -d "${latest:0:4}-${latest:4:2}-${latest:6:2} ${latest:8:2}:${latest:10:2}:${latest:12:2}" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "null"
+  fi
+}
+
+echo "[" > "$OUT"
+first=1
+
+emit() {
+  local name="$1" ip="$2" state="$3" result="$4" last_success="$5"
+  [ "$first" -eq 1 ] && first=0 || echo "," >> "$OUT"
+  if [ "$last_success" = "null" ]; then
+    cat >> "$OUT" <<EOF
+  {"name": "$name", "ip": "$ip", "state": "$state", "result": "$result", "last_success": null}
+EOF
+  else
+    cat >> "$OUT" <<EOF
+  {"name": "$name", "ip": "$ip", "state": "$state", "result": "$result", "last_success": "$last_success"}
+EOF
+  fi
+}
+
+collect() {
+  local name="$1" ip="$2" user="$3" mode="$4" nas_folder="$5"
+  local r state result id last_success
+  r=$(get_plan_status "$ip" "$user" "" "$mode")
+  state="${r%%|*}"
+  rest="${r#*|}"
+  result="${rest%%|*}"
+  id="${rest#*|}"
+  last_success=$(get_last_success "$nas_folder" "$id")
+  emit "$name" "$ip" "$state" "$result" "$last_success"
+}
+
+# Homebridge dropped its in-guest MSP360 agent (repeated, unresolved storage
+# errors) in favor of the cluster-wide Proxmox vzdump job, which snapshots
+# the whole VM disk nightly at 21:00 to the same NAS. Reads the dump
+# directory directly instead of querying an in-guest agent.
+collect_proxmox_vm() {
+  local name="$1" ip="$2" vmid="$3" dump_dir="/mnt/pve/SynologyProx/dump"
+  local latest_dump latest_log ts iso result state
+
+  latest_dump=$(ls "$dump_dir"/vzdump-qemu-"$vmid"-*.vma.zst 2>/dev/null | sort | tail -1)
+  if [ -z "$latest_dump" ]; then
+    emit "$name" "$ip" "Unknown" "unreachable" "null"
+    return
+  fi
+
+  ts=$(basename "$latest_dump" | sed -E 's/vzdump-qemu-[0-9]+-([0-9_]+-[0-9_]+)\.vma\.zst/\1/')
+  iso=$(date -u -d "$(echo "$ts" | sed -E 's/(....)_(..)_(..)-(..)_(..)_(..)/\1-\2-\3 \4:\5:\6/')" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "")
+
+  latest_log="$dump_dir/vzdump-qemu-$vmid-$ts.log"
+  if [ -f "$latest_log" ] && grep -q "Finished Backup" "$latest_log"; then
+    result="Success"
+  else
+    result="Warning"
+  fi
+  state="Proxmox nightly"
+
+  if [ -n "$iso" ]; then
+    emit "$name" "$ip" "$state" "$result" "$iso"
+  else
+    emit "$name" "$ip" "$state" "$result" "null"
+  fi
+}
+
+collect "PVE1" "10.48.200.90" "" "local" "CBB_pve"
+collect "JARVIS" "10.48.200.211" "root" "hop" "CBB_JARVIS-211"
+collect "NovaCPX" "10.48.200.110" "root" "hop" "CBB_NovaCPX-110"
+collect "Jellyfin" "10.48.200.33" "root" "hop" "CBB_Jellyfin-33"
+collect "MediaStack" "10.48.200.35" "root" "hop" "CBB_MediaStack-35"
+collect_proxmox_vm "Homebridge" "10.48.200.18" "118"
+
+echo "]" >> "$OUT"
+
+python3 -c "
+import json
+data = json.load(open('$OUT'))
+data = {'updated': '$NOW', 'hosts': data}
+json.dump(data, open('$OUT', 'w'), indent=2)
+"
+
+sshpass -p "$SSHPW" scp -o StrictHostKeyChecking=no "$OUT" root@10.48.200.110:/home/webacct/public_html/downloads/backup-status.json