#!/usr/bin/env bash
#
# Homelab monitoring, Part 8: Backup monitoring with Proxmox Backup Server
# https://www.zadran.net/guides/homelab-monitoring-8-backups-pbs
# Generated from the guide, last updated 2026-10-13.
#
# NO WARRANTY. This script is provided as is. You are responsible for
# understanding what it does before running it, and for the consequences.
# Take a backup first. See https://www.zadran.net/disclaimer
#
# Destructive steps are left commented out on purpose -- uncomment them
# only once you have understood what they do.
#
# Where the guide writes a file with <<'EOF' and your values are in it,
# this script uses <<EOF so that they expand. Every other $, ` and \
# gets a backslash, so the rest comes out exactly as the guide shows it.

set -euo pipefail

# ---- Set these for your environment ----
# From Part 1
CTID='140'
# The IP of PBS
PBS_IP='192.168.1.30'
# 30 for daily backups, 180 for weekly
MAX_HOURS='30'

proxmox-backup-manager user create prometheus@pbs --comment "Monitoring (read-only)"
proxmox-backup-manager acl update / Audit --auth-id prometheus@pbs
proxmox-backup-manager user generate-token prometheus@pbs exporter
proxmox-backup-manager acl update / Audit --auth-id 'prometheus@pbs!exporter'

pct enter ${CTID}

mkdir -p /opt/monitoring/pbs && chmod 700 /opt/monitoring/pbs
read -rsp "PBS token value: " T && printf '%s\n' "$T" | tr -d ' \r' > /opt/monitoring/pbs/token && unset T && echo
chmod 644 /opt/monitoring/pbs/token
grep -cE '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' /opt/monitoring/pbs/token

cd /opt/monitoring
mkdir -p backup
cp compose.override.yaml backup/compose.override.yaml.$(date +%Y%m%d-%H%M%S) 2>/dev/null
[ -f compose.override.yaml ] || echo "services:" > compose.override.yaml
cat >> compose.override.yaml <<EOF

  pbs-exporter:
    image: ghcr.io/natrontech/pbs-exporter:latest
    container_name: pbs-exporter
    restart: unless-stopped
    user: '65534'
    environment:
      - PBS_ENDPOINT=https://${PBS_IP}:8007
      - PBS_INSECURE=true
      - PBS_USERNAME=prometheus@pbs
      - PBS_API_TOKEN_NAME=exporter
      - PBS_API_TOKEN_FILE=/run/secrets/pbs_token
      - PBS_TIMEOUT=30s
    volumes:
      - ./pbs/token:/run/secrets/pbs_token:ro
EOF
docker compose config -q && docker compose up -d pbs-exporter

P=/opt/monitoring/prometheus/prometheus.yml
cp $P /opt/monitoring/backup/prometheus.yml.$(date +%Y%m%d-%H%M%S)
cat >> $P <<'EOF'

  - job_name: pbs
    scrape_interval: 2m
    scrape_timeout: 60s
    static_configs:
      - targets: ['pbs-exporter:10019']
        labels:
          host: pbs
EOF
docker exec prometheus promtool check config /etc/prometheus/prometheus.yml && curl -fsS -X POST localhost:9090/-/reload && echo reloaded

docker exec prometheus wget -qO- --timeout=60 http://pbs-exporter:10019/metrics | grep -E '^pbs_up |^pbs_size\{|^pbs_snapshot_vm_last_timestamp\{' | cut -c1-110 | head -5

H=${MAX_HOURS}
curl -fsSL https://www.zadran.net/downloads/grafana/dashboards/pbs-backups.json | sed -e "s/ < 30)/ < $H)/; s/ >= 30)/ >= $H)/; s/30 hours/$H hours/g; s/30 h\([ .]\)/$H h\1/g; s/\"value\": 30\$/\"value\": $H/; s/\"value\": 26\$/\"value\": $((H-4))/; s/\"value\": 93600\$/\"value\": $(((H-4)*3600))/; s/\"value\": 108000\$/\"value\": $((H*3600))/" > /opt/monitoring/grafana/provisioning/dashboards/files/Servers/pbs-backups.json
curl -fsSL https://www.zadran.net/downloads/grafana/alerting/pbs.yaml | sed -e "s/params: \[30\]/params: [$H]/" > /opt/monitoring/grafana/provisioning/alerting/pbs.yaml
grep -c "$H" /opt/monitoring/grafana/provisioning/alerting/pbs.yaml
cd /opt/monitoring && docker compose restart grafana